From 5084dc1579f7a5465bfc99a46d35f6e3fb39fb20 Mon Sep 17 00:00:00 2001 From: Disassembler Date: Sun, 17 Nov 2019 15:04:34 +0100 Subject: [PATCH] Rename FIXLAYER to MERGE and update for merging passwd/group/shadow --- build/install-toolchain.sh | 4 +- build/usr/bin/fix-apk | 94 ---------- build/usr/bin/lxcmerge | 175 ++++++++++++++++++ .../lib/python3.6/lxcbuild/imagebuilder.py | 6 +- lxc-apps/crisiscleanup/lxcfile | 2 +- lxc-apps/cts/lxcfile | 2 +- lxc-apps/decidim/lxcfile | 2 +- lxc-apps/gnuhealth/lxcfile | 2 +- lxc-apps/odoo/lxcfile | 2 +- .../opendatakit/opendatakit-build.lxcfile | 2 +- lxc-apps/openmapkit/lxcfile | 2 +- lxc-apps/pandora/lxcfile | 2 +- lxc-apps/sahana/lxcfile | 2 +- lxc-apps/seeddms/lxcfile | 2 +- 14 files changed, 190 insertions(+), 109 deletions(-) delete mode 100755 build/usr/bin/fix-apk create mode 100755 build/usr/bin/lxcmerge diff --git a/build/install-toolchain.sh b/build/install-toolchain.sh index 1530203..7bf2762 100755 --- a/build/install-toolchain.sh +++ b/build/install-toolchain.sh @@ -22,8 +22,8 @@ adduser root abuild cp etc/abuild.conf /etc/abuild.conf # Prepare LXC build toolchain -cp usr/bin/fix-apk /usr/bin/fix-apk -cp usr/bin/lxc-build /usr/bin/lxc-build +cp usr/bin/lxcbuild /usr/bin/lxcbuild +cp usr/bin/lxcmerge /usr/bin/lxcmerge mkdir -p /srv/build/lxc/apps /srv/build/lxc/images # Prepare local APK repository diff --git a/build/usr/bin/fix-apk b/build/usr/bin/fix-apk deleted file mode 100755 index 20f1570..0000000 --- a/build/usr/bin/fix-apk +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- - -import argparse -import os -import shutil -import sys -import tarfile -import tempfile - -parser = argparse.ArgumentParser(description='APK database merge script') -parser.add_argument('layers', help='Path to LXC layers to be merged', nargs=argparse.REMAINDER) - -if len(sys.argv) < 2: - parser.print_usage() - sys.exit(1) -args = parser.parse_args() -layers = args.layers[::-1] - -# /etc/apk/world -world = [] -for layer in layers: - try: - with open(os.path.join(layer, 'etc/apk/world'), 'r') as f: - for line in f: - if line not in world: - world.append(line) - except: - continue -os.makedirs(os.path.join(layers[-1], 'etc/apk'), 0o755, True) -with open(os.path.join(layers[-1], 'etc/apk/world'), 'w') as f: - f.writelines(world) -os.chown(os.path.join(layers[-1], 'etc'), 100000, 100000) -os.chown(os.path.join(layers[-1], 'etc/apk'), 100000, 100000) -os.chown(os.path.join(layers[-1], 'etc/apk/world'), 100000, 100000) - -# /lib/apk/db/installed -installed = [] -for layer in layers: - try: - with open(os.path.join(layer, 'lib/apk/db/installed'), 'r') as f: - buffer = [] - for line in f: - if line.startswith('C:'): - buffer = ''.join(buffer) - if buffer not in installed: - installed.append(buffer) - buffer = [] - buffer.append(line) - buffer = ''.join(buffer) - if buffer not in installed: - installed.append(buffer) - except: - continue -os.makedirs(os.path.join(layers[-1], 'lib/apk/db'), 0o755, True) -with open(os.path.join(layers[-1], 'lib/apk/db/installed'), 'w') as f: - f.writelines(installed) -os.chown(os.path.join(layers[-1], 'lib'), 100000, 100000) -os.chown(os.path.join(layers[-1], 'lib/apk'), 100000, 100000) -os.chown(os.path.join(layers[-1], 'lib/apk/db'), 100000, 100000) -os.chown(os.path.join(layers[-1], 'lib/apk/db/installed'), 100000, 100000) - -# /lib/apk/db/scripts.tar -tmp_tar_path = tempfile.mkstemp()[1] -files_in_tar = [] -with tarfile.open(tmp_tar_path, 'w:') as tmp_tar: - for layer in layers: - tar_path = os.path.join(layer, 'lib/apk/db/scripts.tar') - if os.path.exists(tar_path): - with tarfile.open(tar_path, 'r:') as tar: - for member in tar.getmembers(): - if member.name not in files_in_tar: - buffer = tar.extractfile(member) - tmp_tar.addfile(member, buffer) - files_in_tar.append(member.name) -if files_in_tar: - shutil.move(tmp_tar_path, os.path.join(layers[-1], 'lib/apk/db/scripts.tar')) - os.chown(os.path.join(layers[-1], 'lib/apk/db/scripts.tar'), 100000, 100000) -else: - os.unlink(tmp_tar_path) - -# /lib/apk/db/triggers -triggers = [] -for layer in layers: - try: - with open(os.path.join(layer, 'lib/apk/db/triggers'), 'r') as f: - for line in f: - if line not in triggers: - triggers.append(line) - except: - continue -with open(os.path.join(layers[-1], 'lib/apk/db/triggers'), 'w') as f: - f.writelines(triggers) -os.chown(os.path.join(layers[-1], 'lib/apk/db/triggers'), 100000, 100000) diff --git a/build/usr/bin/lxcmerge b/build/usr/bin/lxcmerge new file mode 100755 index 0000000..234bb04 --- /dev/null +++ b/build/usr/bin/lxcmerge @@ -0,0 +1,175 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +import argparse +import os +import shutil +import sys +import tarfile +import tempfile + +APK_WORLD = 'etc/apk/world' +APK_INSTALLED = 'lib/apk/db/installed' +APK_SCRIPTS = 'lib/apk/db/scripts.tar' +APK_TRIGGERS = 'lib/apk/db/triggers' + +ETC_PASSWD = 'etc/passwd' +ETC_GROUP = 'etc/groups' +ETC_SHADOW = 'etc/shadow' + +def makedirs(path, mode=0o755, uid=100000, gid=100000): + try: + os.mkdir(path, mode) + os.chown(path, uid, gid) + except FileNotFoundError: + makedirs(os.path.dirname(path), mode, uid, gid) + os.mkdir(path, mode) + os.chown(path, uid, gid) + except FileExistsError: + pass + +def merge_apk_world(): + world = [] + for layer in layers: + try: + with open(os.path.join(layer, APK_WORLD), 'r') as f: + for line in f: + if line not in world: + world.append(line) + except: + continue + makedirs(os.path.join(layers[-1], os.path.dirname(APK_WORLD))) + with open(os.path.join(layers[-1], APK_WORLD), 'w') as f: + f.writelines(world) + os.chown(os.path.join(layers[-1], APK_WORLD), 100000, 100000) + +def merge_apk_installed(): + installed = [] + for layer in layers: + try: + with open(os.path.join(layer, APK_INSTALLED), 'r') as f: + buffer = [] + for line in f: + if line.startswith('C:'): + buffer = ''.join(buffer) + if buffer not in installed: + installed.append(buffer) + buffer = [] + buffer.append(line) + buffer = ''.join(buffer) + if buffer not in installed: + installed.append(buffer) + except: + continue + makedirs(os.path.join(layers[-1], os.path.dirname(APK_INSTALLED))) + with open(os.path.join(layers[-1], APK_INSTALLED), 'w') as f: + f.writelines(installed) + os.chown(os.path.join(layers[-1], APK_INSTALLED), 100000, 100000) + +def merge_apk_scripts(): + tmp_tar_path = tempfile.mkstemp()[1] + files_in_tar = [] + with tarfile.open(tmp_tar_path, 'w:') as tmp_tar: + for layer in layers: + tar_path = os.path.join(layer, APK_SCRIPTS) + if os.path.exists(tar_path): + with tarfile.open(tar_path, 'r:') as tar: + for member in tar.getmembers(): + if member.name not in files_in_tar: + buffer = tar.extractfile(member) + tmp_tar.addfile(member, buffer) + files_in_tar.append(member.name) + if files_in_tar: + makedirs(os.path.join(layers[-1], os.path.dirname(APK_SCRIPTS))) + shutil.move(tmp_tar_path, os.path.join(layers[-1], APK_SCRIPTS)) + os.chown(os.path.join(layers[-1], APK_SCRIPTS), 100000, 100000) + else: + os.unlink(tmp_tar_path) + +def merge_apk_triggers(): + triggers = [] + for layer in layers: + try: + with open(os.path.join(layer, APK_TRIGGERS), 'r') as f: + for line in f: + if line not in triggers: + triggers.append(line) + except: + continue + makedirs(os.path.join(layers[-1], os.path.dirname(APK_TRIGGERS))) + with open(os.path.join(layers[-1], APK_TRIGGERS), 'w') as f: + f.writelines(triggers) + os.chown(os.path.join(layers[-1], APK_TRIGGERS), 100000, 100000) + +def merge_etc_passwd(): + passwd = {} + for layer in layers: + try: + with open(os.path.join(layer, ETC_PASSWD), 'r') as f: + for line in f: + passwd[line.split(':')[0]] = line + except: + continue + makedirs(os.path.join(layers[-1], os.path.dirname(ETC_PASSWD))) + with open(os.path.join(layers[-1], ETC_PASSWD), 'w') as f: + f.writelines(passwd.values()) + os.chown(os.path.join(layers[-1], ETC_PASSWD), 100000, 100000) + +def merge_etc_group(): + groups = {} + for layer in layers: + try: + with open(os.path.join(layer, ETC_GROUP), 'r') as f: + for line in f: + name,pwd,gid,users = line.split(':') + name = splitline[0] + users = splitline[3].strip().split(',') + if name not in groups: + groups[name] = [name,pwd,gid,users] + else: + groups[name][1] = pwd + groups[name][2] = gid + for user in users: + if user not in groups[name][3]: + groups[name][3].append(user) + except: + continue + for group in groups.values(): + group[3] = '{}\n'.format(','.join(group[3])) + makedirs(os.path.join(layers[-1], os.path.dirname(ETC_GROUP))) + with open(os.path.join(layers[-1], ETC_GROUP), 'w') as f: + f.writelines([':'.join(group) for group in groups.values()]) + os.chown(os.path.join(layers[-1], ETC_GROUP), 100000, 100000) + +def merge_etc_shadow(): + shadow = {} + for layer in layers: + try: + with open(os.path.join(layer, ETC_SHADOW), 'r') as f: + for line in f: + shadow[line.split(':')[0]] = line + except: + continue + makedirs(os.path.join(layers[-1], os.path.dirname(ETC_SHADOW))) + with open(os.path.join(layers[-1], ETC_SHADOW), 'w') as f: + f.writelines(shadow.values()) + os.chown(os.path.join(layers[-1], ETC_SHADOW), 100000, 100042) + + +parser = argparse.ArgumentParser(description='APK database merge script') +parser.add_argument('layers', help='Path to LXC layers to be merged', nargs=argparse.REMAINDER) + +if len(sys.argv) < 2: + parser.print_usage() + sys.exit(1) +args = parser.parse_args() +layers = args.layers[::-1] + +merge_apk_world() +merge_apk_installed() +merge_apk_scripts() +merge_apk_triggers() + +merge_etc_passwd() +merge_etc_group() +merge_etc_shadow() diff --git a/build/usr/lib/python3.6/lxcbuild/imagebuilder.py b/build/usr/lib/python3.6/lxcbuild/imagebuilder.py index 337a9c5..8681437 100644 --- a/build/usr/lib/python3.6/lxcbuild/imagebuilder.py +++ b/build/usr/lib/python3.6/lxcbuild/imagebuilder.py @@ -41,8 +41,8 @@ class ImageBuilder: self.set_name(args) elif 'LAYER' == directive: self.add_layer(args) - elif 'FIXLAYER' == directive: - self.fix_layer(args.split()) + elif 'MERGE' == directive: + self.merge_layers(args.split()) elif 'COPY' == directive: srcdst = args.split() self.copy_files(srcdst[0], srcdst[1] if len(srcdst) == 2 else '') @@ -92,7 +92,7 @@ class ImageBuilder: raise ImageNotFoundError(layer_path) self.image.conf['layers'].insert(1, name) - def fix_layer(self, cmd): + def merge_layers(self, cmd): layers = [self.get_layer_path(layer) for layer in self.image.conf['layers']] subprocess.run(cmd + layers, check=True) diff --git a/lxc-apps/crisiscleanup/lxcfile b/lxc-apps/crisiscleanup/lxcfile index 9556e2f..373d7ac 100644 --- a/lxc-apps/crisiscleanup/lxcfile +++ b/lxc-apps/crisiscleanup/lxcfile @@ -4,7 +4,7 @@ LAYER alpine3.8_3.8.4-190620 LAYER alpine3.8-ruby2.4_2.4.5-190620 LAYER alpine3.8-nodejs8_8.14.0-190620 -FIXLAYER /usr/bin/fix-apk +MERGE /usr/bin/lxcmerge ENV RAILS_ENV production diff --git a/lxc-apps/cts/lxcfile b/lxc-apps/cts/lxcfile index 7e99e9a..c8163a0 100644 --- a/lxc-apps/cts/lxcfile +++ b/lxc-apps/cts/lxcfile @@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620 LAYER alpine3.9-python2.7_2.7.16-190620 LAYER alpine3.9-nginx_1.14.2-191115 -FIXLAYER /usr/bin/fix-apk +MERGE /usr/bin/lxcmerge RUN EOF # Install runtime dependencies diff --git a/lxc-apps/decidim/lxcfile b/lxc-apps/decidim/lxcfile index a92d29a..21fbff3 100644 --- a/lxc-apps/decidim/lxcfile +++ b/lxc-apps/decidim/lxcfile @@ -5,7 +5,7 @@ LAYER alpine3.9-ruby2.6_2.6.3-190620 LAYER alpine3.9-nodejs10_10.14.2-190620 LAYER alpine3.9-nginx_1.14.2-191115 -FIXLAYER /usr/bin/fix-apk +MERGE /usr/bin/lxcmerge ENV RAILS_ENV production diff --git a/lxc-apps/gnuhealth/lxcfile b/lxc-apps/gnuhealth/lxcfile index cde9b09..0ed42c1 100644 --- a/lxc-apps/gnuhealth/lxcfile +++ b/lxc-apps/gnuhealth/lxcfile @@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620 LAYER alpine3.9-python3.6_3.6.8-190620 LAYER alpine3.9-nodejs10_10.14.2-190620 -FIXLAYER /usr/bin/fix-apk +MERGE /usr/bin/lxcmerge RUN EOF # Install runtime dependencies diff --git a/lxc-apps/odoo/lxcfile b/lxc-apps/odoo/lxcfile index 21bc8af..bc9eff2 100644 --- a/lxc-apps/odoo/lxcfile +++ b/lxc-apps/odoo/lxcfile @@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620 LAYER alpine3.9-python3.6_3.6.8-190620 LAYER alpine3.9-nodejs10_10.14.2-190620 -FIXLAYER /usr/bin/fix-apk +MERGE /usr/bin/lxcmerge RUN EOF # Install runtime dependencies diff --git a/lxc-apps/opendatakit/opendatakit-build.lxcfile b/lxc-apps/opendatakit/opendatakit-build.lxcfile index 492e6d9..1353a09 100644 --- a/lxc-apps/opendatakit/opendatakit-build.lxcfile +++ b/lxc-apps/opendatakit/opendatakit-build.lxcfile @@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620 LAYER alpine3.9-ruby2.4_2.4.5-190620 LAYER alpine3.9-nodejs10_10.14.2-190620 -FIXLAYER /usr/bin/fix-apk +MERGE /usr/bin/lxcmerge RUN EOF # Install runtime dependencies diff --git a/lxc-apps/openmapkit/lxcfile b/lxc-apps/openmapkit/lxcfile index 563fd12..7b6db89 100644 --- a/lxc-apps/openmapkit/lxcfile +++ b/lxc-apps/openmapkit/lxcfile @@ -5,7 +5,7 @@ LAYER alpine3.9-java8_8.212.04-190620 LAYER alpine3.9-python2.7_2.7.16-190620 LAYER alpine3.9-nodejs10_10.14.2-190620 -FIXLAYER /usr/bin/fix-apk +MERGE /usr/bin/lxcmerge RUN EOF # Install build dependencies diff --git a/lxc-apps/pandora/lxcfile b/lxc-apps/pandora/lxcfile index ce920d3..c0a3cd3 100644 --- a/lxc-apps/pandora/lxcfile +++ b/lxc-apps/pandora/lxcfile @@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620 LAYER alpine3.9-python3.6_3.6.8-190620 LAYER alpine3.9-nginx_1.14.2-191115 -FIXLAYER /usr/bin/fix-apk +MERGE /usr/bin/lxcmerge RUN EOF # Install runtime dependencies diff --git a/lxc-apps/sahana/lxcfile b/lxc-apps/sahana/lxcfile index c92402c..10bfec2 100644 --- a/lxc-apps/sahana/lxcfile +++ b/lxc-apps/sahana/lxcfile @@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620 LAYER alpine3.9-python2.7_2.7.16-190620 LAYER alpine3.9-nginx_1.14.2-191115 -FIXLAYER /usr/bin/fix-apk +MERGE /usr/bin/lxcmerge RUN EOF # Install runtime dependencies diff --git a/lxc-apps/seeddms/lxcfile b/lxc-apps/seeddms/lxcfile index 52ea6d6..d1231bf 100644 --- a/lxc-apps/seeddms/lxcfile +++ b/lxc-apps/seeddms/lxcfile @@ -5,7 +5,7 @@ LAYER alpine3.9-nginx_1.14.2-191115 LAYER alpine3.9-php7.2_7.2.19-190620 LAYER alpine3.9-python3.6_3.6.8-190620 -FIXLAYER /usr/bin/fix-apk +MERGE /usr/bin/lxcmerge RUN EOF # Install runtime dependencies