Rename FIXLAYER to MERGE and update for merging passwd/group/shadow

build/install-toolchain.sh

@@ -22,8 +22,8 @@ adduser root abuild
 cp etc/abuild.conf /etc/abuild.conf
 
 # Prepare LXC build toolchain
-cp usr/bin/fix-apk /usr/bin/fix-apk
+cp usr/bin/lxcbuild /usr/bin/lxcbuild
-cp usr/bin/lxc-build /usr/bin/lxc-build
+cp usr/bin/lxcmerge /usr/bin/lxcmerge
 mkdir -p /srv/build/lxc/apps /srv/build/lxc/images
 
 # Prepare local APK repository

build/usr/bin/fix-apk

@@ -1,94 +0,0 @@
-#!/usr/bin/python3
-# -*- coding: utf-8 -*-
-
-import argparse
-import os
-import shutil
-import sys
-import tarfile
-import tempfile
-
-parser = argparse.ArgumentParser(description='APK database merge script')
-parser.add_argument('layers', help='Path to LXC layers to be merged', nargs=argparse.REMAINDER)
-
-if len(sys.argv) < 2:
-    parser.print_usage()
-    sys.exit(1)
-args = parser.parse_args()
-layers = args.layers[::-1]
-
-# /etc/apk/world
-world = []
-for layer in layers:
-    try:
-        with open(os.path.join(layer, 'etc/apk/world'), 'r') as f:
-            for line in f:
-                if line not in world:
-                    world.append(line)
-    except:
-        continue
-os.makedirs(os.path.join(layers[-1], 'etc/apk'), 0o755, True)
-with open(os.path.join(layers[-1], 'etc/apk/world'), 'w') as f:
-    f.writelines(world)
-os.chown(os.path.join(layers[-1], 'etc'), 100000, 100000)
-os.chown(os.path.join(layers[-1], 'etc/apk'), 100000, 100000)
-os.chown(os.path.join(layers[-1], 'etc/apk/world'), 100000, 100000)
-
-# /lib/apk/db/installed
-installed = []
-for layer in layers:
-    try:
-        with open(os.path.join(layer, 'lib/apk/db/installed'), 'r') as f:
-            buffer = []
-            for line in f:
-                if line.startswith('C:'):
-                    buffer = ''.join(buffer)
-                    if buffer not in installed:
-                        installed.append(buffer)
-                    buffer = []
-                buffer.append(line)
-            buffer = ''.join(buffer)
-            if buffer not in installed:
-                installed.append(buffer)
-    except:
-        continue
-os.makedirs(os.path.join(layers[-1], 'lib/apk/db'), 0o755, True)
-with open(os.path.join(layers[-1], 'lib/apk/db/installed'), 'w') as f:
-    f.writelines(installed)
-os.chown(os.path.join(layers[-1], 'lib'), 100000, 100000)
-os.chown(os.path.join(layers[-1], 'lib/apk'), 100000, 100000)
-os.chown(os.path.join(layers[-1], 'lib/apk/db'), 100000, 100000)
-os.chown(os.path.join(layers[-1], 'lib/apk/db/installed'), 100000, 100000)
-
-# /lib/apk/db/scripts.tar
-tmp_tar_path = tempfile.mkstemp()[1]
-files_in_tar = []
-with tarfile.open(tmp_tar_path, 'w:') as tmp_tar:
-    for layer in layers:
-        tar_path = os.path.join(layer, 'lib/apk/db/scripts.tar')
-        if os.path.exists(tar_path):
-            with tarfile.open(tar_path, 'r:') as tar:
-                for member in tar.getmembers():
-                    if member.name not in files_in_tar:
-                        buffer = tar.extractfile(member)
-                        tmp_tar.addfile(member, buffer)
-                        files_in_tar.append(member.name)
-if files_in_tar:
-    shutil.move(tmp_tar_path, os.path.join(layers[-1], 'lib/apk/db/scripts.tar'))
-    os.chown(os.path.join(layers[-1], 'lib/apk/db/scripts.tar'), 100000, 100000)
-else:
-    os.unlink(tmp_tar_path)
-
-# /lib/apk/db/triggers
-triggers = []
-for layer in layers:
-    try:
-        with open(os.path.join(layer, 'lib/apk/db/triggers'), 'r') as f:
-            for line in f:
-                if line not in triggers:
-                    triggers.append(line)
-    except:
-        continue
-with open(os.path.join(layers[-1], 'lib/apk/db/triggers'), 'w') as f:
-    f.writelines(triggers)
-os.chown(os.path.join(layers[-1], 'lib/apk/db/triggers'), 100000, 100000)

build/usr/bin/lxcmerge

@@ -0,0 +1,175 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+import argparse
+import os
+import shutil
+import sys
+import tarfile
+import tempfile
+
+APK_WORLD = 'etc/apk/world'
+APK_INSTALLED = 'lib/apk/db/installed'
+APK_SCRIPTS = 'lib/apk/db/scripts.tar'
+APK_TRIGGERS = 'lib/apk/db/triggers'
+
+ETC_PASSWD = 'etc/passwd'
+ETC_GROUP = 'etc/groups'
+ETC_SHADOW = 'etc/shadow'
+
+def makedirs(path, mode=0o755, uid=100000, gid=100000):
+    try:
+        os.mkdir(path, mode)
+        os.chown(path, uid, gid)
+    except FileNotFoundError:
+        makedirs(os.path.dirname(path), mode, uid, gid)
+        os.mkdir(path, mode)
+        os.chown(path, uid, gid)
+    except FileExistsError:
+        pass
+
+def merge_apk_world():
+    world = []
+    for layer in layers:
+        try:
+            with open(os.path.join(layer, APK_WORLD), 'r') as f:
+                for line in f:
+                    if line not in world:
+                        world.append(line)
+        except:
+            continue
+    makedirs(os.path.join(layers[-1], os.path.dirname(APK_WORLD)))
+    with open(os.path.join(layers[-1], APK_WORLD), 'w') as f:
+        f.writelines(world)
+    os.chown(os.path.join(layers[-1], APK_WORLD), 100000, 100000)
+
+def merge_apk_installed():
+    installed = []
+    for layer in layers:
+        try:
+            with open(os.path.join(layer, APK_INSTALLED), 'r') as f:
+                buffer = []
+                for line in f:
+                    if line.startswith('C:'):
+                        buffer = ''.join(buffer)
+                        if buffer not in installed:
+                            installed.append(buffer)
+                        buffer = []
+                    buffer.append(line)
+                buffer = ''.join(buffer)
+                if buffer not in installed:
+                    installed.append(buffer)
+        except:
+            continue
+    makedirs(os.path.join(layers[-1], os.path.dirname(APK_INSTALLED)))
+    with open(os.path.join(layers[-1], APK_INSTALLED), 'w') as f:
+        f.writelines(installed)
+    os.chown(os.path.join(layers[-1], APK_INSTALLED), 100000, 100000)
+
+def merge_apk_scripts():
+    tmp_tar_path = tempfile.mkstemp()[1]
+    files_in_tar = []
+    with tarfile.open(tmp_tar_path, 'w:') as tmp_tar:
+        for layer in layers:
+            tar_path = os.path.join(layer, APK_SCRIPTS)
+            if os.path.exists(tar_path):
+                with tarfile.open(tar_path, 'r:') as tar:
+                    for member in tar.getmembers():
+                        if member.name not in files_in_tar:
+                            buffer = tar.extractfile(member)
+                            tmp_tar.addfile(member, buffer)
+                            files_in_tar.append(member.name)
+    if files_in_tar:
+        makedirs(os.path.join(layers[-1], os.path.dirname(APK_SCRIPTS)))
+        shutil.move(tmp_tar_path, os.path.join(layers[-1], APK_SCRIPTS))
+        os.chown(os.path.join(layers[-1], APK_SCRIPTS), 100000, 100000)
+    else:
+        os.unlink(tmp_tar_path)
+
+def merge_apk_triggers():
+    triggers = []
+    for layer in layers:
+        try:
+            with open(os.path.join(layer, APK_TRIGGERS), 'r') as f:
+                for line in f:
+                    if line not in triggers:
+                        triggers.append(line)
+        except:
+            continue
+    makedirs(os.path.join(layers[-1], os.path.dirname(APK_TRIGGERS)))
+    with open(os.path.join(layers[-1], APK_TRIGGERS), 'w') as f:
+        f.writelines(triggers)
+    os.chown(os.path.join(layers[-1], APK_TRIGGERS), 100000, 100000)
+
+def merge_etc_passwd():
+    passwd = {}
+    for layer in layers:
+        try:
+            with open(os.path.join(layer, ETC_PASSWD), 'r') as f:
+                for line in f:
+                    passwd[line.split(':')[0]] = line
+        except:
+            continue
+    makedirs(os.path.join(layers[-1], os.path.dirname(ETC_PASSWD)))
+    with open(os.path.join(layers[-1], ETC_PASSWD), 'w') as f:
+        f.writelines(passwd.values())
+    os.chown(os.path.join(layers[-1], ETC_PASSWD), 100000, 100000)
+
+def merge_etc_group():
+    groups = {}
+    for layer in layers:
+        try:
+            with open(os.path.join(layer, ETC_GROUP), 'r') as f:
+                for line in f:
+                    name,pwd,gid,users = line.split(':')
+                    name = splitline[0]
+                    users = splitline[3].strip().split(',')
+                    if name not in groups:
+                        groups[name] = [name,pwd,gid,users]
+                    else:
+                        groups[name][1] = pwd
+                        groups[name][2] = gid
+                        for user in users:
+                            if user not in groups[name][3]:
+                                groups[name][3].append(user)
+        except:
+            continue
+    for group in groups.values():
+        group[3] = '{}\n'.format(','.join(group[3]))
+    makedirs(os.path.join(layers[-1], os.path.dirname(ETC_GROUP)))
+    with open(os.path.join(layers[-1], ETC_GROUP), 'w') as f:
+        f.writelines([':'.join(group) for group in groups.values()])
+    os.chown(os.path.join(layers[-1], ETC_GROUP), 100000, 100000)
+
+def merge_etc_shadow():
+    shadow = {}
+    for layer in layers:
+        try:
+            with open(os.path.join(layer, ETC_SHADOW), 'r') as f:
+                for line in f:
+                    shadow[line.split(':')[0]] = line
+        except:
+            continue
+    makedirs(os.path.join(layers[-1], os.path.dirname(ETC_SHADOW)))
+    with open(os.path.join(layers[-1], ETC_SHADOW), 'w') as f:
+        f.writelines(shadow.values())
+    os.chown(os.path.join(layers[-1], ETC_SHADOW), 100000, 100042)
+
+
+parser = argparse.ArgumentParser(description='APK database merge script')
+parser.add_argument('layers', help='Path to LXC layers to be merged', nargs=argparse.REMAINDER)
+
+if len(sys.argv) < 2:
+    parser.print_usage()
+    sys.exit(1)
+args = parser.parse_args()
+layers = args.layers[::-1]
+
+merge_apk_world()
+merge_apk_installed()
+merge_apk_scripts()
+merge_apk_triggers()
+
+merge_etc_passwd()
+merge_etc_group()
+merge_etc_shadow()

build/usr/lib/python3.6/lxcbuild/imagebuilder.py

@@ -41,8 +41,8 @@ class ImageBuilder:
             self.set_name(args)
         elif 'LAYER' == directive:
             self.add_layer(args)
-        elif 'FIXLAYER' == directive:
+        elif 'MERGE' == directive:
-            self.fix_layer(args.split())
+            self.merge_layers(args.split())
         elif 'COPY' == directive:
             srcdst = args.split()
             self.copy_files(srcdst[0], srcdst[1] if len(srcdst) == 2 else '')
@@ -92,7 +92,7 @@ class ImageBuilder:
             raise ImageNotFoundError(layer_path)
         self.image.conf['layers'].insert(1, name)
 
-    def fix_layer(self, cmd):
+    def merge_layers(self, cmd):
         layers = [self.get_layer_path(layer) for layer in self.image.conf['layers']]
         subprocess.run(cmd + layers, check=True)
 

lxc-apps/crisiscleanup/lxcfile

@@ -4,7 +4,7 @@ LAYER alpine3.8_3.8.4-190620
 LAYER alpine3.8-ruby2.4_2.4.5-190620
 LAYER alpine3.8-nodejs8_8.14.0-190620
 
-FIXLAYER /usr/bin/fix-apk
+MERGE /usr/bin/lxcmerge
 
 ENV RAILS_ENV production
 

lxc-apps/cts/lxcfile

@@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620
 LAYER alpine3.9-python2.7_2.7.16-190620
 LAYER alpine3.9-nginx_1.14.2-191115
 
-FIXLAYER /usr/bin/fix-apk
+MERGE /usr/bin/lxcmerge
 
 RUN EOF
     # Install runtime dependencies

lxc-apps/decidim/lxcfile

@@ -5,7 +5,7 @@ LAYER alpine3.9-ruby2.6_2.6.3-190620
 LAYER alpine3.9-nodejs10_10.14.2-190620
 LAYER alpine3.9-nginx_1.14.2-191115
 
-FIXLAYER /usr/bin/fix-apk
+MERGE /usr/bin/lxcmerge
 
 ENV RAILS_ENV production
 

lxc-apps/gnuhealth/lxcfile

@@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620
 LAYER alpine3.9-python3.6_3.6.8-190620
 LAYER alpine3.9-nodejs10_10.14.2-190620
 
-FIXLAYER /usr/bin/fix-apk
+MERGE /usr/bin/lxcmerge
 
 RUN EOF
     # Install runtime dependencies

lxc-apps/odoo/lxcfile

@@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620
 LAYER alpine3.9-python3.6_3.6.8-190620
 LAYER alpine3.9-nodejs10_10.14.2-190620
 
-FIXLAYER /usr/bin/fix-apk
+MERGE /usr/bin/lxcmerge
 
 RUN EOF
     # Install runtime dependencies

lxc-apps/opendatakit/opendatakit-build.lxcfile

@@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620
 LAYER alpine3.9-ruby2.4_2.4.5-190620
 LAYER alpine3.9-nodejs10_10.14.2-190620
 
-FIXLAYER /usr/bin/fix-apk
+MERGE /usr/bin/lxcmerge
 
 RUN EOF
     # Install runtime dependencies

lxc-apps/openmapkit/lxcfile

@@ -5,7 +5,7 @@ LAYER alpine3.9-java8_8.212.04-190620
 LAYER alpine3.9-python2.7_2.7.16-190620
 LAYER alpine3.9-nodejs10_10.14.2-190620
 
-FIXLAYER /usr/bin/fix-apk
+MERGE /usr/bin/lxcmerge
 
 RUN EOF
     # Install build dependencies

lxc-apps/pandora/lxcfile

@@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620
 LAYER alpine3.9-python3.6_3.6.8-190620
 LAYER alpine3.9-nginx_1.14.2-191115
 
-FIXLAYER /usr/bin/fix-apk
+MERGE /usr/bin/lxcmerge
 
 RUN EOF
     # Install runtime dependencies

lxc-apps/sahana/lxcfile

@@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620
 LAYER alpine3.9-python2.7_2.7.16-190620
 LAYER alpine3.9-nginx_1.14.2-191115
 
-FIXLAYER /usr/bin/fix-apk
+MERGE /usr/bin/lxcmerge
 
 RUN EOF
     # Install runtime dependencies

lxc-apps/seeddms/lxcfile

@@ -5,7 +5,7 @@ LAYER alpine3.9-nginx_1.14.2-191115
 LAYER alpine3.9-php7.2_7.2.19-190620
 LAYER alpine3.9-python3.6_3.6.8-190620
 
-FIXLAYER /usr/bin/fix-apk
+MERGE /usr/bin/lxcmerge
 
 RUN EOF
     # Install runtime dependencies