Implement scratch containers and image/app removal

apk/vmmgr

@@ -1 +1 @@
-Subproject commit 6045349f9c3602d6ba9b081a62d4338b202521d6
+Subproject commit b02fc3f42c65d8833451e41b550f7588c9de2cc2

build/usr/bin/lxcbuild

@@ -8,37 +8,62 @@ from lxcbuild.app import App
 from lxcbuild.image import Image
 
 parser = argparse.ArgumentParser(description='VM application builder and packager')
-parser.add_argument('-f', '--force', action='store_true', help='Force rebuild already built package')
+group = parser.add_mutually_exclusive_group()
-parser.add_argument('buildpath', help='Either specific "lxcfile" or "meta" file or a directory containing at least one')
+group.add_argument('-f', '--force', action='store_true', help='Force rebuild already built package')
+group.add_argument('-s', '--scratch', action='store_true', help='Build container for testing purposes, i.e. without cleanup on failure and packaging')
+group.add_argument('-r', '--remove-image', action='store_true', help='Delete image (including scratch) from build repository')
+group.add_argument('-e', '--remove-app', action='store_true', help='Delete application from build repository')
+parser.add_argument('buildarg', help='Either specific "lxcfile" or "meta" file or a directory containing at least one of them')
 
 if len(sys.argv) < 2:
     parser.print_usage()
     sys.exit(1)
 args = parser.parse_args()
 
-buildpath = os.path.realpath(args.buildpath)
+def build_and_pack_image(args, path):
-if os.path.isfile(buildpath):
+    image = Image()
-    basename = os.path.basename(buildpath)
+    image.force_build = args.force or args.scratch
-    if basename == 'lxcfile' or basename.endswith('.lxcfile'):
+    image.scratch_build = args.scratch
-        image = Image(buildpath)
+    image.build_and_pack(path)
-        image.build_and_pack(args.force)
+
-    elif basename == 'meta' or basename.endswith('.meta'):
+def pack_app(path):
-        app = App(buildpath)
+    app = App()
-        app.pack()
+    app.pack(path)
-    else:
+
-        print('Unknown file {} given, expected "lxcfile" or "meta"'.format(buildpath))
+if args.remove_image:
-        sys.exit(1)
+    image = Image()
+    image.name = args.buildarg
+    image.remove()
+elif args.remove_app:
+    app = App()
+    app.name = args.buildarg
+    app.remove()
 else:
-    valid_dir = False
+    buildpath = os.path.realpath(args.buildarg)
-    for entry in os.scandir(buildpath):
+    # If the buildpath is a file, determine type from filename
-        if entry.is_file() and (entry.name == 'lxcfile' or entry.name.endswith('.lxcfile')):
+    if os.path.isfile(buildpath):
+        basename = os.path.basename(buildpath)
+        if basename == 'lxcfile' or basename.endswith('.lxcfile'):
+            build_and_pack_image(args, buildpath)
+        # Compose files needs to be ignored when performing scratch builds
+        elif not args.scratch and basename == 'meta':
+            pack_app(buildpath)
+        else:
+            print('Unknown file {} given, expected "lxcfile"{}'.format(buildpath, '' if args.scratch else ' or "meta"'))
+            sys.exit(1)
+    # If the buildpath is a directory, build as much as possible, unless scratch build was requested, in which case don't build anything
+    else:
+        if args.scratch:
+            print('Please specify an lxcfile for scratch build')
+            sys.exit(1)
+        valid_dir = False
+        for entry in os.scandir(buildpath):
+            if entry.is_file() and (entry.name == 'lxcfile' or entry.name.endswith('.lxcfile')):
+                valid_dir = True
+                build_and_pack_image(args, entry.path)
+        meta = os.path.join(buildpath, 'meta')
+        if os.path.exists(meta):
             valid_dir = True
-            image = Image(entry.path)
+            pack_app(meta)
-            image.build_and_pack(args.force)
+        if not valid_dir:
-    meta = os.path.join(buildpath, 'meta')
+            print('Directory {} doesn\'t contain anything to build, skipping'.format(buildpath))
-    if os.path.exists(meta):
-        valid_dir = True
-        app = App(meta)
-        app.pack()
-    if not valid_dir:
-        print('Directory {} doesn\'t contain anything to build, skipping'.format(buildpath))

build/usr/lib/python3.6/lxcbuild/app.py

@@ -4,11 +4,16 @@ import json
 import os
 import sys
 
-from .builder import ImageNotFoundError
+from .apppacker import AppPacker
-from .packer import Packer
+from .imagebuilder import ImageNotFoundError
 
 class App:
-    def __init__(self, metafile):
+    def __init__(self):
+        self.name = None
+        self.conf = {}
+        self.build_dir = None
+
+    def load_metafile(self, metafile):
         self.build_dir = os.path.dirname(metafile)
         if os.path.basename(metafile) == 'meta':
             self.name = os.path.basename(self.build_dir)
@@ -17,10 +22,15 @@ class App:
         with open(metafile, 'r') as f:
             self.conf = json.load(f)
 
-    def pack(self):
+    def pack(self, metafile):
-        packer = Packer()
+        self.load_metafile(metafile)
+        packer = AppPacker(self)
         try:
-            packer.pack_app(self)
+            packer.pack()
         except ImageNotFoundError as e:
             print('Image {} not found, can\'t pack {}'.format(e, self.name))
             sys.exit(1)
+
+    def remove(self):
+        packer = AppPacker(self)
+        packer.remove()

build/usr/lib/python3.6/lxcbuild/apppacker.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+import os
+import subprocess
+
+from . import crypto
+from .imagebuilder import ImageNotFoundError
+from .packer import Packer
+from .paths import REPO_APPS_DIR
+
+class AppPacker(Packer):
+    def __init__(self, app):
+        super().__init__()
+        self.app = app
+        # Prepare package file names
+        self.tar_path = os.path.join(REPO_APPS_DIR, '{}.tar'.format(self.app.name))
+        self.xz_path = '{}.xz'.format(self.tar_path)
+
+    def pack(self):
+        # Check if all images used by containers exist
+        for container in self.app.conf['containers']:
+            image = self.app.conf['containers'][container]['image']
+            if image not in self.packages['images']:
+                raise ImageNotFoundError(image)
+        try:
+            os.unlink(self.xz_path)
+        except FileNotFoundError:
+            pass
+        self.create_archive()
+        self.register()
+        self.sign_packages()
+
+    def remove(self):
+        self.unregister()
+        try:
+            os.unlink(self.xz_path)
+        except FileNotFoundError:
+            pass
+
+    def create_archive(self):
+        # Create archive with application setup scripts
+        print('Archiving setup scripts for', self.app.name)
+        scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh')
+        scripts = [s for s in scripts if os.path.exists(os.path.join(self.app.build_dir, s))]
+        subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, '--transform', 's,^,{}/,'.format(self.app.name)] + scripts, cwd=self.app.build_dir)
+        self.compress_archive()
+
+    def register(self):
+        # Register package in global repository metadata file
+        print('Registering package {}'.format(self.app.name))
+        self.packages['apps'][self.app.name] = self.app.conf.copy()
+        self.packages['apps'][self.app.name]['size'] = self.tar_size
+        self.packages['apps'][self.app.name]['pkgsize'] = self.xz_size
+        self.packages['apps'][self.app.name]['sha512'] = crypto.hash_file(self.xz_path)
+        self.save_repo_meta()
+
+    def unregister(self):
+        # Removes package from global repository metadata file
+        if self.app.name in self.packages['apps']:
+            del self.packages['apps'][self.app.name]
+            self.save_repo_meta()

build/usr/lib/python3.6/lxcbuild/image.py

@@ -3,24 +3,30 @@
 import os
 import sys
 
-from .builder import Builder, ImageExistsError, ImageNotFoundError
+from lxcmgr import lxcmgr
-from .packer import Packer, PackageExistsError
+
+from .imagebuilder import ImageBuilder, ImageExistsError, ImageNotFoundError
+from .imagepacker import ImagePacker
+from .packer import PackageExistsError
 
 class Image:
-    def __init__(self, lxcfile):
+    def __init__(self):
         self.name = None
-        self.path = None
         self.conf = {}
+        self.lxcfile = None
+        self.build_dir = None
+        self.force_build = False
+        self.scratch_build = False
+
+    def build_and_pack(self, lxcfile):
         self.lxcfile = lxcfile
         self.build_dir = os.path.dirname(lxcfile)
-
-    def build_and_pack(self, force):
         self.conf['build'] = True
         try:
-            builder = Builder()
+            builder = ImageBuilder(self)
-            builder.build(self, force)
+            builder.build()
             # In case of successful build, packaging needs to happen in all cases to prevent outdated packages
-            force = True
+            self.force_build = True
         except ImageExistsError as e:
             print('Image {} already exists, skipping build tasks'.format(e))
         except ImageNotFoundError as e:
@@ -28,11 +34,22 @@ class Image:
             builder.clean()
             sys.exit(1)
         except:
-            builder.clean()
+            if not self.scratch_build:
+                builder.clean()
             raise
         del self.conf['build']
-        try:
+        # If we're doing a scratch build, regenerate the final LXC container configuration including ephemeral layer
-            packer = Packer()
+        if self.scratch_build:
-            packer.pack_image(self, force)
+            lxcmgr.create_container(self.name, self.conf)
-        except PackageExistsError as e:
+        else:
-            print('Package {} already exists, skipping packaging tasks'.format(e))
+            try:
+                packer = ImagePacker(self)
+                packer.pack()
+            except PackageExistsError as e:
+                print('Package {} already exists, skipping packaging tasks'.format(e))
+
+    def remove(self):
+        builder = ImageBuilder(self)
+        builder.clean()
+        packer = ImagePacker(self)
+        packer.remove()

build/usr/lib/python3.6/lxcbuild/imagebuilder.py

@@ -14,16 +14,13 @@ class ImageExistsError(Exception):
 class ImageNotFoundError(Exception):
     pass
 
-class Builder:
+class ImageBuilder:
-    def __init__(self):
+    def __init__(self, image):
-        self.image = None
+        self.image = image
         self.script = []
         self.script_eof = None
-        self.force = False
 
-    def build(self, image, force=False):
+    def build(self):
-        self.image = image
-        self.force = force
         with open(self.image.lxcfile, 'r') as f:
             for line in f:
                 line = line.strip()
@@ -67,26 +64,27 @@ class Builder:
 
     def run_script(self, script):
         lxcmgr.create_container(self.image.name, self.image.conf)
-        sh = os.path.join(self.image.path, 'run.sh')
+        sh = os.path.join(LXC_STORAGE_DIR, self.image.name, 'run.sh')
         with open(sh, 'w') as f:
             f.write('#!/bin/sh\nset -ev\n\n{}\n'.format('\n'.join(script)))
         os.chmod(sh, 0o700)
         os.chown(sh, 100000, 100000)
         subprocess.run(['lxc-execute', self.image.name, '--', '/bin/sh', '-lc', '/run.sh'], check=True)
         os.unlink(sh)
-        lxcmgr.destroy_container(self.image.name)
+        if not self.image.scratch_build:
+            lxcmgr.destroy_container(self.image.name)
 
     def set_name(self, name):
         self.image.name = name
-        self.image.path = self.get_layer_path(name)
         self.image.conf['layers'] = [name]
-        if os.path.exists(self.image.path):
+        image_path = self.get_layer_path(name)
-            if self.force:
+        if os.path.exists(image_path):
+            if self.image.force_build:
                 self.clean()
             else:
-                raise ImageExistsError(self.image.path)
+                raise ImageExistsError(image_path)
-        os.makedirs(self.image.path, 0o755, True)
+        os.makedirs(image_path, 0o755, True)
-        os.chown(self.image.path, 100000, 100000)
+        os.chown(image_path, 100000, 100000)
 
     def add_layer(self, name):
         layer_path = self.get_layer_path(name)
@@ -99,7 +97,7 @@ class Builder:
         subprocess.run(cmd + layers, check=True)
 
     def copy_files(self, src, dst):
-        dst = os.path.join(self.image.path, dst)
+        dst = os.path.join(LXC_STORAGE_DIR, self.image.name, dst)
         if src.startswith('http://') or src.startswith('https://'):
             unpack_http_archive(src, dst)
         else:
@@ -128,8 +126,8 @@ class Builder:
         self.image.conf['ready'] = cmd
 
     def clean(self):
-        shutil.rmtree(self.image.path)
         lxcmgr.destroy_container(self.image.name)
+        shutil.rmtree(self.get_layer_path(self.image.name))
 
 def unpack_http_archive(src, dst):
         xf = 'xzf'

build/usr/lib/python3.6/lxcbuild/imagepacker.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+
+import os
+import subprocess
+
+from lxcmgr.paths import LXC_STORAGE_DIR
+from lxcmgr.pkgmgr import PkgMgr
+
+from . import crypto
+from .packer import Packer
+from .paths import REPO_IMAGES_DIR
+
+class ImagePacker(Packer):
+    def __init__(self, image):
+        super().__init__()
+        self.image = image
+        # Prepare package file names
+        self.tar_path = os.path.join(REPO_IMAGES_DIR, '{}.tar'.format(self.image.name))
+        self.xz_path = '{}.xz'.format(self.tar_path)
+
+    def pack(self):
+        if self.image.force_build:
+            self.unregister()
+            try:
+                os.unlink(self.xz_path)
+            except FileNotFoundError:
+                pass
+        elif os.path.exists(self.xz_path):
+            raise PackageExistsError(self.xz_path)
+        self.create_archive()
+        self.register()
+        self.sign_packages()
+
+    def remove(self):
+        self.unregister()
+        try:
+            os.unlink(self.xz_path)
+        except FileNotFoundError:
+            pass
+
+    def create_archive(self):
+        # Create archive
+        print('Archiving', self.image.path)
+        subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, self.image.name], cwd=LXC_STORAGE_DIR)
+        self.compress_archive()
+
+    def register(self):
+        # Register image in global repository metadata file
+        print('Registering package {}'.format(self.image.name))
+        self.packages['images'][self.image.name] = self.image.conf.copy()
+        self.packages['images'][self.image.name]['size'] = self.tar_size
+        self.packages['images'][self.image.name]['pkgsize'] = self.xz_size
+        self.packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path)
+        self.save_repo_meta()
+        # Register the image also to locally installed images for package manager
+        pm = PkgMgr()
+        pm.register_image(self.image.name, self.packages['images'][self.image.name])
+
+    def unregister(self):
+        # Removes package from global repository metadata file
+        if self.image.name in self.packages['images']:
+            del self.packages['images'][self.image.name]
+            self.save_repo_meta()
+        # Unregister the image also from locally installed images for package manager
+        pm = PkgMgr()
+        pm.unregister_image(self.image.name)

build/usr/lib/python3.6/lxcbuild/packer.py

@@ -3,22 +3,15 @@
 import json
 import os
 import subprocess
-import sys
-
-from lxcmgr.paths import LXC_STORAGE_DIR
-from lxcmgr.pkgmgr import PkgMgr
 
 from . import crypto
-from .builder import ImageNotFoundError
+from .paths import PRIVATE_KEY, REPO_META_FILE, REPO_SIG_FILE
-from .paths import PRIVATE_KEY, REPO_APPS_DIR, REPO_IMAGES_DIR, REPO_META_FILE, REPO_SIG_FILE
 
 class PackageExistsError(Exception):
     pass
 
 class Packer:
     def __init__(self):
-        self.app = None
-        self.image = None
         self.tar_path = None
         self.tar_size = 0
         self.xz_path = None
@@ -33,29 +26,6 @@ class Packer:
         with open(REPO_META_FILE, 'w') as f:
             json.dump(self.packages, f, sort_keys=True, indent=4)
 
-    def pack_image(self, image, force):
-        self.image = image
-        # Prepare package file names
-        self.tar_path = os.path.join(REPO_IMAGES_DIR, '{}.tar'.format(self.image.name))
-        self.xz_path = '{}.xz'.format(self.tar_path)
-        if force:
-            self.unregister_image()
-            try:
-                os.unlink(self.xz_path)
-            except FileNotFoundError:
-                pass
-        elif os.path.exists(self.xz_path):
-            raise PackageExistsError(self.xz_path)
-        self.create_image_archive()
-        self.register_image()
-        self.sign_packages()
-
-    def create_image_archive(self):
-        # Create archive
-        print('Archiving', self.image.path)
-        subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, self.image.name], cwd=LXC_STORAGE_DIR)
-        self.compress_archive()
-
     def compress_archive(self):
         # Compress the tarball with xz (LZMA2)
         self.tar_size = os.path.getsize(self.tar_path)
@@ -64,60 +34,7 @@ class Packer:
         self.xz_size = os.path.getsize(self.xz_path)
         print('Compressed ', self.xz_path, '({:.2f} MB)'.format(self.xz_size/1048576))
 
-    def register_image(self):
-        # Register image in global repository metadata file
-        print('Registering package {}'.format(self.image.name))
-        self.packages['images'][self.image.name] = self.image.conf.copy()
-        self.packages['images'][self.image.name]['size'] = self.tar_size
-        self.packages['images'][self.image.name]['pkgsize'] = self.xz_size
-        self.packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path)
-        self.save_repo_meta()
-        # Register the image also to locally installed images for package manager
-        pm = PkgMgr()
-        pm.register_image(self.image.name, self.packages['images'][self.image.name])
-
     def sign_packages(self):
         signature = crypto.sign_file(PRIVATE_KEY, REPO_META_FILE)
         with open(REPO_SIG_FILE, 'wb') as f:
             f.write(signature)
-
-    def unregister_image(self):
-        # Removes package from global repository metadata file
-        if self.image.name in self.packages['images']:
-            del self.packages['images'][self.image.name]
-            self.save_repo_meta()
-
-    def pack_app(self, app):
-        self.app = app
-        # Check if all images exist
-        for container in app.conf['containers']:
-            image = app.conf['containers'][container]['image']
-            if image not in self.packages['images']:
-                raise ImageNotFoundError(image)
-        # Prepare package file names
-        self.tar_path = os.path.join(REPO_APPS_DIR, '{}.tar'.format(self.app.name))
-        self.xz_path = '{}.xz'.format(self.tar_path)
-        try:
-            os.unlink(self.xz_path)
-        except FileNotFoundError:
-            pass
-        self.create_app_archive()
-        self.register_app()
-        self.sign_packages()
-
-    def create_app_archive(self):
-        # Create archive with application setup scripts
-        print('Archiving setup scripts for', self.app.name)
-        scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh')
-        scripts = [s for s in scripts if os.path.exists(os.path.join(self.app.build_dir, s))]
-        subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, '--transform', 's,^,{}/,'.format(self.app.name)] + scripts, cwd=self.app.build_dir)
-        self.compress_archive()
-
-    def register_app(self):
-        # Register package in global repository metadata file
-        print('Registering package {}'.format(self.app.name))
-        self.packages['apps'][self.app.name] = self.app.conf.copy()
-        self.packages['apps'][self.app.name]['size'] = self.tar_size
-        self.packages['apps'][self.app.name]['pkgsize'] = self.xz_size
-        self.packages['apps'][self.app.name]['sha512'] = crypto.hash_file(self.xz_path)
-        self.save_repo_meta()