Allow lxcbuilder to pack meta files

apk/vmmgr

@@ -1 +1 @@
-Subproject commit 972ca0b6967edd56af96a7de159950ac9fcbc4a6
+Subproject commit c3b711850e02a6e228c4eb64ed82a4d1bc889ae9

build/build-all.sh

@@ -66,14 +66,15 @@ cd ${ROOT}/lxc-services
 lxc-build activemq
 lxc-build mariadb
 lxc-build postgres
+lxc-build postgis
 lxc-build rabbitmq
 lxc-build redis
 lxc-build solr
 
 # Build applications
 cd ${ROOT}/lxc-apps
-lxc-build ckan-datapusher
 lxc-build ckan
+lxc-build ckan-datapusher
 lxc-build crisiscleanup
 lxc-build cts
 lxc-build ecogis

build/install-toolchain.sh

@@ -24,7 +24,7 @@ cp etc/abuild.conf /etc/abuild.conf
 # Prepare LXC build toolchain
 cp usr/bin/fix-apk /usr/bin/fix-apk
 cp usr/bin/lxc-build /usr/bin/lxc-build
-cp usr/bin/lxc-pack /usr/bin/lxc-pack
+mkdir -p /srv/build/lxc/apps /srv/build/lxc/images
 
 # Prepare local APK repository
 cp etc/nginx/conf.d/apkrepo.conf /etc/nginx/conf.d/apkrepo.conf
@@ -36,4 +36,4 @@ service nginx reload
 
 # Supply LXC build key
 # openssl ecparam -genkey -name secp384r1 -out /srv/build/packages.key
-# openssl ec -in /srv/build/packages.key -pubout -out /srv/build/packages.pub
+# openssl ec -in /srv/build/packages.key -pubout -out /srv/build/lxc/packages.pub

build/usr/bin/lxc-build

@@ -1,12 +0,0 @@
-#!/usr/bin/python3
-# -*- coding: utf-8 -*-
-
-import sys
-from lxcbuild.lxcimage import LXCImage
-
-if __name__ == '__main__':
-    if len(sys.argv) != 2 or sys.argv[1] in ('-h', '--help'):
-        print('Usage: lxc-build <buildpath>\n  where the buildpath can be either specific lxcfile or a directory containing one')
-    else:
-        image = LXCImage(sys.argv[1])
-        image.build_and_pack()

build/usr/bin/lxcbuild

@@ -0,0 +1,43 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+import argparse
+import sys
+from lxcbuild.app import App
+from lxcbuild.image import Image
+
+parser = argparse.ArgumentParser(description='VM application builder and packager')
+parser.add_argument('-f', '--force', action='store_true', help='Force rebuild already built package')
+parser.add_argument('buildpath', help='Either specific "lxcfile" or "meta" file or a directory containing one')
+
+if len(sys.argv) < 2:
+    parser.print_usage()
+    sys.exit(1)
+args = parser.parse_args()
+
+buildpath = os.path.realpath(args.buildpath)
+if os.path.isfile(buildpath):
+    basename = os.path.basename(buildpath)
+    if basename == 'lxcfile' or basename.endswith('.lxcfile'):
+        image = Image(buildpath)
+        image.build_and_pack(args.force)
+    elif basename == 'meta' or basename.endswith('.meta'):
+        app = App(buildpath)
+        app.build_and_pack()
+    else:
+        print('Unknown file {} given, expected "lxcfile" or "meta"'.format(buildpath))
+        sys.exit(1)
+else:
+    valid_dir = False
+    lxcfile = os.path.join(buildpath, 'lxcfile')
+    meta = os.path.join(buildpath, 'meta')
+    if os.path.exists(lxcfile):
+        valid_dir = True
+        image = Image(lxcfile)
+        image.build_and_pack(args.force)
+    if os.path.exists(meta):
+        valid_dir = True
+        app = App(buildpath)
+        app.pack()
+    if not valid_dir:
+        print('Directory {} doesn\'t contain anything to build, skipping'.format(buildpath))

build/usr/lib/python3.6/lxcbuild/app.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+import os
+
+from .packer import Packer
+
+class App:
+    def __init__(self, metafile):
+        self.build_dir = os.path.dirname(metafile)
+        self.name = os.path.basename(self.build_dir)
+        with open(metafile, 'r') as f:
+            self.conf = json.load(f)
+
+    def pack(self):
+        packer = Packer()
+        packer.pack_app(self)

build/usr/lib/python3.6/lxcbuild/builder.py

@@ -4,25 +4,26 @@ import os
 import shutil
 import subprocess
 import sys
-from vmmgr import lxcmgr
 
-LXC_ROOT = '/var/lib/lxc'
+from lxcmgr import lxcmgr
+from lxcmgr.paths import PKG_STORAGE_DIR
 
-class LXCBuilder:
+class ImageExistsError(Exception):
-    def __init__(self, image):
+    pass
-        self.image = image
+
+class ImageNotFoundError(Exception):
+    pass
+
+class Builder:
+    def __init__(self):
+        self.image = None
         self.script = []
         self.script_eof = None
+        self.force = False
 
-    def build(self):
+    def build(self, image, force=False):
-        try:
+        self.image = image
-            self.image.conf['build'] = True
+        self.force = force
-            self.process_file()
-        except FileExistsError as e:
-            print(e)
-        del self.image.conf['build']
-
-    def process_file(self):
         with open(self.image.lxcfile, 'r') as f:
             for line in f:
                 line = line.strip()
@@ -62,11 +63,11 @@ class LXCBuilder:
             self.set_ready(args)
 
     def get_layer_path(self, layer):
-        return os.path.join(LXC_ROOT, 'storage', layer)
+        return os.path.join(PKG_STORAGE_DIR, layer)
 
     def run_script(self, script):
         lxcmgr.register_container(self.image.name, self.image.conf)
-        sh = os.path.join(self.get_layer_path(self.image.name), 'run.sh')
+        sh = os.path.join(self.image.path, 'run.sh')
         with open(sh, 'w') as f:
             f.write('#!/bin/sh\nset -ev\n\n{}\n'.format('\n'.join(script)))
         os.chmod(sh, 0o700)
@@ -77,12 +78,20 @@ class LXCBuilder:
 
     def set_name(self, name):
         self.image.name = name
-        self.image.conf['layers'] = [self.image.name]
+        self.image.path = self.get_layer_path(name)
-        image_path = self.get_layer_path(self.image.name)
+        self.image.conf['layers'] = [name]
-        os.makedirs(image_path, 0o755, True)
+        if os.path.exists(self.image.path):
-        os.chown(image_path, 100000, 100000)
+            if self.force:
+                self.clean()
+            else:
+                raise ImageExistsError(self.image.path)
+        os.makedirs(self.image.path, 0o755, True)
+        os.chown(self.image.path, 100000, 100000)
 
     def add_layer(self, name):
+        layer_path = self.get_layer_path(name)
+        if not os.path.exists(layer_path):
+            raise ImageNotFoundError(layer_path)
         self.image.conf['layers'].insert(0, name)
 
     def fix_layer(self, cmd):
@@ -90,17 +99,17 @@ class LXCBuilder:
         subprocess.run([cmd] + layers, check=True)
 
     def copy_files(self, src, dst):
-        dst = os.path.join(self.get_layer_path(self.image.name), dst)
+        dst = os.path.join(self.image.path, dst)
         if src.startswith('http://') or src.startswith('https://'):
             unpack_http_archive(src, dst)
         else:
-            copy_tree(os.path.join(self.build_dir, src), dst)
+            copy_tree(os.path.join(self.image.build_dir, src), dst)
         shift_uid(dst)
 
-    def add_env(self, args):
+    def add_env(self, key, value):
         if 'env' not in self.image.conf:
             self.image.conf['env'] = []
-        self.image.conf['env'].append(args)
+        self.image.conf['env'].append('{}={}'.format(key, value))
 
     def set_user(self, uid, gid):
         self.image.conf['uid'] = uid
@@ -118,6 +127,9 @@ class LXCBuilder:
     def set_ready(self, cmd):
         self.image.conf['ready'] = cmd
 
+    def clean(self):
+        shutil.rmtree(self.image.path)
+
 def unpack_http_archive(src, dst):
         xf = 'xzf'
         if src.endswith('.bz2'):

build/usr/lib/python3.6/lxcbuild/crypto.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+import hashlib
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.serialization import load_pem_private_key
+
+def sign_file(private_key, input_path):
+    # Generate SHA512 signature of a file using EC private key
+    print('Signing packages')
+    with open(private_key, 'rb') as f:
+        priv_key = load_pem_private_key(f.read(), None, default_backend())
+    with open(input_path, 'rb') as f:
+        data = f.read()
+    return priv_key.sign(data, ec.ECDSA(hashes.SHA512()))
+
+def hash_file(file_path):
+    # Calculate SHA512 hash of a file
+    sha512 = hashlib.sha512()
+    with open(file_path, 'rb') as f:
+        while True:
+            data = f.read(65536)
+            if not data:
+                break
+            sha512.update(data)
+    return sha512.hexdigest()

build/usr/lib/python3.6/lxcbuild/image.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+
+from .builder import Builder, ImageExistsError, ImageNotFoundError
+from .packer import Packer, PackageExistsError
+
+class Image:
+    def __init__(self, lxcfile):
+        self.name = None
+        self.path = None
+        self.conf = {}
+        self.lxcfile = lxcfile
+        self.build_dir = os.path.dirname(lxcfile)
+
+    def build_and_pack(self, force=False):
+        self.conf['build'] = True
+        try:
+            builder = Builder()
+            builder.build(self, force)
+            # In case of successful build, packaging needs to be forced to prevent outdated packages
+            force = True
+        except ImageExistsError as e:
+            print('Image {} already exists, skipping build tasks'.format(e))
+        except ImageNotFoundError as e:
+            print('Image {} not found, can\'t build {}'.format(e, self.name))
+            builder.clean()
+            sys.exit(1)
+        except:
+            builder.clean()
+            raise
+        try:
+            packer = Packer()
+            packer.pack_image(self, force)
+        except PackageExistsError as e:
+            print('Package {} already exists, skipping packaging tasks'.format(e))
+        del self.conf['build']

build/usr/lib/python3.6/lxcbuild/lxcimage.py

@@ -1,24 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-
-from .lxcbuilder import LXCBuilder
-from .lxcpacker import LXCPacker
-
-class LXCImage:
-    def __init__(self, build_path):
-        self.name = None
-        self.conf = {}
-
-        if os.path.isfile(build_path):
-            self.lxcfile = os.path.realpath(build_path)
-            self.build_dir = os.path.dirname(self.lxcfile)
-        else:
-            self.build_dir = os.path.realpath(build_path)
-            self.lxcfile = os.path.join(self.build_dir, 'lxcfile')
-
-    def build_and_pack(self):
-        builder = LXCBuilder(self)
-        builder.build()
-        packer = LXCPacker(self)
-        packer.pack()

build/usr/lib/python3.6/lxcbuild/lxcpacker.py

@@ -1,83 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import hashlib
-import json
-import os
-import subprocess
-import sys
-
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import hashes
-from cryptography.hazmat.primitives.asymmetric import ec
-from cryptography.hazmat.primitives.serialization import load_pem_private_key
-
-PKG_ROOT = '/srv/build/lxc'
-PRIVATE_KEY = '/srv/build/packages.key'
-LXC_STORAGE = '/var/lib/lxc/storage'
-
-class LXCPacker:
-    def __init__(self, image):
-        self.image = image
-        self.tar_path = None
-        self.xz_path = None
-
-    def pack(self):
-        # Prepare package file names
-        self.tar_path = os.path.join(PKG_ROOT, '{}.tar'.format(self.image.name))
-        self.xz_path = '{}.xz'.format(self.tar_path)
-        if os.path.exists(self.xz_path):
-            print('Package {} already exists, skipping packaging tasks'.format(self.xz_path))
-            return
-        os.makedirs(PKG_ROOT, 0o755, True)
-        self.create_archive()
-        self.register_package()
-        self.sign_packages()
-
-    def create_archive(self):
-        # Create archive
-        print('Archiving', self.image.name)
-        subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, os.path.join(LXC_STORAGE, self.image.name)], cwd='/')
-        # Add install/upgrade/uninstall scripts
-        # TODO: skripty balit jen s aplikacemi, ne s imagi
-        scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh')
-        scripts = [s for s in scripts if os.path.exists(os.path.join(self.image.build_dir, s))]
-        subprocess.run(['tar', '--transform', 's|^|srv/{}/|'.format(self.image.name), '-rpf', self.tar_path] + scripts, cwd=self.image.build_dir)
-        # Compress the tarball with xz (LZMA2)
-        print('Compressing', self.tar_path, '({:.2f} MB)'.format(os.path.getsize(self.tar_path)/1048576))
-        subprocess.run(['xz', '-9', self.tar_path])
-        print('Compressed ', self.xz_path, '({:.2f} MB)'.format(os.path.getsize(self.xz_path)/1048576))
-
-    def register_package(self):
-        # Register package
-        print('Registering package')
-        packages_file = os.path.join(PKG_ROOT, 'packages')
-        if os.path.exists(packages_file):
-            with open(packages_file, 'r') as f:
-                packages = json.load(f)
-        else:
-            packages = {'apps': {}, 'images': {}}
-        packages['images'][self.image.name] = self.image.conf.copy()
-        packages['images'][self.image.name]['size'] = os.path.getsize(self.xz_path)
-        packages['images'][self.image.name]['sha512'] = hash_file(self.xz_path)
-        with open(packages_file, 'w') as f:
-            json.dump(packages, f, sort_keys=True, indent=4)
-
-    def sign_packages(self):
-        # Sign packages file
-        print('Signing packages')
-        with open(PRIVATE_KEY, 'rb') as f:
-            priv_key = load_pem_private_key(f.read(), None, default_backend())
-        with open(os.path.join(PKG_ROOT, 'packages'), 'rb') as f:
-            data = f.read()
-        with open(os.path.join(PKG_ROOT, 'packages.sig'), 'wb') as f:
-            f.write(priv_key.sign(data, ec.ECDSA(hashes.SHA512())))
-
-def hash_file(file_path):
-    sha512 = hashlib.sha512()
-    with open(file_path, 'rb') as f:
-        while True:
-            data = f.read(65536)
-            if not data:
-                break
-            sha512.update(data)
-    return sha512.hexdigest()

build/usr/lib/python3.6/lxcbuild/packer.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+
+import json
+import os
+import subprocess
+import sys
+
+from lxcmgr.paths import LXC_STORAGE_DIR
+
+from . import crypto
+from .paths import APP_DIR, IMAGE_DIR, META_FILE, PRIVATE_KEY, ROOT_DIR, SIGNATURE_FILE
+
+class PackageExistsError(Exception):
+    pass
+
+class Packer:
+    def __init__(self):
+        self.app = None
+        self.image = None
+        self.tar_path = None
+        self.xz_path = None
+
+    def load_packages_meta(self):
+        if os.path.exists(PKG_META):
+            with open(PKG_META, 'r') as f:
+                return json.load(f)
+        else:
+            return {'apps': {}, 'images': {}}
+
+    def save_packages_meta(self, packages):
+        with open(PKG_META, 'w') as f:
+            json.dump(packages, f, sort_keys=True, indent=4)
+
+
+    def pack_image(self, image, force):
+        self.image = image
+        # Prepare package file names
+        self.tar_path = os.path.join(IMAGE_DIR, '{}.tar'.format(self.image.name))
+        self.xz_path = '{}.xz'.format(self.tar_path)
+        if os.path.exists(self.xz_path):
+            if force:
+                self.unregister_image()
+                os.unlink(self.xz_path)
+            else:
+                raise PackageExistsError(self.xz_path)
+        self.create_image_archive()
+        self.register_image()
+        self.sign_packages()
+
+    def create_image_archive(self):
+        # Create archive
+        print('Archiving', self.image.path)
+        subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, self.image.name], cwd=LXC_STORAGE_DIR)
+        self.compress_archive()
+
+    def compress_archive(self):
+        # Compress the tarball with xz (LZMA2)
+        print('Compressing', self.tar_path, '({:.2f} MB)'.format(os.path.getsize(self.tar_path)/1048576))
+        subprocess.run(['xz', '-9', self.tar_path])
+        print('Compressed ', self.xz_path, '({:.2f} MB)'.format(os.path.getsize(self.xz_path)/1048576))
+
+    def register_image(self):
+        # Register package in global repository metadata file
+        print('Registering package {}'.format(self.image.name))
+        packages = self.load_packages_meta()
+        packages['images'][self.image.name] = self.image.conf.copy()
+        packages['images'][self.image.name]['size'] = os.path.getsize(self.xz_path)
+        packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path)
+        self.save_packages_meta(packages)
+
+    def sign_packages(self):
+        signature = crypto.sign_file(PRIVATE_KEY, META_FILE)
+        with open(SIGNATURE_FILE, 'wb') as f:
+            f.write(signature)
+
+    def unregister_image(self):
+        # Removes package from global repository metadata file
+        packages = self.load_packages_meta()
+        if self.image.name in packages['images']:
+            del packages['images'][self.image.name]
+            self.save_packages_meta(packages)
+
+    def pack_app(self, app):
+        self.app = app
+        # Prepare package file names
+        self.tar_path = os.path.join(APP_DIR, '{}.tar'.format(self.image.name))
+        self.xz_path = '{}.xz'.format(self.tar_path)
+        if os.path.exists(self.xz_path):
+            os.unlink(self.xz_path)
+        self.create_app_archive()
+        self.register_app()
+        self.sign_packages()
+
+    def create_app_archive(self):
+        # Create archive with application setup scripts
+        print('Archiving setup scripts for', self.app.name)
+        scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh')
+        scripts = [s for s in scripts if os.path.exists(os.path.join(self.app.build_dir, s))]
+        subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path] + scripts, cwd=self.app.build_dir)
+        self.compress_archive()
+
+    def register_app(self):
+        # Register package in global repository metadata file
+        print('Registering package {}'.format(self.app.name))
+        packages = self.load_packages_meta()
+        packages['apps'][self.image.name] = self.app.conf.copy()
+        packages['apps'][self.image.name]['size'] = os.path.getsize(self.xz_path)
+        packages['apps'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path)
+        self.save_packages_meta(packages)

build/usr/lib/python3.6/lxcbuild/paths.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+
+ROOT_DIR = '/srv/build/lxc'
+IMAGE_DIR = os.path.join(ROOT_DIR, 'images')
+APP_DIR = os.path.join(ROOT_DIR, 'apps')
+META_FILE = os.path.join(ROOT_DIR, 'packages')
+SIGNATURE_FILE = os.path.join(ROOT_DIR, 'packages.sig')
+PRIVATE_KEY = '/srv/build/packages.key'