Split to lxcmgr

usr/bin/lxcmgr

@@ -0,0 +1,118 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+import argparse
+
+from lxcmgr import lxcmgr
+from lxcmgr.pkgmgr import App, PkgMgr
+
+parser = argparse.ArgumentParser(description='LXC container and package manager')
+subparsers = parser.add_subparsers()
+
+parser_list = subparsers.add_parser('list')
+subparsers_list = parser_list.add_subparsers()
+parser_list_installed = subparsers_list.add_parser('installed')
+parser_list_installed.set_defaults(action='list-installed')
+parser_list_online = subparsers_list.add_parser('online')
+parser_list_online.set_defaults(action='list-online')
+parser_list_updates = subparsers_list.add_parser('updates')
+parser_list_updates.set_defaults(action='list-updates')
+
+parser_install = subparsers.add_parser('install')
+parser_install.set_defaults(action='install')
+parser_install.add_argument('app', help='Application to install')
+
+parser_update = subparsers.add_parser('update')
+parser_update.set_defaults(action='update')
+parser_update.add_argument('app', help='Application to update')
+
+parser_uninstall = subparsers.add_parser('uninstall')
+parser_uninstall.set_defaults(action='uninstall')
+parser_uninstall.add_argument('app', help='Application to uninstall')
+
+parser_container = subparsers.add_parser('container')
+subparsers_container = parser_container.add_subparsers()
+
+parser_container_prepare = subparsers_container.add_parser('prepare')
+parser_container_prepare.set_defaults(action='container-prepare')
+parser_container_prepare.add_argument('layers', help='OverlayFS LXC rootfs layers')
+parser_container_prepare.add_argument('container', help='Container name')
+parser_container_prepare.add_argument('lxc', nargs=argparse.REMAINDER)
+
+parser_container_cleanup = subparsers_container.add_parser('cleanup')
+parser_container_cleanup.set_defaults(action='container-cleanup')
+parser_container_cleanup.add_argument('container', help='Container name')
+parser_container_cleanup.add_argument('lxc', nargs=argparse.REMAINDER)
+
+parser_container_create = subparsers_container.add_parser('create')
+parser_container_create.set_defaults(action='container-create')
+parser_container_create.add_argument('container', help='Container name')
+parser_container_create.add_argument('lxc', nargs=argparse.REMAINDER)
+
+parser_container_destroy = subparsers_container.add_parser('destroy')
+parser_container_destroy.set_defaults(action='container-destroy')
+parser_container_destroy.add_argument('container', help='Container name')
+parser_container_destroy.add_argument('lxc', nargs=argparse.REMAINDER)
+
+def print_apps(packages):
+    for app, meta in packages.items():
+        print('{} {}'.format(app, meta['version']))
+        for key, value in meta['meta'].items():
+            print('  {}: {}'.format(key, value))
+
+def list_online():
+    pm = PkgMgr()
+    pm.fetch_online_packages()
+    print_apps(pm.online_packages['apps'])
+
+def list_installed():
+    pm = PkgMgr()
+    pm.load_installed_packages()
+    print_apps(pm.installed_packages['apps'])
+
+def list_updates():
+    pm = PkgMgr()
+    pm.load_installed_packages()
+    apps = [app for app in pm.installed_packages if pm.has_update(app)]
+    updates = {name: meta for (name, meta) in pm.online_packages['apps'].items() if name in apps}
+    print_apps(updates)
+
+def install_app(app):
+    pm = PkgMgr()
+    app = App(app)
+    pm.install_app(app)
+    # TODO: periodicky vypisovat output
+
+def update_app(app):
+    pm = PkgMgr()
+    pm.update_app(app)
+
+def uninstall_app(app):
+    pm = PkgMgr()
+    pm.uninstall_app(app)
+
+args = parser.parse_args()
+if args.action == 'list-installed':
+    list_installed()
+elif args.action == 'list-online':
+    list_online()
+elif args.action == 'list-updates':
+    list_updates()
+elif args.action == 'install':
+    install_app(args.app)
+elif args.action == 'update':
+    update_app(args.app)
+elif args.action == 'uninstall':
+    uninstall_app(args.app)
+elif args.action == 'container-prepare':
+    # Used with LXC hooks on container startup
+    lxcmgr.prepare_container(args.container, args.layers)
+elif args.action == 'container-cleanup':
+    # Used with LXC hooks on container stop
+    lxcmgr.cleanup_container(args.container)
+elif args.action == 'container-create':
+    # Used by package installer and builder
+    lxcmgr.register_container(args.container)
+elif args.action == 'container-destroy':
+    # Used by package installer and builder
+    lxcmgr.unregister_container(args.container)

usr/bin/vmmgr

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 import argparse
-from vmmgr import lxcmgr
+
 from vmmgr.config import Config
 from vmmgr.vmmgr import VMMgr
 
@@ -23,23 +23,6 @@ parser_unregister_app.add_argument('app', help='Application name')
 parser_rebuild_issue = subparsers.add_parser('rebuild-issue')
 parser_rebuild_issue.set_defaults(action='rebuild-issue')
 
-parser_prepare_container = subparsers.add_parser('prepare-container')
-parser_prepare_container.set_defaults(action='prepare-container')
-parser_prepare_container.add_argument('layers', help='OverlayFS LXC rootfs layers')
-parser_prepare_container.add_argument('lxc', nargs=argparse.REMAINDER)
-
-parser_cleanup_container = subparsers.add_parser('cleanup-container')
-parser_cleanup_container.set_defaults(action='cleanup-container')
-parser_cleanup_container.add_argument('lxc', nargs=argparse.REMAINDER)
-
-parser_register_container = subparsers.add_parser('register-container')
-parser_register_container.set_defaults(action='register-container')
-parser_register_container.add_argument('lxc', nargs=argparse.REMAINDER)
-
-parser_unregister_container = subparsers.add_parser('unregister-container')
-parser_unregister_container.set_defaults(action='unregister-container')
-parser_unregister_container.add_argument('lxc', nargs=argparse.REMAINDER)
-
 parser_register_proxy = subparsers.add_parser('register-proxy')
 parser_register_proxy.set_defaults(action='register-proxy')
 parser_register_proxy.add_argument('app', help='Application name')
@@ -59,18 +42,6 @@ elif args.action == 'unregister-app':
 elif args.action == 'rebuild-issue':
     # Used by inittab on VM startup
     vmmgr.rebuild_issue()
-elif args.action == 'prepare-container':
-    # Used with LXC hooks on container startup
-    lxcmgr.prepare_container(args.layers)
-elif args.action == 'cleanup-container':
-    # Used with LXC hooks on container stop
-    lxcmgr.cleanup_container()
-elif args.action == 'register-container':
-    # Used by package installer and builder
-    lxcmgr.register_container()
-elif args.action == 'unregister-container':
-    # Used by package installer and builder
-    lxcmgr.unregister_container()
 elif args.action == 'register-proxy':
     # Used in init scripts on application startup
     vmmgr.register_proxy(args.app)

usr/lib/python3.6/lxcmgr/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

usr/lib/python3.6/lxcmgr/crypto.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+import hashlib
+
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import ec
+
+from .paths import REPO_SIG_FILE
+
+def verify_signature(file, signature):
+    # Verifies ECDSA HMAC SHA512 signature of a file
+    with open(REPO_SIG_FILE, 'rb') as f:
+        pub_key = serialization.load_pem_public_key(f.read(), default_backend())
+    pub_key.verify(signature, file, ec.ECDSA(hashes.SHA512()))
+
+def verify_hash(file, expected_hash):
+    # Verifies SHA512 hash of a file against expected hash
+    sha512 = hashlib.sha512()
+    with open(file, 'rb') as f:
+        while True:
+            data = f.read(65536)
+            if not data:
+                break
+            sha512.update(data)
+    if sha512.hexdigest() != expected_hash:
+        raise InvalidSignature(file)

usr/lib/python3.6/lxcmgr/flock.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+
+import fcntl
+
+def flock_ex(lock_file):
+    def decorator(target):
+        def wrapper(*args, **kwargs):
+            with open(lock_file, 'w') as lock:
+                fcntl.lockf(lock, fcntl.LOCK_EX)
+                return target(*args, **kwargs)
+        return wrapper
+    return decorator

usr/lib/python3.6/lxcmgr/lxcmgr.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+
+import fcntl
+import os
+import shutil
+import subprocess
+
+from . import flock
+from .paths import HOSTS_FILE, HOSTS_LOCK, LXC_ROOT
+from .templates import LXC_CONTAINER
+
+def prepare_container(container, layers):
+    # Remove ephemeral layer data
+    clean_ephemeral_layer(container)
+    # Prepare and mount overlayfs
+    rootfs = os.path.join(LXC_ROOT, container, 'rootfs')
+    # Unmount rootfs in case it remained mounted for whatever reason
+    subprocess.run(['umount', rootfs])
+    layers = layers.split(',')
+    if len(layers) == 1:
+        # We have only single layer, no overlay needed
+        subprocess.run(['mount', '--bind', layers[0], rootfs])
+    else:
+        olwork = os.path.join(LXC_ROOT, container, 'olwork')
+        subprocess.run(['mount', '-t', 'overlay', '-o', 'upperdir={},lowerdir={},workdir={}'.format(layers[-1], ':'.join(layers[:-1]), olwork), 'none', rootfs])
+
+def clean_ephemeral_layer(container):
+    # Cleans containers ephemeral layer. Called in lxc.hook.post-stop and lxc.hook.pre-start in case of unclean shutdown
+    # This is done early in the container start process, so the inode of the ephemeral directory must remain unchanged
+    ephemeral = os.path.join(LXC_ROOT, container, 'ephemeral')
+    for item in os.scandir(ephemeral):
+        shutil.rmtree(item.path) if item.is_dir() else os.unlink(item.path)
+
+def cleanup_container(container):
+    # Unmount rootfs
+    rootfs = os.path.join(LXC_ROOT, container, 'rootfs')
+    subprocess.run(['umount', rootfs])
+    # Remove ephemeral layer data
+    clean_ephemeral_layer(container)
+
+def create_container(container, image):
+    # Create directories after container installation
+    rootfs = os.path.join(LXC_ROOT, container, 'rootfs')
+    olwork = os.path.join(LXC_ROOT, container, 'olwork')
+    ephemeral = os.path.join(LXC_ROOT, container, 'ephemeral')
+    os.makedirs(rootfs, 0o755, True)
+    os.makedirs(olwork, 0o755, True)
+    os.makedirs(ephemeral, 0o755, True)
+    os.chown(ephemeral, 100000, 100000)
+    # Create container configuration file
+    layers = ','.join([os.path.join(LXC_STORAGE_DIR, layer) for layer in image['layers']])
+    if 'build' not in image:
+        layers = '{},{}'.format(layer, ephemeral)
+    mounts = '\n'.join(['lxc.mount.entry = {} {} none bind,create={} 0 0'.format(m[1], m[2].lstrip('/'), m[0].lower()) for m in image['mounts']]) if 'mounts' in image else ''
+    env = '\n'.join(['lxc.environment = {}={}'.format(e[0], e[1]) for e in image['env']]) if 'env' in image else ''
+    uid = image['uid'] if 'uid' in image else '0'
+    gid = image['gid'] if 'gid' in image else '0'
+    cmd = image['cmd'] if 'cmd' in image else '/bin/sh'
+    cwd = image['cwd'] if 'cwd' in image else '/'
+    halt = image['halt'] if 'halt' in image else 'SIGINT'
+    # Lease the first unused IP to the container
+    ipv4 = update_hosts_lease(container, True)
+    # Create the config file
+    with open(os.path.join(LXC_ROOT, container, 'config'), 'w') as f:
+        f.write(LXC_CONTAINER.format(name=container, ipv4=ipv4, layers=layers, mounts=mounts, env=env, uid=uid, gid=gid, cmd=cmd, cwd=cwd, halt=halt))
+
+def destroy_container(container):
+    # Remove container configuration and directories
+    shutil.rmtree(os.path.join(LXC_ROOT, container))
+    # Release the IP address
+    update_hosts_lease(container, False)
+
+@flock.flock_ex(HOSTS_LOCK)
+def update_hosts_lease(container, is_request):
+    # This is a poor man's DHCP server which uses /etc/hosts as lease database
+    # Leases the first unused IP from range 172.17.0.0/16
+    # Uses file lock as interprocess mutex
+    ip = None
+    # Load all existing records
+    with open(HOSTS_FILE, 'r') as f:
+        leases = [l.strip().split(' ', 1) for l in f]
+    # If this call is a request for lease, find the first unassigned IP
+    if is_request:
+        used_ips = [l[0] for l in leases]
+        for i in range(2, 65278): # Reserve last /24 subnet for VPN
+            ip = '172.17.{}.{}'. format(i // 256, i % 256)
+            if ip not in used_ips:
+                leases.append([ip, container])
+                break
+    # Otherwise it is a release in which case we just delete the record
+    else:
+        leases = [l for l in leases if l[1] != container]
+    # Write the contents back to the file
+    with open(HOSTS_FILE, 'w') as f:
+        for lease in leases:
+            f.write('{} {}\n'.format(lease[0], lease[1]))
+    return ip

usr/lib/python3.6/lxcmgr/paths.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+
+# Package manager
+REPO_CACHE_DIR = '/var/lib/lxcmgr/cache'
+REPO_LOCAL_FILE = '/var/lib/lxcmgr/packages'
+REPO_LOCK = '/var/lock/lxcmgr-repo.lock'
+REPO_SIG_FILE = '/var/lib/lxcmgr/packages.pub'
+
+# LXC
+HOSTS_FILE = '/etc/hosts'
+HOSTS_LOCK = '/var/lock/lxcmgr-hosts.lock'
+LXC_ROOT = '/var/lib/lxc'
+LXC_STORAGE_DIR = '/var/lib/lxcmgr/storage'

usr/lib/python3.6/lxcmgr/pkgmgr.py

@@ -0,0 +1,229 @@
+# -*- coding: utf-8 -*-
+
+import json
+import os
+import requests
+import shutil
+import subprocess
+
+from enum import Enum
+from pkg_resources import parse_version
+
+from . import crypto
+from . import flock
+from . import lxcmgr
+from .paths import LXC_STORAGE_DIR, REPO_CACHE_DIR, REPO_LOCAL_FILE, REPO_LOCK
+
+class Stage(Enum):
+    QUEUED = 1
+    DOWNLOAD = 2
+    UNPACK = 3
+    INSTALL = 4
+    UNINSTALL = 5
+    DONE = 6
+
+class RepoUnauthorized(Exception):
+    pass
+
+class RepoFileNotFound(Exception):
+    pass
+
+class RepoBadRequest(Exception):
+    pass
+
+class AppInstall:
+    def __init__(self, name):
+        self.name = name
+        self.stage = Stage.QUEUED
+        self.bytes_total = 1
+        self.bytes_downloaded = 0
+
+    @property
+    def percent_downloaded(self):
+        # Limit the displayed percentage to 0 - 99
+        return min(99, round(self.bytes_downloaded / self.bytes_total * 100))
+
+class PkgMgr:
+    def __init__(self, repo_url, repo_auth=None):
+        self.repo_url = repo_url
+        self.repo_auth = repo_auth
+        self.installed_packages = None
+        self.online_packages = None
+
+    def load_installed_packages(self):
+        with open(REPO_LOCAL_FILE, 'r') as f:
+            self.installed_packages = json.load(f)
+
+    def save_installed_packages(self, packages):
+        with open(REPO_LOCAL_FILE, 'w') as f:
+            json.dump(packages, f, sort_keys=True, indent=4)
+
+    def get_repo_resource(self, resource_url, stream=False):
+        # Download requested repository resource
+        r = requests.get('{}/{}'.format(self.repo_url, resource_url), auth=self.repo_auth, timeout=5, stream=stream)
+        if r.status_code == 401:
+            raise RepoUnauthorized(r.url)
+        elif r.status_code == 404:
+            raise RepoFileNotFound(r.url)
+        elif r.status_code != 200:
+            raise RepoBadRequest(r.url)
+        return r
+
+    def fetch_online_packages(self):
+        # Fetches and verifies online packages. Can raise InvalidSignature
+        packages = self.get_repo_resource('packages').content
+        packages_sig = self.get_repo_resource('packages.sig').content
+        crypto.verify_signature(packages, packages_sig)
+        self.online_packages = json.loads(packages)
+
+    @flock.flock_ex(REPO_LOCK)
+    def install_app(self, app):
+        # Main installation function. Wrapper for download, registration and install script
+        if not self.installed_packages:
+            self.load_installed_packages()
+        # Request for installation of already installed app immediately returns with success
+        if app.name in self.installed_packages['apps']:
+            app.stage = Stage.DONE
+            return
+        if not self.online_packages:
+            self.fetch_online_packages()
+        # Get all packages on which the app depends and which have not been installed yet
+        layers = []
+        images = [container['image'] for container in self.online_packages['apps'][app]['containers'].values()]
+        for image in images:
+            layers.extend(self.online_packages['images'][image]['layers'])
+        layers = [layer for layer in set(layers) if layer not in self.installed_packages['images']]
+        # Calculate bytes to download
+        app.bytes_total = sum(self.online_packages['images'][layer]['size'] for layer in layers) + self.online_packages['apps'][app.name]['size']
+        # Download layers and setup script files
+        app.stage = Stage.DOWNLOAD
+        for layer in layers:
+            self.download_layer(app, layer)
+        self.download_scripts(app)
+        # Purge old data (to clean previous failed installation) and unpack
+        app.stage = Stage.UNPACK
+        for layer in layers:
+            self.purge_layer(layer)
+            self.unpack_layer(layer)
+        self.purge_scripts(app.name)
+        self.unpack_scripts(app.name)
+        # Run setup scripts
+        app.stage = Stage.INSTALL
+        self.run_uninstall_script(app.name)
+        # Build containers and services
+        self.create_containers(app.name)
+        # Run install script and finish the installation
+        self.run_install_script(app.name)
+        self.installed_packages['apps'][app.name] = self.online_packages['apps'][app.name]
+        self.save_installed_packages()
+        app.stage = Stage.DONE
+
+    def download_layer(self, app, layer):
+        pkg_archive = 'images/{}.tar.xz'.format(layer)
+        self.download_archive(app, pkg_archive)
+        # Verify hash
+        crypto.verify_hash(tmp_archive, self.online_packages['images'][layer]['sha512'])
+
+    def download_scripts(self, app):
+        pkg_archive = 'apps/{}.tar.xz'.format(app.name)
+        self.download_archive(app, pkg_archive)
+        # Verify hash
+        crypto.verify_hash(tmp_archive, self.online_packages['apps'][app.name]['sha512'])
+
+    def download_archive(self, app, archive):
+        # Download the archive
+        tmp_archive = os.path.join(REPO_CACHE_DIR, pkg_archive)
+        res = self.get_repo_resource('{}/{}'.format(type, pkg_archive), True)
+        with open(tmp_archive, 'wb') as f:
+            for chunk in res.iter_content(chunk_size=65536):
+                if chunk:
+                    app.bytes_downloaded += f.write(chunk)
+
+    def purge_layer(self, layer):
+        # Delete layer files from storage directory
+        shutil.rmtree(os.path.join(LXC_STORAGE_DIR, layer))
+        if layer in self.installed_packages['images']:
+            del self.installed_packages['images'][layer]
+            self.save_installed_packages()
+
+    def unpack_layer(self, layer):
+        # Unpack layer archive
+        tmp_archive = os.path.join(REPO_CACHE_DIR, 'images/{}.tar.xz'.format(layer))
+        subprocess.run(['tar', 'xJf', tmp_archive], cwd=LXC_STORAGE_DIR, check=True)
+        os.unlink(tmp_archive)
+        self.installed_packages['images'][layer] = self.online_packages['images'][layer]
+        self.save_installed_packages()
+
+    def purge_scripts(self, app):
+        # Delete application setup scripts from storage directory
+        shutil.rmtree(os.path.join(REPO_CACHE_DIR, 'apps', app))
+        del self.installed_packages['apps'][app]
+        self.save_installed_packages()
+
+    def unpack_scripts(self, app):
+        # Unpack setup scripts archive
+        tmp_archive = os.path.join(REPO_CACHE_DIR, 'apps/{}.tar.xz'.format(app))
+        subprocess.run(['tar', 'xJf', tmp_archive], cwd=os.path.join(REPO_CACHE_DIR, 'apps'), check=True)
+        os.unlink(tmp_archive)
+
+    def run_uninstall_script(self, app):
+        # Runs uninstall.sh for an app, if the script is present
+        self.run_script(app, 'uninstall.sh')
+
+    def run_install_script(self, app):
+        # Runs install.sh for a package, if the script is present
+        self.run_script(app, 'install.sh')
+
+    def run_script(self, app, script):
+        # Runs script for an app, if the script is present
+        script_path = os.path.join(REPO_CACHE_DIR, app, script)
+        if os.path.exists(script_path):
+            subprocess.run(script_path, check=True)
+
+    def create_containers(self, app):
+        # Create LXC containers from image and app metadata
+        for container in self.online_packages['apps'][app]['containers']:
+            image = self.online_packages['apps'][app]['containers'][container]['image']
+            image = self.online_packages['images'][image]['containers'].copy()
+            if 'mounts' in self.online_packages['apps'][app]['containers'][container]:
+                image['mounts'] = self.online_packages['apps'][app]['containers'][container]['mounts']
+            lxcmgr.create_container(container, image)
+
+    @flock.flock_ex(REPO_LOCK)
+    def uninstall_app(self, app):
+        # Main uninstallation function. Wrapper for uninstall script and filesystem purge
+        self.run_uninstall_script(app)
+        self.destroy_containers(app)
+        self.purge_scripts(app)
+        self.purge_unused_layers()
+
+    def destroy_containers(self, app):
+        # Destroy LXC containers
+        for container in self.installed_packages['apps'][app]['containers']:
+            lxcmgr.destroy_container(container)
+
+    def purge_unused_layers(self):
+        # Remove layers which are no longer used by any installed application
+        layers = set(os.list(LXC_STORAGE_DIR))
+        for app in self.installed_packages['apps']:
+            for container in self.installed_packages['apps'][app]['containers']:
+                image = self.installed_packages['apps'][app]['containers'][container]['image']
+                for layer in self.installed_packages['images'][image]['layers']:
+                    if layer in layers:
+                        del layers[layer]
+        for layer in layers:
+            self.purge_layer(layer)
+
+    @flock.flock_ex(REPO_LOCK)
+    def update_app(self, app, item):
+        # Main update function.
+        # TODO: Implement actual update
+        #uninstall_app(app)
+        #install_app(app, item)
+
+    def has_update(self, app):
+        if not self.installed_packages:
+            self.load_installed_packages()
+        if not self.online_packages:
+            self.fetch_online_packages()
+        return parse_version(self.installed_packages['apps'][app]['version']) < parse_version(self.online_packages['apps'][app]['version'])

usr/lib/python3.6/lxcmgr/templates.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+
+LXC_CONTAINER = '''# Image name
+lxc.uts.name = {name}
+
+# Network
+lxc.net.0.type = veth
+lxc.net.0.link = lxcbr0
+lxc.net.0.flags = up
+lxc.net.0.ipv4.address = {ipv4}/16
+lxc.net.0.ipv4.gateway = 172.17.0.1
+
+# Volumes
+lxc.rootfs.path = /var/lib/lxc/{name}/rootfs
+
+# Mounts
+lxc.mount.entry = shm dev/shm tmpfs rw,nodev,noexec,nosuid,relatime,mode=1777,create=dir 0 0
+lxc.mount.entry = /etc/hosts etc/hosts none bind,create=file 0 0
+lxc.mount.entry = /etc/resolv.conf etc/resolv.conf none bind,create=file 0 0
+{mounts}
+
+# Init
+lxc.init.uid = {uid}
+lxc.init.gid = {gid}
+lxc.init.cwd = {cwd}
+
+# Environment
+lxc.environment = PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+{env}
+
+# Halt
+lxc.signal.halt = {halt}
+
+# Log
+lxc.console.size = 1MB
+lxc.console.logfile = /var/log/lxc/{name}.log
+
+# ID map
+lxc.idmap = u 0 100000 65536
+lxc.idmap = g 0 100000 65536
+
+# Hooks
+lxc.hook.pre-start = /usr/bin/vmmgr prepare-container {layers}
+lxc.hook.post-stop = /usr/bin/vmmgr cleanup-container
+
+# Other
+lxc.arch = linux64
+lxc.cap.drop = sys_admin
+lxc.include = /usr/share/lxc/config/common.conf
+lxc.include = /usr/share/lxc/config/userns.conf
+'''

usr/lib/python3.6/vmmgr/crypto.py

@@ -2,35 +2,15 @@
 
 import bcrypt
 import datetime
-import hashlib
 import os
 
 from cryptography import x509
-from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import ec
 from cryptography.x509.oid import NameOID, ExtendedKeyUsageOID
 
-from .paths import ACME_CRON, CERT_PUB_FILE, CERT_KEY_FILE, PKG_SIG_FILE
+from .paths import ACME_CRON, CERT_PUB_FILE, CERT_KEY_FILE
-
-def verify_signature(file, signature):
-    # Verifies ECDSA HMAC SHA512 signature of a file
-    with open(PKG_SIG_FILE, 'rb') as f:
-        pub_key = serialization.load_pem_public_key(f.read(), default_backend())
-    pub_key.verify(signature, file, ec.ECDSA(hashes.SHA512()))
-
-def verify_hash(file, expected_hash):
-    # Verifies SHA512 hash of a file against expected hash
-    sha512 = hashlib.sha512()
-    with open(file, 'rb') as f:
-        while True:
-            data = f.read(65536)
-            if not data:
-                break
-            sha512.update(data)
-    if sha512.hexdigest() != expected_hash:
-        raise InvalidSignature(file)
 
 def create_selfsigned_cert(domain):
     # Create selfsigned certificate with wildcard alternative subject name

usr/lib/python3.6/vmmgr/lxcmgr.py

@@ -1,121 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import fcntl
-import os
-import shutil
-import subprocess
-
-from .config import Config
-from .paths import HOSTS_FILE, HOSTS_LOCK, LXC_ROOT
-from .templates import LXC_CONTAINER
-
-def prepare_container(layers):
-    # Extract the variables from values given via lxc.hook.pre-start hook
-    app = os.environ['LXC_NAME']
-    # Remove ephemeral layer data
-    clean_ephemeral_layer(app)
-    # Prepare and mount overlayfs
-    prepare_overlayfs(app, layers)
-    # Configure host and common params used in the app
-    configure_app(app)
-
-def clean_ephemeral_layer(app):
-    # Cleans containers ephemeral layer. Called in lxc.hook.post-stop and lxc.hook.pre-start in case of unclean shutdown
-    # This is done early in the container start process, so the inode of the ephemeral directory must remain unchanged
-    ephemeral = os.path.join(LXC_ROOT, app, 'ephemeral')
-    for item in os.scandir(ephemeral):
-        shutil.rmtree(item.path) if item.is_dir() else os.unlink(item.path)
-
-def prepare_overlayfs(app, layers):
-    # Prepare and mount overlayfs
-    rootfs = os.path.join(LXC_ROOT, app, 'rootfs')
-    # Unmount rootfs in case it remained mounted for whatever reason
-    subprocess.run(['umount', rootfs])
-    layers = layers.split(',')
-    if len(layers) == 1:
-        # We have only single layer, no overlay needed
-        subprocess.run(['mount', '--bind', layers[0], rootfs])
-    else:
-        olwork = os.path.join(LXC_ROOT, app, 'olwork')
-        subprocess.run(['mount', '-t', 'overlay', '-o', 'upperdir={},lowerdir={},workdir={}'.format(layers[-1], ','.join(layers[:-1]), olwork), 'none', rootfs])
-
-def configure_app(app):
-    # Supply common configuration for the application. Done as part of container preparation during service startup
-    script = os.path.join('/srv', app, 'update-conf.sh')
-    if os.path.exists(script):
-        conf = Config()
-        setup_env = os.environ.copy()
-        setup_env['DOMAIN'] = conf['host']['domain']
-        setup_env['PORT'] = conf['host']['port']
-        setup_env['EMAIL'] = conf['common']['email']
-        setup_env['GMAPS_API_KEY'] = conf['common']['gmaps-api-key']
-        subprocess.run([script], env=setup_env, check=True)
-
-def cleanup_container():
-    # Extract the variables from values given via lxc.hook.post-stop hook
-    app = os.environ['LXC_NAME']
-    # Unmount rootfs
-    rootfs = os.path.join(LXC_ROOT, app, 'rootfs')
-    subprocess.run(['umount', rootfs])
-    # Remove ephemeral layer data
-    clean_ephemeral_layer(app)
-
-def register_container(app, image):
-    # Create directories after container installation
-    rootfs = os.path.join(LXC_ROOT, app, 'rootfs')
-    olwork = os.path.join(LXC_ROOT, app, 'olwork')
-    ephemeral = os.path.join(LXC_ROOT, app, 'ephemeral')
-    os.makedirs(rootfs, 0o755, True)
-    os.makedirs(olwork, 0o755, True)
-    os.makedirs(ephemeral, 0o755, True)
-    os.chown(ephemeral, 100000, 100000)
-    # Create container configuration file
-    layers = ','.join([os.path.join(LXC_ROOT, 'storage', layer) for layer in image['layers']])
-    if 'build' not in image:
-        layers = '{},{}'.format(layer, ephemeral)
-    mounts = '\n'.join(['lxc.mount.entry = {} {} none bind,create={} 0 0'.format(m[1], m[2], m[0].lower()) for m in image['mounts']]) if 'mounts' in image else ''
-    env = '\n'.join(['lxc.environment = {}={}'.format(e[0], e[1]) for e in image['env']]) if 'env' in image else ''
-    uid = image['uid'] if 'uid' in image else '0'
-    gid = image['gid'] if 'gid' in image else '0'
-    cmd = image['cmd'] if 'cmd' in image else '/bin/sh'
-    cwd = image['cwd'] if 'cwd' in image else '/'
-    halt = image['halt'] if 'halt' in image else 'SIGINT'
-    # Lease the first unused IP to the container
-    ipv4 = update_hosts_lease(app, True)
-    # Create the config file
-    with open(os.path.join(LXC_ROOT, app, 'config'), 'w') as f:
-        f.write(LXC_CONTAINER.format(name=app, ipv4=ipv4, layers=layers, mounts=mounts, env=env, uid=uid, gid=gid, cmd=cmd, cwd=cwd, halt=halt))
-
-def unregister_container(app):
-    # Remove container configuration and directories
-    # TODO: Duplicated with what pkgmgr does, ale zustane to tady, protoze unregister se pouziva pri buildu
-    shutil.rmtree(os.path.join(LXC_ROOT, app))
-    # Release the IP address
-    update_hosts_lease(app, False)
-
-def update_hosts_lease(app, is_request):
-    # This is a poor man's DHCP server which uses /etc/hosts as lease database
-    # Leases the first unused IP from range 172.17.0.0/16
-    # Uses file lock as interprocess mutex
-    ip = None
-    with open(HOSTS_LOCK, 'w') as lock:
-        fcntl.lockf(lock, fcntl.LOCK_EX)
-        # Load all existing records
-        with open(HOSTS_FILE, 'r') as f:
-            leases = [l.strip().split(' ', 1) for l in f]
-        # If this call is a request for lease, find the first unassigned IP
-        if is_request:
-            used_ips = [l[0] for l in leases]
-            for i in range(2, 65278): # Reserve last /24 subnet for VPN
-                ip = '172.17.{}.{}'. format(i // 256, i % 256)
-                if ip not in used_ips:
-                    leases.append([ip, app])
-                    break
-        # Otherwise it is a release in which case we just delete the record
-        else:
-            leases = [l for l in leases if l[1] != app]
-        # Write the contents back to the file
-        with open(HOSTS_FILE, 'w') as f:
-            for lease in leases:
-                f.write('{} {}\n'.format(lease[0], lease[1]))
-    return ip

usr/lib/python3.6/vmmgr/paths.py

@@ -11,13 +11,15 @@ CERT_KEY_FILE = '/etc/ssl/services.key'
 CERT_PUB_FILE = '/etc/ssl/services.pem'
 
 # Package manager
-PKG_SIG_FILE = '/etc/vmmgr/packages.pub'
+REPO_LOCAL_FILE = '/var/lib/lxc-pkg/packages'
-PKG_TEMP_DIR = '/var/cache/vmmgr'
+REPO_SIG_FILE = '/etc/vmmgr/packages.pub'
+REPO_CACHE_DIR = '/var/lib/lxc-pkg/cache'
 
 # LXC
 HOSTS_FILE = '/etc/hosts'
 HOSTS_LOCK = '/var/lock/vmmgr-hosts.lock'
 LXC_ROOT = '/var/lib/lxc'
+LXC_STORAGE_DIR = '/var/lib/lxc-pkg/storage'
 
 # OS
 ISSUE_FILE = '/etc/issue'

usr/lib/python3.6/vmmgr/pkgmgr.py

@@ -1,178 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import json
-import os
-import requests
-import shutil
-import subprocess
-
-from enum import Enum
-from pkg_resources import parse_version
-from werkzeug.exceptions import BadRequest, NotFound, Unauthorized
-
-from . import crypto
-from .paths import LXC_ROOT, PKG_TEMP_DIR
-
-class Stage(Enum):
-    DOWNLOAD = 1
-    UNPACK = 2
-    INSTALL_DEPS = 3
-    INSTALL_APP = 4
-
-class Pkg:
-    def __init__(self):
-        self.stage = Stage.DOWNLOAD
-        self.bytes_total = 1
-        self.bytes_downloaded = 0
-
-    @property
-    def percent_downloaded(self):
-        # Limit the displayed percentage to 0 - 99
-        return min(99, round(self.bytes_downloaded / self.bytes_total * 100))
-
-class PkgMgr:
-    def __init__(self, conf):
-        self.conf = conf
-        self.online_packages = {}
-
-    def get_repo_resource(self, resource_url, stream=False):
-        # Download requested repository resource
-        r = requests.get('{}/{}'.format(self.conf['repo']['url'], resource_url), auth=(self.conf['repo']['user'], self.conf['repo']['pwd']), timeout=5, stream=stream)
-        if r.status_code == 401:
-            raise Unauthorized(r.url)
-        elif r.status_code == 404:
-            raise NotFound(r.url)
-        elif r.status_code != 200:
-            raise BadRequest(r.url)
-        return r
-
-    def fetch_online_packages(self):
-        # Fetches and verifies online packages. Can raise InvalidSignature
-        packages = self.get_repo_resource('packages').content
-        packages_sig = self.get_repo_resource('packages.sig').content
-        crypto.verify_signature(packages, packages_sig)
-        self.online_packages = json.loads(packages)
-
-    def install_app(self, app, item):
-        # Main installation function. Wrapper for download, registration and install script
-        self.fetch_online_packages()
-        # Get all packages on which the app depends and which have not been installed yet
-        deps = [d for d in self.get_install_deps(app) if d not in self.conf['packages']]
-        item.bytes_total = sum(self.online_packages[d]['size'] for d in deps)
-        for dep in deps:
-            self.download_package(dep, item)
-        for dep in deps:
-            # Purge old data before unpacking to clean previous failed installation
-            item.stage = Stage.UNPACK
-            self.purge_package(dep)
-            self.unpack_package(dep)
-        for dep in deps:
-            # Set stage to INSTALLING_DEPS or INSTALLING based on which package in sequence is being installed
-            item.stage = Stage.INSTALL_APP if dep == deps[-1] else Stage.INSTALL_DEPS
-            # Run uninstall script before installation to clean previous failed installation
-            self.run_uninstall_script(dep)
-            self.run_install_script(dep)
-            self.register_package(dep)
-
-    def uninstall_app(self, app):
-        # Main uninstallation function. Wrapper for uninstall script, filesystem purge and unregistration
-        deps = self.get_install_deps(app, False)[::-1]
-        for dep in deps:
-            if dep not in self.get_uninstall_deps():
-                self.run_uninstall_script(dep)
-                self.purge_package(dep)
-                self.unregister_package(dep)
-
-    def update_app(self, app, item):
-        # Main update function.
-        # TODO: Implement actual update
-        uninstall_app(app)
-        install_app(app, item)
-
-    def download_package(self, name, item):
-        # Download tar.xz package and verify its hash. Can raise InvalidSignature
-        pkg_archive = '{}_{}-{}.tar.xz'.format(name, self.online_packages[name]['version'], self.online_packages[name]['release'])
-        tmp_archive = os.path.join(PKG_TEMP_DIR, pkg_archive)
-        os.makedirs(PKG_TEMP_DIR, 0o700, True)
-        # If the archive already exists in temp (presumably because the previous installation was interrupted), it was already verified and can be reused
-        if os.path.exists(tmp_archive):
-            item.bytes_downloaded += os.path.getsize(tmp_archive)
-            return
-        # Download the archive
-        partial_archive = '{}.partial'.format(tmp_archive)
-        res = self.get_repo_resource(pkg_archive, True)
-        with open(partial_archive, 'wb') as f:
-            for chunk in res.iter_content(chunk_size=65536):
-                if chunk:
-                    item.bytes_downloaded += f.write(chunk)
-        # Verify hash
-        crypto.verify_hash(partial_archive, self.online_packages[name]['sha512'])
-        # Remove ".partial" extension
-        os.rename(partial_archive, tmp_archive)
-
-    def unpack_package(self, name):
-        # Unpack archive
-        pkg_archive = '{}_{}-{}.tar.xz'.format(name, self.online_packages[name]['version'], self.online_packages[name]['release'])
-        tmp_archive = os.path.join(PKG_TEMP_DIR, pkg_archive)
-        subprocess.run(['tar', 'xJf', tmp_archive], cwd='/', check=True)
-        os.unlink(tmp_archive)
-
-    def purge_package(self, name):
-        # Removes package and shared data from filesystem
-        lxcpath = self.conf['packages'][name]['lxcpath'] if name in self.conf['packages'] else self.online_packages[name]['lxcpath']
-        lxc_dir = os.path.join(LXC_ROOT, lxcpath)
-        if os.path.exists(lxc_dir):
-            shutil.rmtree(lxc_dir)
-        srv_dir = os.path.join('/srv/', name)
-        if os.path.exists(srv_dir):
-            shutil.rmtree(srv_dir)
-        lxc_log = '/var/log/lxc/{}.log'.format(name)
-        if os.path.exists(lxc_log):
-            os.unlink(lxc_log)
-
-    def register_package(self, name):
-        # Registers a package in installed packages
-        metadata = self.online_packages[name].copy()
-        del metadata['sha512']
-        del metadata['size']
-        self.conf['packages'][name] = metadata
-        self.conf.save()
-
-    def unregister_package(self, name):
-        # Removes a package from installed packages
-        del self.conf['packages'][name]
-        self.conf.save()
-
-    def run_install_script(self, name):
-        # Runs install.sh for a package, if the script is present
-        install_script = os.path.join('/srv/', name, 'install.sh')
-        if os.path.exists(install_script):
-            subprocess.run(install_script, check=True)
-
-    def run_uninstall_script(self, name):
-        # Runs uninstall.sh for a package, if the script is present
-        uninstall_script = os.path.join('/srv/', name, 'uninstall.sh')
-        if os.path.exists(uninstall_script):
-            subprocess.run(uninstall_script, check=True)
-
-    def get_install_deps(self, name, online=True):
-        # Flatten dependency tree for a package while preserving the dependency order
-        packages = self.online_packages if online else self.conf['packages']
-        deps = packages[name]['depends'].copy()
-        for dep in deps[::-1]:
-            deps[:0] = [d for d in self.get_install_deps(dep, online)]
-        deps = list(dict.fromkeys(deps + [name]))
-        return deps
-
-    def get_uninstall_deps(self):
-        # Create reverse dependency tree for all installed packages
-        deps = {}
-        for name in self.conf['packages'].copy():
-            for d in self.conf['packages'][name]['depends']:
-                deps.setdefault(d, []).append(name)
-        return deps
-
-    def has_update(self, app):
-        if not self.online_packages:
-            return False
-        return parse_version(self.conf['packages'][app]['version']) < parse_version(self.online_packages[app]['version'])

usr/lib/python3.6/vmmgr/templates.py

@@ -112,53 +112,3 @@ ISSUE = '''
    - \x1b[1m{url}\x1b[0m
    - \x1b[1m{ip}\x1b[0m\x1b[?1c
 '''
-
-LXC_CONTAINER = '''# Image name
-lxc.uts.name = {name}
-
-# Network
-lxc.net.0.type = veth
-lxc.net.0.link = lxcbr0
-lxc.net.0.flags = up
-lxc.net.0.ipv4.address = {ipv4}/16
-lxc.net.0.ipv4.gateway = 172.17.0.1
-
-# Volumes
-lxc.rootfs.path = /var/lib/lxc/{name}/rootfs
-
-# Mounts
-lxc.mount.entry = shm dev/shm tmpfs rw,nodev,noexec,nosuid,relatime,mode=1777,create=dir 0 0
-lxc.mount.entry = /etc/hosts etc/hosts none bind,create=file 0 0
-lxc.mount.entry = /etc/resolv.conf etc/resolv.conf none bind,create=file 0 0
-{mounts}
-
-# Init
-lxc.init.uid = {uid}
-lxc.init.gid = {gid}
-lxc.init.cwd = {cwd}
-
-# Environment
-lxc.environment = PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-{env}
-
-# Halt
-lxc.signal.halt = {halt}
-
-# Log
-lxc.console.size = 1MB
-lxc.console.logfile = /var/log/lxc/{name}.log
-
-# ID map
-lxc.idmap = u 0 100000 65536
-lxc.idmap = g 0 100000 65536
-
-# Hooks
-lxc.hook.pre-start = /usr/bin/vmmgr prepare-container {layers}
-lxc.hook.post-stop = /usr/bin/vmmgr cleanup-container
-
-# Other
-lxc.arch = linux64
-lxc.cap.drop = sys_admin
-lxc.include = /usr/share/lxc/config/common.conf
-lxc.include = /usr/share/lxc/config/userns.conf
-'''