From c3b711850e02a6e228c4eb64ed82a4d1bc889ae9 Mon Sep 17 00:00:00 2001 From: Disassembler Date: Fri, 20 Sep 2019 10:10:25 +0200 Subject: [PATCH] Split to lxcmgr --- usr/bin/lxcmgr | 118 +++++++++++++ usr/bin/vmmgr | 31 +--- usr/lib/python3.6/lxcmgr/__init__.py | 1 + usr/lib/python3.6/lxcmgr/crypto.py | 28 ++++ usr/lib/python3.6/lxcmgr/flock.py | 12 ++ usr/lib/python3.6/lxcmgr/lxcmgr.py | 97 +++++++++++ usr/lib/python3.6/lxcmgr/paths.py | 13 ++ usr/lib/python3.6/lxcmgr/pkgmgr.py | 229 ++++++++++++++++++++++++++ usr/lib/python3.6/lxcmgr/templates.py | 51 ++++++ usr/lib/python3.6/vmmgr/crypto.py | 22 +-- usr/lib/python3.6/vmmgr/lxcmgr.py | 121 -------------- usr/lib/python3.6/vmmgr/paths.py | 6 +- usr/lib/python3.6/vmmgr/pkgmgr.py | 178 -------------------- usr/lib/python3.6/vmmgr/templates.py | 50 ------ 14 files changed, 555 insertions(+), 402 deletions(-) create mode 100644 usr/bin/lxcmgr create mode 100644 usr/lib/python3.6/lxcmgr/__init__.py create mode 100644 usr/lib/python3.6/lxcmgr/crypto.py create mode 100644 usr/lib/python3.6/lxcmgr/flock.py create mode 100644 usr/lib/python3.6/lxcmgr/lxcmgr.py create mode 100644 usr/lib/python3.6/lxcmgr/paths.py create mode 100644 usr/lib/python3.6/lxcmgr/pkgmgr.py create mode 100644 usr/lib/python3.6/lxcmgr/templates.py delete mode 100644 usr/lib/python3.6/vmmgr/lxcmgr.py delete mode 100644 usr/lib/python3.6/vmmgr/pkgmgr.py diff --git a/usr/bin/lxcmgr b/usr/bin/lxcmgr new file mode 100644 index 0000000..5be0a13 --- /dev/null +++ b/usr/bin/lxcmgr @@ -0,0 +1,118 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +import argparse + +from lxcmgr import lxcmgr +from lxcmgr.pkgmgr import App, PkgMgr + +parser = argparse.ArgumentParser(description='LXC container and package manager') +subparsers = parser.add_subparsers() + +parser_list = subparsers.add_parser('list') +subparsers_list = parser_list.add_subparsers() +parser_list_installed = subparsers_list.add_parser('installed') +parser_list_installed.set_defaults(action='list-installed') +parser_list_online = subparsers_list.add_parser('online') +parser_list_online.set_defaults(action='list-online') +parser_list_updates = subparsers_list.add_parser('updates') +parser_list_updates.set_defaults(action='list-updates') + +parser_install = subparsers.add_parser('install') +parser_install.set_defaults(action='install') +parser_install.add_argument('app', help='Application to install') + +parser_update = subparsers.add_parser('update') +parser_update.set_defaults(action='update') +parser_update.add_argument('app', help='Application to update') + +parser_uninstall = subparsers.add_parser('uninstall') +parser_uninstall.set_defaults(action='uninstall') +parser_uninstall.add_argument('app', help='Application to uninstall') + +parser_container = subparsers.add_parser('container') +subparsers_container = parser_container.add_subparsers() + +parser_container_prepare = subparsers_container.add_parser('prepare') +parser_container_prepare.set_defaults(action='container-prepare') +parser_container_prepare.add_argument('layers', help='OverlayFS LXC rootfs layers') +parser_container_prepare.add_argument('container', help='Container name') +parser_container_prepare.add_argument('lxc', nargs=argparse.REMAINDER) + +parser_container_cleanup = subparsers_container.add_parser('cleanup') +parser_container_cleanup.set_defaults(action='container-cleanup') +parser_container_cleanup.add_argument('container', help='Container name') +parser_container_cleanup.add_argument('lxc', nargs=argparse.REMAINDER) + +parser_container_create = subparsers_container.add_parser('create') +parser_container_create.set_defaults(action='container-create') +parser_container_create.add_argument('container', help='Container name') +parser_container_create.add_argument('lxc', nargs=argparse.REMAINDER) + +parser_container_destroy = subparsers_container.add_parser('destroy') +parser_container_destroy.set_defaults(action='container-destroy') +parser_container_destroy.add_argument('container', help='Container name') +parser_container_destroy.add_argument('lxc', nargs=argparse.REMAINDER) + +def print_apps(packages): + for app, meta in packages.items(): + print('{} {}'.format(app, meta['version'])) + for key, value in meta['meta'].items(): + print(' {}: {}'.format(key, value)) + +def list_online(): + pm = PkgMgr() + pm.fetch_online_packages() + print_apps(pm.online_packages['apps']) + +def list_installed(): + pm = PkgMgr() + pm.load_installed_packages() + print_apps(pm.installed_packages['apps']) + +def list_updates(): + pm = PkgMgr() + pm.load_installed_packages() + apps = [app for app in pm.installed_packages if pm.has_update(app)] + updates = {name: meta for (name, meta) in pm.online_packages['apps'].items() if name in apps} + print_apps(updates) + +def install_app(app): + pm = PkgMgr() + app = App(app) + pm.install_app(app) + # TODO: periodicky vypisovat output + +def update_app(app): + pm = PkgMgr() + pm.update_app(app) + +def uninstall_app(app): + pm = PkgMgr() + pm.uninstall_app(app) + +args = parser.parse_args() +if args.action == 'list-installed': + list_installed() +elif args.action == 'list-online': + list_online() +elif args.action == 'list-updates': + list_updates() +elif args.action == 'install': + install_app(args.app) +elif args.action == 'update': + update_app(args.app) +elif args.action == 'uninstall': + uninstall_app(args.app) +elif args.action == 'container-prepare': + # Used with LXC hooks on container startup + lxcmgr.prepare_container(args.container, args.layers) +elif args.action == 'container-cleanup': + # Used with LXC hooks on container stop + lxcmgr.cleanup_container(args.container) +elif args.action == 'container-create': + # Used by package installer and builder + lxcmgr.register_container(args.container) +elif args.action == 'container-destroy': + # Used by package installer and builder + lxcmgr.unregister_container(args.container) diff --git a/usr/bin/vmmgr b/usr/bin/vmmgr index b668b33..edf411f 100755 --- a/usr/bin/vmmgr +++ b/usr/bin/vmmgr @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- import argparse -from vmmgr import lxcmgr + from vmmgr.config import Config from vmmgr.vmmgr import VMMgr @@ -23,23 +23,6 @@ parser_unregister_app.add_argument('app', help='Application name') parser_rebuild_issue = subparsers.add_parser('rebuild-issue') parser_rebuild_issue.set_defaults(action='rebuild-issue') -parser_prepare_container = subparsers.add_parser('prepare-container') -parser_prepare_container.set_defaults(action='prepare-container') -parser_prepare_container.add_argument('layers', help='OverlayFS LXC rootfs layers') -parser_prepare_container.add_argument('lxc', nargs=argparse.REMAINDER) - -parser_cleanup_container = subparsers.add_parser('cleanup-container') -parser_cleanup_container.set_defaults(action='cleanup-container') -parser_cleanup_container.add_argument('lxc', nargs=argparse.REMAINDER) - -parser_register_container = subparsers.add_parser('register-container') -parser_register_container.set_defaults(action='register-container') -parser_register_container.add_argument('lxc', nargs=argparse.REMAINDER) - -parser_unregister_container = subparsers.add_parser('unregister-container') -parser_unregister_container.set_defaults(action='unregister-container') -parser_unregister_container.add_argument('lxc', nargs=argparse.REMAINDER) - parser_register_proxy = subparsers.add_parser('register-proxy') parser_register_proxy.set_defaults(action='register-proxy') parser_register_proxy.add_argument('app', help='Application name') @@ -59,18 +42,6 @@ elif args.action == 'unregister-app': elif args.action == 'rebuild-issue': # Used by inittab on VM startup vmmgr.rebuild_issue() -elif args.action == 'prepare-container': - # Used with LXC hooks on container startup - lxcmgr.prepare_container(args.layers) -elif args.action == 'cleanup-container': - # Used with LXC hooks on container stop - lxcmgr.cleanup_container() -elif args.action == 'register-container': - # Used by package installer and builder - lxcmgr.register_container() -elif args.action == 'unregister-container': - # Used by package installer and builder - lxcmgr.unregister_container() elif args.action == 'register-proxy': # Used in init scripts on application startup vmmgr.register_proxy(args.app) diff --git a/usr/lib/python3.6/lxcmgr/__init__.py b/usr/lib/python3.6/lxcmgr/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/usr/lib/python3.6/lxcmgr/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/usr/lib/python3.6/lxcmgr/crypto.py b/usr/lib/python3.6/lxcmgr/crypto.py new file mode 100644 index 0000000..00de575 --- /dev/null +++ b/usr/lib/python3.6/lxcmgr/crypto.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +import hashlib + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec + +from .paths import REPO_SIG_FILE + +def verify_signature(file, signature): + # Verifies ECDSA HMAC SHA512 signature of a file + with open(REPO_SIG_FILE, 'rb') as f: + pub_key = serialization.load_pem_public_key(f.read(), default_backend()) + pub_key.verify(signature, file, ec.ECDSA(hashes.SHA512())) + +def verify_hash(file, expected_hash): + # Verifies SHA512 hash of a file against expected hash + sha512 = hashlib.sha512() + with open(file, 'rb') as f: + while True: + data = f.read(65536) + if not data: + break + sha512.update(data) + if sha512.hexdigest() != expected_hash: + raise InvalidSignature(file) diff --git a/usr/lib/python3.6/lxcmgr/flock.py b/usr/lib/python3.6/lxcmgr/flock.py new file mode 100644 index 0000000..559e0f9 --- /dev/null +++ b/usr/lib/python3.6/lxcmgr/flock.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- + +import fcntl + +def flock_ex(lock_file): + def decorator(target): + def wrapper(*args, **kwargs): + with open(lock_file, 'w') as lock: + fcntl.lockf(lock, fcntl.LOCK_EX) + return target(*args, **kwargs) + return wrapper + return decorator diff --git a/usr/lib/python3.6/lxcmgr/lxcmgr.py b/usr/lib/python3.6/lxcmgr/lxcmgr.py new file mode 100644 index 0000000..f50b2d0 --- /dev/null +++ b/usr/lib/python3.6/lxcmgr/lxcmgr.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +import fcntl +import os +import shutil +import subprocess + +from . import flock +from .paths import HOSTS_FILE, HOSTS_LOCK, LXC_ROOT +from .templates import LXC_CONTAINER + +def prepare_container(container, layers): + # Remove ephemeral layer data + clean_ephemeral_layer(container) + # Prepare and mount overlayfs + rootfs = os.path.join(LXC_ROOT, container, 'rootfs') + # Unmount rootfs in case it remained mounted for whatever reason + subprocess.run(['umount', rootfs]) + layers = layers.split(',') + if len(layers) == 1: + # We have only single layer, no overlay needed + subprocess.run(['mount', '--bind', layers[0], rootfs]) + else: + olwork = os.path.join(LXC_ROOT, container, 'olwork') + subprocess.run(['mount', '-t', 'overlay', '-o', 'upperdir={},lowerdir={},workdir={}'.format(layers[-1], ':'.join(layers[:-1]), olwork), 'none', rootfs]) + +def clean_ephemeral_layer(container): + # Cleans containers ephemeral layer. Called in lxc.hook.post-stop and lxc.hook.pre-start in case of unclean shutdown + # This is done early in the container start process, so the inode of the ephemeral directory must remain unchanged + ephemeral = os.path.join(LXC_ROOT, container, 'ephemeral') + for item in os.scandir(ephemeral): + shutil.rmtree(item.path) if item.is_dir() else os.unlink(item.path) + +def cleanup_container(container): + # Unmount rootfs + rootfs = os.path.join(LXC_ROOT, container, 'rootfs') + subprocess.run(['umount', rootfs]) + # Remove ephemeral layer data + clean_ephemeral_layer(container) + +def create_container(container, image): + # Create directories after container installation + rootfs = os.path.join(LXC_ROOT, container, 'rootfs') + olwork = os.path.join(LXC_ROOT, container, 'olwork') + ephemeral = os.path.join(LXC_ROOT, container, 'ephemeral') + os.makedirs(rootfs, 0o755, True) + os.makedirs(olwork, 0o755, True) + os.makedirs(ephemeral, 0o755, True) + os.chown(ephemeral, 100000, 100000) + # Create container configuration file + layers = ','.join([os.path.join(LXC_STORAGE_DIR, layer) for layer in image['layers']]) + if 'build' not in image: + layers = '{},{}'.format(layer, ephemeral) + mounts = '\n'.join(['lxc.mount.entry = {} {} none bind,create={} 0 0'.format(m[1], m[2].lstrip('/'), m[0].lower()) for m in image['mounts']]) if 'mounts' in image else '' + env = '\n'.join(['lxc.environment = {}={}'.format(e[0], e[1]) for e in image['env']]) if 'env' in image else '' + uid = image['uid'] if 'uid' in image else '0' + gid = image['gid'] if 'gid' in image else '0' + cmd = image['cmd'] if 'cmd' in image else '/bin/sh' + cwd = image['cwd'] if 'cwd' in image else '/' + halt = image['halt'] if 'halt' in image else 'SIGINT' + # Lease the first unused IP to the container + ipv4 = update_hosts_lease(container, True) + # Create the config file + with open(os.path.join(LXC_ROOT, container, 'config'), 'w') as f: + f.write(LXC_CONTAINER.format(name=container, ipv4=ipv4, layers=layers, mounts=mounts, env=env, uid=uid, gid=gid, cmd=cmd, cwd=cwd, halt=halt)) + +def destroy_container(container): + # Remove container configuration and directories + shutil.rmtree(os.path.join(LXC_ROOT, container)) + # Release the IP address + update_hosts_lease(container, False) + +@flock.flock_ex(HOSTS_LOCK) +def update_hosts_lease(container, is_request): + # This is a poor man's DHCP server which uses /etc/hosts as lease database + # Leases the first unused IP from range 172.17.0.0/16 + # Uses file lock as interprocess mutex + ip = None + # Load all existing records + with open(HOSTS_FILE, 'r') as f: + leases = [l.strip().split(' ', 1) for l in f] + # If this call is a request for lease, find the first unassigned IP + if is_request: + used_ips = [l[0] for l in leases] + for i in range(2, 65278): # Reserve last /24 subnet for VPN + ip = '172.17.{}.{}'. format(i // 256, i % 256) + if ip not in used_ips: + leases.append([ip, container]) + break + # Otherwise it is a release in which case we just delete the record + else: + leases = [l for l in leases if l[1] != container] + # Write the contents back to the file + with open(HOSTS_FILE, 'w') as f: + for lease in leases: + f.write('{} {}\n'.format(lease[0], lease[1])) + return ip diff --git a/usr/lib/python3.6/lxcmgr/paths.py b/usr/lib/python3.6/lxcmgr/paths.py new file mode 100644 index 0000000..da93de3 --- /dev/null +++ b/usr/lib/python3.6/lxcmgr/paths.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +# Package manager +REPO_CACHE_DIR = '/var/lib/lxcmgr/cache' +REPO_LOCAL_FILE = '/var/lib/lxcmgr/packages' +REPO_LOCK = '/var/lock/lxcmgr-repo.lock' +REPO_SIG_FILE = '/var/lib/lxcmgr/packages.pub' + +# LXC +HOSTS_FILE = '/etc/hosts' +HOSTS_LOCK = '/var/lock/lxcmgr-hosts.lock' +LXC_ROOT = '/var/lib/lxc' +LXC_STORAGE_DIR = '/var/lib/lxcmgr/storage' diff --git a/usr/lib/python3.6/lxcmgr/pkgmgr.py b/usr/lib/python3.6/lxcmgr/pkgmgr.py new file mode 100644 index 0000000..be49552 --- /dev/null +++ b/usr/lib/python3.6/lxcmgr/pkgmgr.py @@ -0,0 +1,229 @@ +# -*- coding: utf-8 -*- + +import json +import os +import requests +import shutil +import subprocess + +from enum import Enum +from pkg_resources import parse_version + +from . import crypto +from . import flock +from . import lxcmgr +from .paths import LXC_STORAGE_DIR, REPO_CACHE_DIR, REPO_LOCAL_FILE, REPO_LOCK + +class Stage(Enum): + QUEUED = 1 + DOWNLOAD = 2 + UNPACK = 3 + INSTALL = 4 + UNINSTALL = 5 + DONE = 6 + +class RepoUnauthorized(Exception): + pass + +class RepoFileNotFound(Exception): + pass + +class RepoBadRequest(Exception): + pass + +class AppInstall: + def __init__(self, name): + self.name = name + self.stage = Stage.QUEUED + self.bytes_total = 1 + self.bytes_downloaded = 0 + + @property + def percent_downloaded(self): + # Limit the displayed percentage to 0 - 99 + return min(99, round(self.bytes_downloaded / self.bytes_total * 100)) + +class PkgMgr: + def __init__(self, repo_url, repo_auth=None): + self.repo_url = repo_url + self.repo_auth = repo_auth + self.installed_packages = None + self.online_packages = None + + def load_installed_packages(self): + with open(REPO_LOCAL_FILE, 'r') as f: + self.installed_packages = json.load(f) + + def save_installed_packages(self, packages): + with open(REPO_LOCAL_FILE, 'w') as f: + json.dump(packages, f, sort_keys=True, indent=4) + + def get_repo_resource(self, resource_url, stream=False): + # Download requested repository resource + r = requests.get('{}/{}'.format(self.repo_url, resource_url), auth=self.repo_auth, timeout=5, stream=stream) + if r.status_code == 401: + raise RepoUnauthorized(r.url) + elif r.status_code == 404: + raise RepoFileNotFound(r.url) + elif r.status_code != 200: + raise RepoBadRequest(r.url) + return r + + def fetch_online_packages(self): + # Fetches and verifies online packages. Can raise InvalidSignature + packages = self.get_repo_resource('packages').content + packages_sig = self.get_repo_resource('packages.sig').content + crypto.verify_signature(packages, packages_sig) + self.online_packages = json.loads(packages) + + @flock.flock_ex(REPO_LOCK) + def install_app(self, app): + # Main installation function. Wrapper for download, registration and install script + if not self.installed_packages: + self.load_installed_packages() + # Request for installation of already installed app immediately returns with success + if app.name in self.installed_packages['apps']: + app.stage = Stage.DONE + return + if not self.online_packages: + self.fetch_online_packages() + # Get all packages on which the app depends and which have not been installed yet + layers = [] + images = [container['image'] for container in self.online_packages['apps'][app]['containers'].values()] + for image in images: + layers.extend(self.online_packages['images'][image]['layers']) + layers = [layer for layer in set(layers) if layer not in self.installed_packages['images']] + # Calculate bytes to download + app.bytes_total = sum(self.online_packages['images'][layer]['size'] for layer in layers) + self.online_packages['apps'][app.name]['size'] + # Download layers and setup script files + app.stage = Stage.DOWNLOAD + for layer in layers: + self.download_layer(app, layer) + self.download_scripts(app) + # Purge old data (to clean previous failed installation) and unpack + app.stage = Stage.UNPACK + for layer in layers: + self.purge_layer(layer) + self.unpack_layer(layer) + self.purge_scripts(app.name) + self.unpack_scripts(app.name) + # Run setup scripts + app.stage = Stage.INSTALL + self.run_uninstall_script(app.name) + # Build containers and services + self.create_containers(app.name) + # Run install script and finish the installation + self.run_install_script(app.name) + self.installed_packages['apps'][app.name] = self.online_packages['apps'][app.name] + self.save_installed_packages() + app.stage = Stage.DONE + + def download_layer(self, app, layer): + pkg_archive = 'images/{}.tar.xz'.format(layer) + self.download_archive(app, pkg_archive) + # Verify hash + crypto.verify_hash(tmp_archive, self.online_packages['images'][layer]['sha512']) + + def download_scripts(self, app): + pkg_archive = 'apps/{}.tar.xz'.format(app.name) + self.download_archive(app, pkg_archive) + # Verify hash + crypto.verify_hash(tmp_archive, self.online_packages['apps'][app.name]['sha512']) + + def download_archive(self, app, archive): + # Download the archive + tmp_archive = os.path.join(REPO_CACHE_DIR, pkg_archive) + res = self.get_repo_resource('{}/{}'.format(type, pkg_archive), True) + with open(tmp_archive, 'wb') as f: + for chunk in res.iter_content(chunk_size=65536): + if chunk: + app.bytes_downloaded += f.write(chunk) + + def purge_layer(self, layer): + # Delete layer files from storage directory + shutil.rmtree(os.path.join(LXC_STORAGE_DIR, layer)) + if layer in self.installed_packages['images']: + del self.installed_packages['images'][layer] + self.save_installed_packages() + + def unpack_layer(self, layer): + # Unpack layer archive + tmp_archive = os.path.join(REPO_CACHE_DIR, 'images/{}.tar.xz'.format(layer)) + subprocess.run(['tar', 'xJf', tmp_archive], cwd=LXC_STORAGE_DIR, check=True) + os.unlink(tmp_archive) + self.installed_packages['images'][layer] = self.online_packages['images'][layer] + self.save_installed_packages() + + def purge_scripts(self, app): + # Delete application setup scripts from storage directory + shutil.rmtree(os.path.join(REPO_CACHE_DIR, 'apps', app)) + del self.installed_packages['apps'][app] + self.save_installed_packages() + + def unpack_scripts(self, app): + # Unpack setup scripts archive + tmp_archive = os.path.join(REPO_CACHE_DIR, 'apps/{}.tar.xz'.format(app)) + subprocess.run(['tar', 'xJf', tmp_archive], cwd=os.path.join(REPO_CACHE_DIR, 'apps'), check=True) + os.unlink(tmp_archive) + + def run_uninstall_script(self, app): + # Runs uninstall.sh for an app, if the script is present + self.run_script(app, 'uninstall.sh') + + def run_install_script(self, app): + # Runs install.sh for a package, if the script is present + self.run_script(app, 'install.sh') + + def run_script(self, app, script): + # Runs script for an app, if the script is present + script_path = os.path.join(REPO_CACHE_DIR, app, script) + if os.path.exists(script_path): + subprocess.run(script_path, check=True) + + def create_containers(self, app): + # Create LXC containers from image and app metadata + for container in self.online_packages['apps'][app]['containers']: + image = self.online_packages['apps'][app]['containers'][container]['image'] + image = self.online_packages['images'][image]['containers'].copy() + if 'mounts' in self.online_packages['apps'][app]['containers'][container]: + image['mounts'] = self.online_packages['apps'][app]['containers'][container]['mounts'] + lxcmgr.create_container(container, image) + + @flock.flock_ex(REPO_LOCK) + def uninstall_app(self, app): + # Main uninstallation function. Wrapper for uninstall script and filesystem purge + self.run_uninstall_script(app) + self.destroy_containers(app) + self.purge_scripts(app) + self.purge_unused_layers() + + def destroy_containers(self, app): + # Destroy LXC containers + for container in self.installed_packages['apps'][app]['containers']: + lxcmgr.destroy_container(container) + + def purge_unused_layers(self): + # Remove layers which are no longer used by any installed application + layers = set(os.list(LXC_STORAGE_DIR)) + for app in self.installed_packages['apps']: + for container in self.installed_packages['apps'][app]['containers']: + image = self.installed_packages['apps'][app]['containers'][container]['image'] + for layer in self.installed_packages['images'][image]['layers']: + if layer in layers: + del layers[layer] + for layer in layers: + self.purge_layer(layer) + + @flock.flock_ex(REPO_LOCK) + def update_app(self, app, item): + # Main update function. + # TODO: Implement actual update + #uninstall_app(app) + #install_app(app, item) + + def has_update(self, app): + if not self.installed_packages: + self.load_installed_packages() + if not self.online_packages: + self.fetch_online_packages() + return parse_version(self.installed_packages['apps'][app]['version']) < parse_version(self.online_packages['apps'][app]['version']) diff --git a/usr/lib/python3.6/lxcmgr/templates.py b/usr/lib/python3.6/lxcmgr/templates.py new file mode 100644 index 0000000..a170447 --- /dev/null +++ b/usr/lib/python3.6/lxcmgr/templates.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +LXC_CONTAINER = '''# Image name +lxc.uts.name = {name} + +# Network +lxc.net.0.type = veth +lxc.net.0.link = lxcbr0 +lxc.net.0.flags = up +lxc.net.0.ipv4.address = {ipv4}/16 +lxc.net.0.ipv4.gateway = 172.17.0.1 + +# Volumes +lxc.rootfs.path = /var/lib/lxc/{name}/rootfs + +# Mounts +lxc.mount.entry = shm dev/shm tmpfs rw,nodev,noexec,nosuid,relatime,mode=1777,create=dir 0 0 +lxc.mount.entry = /etc/hosts etc/hosts none bind,create=file 0 0 +lxc.mount.entry = /etc/resolv.conf etc/resolv.conf none bind,create=file 0 0 +{mounts} + +# Init +lxc.init.uid = {uid} +lxc.init.gid = {gid} +lxc.init.cwd = {cwd} + +# Environment +lxc.environment = PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +{env} + +# Halt +lxc.signal.halt = {halt} + +# Log +lxc.console.size = 1MB +lxc.console.logfile = /var/log/lxc/{name}.log + +# ID map +lxc.idmap = u 0 100000 65536 +lxc.idmap = g 0 100000 65536 + +# Hooks +lxc.hook.pre-start = /usr/bin/vmmgr prepare-container {layers} +lxc.hook.post-stop = /usr/bin/vmmgr cleanup-container + +# Other +lxc.arch = linux64 +lxc.cap.drop = sys_admin +lxc.include = /usr/share/lxc/config/common.conf +lxc.include = /usr/share/lxc/config/userns.conf +''' diff --git a/usr/lib/python3.6/vmmgr/crypto.py b/usr/lib/python3.6/vmmgr/crypto.py index b048bcc..38ae501 100644 --- a/usr/lib/python3.6/vmmgr/crypto.py +++ b/usr/lib/python3.6/vmmgr/crypto.py @@ -2,35 +2,15 @@ import bcrypt import datetime -import hashlib import os from cryptography import x509 -from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import ec from cryptography.x509.oid import NameOID, ExtendedKeyUsageOID -from .paths import ACME_CRON, CERT_PUB_FILE, CERT_KEY_FILE, PKG_SIG_FILE - -def verify_signature(file, signature): - # Verifies ECDSA HMAC SHA512 signature of a file - with open(PKG_SIG_FILE, 'rb') as f: - pub_key = serialization.load_pem_public_key(f.read(), default_backend()) - pub_key.verify(signature, file, ec.ECDSA(hashes.SHA512())) - -def verify_hash(file, expected_hash): - # Verifies SHA512 hash of a file against expected hash - sha512 = hashlib.sha512() - with open(file, 'rb') as f: - while True: - data = f.read(65536) - if not data: - break - sha512.update(data) - if sha512.hexdigest() != expected_hash: - raise InvalidSignature(file) +from .paths import ACME_CRON, CERT_PUB_FILE, CERT_KEY_FILE def create_selfsigned_cert(domain): # Create selfsigned certificate with wildcard alternative subject name diff --git a/usr/lib/python3.6/vmmgr/lxcmgr.py b/usr/lib/python3.6/vmmgr/lxcmgr.py deleted file mode 100644 index de64c0f..0000000 --- a/usr/lib/python3.6/vmmgr/lxcmgr.py +++ /dev/null @@ -1,121 +0,0 @@ -# -*- coding: utf-8 -*- - -import fcntl -import os -import shutil -import subprocess - -from .config import Config -from .paths import HOSTS_FILE, HOSTS_LOCK, LXC_ROOT -from .templates import LXC_CONTAINER - -def prepare_container(layers): - # Extract the variables from values given via lxc.hook.pre-start hook - app = os.environ['LXC_NAME'] - # Remove ephemeral layer data - clean_ephemeral_layer(app) - # Prepare and mount overlayfs - prepare_overlayfs(app, layers) - # Configure host and common params used in the app - configure_app(app) - -def clean_ephemeral_layer(app): - # Cleans containers ephemeral layer. Called in lxc.hook.post-stop and lxc.hook.pre-start in case of unclean shutdown - # This is done early in the container start process, so the inode of the ephemeral directory must remain unchanged - ephemeral = os.path.join(LXC_ROOT, app, 'ephemeral') - for item in os.scandir(ephemeral): - shutil.rmtree(item.path) if item.is_dir() else os.unlink(item.path) - -def prepare_overlayfs(app, layers): - # Prepare and mount overlayfs - rootfs = os.path.join(LXC_ROOT, app, 'rootfs') - # Unmount rootfs in case it remained mounted for whatever reason - subprocess.run(['umount', rootfs]) - layers = layers.split(',') - if len(layers) == 1: - # We have only single layer, no overlay needed - subprocess.run(['mount', '--bind', layers[0], rootfs]) - else: - olwork = os.path.join(LXC_ROOT, app, 'olwork') - subprocess.run(['mount', '-t', 'overlay', '-o', 'upperdir={},lowerdir={},workdir={}'.format(layers[-1], ','.join(layers[:-1]), olwork), 'none', rootfs]) - -def configure_app(app): - # Supply common configuration for the application. Done as part of container preparation during service startup - script = os.path.join('/srv', app, 'update-conf.sh') - if os.path.exists(script): - conf = Config() - setup_env = os.environ.copy() - setup_env['DOMAIN'] = conf['host']['domain'] - setup_env['PORT'] = conf['host']['port'] - setup_env['EMAIL'] = conf['common']['email'] - setup_env['GMAPS_API_KEY'] = conf['common']['gmaps-api-key'] - subprocess.run([script], env=setup_env, check=True) - -def cleanup_container(): - # Extract the variables from values given via lxc.hook.post-stop hook - app = os.environ['LXC_NAME'] - # Unmount rootfs - rootfs = os.path.join(LXC_ROOT, app, 'rootfs') - subprocess.run(['umount', rootfs]) - # Remove ephemeral layer data - clean_ephemeral_layer(app) - -def register_container(app, image): - # Create directories after container installation - rootfs = os.path.join(LXC_ROOT, app, 'rootfs') - olwork = os.path.join(LXC_ROOT, app, 'olwork') - ephemeral = os.path.join(LXC_ROOT, app, 'ephemeral') - os.makedirs(rootfs, 0o755, True) - os.makedirs(olwork, 0o755, True) - os.makedirs(ephemeral, 0o755, True) - os.chown(ephemeral, 100000, 100000) - # Create container configuration file - layers = ','.join([os.path.join(LXC_ROOT, 'storage', layer) for layer in image['layers']]) - if 'build' not in image: - layers = '{},{}'.format(layer, ephemeral) - mounts = '\n'.join(['lxc.mount.entry = {} {} none bind,create={} 0 0'.format(m[1], m[2], m[0].lower()) for m in image['mounts']]) if 'mounts' in image else '' - env = '\n'.join(['lxc.environment = {}={}'.format(e[0], e[1]) for e in image['env']]) if 'env' in image else '' - uid = image['uid'] if 'uid' in image else '0' - gid = image['gid'] if 'gid' in image else '0' - cmd = image['cmd'] if 'cmd' in image else '/bin/sh' - cwd = image['cwd'] if 'cwd' in image else '/' - halt = image['halt'] if 'halt' in image else 'SIGINT' - # Lease the first unused IP to the container - ipv4 = update_hosts_lease(app, True) - # Create the config file - with open(os.path.join(LXC_ROOT, app, 'config'), 'w') as f: - f.write(LXC_CONTAINER.format(name=app, ipv4=ipv4, layers=layers, mounts=mounts, env=env, uid=uid, gid=gid, cmd=cmd, cwd=cwd, halt=halt)) - -def unregister_container(app): - # Remove container configuration and directories - # TODO: Duplicated with what pkgmgr does, ale zustane to tady, protoze unregister se pouziva pri buildu - shutil.rmtree(os.path.join(LXC_ROOT, app)) - # Release the IP address - update_hosts_lease(app, False) - -def update_hosts_lease(app, is_request): - # This is a poor man's DHCP server which uses /etc/hosts as lease database - # Leases the first unused IP from range 172.17.0.0/16 - # Uses file lock as interprocess mutex - ip = None - with open(HOSTS_LOCK, 'w') as lock: - fcntl.lockf(lock, fcntl.LOCK_EX) - # Load all existing records - with open(HOSTS_FILE, 'r') as f: - leases = [l.strip().split(' ', 1) for l in f] - # If this call is a request for lease, find the first unassigned IP - if is_request: - used_ips = [l[0] for l in leases] - for i in range(2, 65278): # Reserve last /24 subnet for VPN - ip = '172.17.{}.{}'. format(i // 256, i % 256) - if ip not in used_ips: - leases.append([ip, app]) - break - # Otherwise it is a release in which case we just delete the record - else: - leases = [l for l in leases if l[1] != app] - # Write the contents back to the file - with open(HOSTS_FILE, 'w') as f: - for lease in leases: - f.write('{} {}\n'.format(lease[0], lease[1])) - return ip diff --git a/usr/lib/python3.6/vmmgr/paths.py b/usr/lib/python3.6/vmmgr/paths.py index f028326..e8dd368 100644 --- a/usr/lib/python3.6/vmmgr/paths.py +++ b/usr/lib/python3.6/vmmgr/paths.py @@ -11,13 +11,15 @@ CERT_KEY_FILE = '/etc/ssl/services.key' CERT_PUB_FILE = '/etc/ssl/services.pem' # Package manager -PKG_SIG_FILE = '/etc/vmmgr/packages.pub' -PKG_TEMP_DIR = '/var/cache/vmmgr' +REPO_LOCAL_FILE = '/var/lib/lxc-pkg/packages' +REPO_SIG_FILE = '/etc/vmmgr/packages.pub' +REPO_CACHE_DIR = '/var/lib/lxc-pkg/cache' # LXC HOSTS_FILE = '/etc/hosts' HOSTS_LOCK = '/var/lock/vmmgr-hosts.lock' LXC_ROOT = '/var/lib/lxc' +LXC_STORAGE_DIR = '/var/lib/lxc-pkg/storage' # OS ISSUE_FILE = '/etc/issue' diff --git a/usr/lib/python3.6/vmmgr/pkgmgr.py b/usr/lib/python3.6/vmmgr/pkgmgr.py deleted file mode 100644 index adfdbf9..0000000 --- a/usr/lib/python3.6/vmmgr/pkgmgr.py +++ /dev/null @@ -1,178 +0,0 @@ -# -*- coding: utf-8 -*- - -import json -import os -import requests -import shutil -import subprocess - -from enum import Enum -from pkg_resources import parse_version -from werkzeug.exceptions import BadRequest, NotFound, Unauthorized - -from . import crypto -from .paths import LXC_ROOT, PKG_TEMP_DIR - -class Stage(Enum): - DOWNLOAD = 1 - UNPACK = 2 - INSTALL_DEPS = 3 - INSTALL_APP = 4 - -class Pkg: - def __init__(self): - self.stage = Stage.DOWNLOAD - self.bytes_total = 1 - self.bytes_downloaded = 0 - - @property - def percent_downloaded(self): - # Limit the displayed percentage to 0 - 99 - return min(99, round(self.bytes_downloaded / self.bytes_total * 100)) - -class PkgMgr: - def __init__(self, conf): - self.conf = conf - self.online_packages = {} - - def get_repo_resource(self, resource_url, stream=False): - # Download requested repository resource - r = requests.get('{}/{}'.format(self.conf['repo']['url'], resource_url), auth=(self.conf['repo']['user'], self.conf['repo']['pwd']), timeout=5, stream=stream) - if r.status_code == 401: - raise Unauthorized(r.url) - elif r.status_code == 404: - raise NotFound(r.url) - elif r.status_code != 200: - raise BadRequest(r.url) - return r - - def fetch_online_packages(self): - # Fetches and verifies online packages. Can raise InvalidSignature - packages = self.get_repo_resource('packages').content - packages_sig = self.get_repo_resource('packages.sig').content - crypto.verify_signature(packages, packages_sig) - self.online_packages = json.loads(packages) - - def install_app(self, app, item): - # Main installation function. Wrapper for download, registration and install script - self.fetch_online_packages() - # Get all packages on which the app depends and which have not been installed yet - deps = [d for d in self.get_install_deps(app) if d not in self.conf['packages']] - item.bytes_total = sum(self.online_packages[d]['size'] for d in deps) - for dep in deps: - self.download_package(dep, item) - for dep in deps: - # Purge old data before unpacking to clean previous failed installation - item.stage = Stage.UNPACK - self.purge_package(dep) - self.unpack_package(dep) - for dep in deps: - # Set stage to INSTALLING_DEPS or INSTALLING based on which package in sequence is being installed - item.stage = Stage.INSTALL_APP if dep == deps[-1] else Stage.INSTALL_DEPS - # Run uninstall script before installation to clean previous failed installation - self.run_uninstall_script(dep) - self.run_install_script(dep) - self.register_package(dep) - - def uninstall_app(self, app): - # Main uninstallation function. Wrapper for uninstall script, filesystem purge and unregistration - deps = self.get_install_deps(app, False)[::-1] - for dep in deps: - if dep not in self.get_uninstall_deps(): - self.run_uninstall_script(dep) - self.purge_package(dep) - self.unregister_package(dep) - - def update_app(self, app, item): - # Main update function. - # TODO: Implement actual update - uninstall_app(app) - install_app(app, item) - - def download_package(self, name, item): - # Download tar.xz package and verify its hash. Can raise InvalidSignature - pkg_archive = '{}_{}-{}.tar.xz'.format(name, self.online_packages[name]['version'], self.online_packages[name]['release']) - tmp_archive = os.path.join(PKG_TEMP_DIR, pkg_archive) - os.makedirs(PKG_TEMP_DIR, 0o700, True) - # If the archive already exists in temp (presumably because the previous installation was interrupted), it was already verified and can be reused - if os.path.exists(tmp_archive): - item.bytes_downloaded += os.path.getsize(tmp_archive) - return - # Download the archive - partial_archive = '{}.partial'.format(tmp_archive) - res = self.get_repo_resource(pkg_archive, True) - with open(partial_archive, 'wb') as f: - for chunk in res.iter_content(chunk_size=65536): - if chunk: - item.bytes_downloaded += f.write(chunk) - # Verify hash - crypto.verify_hash(partial_archive, self.online_packages[name]['sha512']) - # Remove ".partial" extension - os.rename(partial_archive, tmp_archive) - - def unpack_package(self, name): - # Unpack archive - pkg_archive = '{}_{}-{}.tar.xz'.format(name, self.online_packages[name]['version'], self.online_packages[name]['release']) - tmp_archive = os.path.join(PKG_TEMP_DIR, pkg_archive) - subprocess.run(['tar', 'xJf', tmp_archive], cwd='/', check=True) - os.unlink(tmp_archive) - - def purge_package(self, name): - # Removes package and shared data from filesystem - lxcpath = self.conf['packages'][name]['lxcpath'] if name in self.conf['packages'] else self.online_packages[name]['lxcpath'] - lxc_dir = os.path.join(LXC_ROOT, lxcpath) - if os.path.exists(lxc_dir): - shutil.rmtree(lxc_dir) - srv_dir = os.path.join('/srv/', name) - if os.path.exists(srv_dir): - shutil.rmtree(srv_dir) - lxc_log = '/var/log/lxc/{}.log'.format(name) - if os.path.exists(lxc_log): - os.unlink(lxc_log) - - def register_package(self, name): - # Registers a package in installed packages - metadata = self.online_packages[name].copy() - del metadata['sha512'] - del metadata['size'] - self.conf['packages'][name] = metadata - self.conf.save() - - def unregister_package(self, name): - # Removes a package from installed packages - del self.conf['packages'][name] - self.conf.save() - - def run_install_script(self, name): - # Runs install.sh for a package, if the script is present - install_script = os.path.join('/srv/', name, 'install.sh') - if os.path.exists(install_script): - subprocess.run(install_script, check=True) - - def run_uninstall_script(self, name): - # Runs uninstall.sh for a package, if the script is present - uninstall_script = os.path.join('/srv/', name, 'uninstall.sh') - if os.path.exists(uninstall_script): - subprocess.run(uninstall_script, check=True) - - def get_install_deps(self, name, online=True): - # Flatten dependency tree for a package while preserving the dependency order - packages = self.online_packages if online else self.conf['packages'] - deps = packages[name]['depends'].copy() - for dep in deps[::-1]: - deps[:0] = [d for d in self.get_install_deps(dep, online)] - deps = list(dict.fromkeys(deps + [name])) - return deps - - def get_uninstall_deps(self): - # Create reverse dependency tree for all installed packages - deps = {} - for name in self.conf['packages'].copy(): - for d in self.conf['packages'][name]['depends']: - deps.setdefault(d, []).append(name) - return deps - - def has_update(self, app): - if not self.online_packages: - return False - return parse_version(self.conf['packages'][app]['version']) < parse_version(self.online_packages[app]['version']) diff --git a/usr/lib/python3.6/vmmgr/templates.py b/usr/lib/python3.6/vmmgr/templates.py index 4d81da1..9fa8e9c 100644 --- a/usr/lib/python3.6/vmmgr/templates.py +++ b/usr/lib/python3.6/vmmgr/templates.py @@ -112,53 +112,3 @@ ISSUE = ''' - \x1b[1m{url}\x1b[0m - \x1b[1m{ip}\x1b[0m\x1b[?1c ''' - -LXC_CONTAINER = '''# Image name -lxc.uts.name = {name} - -# Network -lxc.net.0.type = veth -lxc.net.0.link = lxcbr0 -lxc.net.0.flags = up -lxc.net.0.ipv4.address = {ipv4}/16 -lxc.net.0.ipv4.gateway = 172.17.0.1 - -# Volumes -lxc.rootfs.path = /var/lib/lxc/{name}/rootfs - -# Mounts -lxc.mount.entry = shm dev/shm tmpfs rw,nodev,noexec,nosuid,relatime,mode=1777,create=dir 0 0 -lxc.mount.entry = /etc/hosts etc/hosts none bind,create=file 0 0 -lxc.mount.entry = /etc/resolv.conf etc/resolv.conf none bind,create=file 0 0 -{mounts} - -# Init -lxc.init.uid = {uid} -lxc.init.gid = {gid} -lxc.init.cwd = {cwd} - -# Environment -lxc.environment = PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -{env} - -# Halt -lxc.signal.halt = {halt} - -# Log -lxc.console.size = 1MB -lxc.console.logfile = /var/log/lxc/{name}.log - -# ID map -lxc.idmap = u 0 100000 65536 -lxc.idmap = g 0 100000 65536 - -# Hooks -lxc.hook.pre-start = /usr/bin/vmmgr prepare-container {layers} -lxc.hook.post-stop = /usr/bin/vmmgr cleanup-container - -# Other -lxc.arch = linux64 -lxc.cap.drop = sys_admin -lxc.include = /usr/share/lxc/config/common.conf -lxc.include = /usr/share/lxc/config/userns.conf -'''