From 4c2616887f52b1265aa5943d1b6bd0f7d5e9f008 Mon Sep 17 00:00:00 2001 From: Disassembler Date: Fri, 20 Sep 2019 15:41:56 +0200 Subject: [PATCH] Separate repo conf, build should now be almost complete --- etc/{vmmgr => lxcmgr}/packages.pub | 0 etc/lxcmgr/repo.json | 5 ++ etc/vmmgr/config.default.json | 6 -- usr/bin/lxcmgr | 16 ---- usr/lib/python3.6/lxcmgr/crypto.py | 12 +-- usr/lib/python3.6/lxcmgr/lxcmgr.py | 3 +- usr/lib/python3.6/lxcmgr/paths.py | 4 +- usr/lib/python3.6/lxcmgr/pkgmgr.py | 110 ++++++++++++++++---------- usr/lib/python3.6/lxcmgr/templates.py | 4 +- var/lib/lxcmgr/packages | 1 + 10 files changed, 87 insertions(+), 74 deletions(-) rename etc/{vmmgr => lxcmgr}/packages.pub (100%) create mode 100644 etc/lxcmgr/repo.json create mode 100644 var/lib/lxcmgr/packages diff --git a/etc/vmmgr/packages.pub b/etc/lxcmgr/packages.pub similarity index 100% rename from etc/vmmgr/packages.pub rename to etc/lxcmgr/packages.pub diff --git a/etc/lxcmgr/repo.json b/etc/lxcmgr/repo.json new file mode 100644 index 0000000..41164e3 --- /dev/null +++ b/etc/lxcmgr/repo.json @@ -0,0 +1,5 @@ +{ + "url": "https://repo.spotter.cz/lxc", + "user": "", + "pwd": "" +} diff --git a/etc/vmmgr/config.default.json b/etc/vmmgr/config.default.json index fc14cdf..03c8796 100644 --- a/etc/vmmgr/config.default.json +++ b/etc/vmmgr/config.default.json @@ -8,11 +8,5 @@ "adminpwd": "${ADMINPWD}", "domain": "spotter.vm", "port": "443" - }, - "packages": {}, - "repo": { - "pwd": "", - "url": "https://repo.spotter.cz/lxc", - "user": "" } } diff --git a/usr/bin/lxcmgr b/usr/bin/lxcmgr index 5be0a13..9627c8a 100644 --- a/usr/bin/lxcmgr +++ b/usr/bin/lxcmgr @@ -44,16 +44,6 @@ parser_container_cleanup.set_defaults(action='container-cleanup') parser_container_cleanup.add_argument('container', help='Container name') parser_container_cleanup.add_argument('lxc', nargs=argparse.REMAINDER) -parser_container_create = subparsers_container.add_parser('create') -parser_container_create.set_defaults(action='container-create') -parser_container_create.add_argument('container', help='Container name') -parser_container_create.add_argument('lxc', nargs=argparse.REMAINDER) - -parser_container_destroy = subparsers_container.add_parser('destroy') -parser_container_destroy.set_defaults(action='container-destroy') -parser_container_destroy.add_argument('container', help='Container name') -parser_container_destroy.add_argument('lxc', nargs=argparse.REMAINDER) - def print_apps(packages): for app, meta in packages.items(): print('{} {}'.format(app, meta['version'])) @@ -110,9 +100,3 @@ elif args.action == 'container-prepare': elif args.action == 'container-cleanup': # Used with LXC hooks on container stop lxcmgr.cleanup_container(args.container) -elif args.action == 'container-create': - # Used by package installer and builder - lxcmgr.register_container(args.container) -elif args.action == 'container-destroy': - # Used by package installer and builder - lxcmgr.unregister_container(args.container) diff --git a/usr/lib/python3.6/lxcmgr/crypto.py b/usr/lib/python3.6/lxcmgr/crypto.py index 00de575..02acd43 100644 --- a/usr/lib/python3.6/lxcmgr/crypto.py +++ b/usr/lib/python3.6/lxcmgr/crypto.py @@ -9,20 +9,20 @@ from cryptography.hazmat.primitives.asymmetric import ec from .paths import REPO_SIG_FILE -def verify_signature(file, signature): +def verify_signature(public_key_path, input_data, signature_data): # Verifies ECDSA HMAC SHA512 signature of a file - with open(REPO_SIG_FILE, 'rb') as f: + with open(public_key_path, 'rb') as f: pub_key = serialization.load_pem_public_key(f.read(), default_backend()) - pub_key.verify(signature, file, ec.ECDSA(hashes.SHA512())) + pub_key.verify(signature_data, input_data, ec.ECDSA(hashes.SHA512())) -def verify_hash(file, expected_hash): +def verify_hash(input_path, expected_hash): # Verifies SHA512 hash of a file against expected hash sha512 = hashlib.sha512() - with open(file, 'rb') as f: + with open(input_path, 'rb') as f: while True: data = f.read(65536) if not data: break sha512.update(data) if sha512.hexdigest() != expected_hash: - raise InvalidSignature(file) + raise InvalidSignature(input_path) diff --git a/usr/lib/python3.6/lxcmgr/lxcmgr.py b/usr/lib/python3.6/lxcmgr/lxcmgr.py index f50b2d0..3ed7886 100644 --- a/usr/lib/python3.6/lxcmgr/lxcmgr.py +++ b/usr/lib/python3.6/lxcmgr/lxcmgr.py @@ -6,7 +6,7 @@ import shutil import subprocess from . import flock -from .paths import HOSTS_FILE, HOSTS_LOCK, LXC_ROOT +from .paths import HOSTS_FILE, HOSTS_LOCK, LXC_LOGS, LXC_ROOT, LXC_STORAGE_DIR from .templates import LXC_CONTAINER def prepare_container(container, layers): @@ -67,6 +67,7 @@ def create_container(container, image): def destroy_container(container): # Remove container configuration and directories shutil.rmtree(os.path.join(LXC_ROOT, container)) + os.unlink(os.path.join(LXC_LOGS, '{}.log'.format(container))) # Release the IP address update_hosts_lease(container, False) diff --git a/usr/lib/python3.6/lxcmgr/paths.py b/usr/lib/python3.6/lxcmgr/paths.py index da93de3..1ed139b 100644 --- a/usr/lib/python3.6/lxcmgr/paths.py +++ b/usr/lib/python3.6/lxcmgr/paths.py @@ -2,12 +2,14 @@ # Package manager REPO_CACHE_DIR = '/var/lib/lxcmgr/cache' +REPO_CONF_FILE = '/etc/lxcmgr/repo.json' REPO_LOCAL_FILE = '/var/lib/lxcmgr/packages' REPO_LOCK = '/var/lock/lxcmgr-repo.lock' -REPO_SIG_FILE = '/var/lib/lxcmgr/packages.pub' +REPO_SIG_FILE = '/etc/lxcmgr/packages.pub' # LXC HOSTS_FILE = '/etc/hosts' HOSTS_LOCK = '/var/lock/lxcmgr-hosts.lock' +LXC_LOGS = '/var/log/lxc' LXC_ROOT = '/var/lib/lxc' LXC_STORAGE_DIR = '/var/lib/lxcmgr/storage' diff --git a/usr/lib/python3.6/lxcmgr/pkgmgr.py b/usr/lib/python3.6/lxcmgr/pkgmgr.py index be49552..a2ee014 100644 --- a/usr/lib/python3.6/lxcmgr/pkgmgr.py +++ b/usr/lib/python3.6/lxcmgr/pkgmgr.py @@ -12,7 +12,7 @@ from pkg_resources import parse_version from . import crypto from . import flock from . import lxcmgr -from .paths import LXC_STORAGE_DIR, REPO_CACHE_DIR, REPO_LOCAL_FILE, REPO_LOCK +from .paths import LXC_STORAGE_DIR, REPO_CACHE_DIR, REPO_CONF_FILE, REPO_LOCAL_FILE, REPO_LOCK, REPO_SIG_FILE class Stage(Enum): QUEUED = 1 @@ -31,7 +31,7 @@ class RepoFileNotFound(Exception): class RepoBadRequest(Exception): pass -class AppInstall: +class App: def __init__(self, name): self.name = name self.stage = Stage.QUEUED @@ -44,22 +44,30 @@ class AppInstall: return min(99, round(self.bytes_downloaded / self.bytes_total * 100)) class PkgMgr: - def __init__(self, repo_url, repo_auth=None): - self.repo_url = repo_url - self.repo_auth = repo_auth - self.installed_packages = None + def __init__(self): + self.repo_url = None + self.repo_auth = None self.online_packages = None - - def load_installed_packages(self): with open(REPO_LOCAL_FILE, 'r') as f: self.installed_packages = json.load(f) - def save_installed_packages(self, packages): + def save_installed_packages(self): with open(REPO_LOCAL_FILE, 'w') as f: - json.dump(packages, f, sort_keys=True, indent=4) + json.dump(self.installed_packages, f, sort_keys=True, indent=4) + + def load_repo_conf(self): + with open(REPO_CONF_FILE, 'r') as f: + conf = json.load(f) + self.repo_url = conf['url'] + user = conf['user'] if 'user' in conf and conf['user'] else None + pwd = conf['pwd'] if 'pwd' in conf and conf['pwd'] else None + self.repo_auth = (user, pwd) if user else None def get_repo_resource(self, resource_url, stream=False): # Download requested repository resource + if not self.repo_url: + self.load_repo_conf() + # Make a HTTP request r = requests.get('{}/{}'.format(self.repo_url, resource_url), auth=self.repo_auth, timeout=5, stream=stream) if r.status_code == 401: raise RepoUnauthorized(r.url) @@ -73,65 +81,65 @@ class PkgMgr: # Fetches and verifies online packages. Can raise InvalidSignature packages = self.get_repo_resource('packages').content packages_sig = self.get_repo_resource('packages.sig').content - crypto.verify_signature(packages, packages_sig) + crypto.verify_signature(REPO_SIG_FILE, packages, packages_sig) self.online_packages = json.loads(packages) @flock.flock_ex(REPO_LOCK) def install_app(self, app): # Main installation function. Wrapper for download, registration and install script - if not self.installed_packages: - self.load_installed_packages() - # Request for installation of already installed app immediately returns with success if app.name in self.installed_packages['apps']: app.stage = Stage.DONE return if not self.online_packages: self.fetch_online_packages() # Get all packages on which the app depends and which have not been installed yet + #TODO: flatten and change name to "images" layers = [] images = [container['image'] for container in self.online_packages['apps'][app]['containers'].values()] for image in images: layers.extend(self.online_packages['images'][image]['layers']) layers = [layer for layer in set(layers) if layer not in self.installed_packages['images']] # Calculate bytes to download - app.bytes_total = sum(self.online_packages['images'][layer]['size'] for layer in layers) + self.online_packages['apps'][app.name]['size'] + app.bytes_total = sum(self.online_packages['images'][image]['size'] for image in layers) + self.online_packages['apps'][app.name]['size'] # Download layers and setup script files app.stage = Stage.DOWNLOAD - for layer in layers: - self.download_layer(app, layer) + for image in layers: + self.download_image(app, image) self.download_scripts(app) - # Purge old data (to clean previous failed installation) and unpack + # Purge old data to clean previous failed installation and unpack downloaded archives app.stage = Stage.UNPACK - for layer in layers: - self.purge_layer(layer) - self.unpack_layer(layer) + for image in layers: + self.purge_image(image) + self.unpack_image(image) + self.register_image(image, self.online_packages['images'][image]) self.purge_scripts(app.name) self.unpack_scripts(app.name) # Run setup scripts app.stage = Stage.INSTALL + # Run uninstall script to clean previous failed installation self.run_uninstall_script(app.name) # Build containers and services self.create_containers(app.name) - # Run install script and finish the installation + # TODO: Create services + # Run install script and register the app self.run_install_script(app.name) - self.installed_packages['apps'][app.name] = self.online_packages['apps'][app.name] - self.save_installed_packages() + self.register_app(app.name, self.online_packages['apps'][app.name]) app.stage = Stage.DONE - def download_layer(self, app, layer): - pkg_archive = 'images/{}.tar.xz'.format(layer) + def download_image(self, app, image): + # Download image archive and verify hash + pkg_archive = 'images/{}.tar.xz'.format(image) self.download_archive(app, pkg_archive) - # Verify hash - crypto.verify_hash(tmp_archive, self.online_packages['images'][layer]['sha512']) + crypto.verify_hash(tmp_archive, self.online_packages['images'][image]['sha512']) def download_scripts(self, app): + # Download scripts archive and verify hash pkg_archive = 'apps/{}.tar.xz'.format(app.name) self.download_archive(app, pkg_archive) - # Verify hash crypto.verify_hash(tmp_archive, self.online_packages['apps'][app.name]['sha512']) def download_archive(self, app, archive): - # Download the archive + # Download the archive from online repository tmp_archive = os.path.join(REPO_CACHE_DIR, pkg_archive) res = self.get_repo_resource('{}/{}'.format(type, pkg_archive), True) with open(tmp_archive, 'wb') as f: @@ -139,26 +147,30 @@ class PkgMgr: if chunk: app.bytes_downloaded += f.write(chunk) - def purge_layer(self, layer): + def purge_image(self, image): # Delete layer files from storage directory - shutil.rmtree(os.path.join(LXC_STORAGE_DIR, layer)) - if layer in self.installed_packages['images']: - del self.installed_packages['images'][layer] - self.save_installed_packages() + shutil.rmtree(os.path.join(LXC_STORAGE_DIR, image)) - def unpack_layer(self, layer): + def unpack_image(self, image): # Unpack layer archive - tmp_archive = os.path.join(REPO_CACHE_DIR, 'images/{}.tar.xz'.format(layer)) + tmp_archive = os.path.join(REPO_CACHE_DIR, 'images/{}.tar.xz'.format(image)) subprocess.run(['tar', 'xJf', tmp_archive], cwd=LXC_STORAGE_DIR, check=True) os.unlink(tmp_archive) - self.installed_packages['images'][layer] = self.online_packages['images'][layer] + + def register_image(self, image, metadata): + # Add installed layer to list of installed images + self.installed_packages['images'][image] = metadata self.save_installed_packages() + def unregister_image(self, image): + # Remove image from list of installed images + if image in self.installed_packages['images']: + del self.installed_packages['images'][image] + self.save_installed_packages() + def purge_scripts(self, app): # Delete application setup scripts from storage directory shutil.rmtree(os.path.join(REPO_CACHE_DIR, 'apps', app)) - del self.installed_packages['apps'][app] - self.save_installed_packages() def unpack_scripts(self, app): # Unpack setup scripts archive @@ -180,6 +192,17 @@ class PkgMgr: if os.path.exists(script_path): subprocess.run(script_path, check=True) + def register_app(self, app, metadata): + # Register installed app in list of installed apps + self.installed_packages['apps'][app] = metadata + self.save_installed_packages() + + def unregister_app(self, app): + # Remove app from list of installed apps + if app in self.installed_packages['apps']: + del self.installed_packages['apps'][app] + self.save_installed_packages() + def create_containers(self, app): # Create LXC containers from image and app metadata for container in self.online_packages['apps'][app]['containers']: @@ -188,10 +211,13 @@ class PkgMgr: if 'mounts' in self.online_packages['apps'][app]['containers'][container]: image['mounts'] = self.online_packages['apps'][app]['containers'][container]['mounts'] lxcmgr.create_container(container, image) + # TODO: Create services @flock.flock_ex(REPO_LOCK) def uninstall_app(self, app): # Main uninstallation function. Wrapper for uninstall script and filesystem purge + if app not in self.installed_packages['apps']: + return self.run_uninstall_script(app) self.destroy_containers(app) self.purge_scripts(app) @@ -218,8 +244,8 @@ class PkgMgr: def update_app(self, app, item): # Main update function. # TODO: Implement actual update - #uninstall_app(app) - #install_app(app, item) + uninstall_app(app) + install_app(app, item) def has_update(self, app): if not self.installed_packages: diff --git a/usr/lib/python3.6/lxcmgr/templates.py b/usr/lib/python3.6/lxcmgr/templates.py index a170447..7eeb32e 100644 --- a/usr/lib/python3.6/lxcmgr/templates.py +++ b/usr/lib/python3.6/lxcmgr/templates.py @@ -40,8 +40,8 @@ lxc.idmap = u 0 100000 65536 lxc.idmap = g 0 100000 65536 # Hooks -lxc.hook.pre-start = /usr/bin/vmmgr prepare-container {layers} -lxc.hook.post-stop = /usr/bin/vmmgr cleanup-container +lxc.hook.pre-start = /usr/bin/lxcmgr container prepare {layers} +lxc.hook.post-stop = /usr/bin/lxcmgr container cleanup # Other lxc.arch = linux64 diff --git a/var/lib/lxcmgr/packages b/var/lib/lxcmgr/packages new file mode 100644 index 0000000..a6d386a --- /dev/null +++ b/var/lib/lxcmgr/packages @@ -0,0 +1 @@ +{"apps": {}, "images": {}}