From ce3fec436459961a6522e263045c51cf2d260825 Mon Sep 17 00:00:00 2001 From: Disassembler Date: Mon, 23 Sep 2019 15:30:06 +0200 Subject: [PATCH] Fix install, print CLI installation status --- usr/bin/lxcmgr | 61 +++++++++++++++++++++++----- usr/lib/python3.6/lxcmgr/lxcmgr.py | 12 ++++-- usr/lib/python3.6/lxcmgr/pkgmgr.py | 65 +++++++++++++++++------------- usr/lib/python3.6/vmmgr/vmmgr.py | 7 +++- usr/lib/python3.6/vmmgr/wsgiapp.py | 2 +- 5 files changed, 102 insertions(+), 45 deletions(-) diff --git a/usr/bin/lxcmgr b/usr/bin/lxcmgr index 9627c8a..42d9f29 100644 --- a/usr/bin/lxcmgr +++ b/usr/bin/lxcmgr @@ -2,9 +2,13 @@ # -*- coding: utf-8 -*- import argparse +import time +import sys + +from concurrent.futures import ThreadPoolExecutor from lxcmgr import lxcmgr -from lxcmgr.pkgmgr import App, PkgMgr +from lxcmgr.pkgmgr import App, Stage, PkgMgr parser = argparse.ArgumentParser(description='LXC container and package manager') subparsers = parser.add_subparsers() @@ -53,25 +57,58 @@ def print_apps(packages): def list_online(): pm = PkgMgr() pm.fetch_online_packages() - print_apps(pm.online_packages['apps']) + apps = pm.online_packages['apps'] + if apps: + print_apps(apps) + else: + print('Repository lists no applications packages.') def list_installed(): pm = PkgMgr() - pm.load_installed_packages() - print_apps(pm.installed_packages['apps']) + apps = pm.installed_packages['apps'] + if apps: + print_apps(apps) + else: + print('No applications packages installed.') def list_updates(): pm = PkgMgr() - pm.load_installed_packages() - apps = [app for app in pm.installed_packages if pm.has_update(app)] - updates = {name: meta for (name, meta) in pm.online_packages['apps'].items() if name in apps} - print_apps(updates) + apps = pm.installed_packages['apps'] + if apps: + updateable_apps = [app for app in apps if pm.has_update(app)] + if updateable_apps: + updates = {name: meta for (name, meta) in pm.online_packages['apps'].items() if name in updateable_apps} + print_apps(updates) + else: + print('All installed application packages are up-to-date.') + else: + print('No applications packages installed.') def install_app(app): pm = PkgMgr() app = App(app) - pm.install_app(app) - # TODO: periodicky vypisovat output + with ThreadPoolExecutor() as executor: + future = executor.submit(pm.install_app, app) + while not future.done(): + time.sleep(1) + print_install_status(app) + # Get the result of the future and let it raise exception, if there was any + data = future.result() + print_install_status(app) + +def print_install_status(app): + if app.stage == Stage.QUEUED: + print('\x1b[KQueued...', end='\r') + elif app.stage == Stage.DOWNLOAD: + print('\x1b[KDownloading... {} % ({} / {} bytes)'.format(app.percent_processed, app.bytes_processed, app.bytes_total), end='\r') + elif app.stage == Stage.UNPACK: + print('\x1b[KUnpacking...', end='\r') + elif app.stage == Stage.INSTALL: + print('\x1b[KInstalling...', end='\r') + elif app.stage == Stage.UNINSTALL: + print('\x1b[KUninstalling...', end='\r') + elif app.stage == Stage.DONE: + print('\x1b[KDone.') def update_app(app): pm = PkgMgr() @@ -82,6 +119,10 @@ def uninstall_app(app): pm.uninstall_app(app) args = parser.parse_args() +if not hasattr(args, 'action'): + parser.print_usage() + sys.exit(1) + if args.action == 'list-installed': list_installed() elif args.action == 'list-online': diff --git a/usr/lib/python3.6/lxcmgr/lxcmgr.py b/usr/lib/python3.6/lxcmgr/lxcmgr.py index 3ed7886..f066a41 100644 --- a/usr/lib/python3.6/lxcmgr/lxcmgr.py +++ b/usr/lib/python3.6/lxcmgr/lxcmgr.py @@ -50,7 +50,7 @@ def create_container(container, image): # Create container configuration file layers = ','.join([os.path.join(LXC_STORAGE_DIR, layer) for layer in image['layers']]) if 'build' not in image: - layers = '{},{}'.format(layer, ephemeral) + layers = '{},{}'.format(layers, ephemeral) mounts = '\n'.join(['lxc.mount.entry = {} {} none bind,create={} 0 0'.format(m[1], m[2].lstrip('/'), m[0].lower()) for m in image['mounts']]) if 'mounts' in image else '' env = '\n'.join(['lxc.environment = {}={}'.format(e[0], e[1]) for e in image['env']]) if 'env' in image else '' uid = image['uid'] if 'uid' in image else '0' @@ -66,8 +66,14 @@ def create_container(container, image): def destroy_container(container): # Remove container configuration and directories - shutil.rmtree(os.path.join(LXC_ROOT, container)) - os.unlink(os.path.join(LXC_LOGS, '{}.log'.format(container))) + try: + shutil.rmtree(os.path.join(LXC_ROOT, container)) + except FileNotFoundError: + pass + try: + os.unlink(os.path.join(LXC_LOGS, '{}.log'.format(container))) + except FileNotFoundError: + pass # Release the IP address update_hosts_lease(container, False) diff --git a/usr/lib/python3.6/lxcmgr/pkgmgr.py b/usr/lib/python3.6/lxcmgr/pkgmgr.py index a2ee014..75f7318 100644 --- a/usr/lib/python3.6/lxcmgr/pkgmgr.py +++ b/usr/lib/python3.6/lxcmgr/pkgmgr.py @@ -36,12 +36,12 @@ class App: self.name = name self.stage = Stage.QUEUED self.bytes_total = 1 - self.bytes_downloaded = 0 + self.bytes_processed = 0 @property - def percent_downloaded(self): + def percent_processed(self): # Limit the displayed percentage to 0 - 99 - return min(99, round(self.bytes_downloaded / self.bytes_total * 100)) + return min(99, round(self.bytes_processed / self.bytes_total * 100)) class PkgMgr: def __init__(self): @@ -92,23 +92,22 @@ class PkgMgr: return if not self.online_packages: self.fetch_online_packages() - # Get all packages on which the app depends and which have not been installed yet - #TODO: flatten and change name to "images" - layers = [] - images = [container['image'] for container in self.online_packages['apps'][app]['containers'].values()] - for image in images: - layers.extend(self.online_packages['images'][image]['layers']) - layers = [layer for layer in set(layers) if layer not in self.installed_packages['images']] + # Get all packages on which the app and its containers depend and which have not been installed yet + images = [] + image_deps = [container['image'] for container in self.online_packages['apps'][app.name]['containers'].values()] + for image in image_deps: + images.extend(self.online_packages['images'][image]['layers']) + images = [image for image in set(images) if image not in self.installed_packages['images']] # Calculate bytes to download - app.bytes_total = sum(self.online_packages['images'][image]['size'] for image in layers) + self.online_packages['apps'][app.name]['size'] + app.bytes_total = sum(self.online_packages['images'][image]['size'] for image in images) + self.online_packages['apps'][app.name]['size'] # Download layers and setup script files app.stage = Stage.DOWNLOAD - for image in layers: + for image in images: self.download_image(app, image) self.download_scripts(app) # Purge old data to clean previous failed installation and unpack downloaded archives app.stage = Stage.UNPACK - for image in layers: + for image in images: self.purge_image(image) self.unpack_image(image) self.register_image(image, self.online_packages['images'][image]) @@ -128,28 +127,30 @@ class PkgMgr: def download_image(self, app, image): # Download image archive and verify hash - pkg_archive = 'images/{}.tar.xz'.format(image) - self.download_archive(app, pkg_archive) - crypto.verify_hash(tmp_archive, self.online_packages['images'][image]['sha512']) + archive = 'images/{}.tar.xz'.format(image) + self.download_archive(app, archive, self.online_packages['images'][image]['sha512']) def download_scripts(self, app): # Download scripts archive and verify hash - pkg_archive = 'apps/{}.tar.xz'.format(app.name) - self.download_archive(app, pkg_archive) - crypto.verify_hash(tmp_archive, self.online_packages['apps'][app.name]['sha512']) + archive = 'apps/{}.tar.xz'.format(app.name) + self.download_archive(app, archive, self.online_packages['apps'][app.name]['sha512']) - def download_archive(self, app, archive): + def download_archive(self, app, archive, hash): # Download the archive from online repository - tmp_archive = os.path.join(REPO_CACHE_DIR, pkg_archive) - res = self.get_repo_resource('{}/{}'.format(type, pkg_archive), True) + tmp_archive = os.path.join(REPO_CACHE_DIR, archive) + res = self.get_repo_resource(archive, True) with open(tmp_archive, 'wb') as f: for chunk in res.iter_content(chunk_size=65536): if chunk: - app.bytes_downloaded += f.write(chunk) + app.bytes_processed += f.write(chunk) + crypto.verify_hash(tmp_archive, hash) def purge_image(self, image): # Delete layer files from storage directory - shutil.rmtree(os.path.join(LXC_STORAGE_DIR, image)) + try: + shutil.rmtree(os.path.join(LXC_STORAGE_DIR, image)) + except FileNotFoundError: + pass def unpack_image(self, image): # Unpack layer archive @@ -170,7 +171,10 @@ class PkgMgr: def purge_scripts(self, app): # Delete application setup scripts from storage directory - shutil.rmtree(os.path.join(REPO_CACHE_DIR, 'apps', app)) + try: + shutil.rmtree(os.path.join(REPO_CACHE_DIR, 'apps', app)) + except FileNotFoundError: + pass def unpack_scripts(self, app): # Unpack setup scripts archive @@ -188,7 +192,7 @@ class PkgMgr: def run_script(self, app, script): # Runs script for an app, if the script is present - script_path = os.path.join(REPO_CACHE_DIR, app, script) + script_path = os.path.join(REPO_CACHE_DIR, 'apps', app, script) if os.path.exists(script_path): subprocess.run(script_path, check=True) @@ -207,7 +211,7 @@ class PkgMgr: # Create LXC containers from image and app metadata for container in self.online_packages['apps'][app]['containers']: image = self.online_packages['apps'][app]['containers'][container]['image'] - image = self.online_packages['images'][image]['containers'].copy() + image = self.online_packages['images'][image].copy() if 'mounts' in self.online_packages['apps'][app]['containers'][container]: image['mounts'] = self.online_packages['apps'][app]['containers'][container]['mounts'] lxcmgr.create_container(container, image) @@ -248,8 +252,11 @@ class PkgMgr: install_app(app, item) def has_update(self, app): - if not self.installed_packages: - self.load_installed_packages() + # Check if online repository list a newer version of app if not self.online_packages: self.fetch_online_packages() + if app not in self.online_packages['apps']: + # Application has been removed from online repo + return False + # Compare version strings return parse_version(self.installed_packages['apps'][app]['version']) < parse_version(self.online_packages['apps'][app]['version']) diff --git a/usr/lib/python3.6/vmmgr/vmmgr.py b/usr/lib/python3.6/vmmgr/vmmgr.py index c75a008..5cec21c 100644 --- a/usr/lib/python3.6/vmmgr/vmmgr.py +++ b/usr/lib/python3.6/vmmgr/vmmgr.py @@ -51,8 +51,11 @@ class VMMgr: def unregister_proxy(self, app): # Remove proxy configuration and reload nginx - os.unlink(os.path.join(NGINX_DIR, '{}.conf'.format(app))) - self.reload_nginx() + try: + os.unlink(os.path.join(NGINX_DIR, '{}.conf'.format(app))) + self.reload_nginx() + except FileNotFoundError: + pass def update_host(self, domain, port): # Update domain and port, rebuild all configuration and restart nginx diff --git a/usr/lib/python3.6/vmmgr/wsgiapp.py b/usr/lib/python3.6/vmmgr/wsgiapp.py index f09dcf7..8628752 100644 --- a/usr/lib/python3.6/vmmgr/wsgiapp.py +++ b/usr/lib/python3.6/vmmgr/wsgiapp.py @@ -229,7 +229,7 @@ class WSGIApp: actions = None else: if item.data.stage == Stage.DOWNLOAD: - status = '{} ({} %)'.format(lang.status_downloading(), item.data.percent_downloaded) + status = '{} ({} %)'.format(lang.status_downloading(), item.data.percent_processed) elif item.data.stage == Stage.UNPACK: status = lang.status_unpacking() elif item.data.stage == Stage.INSTALL_DEPS: