# -*- coding: utf-8 -*- import json import os import requests import shutil import subprocess from enum import Enum from pkg_resources import parse_version from . import crypto from . import flock from . import lxcmgr from .paths import LXC_STORAGE_DIR, REPO_CACHE_DIR, REPO_CONF_FILE, REPO_LOCAL_FILE, REPO_LOCK, REPO_SIG_FILE class Stage(Enum): QUEUED = 1 DOWNLOAD = 2 UNPACK = 3 INSTALL = 4 UNINSTALL = 5 DONE = 6 class RepoUnauthorized(Exception): pass class RepoFileNotFound(Exception): pass class RepoBadRequest(Exception): pass class App: def __init__(self, name): self.name = name self.stage = Stage.QUEUED self.bytes_total = 1 self.bytes_processed = 0 @property def percent_processed(self): # Limit the displayed percentage to 0 - 99 return min(99, round(self.bytes_processed / self.bytes_total * 100)) class PkgMgr: def __init__(self): self.repo_url = None self.repo_auth = None self.online_packages = None with open(REPO_LOCAL_FILE, 'r') as f: self.installed_packages = json.load(f) def save_installed_packages(self): with open(REPO_LOCAL_FILE, 'w') as f: json.dump(self.installed_packages, f, sort_keys=True, indent=4) def load_repo_conf(self): with open(REPO_CONF_FILE, 'r') as f: conf = json.load(f) self.repo_url = conf['url'] user = conf['user'] if 'user' in conf and conf['user'] else None pwd = conf['pwd'] if 'pwd' in conf and conf['pwd'] else None self.repo_auth = (user, pwd) if user else None def get_repo_resource(self, resource_url, stream=False): # Download requested repository resource if not self.repo_url: self.load_repo_conf() # Make a HTTP request r = requests.get('{}/{}'.format(self.repo_url, resource_url), auth=self.repo_auth, timeout=5, stream=stream) if r.status_code == 401: raise RepoUnauthorized(r.url) elif r.status_code == 404: raise RepoFileNotFound(r.url) elif r.status_code != 200: raise RepoBadRequest(r.url) return r def fetch_online_packages(self): # Fetches and verifies online packages. Can raise InvalidSignature packages = self.get_repo_resource('packages').content packages_sig = self.get_repo_resource('packages.sig').content crypto.verify_signature(REPO_SIG_FILE, packages, packages_sig) self.online_packages = json.loads(packages) @flock.flock_ex(REPO_LOCK) def install_app(self, app): # Main installation function. Wrapper for download, registration and install script if app.name in self.installed_packages['apps']: app.stage = Stage.DONE return if not self.online_packages: self.fetch_online_packages() # Get all packages on which the app and its containers depend and which have not been installed yet images = [] image_deps = [container['image'] for container in self.online_packages['apps'][app.name]['containers'].values()] for image in image_deps: images.extend(self.online_packages['images'][image]['layers']) images = [image for image in set(images) if image not in self.installed_packages['images']] # Calculate bytes to download app.bytes_total = sum(self.online_packages['images'][image]['size'] for image in images) + self.online_packages['apps'][app.name]['size'] # Download layers and setup script files app.stage = Stage.DOWNLOAD for image in images: self.download_image(app, image) self.download_scripts(app) # Purge old data to clean previous failed installation and unpack downloaded archives app.stage = Stage.UNPACK for image in images: self.purge_image(image) self.unpack_image(image) self.register_image(image, self.online_packages['images'][image]) self.purge_scripts(app.name) self.unpack_scripts(app.name) # Run setup scripts app.stage = Stage.INSTALL # Run uninstall script to clean previous failed installation self.run_uninstall_script(app.name) # Build containers and services self.create_containers(app.name) # TODO: Create services # Run install script and register the app self.run_install_script(app.name) self.register_app(app.name, self.online_packages['apps'][app.name]) app.stage = Stage.DONE def download_image(self, app, image): # Download image archive and verify hash archive = 'images/{}.tar.xz'.format(image) self.download_archive(app, archive, self.online_packages['images'][image]['sha512']) def download_scripts(self, app): # Download scripts archive and verify hash archive = 'apps/{}.tar.xz'.format(app.name) self.download_archive(app, archive, self.online_packages['apps'][app.name]['sha512']) def download_archive(self, app, archive, hash): # Download the archive from online repository tmp_archive = os.path.join(REPO_CACHE_DIR, archive) res = self.get_repo_resource(archive, True) with open(tmp_archive, 'wb') as f: for chunk in res.iter_content(chunk_size=65536): if chunk: app.bytes_processed += f.write(chunk) crypto.verify_hash(tmp_archive, hash) def purge_image(self, image): # Delete layer files from storage directory try: shutil.rmtree(os.path.join(LXC_STORAGE_DIR, image)) except FileNotFoundError: pass def unpack_image(self, image): # Unpack layer archive tmp_archive = os.path.join(REPO_CACHE_DIR, 'images/{}.tar.xz'.format(image)) subprocess.run(['tar', 'xJf', tmp_archive], cwd=LXC_STORAGE_DIR, check=True) os.unlink(tmp_archive) def register_image(self, image, metadata): # Add installed layer to list of installed images self.installed_packages['images'][image] = metadata self.save_installed_packages() def unregister_image(self, image): # Remove image from list of installed images if image in self.installed_packages['images']: del self.installed_packages['images'][image] self.save_installed_packages() def purge_scripts(self, app): # Delete application setup scripts from storage directory try: shutil.rmtree(os.path.join(REPO_CACHE_DIR, 'apps', app)) except FileNotFoundError: pass def unpack_scripts(self, app): # Unpack setup scripts archive tmp_archive = os.path.join(REPO_CACHE_DIR, 'apps/{}.tar.xz'.format(app)) subprocess.run(['tar', 'xJf', tmp_archive], cwd=os.path.join(REPO_CACHE_DIR, 'apps'), check=True) os.unlink(tmp_archive) def run_uninstall_script(self, app): # Runs uninstall.sh for an app, if the script is present self.run_script(app, 'uninstall.sh') def run_install_script(self, app): # Runs install.sh for a package, if the script is present self.run_script(app, 'install.sh') def run_script(self, app, script): # Runs script for an app, if the script is present script_path = os.path.join(REPO_CACHE_DIR, 'apps', app, script) if os.path.exists(script_path): subprocess.run(script_path, check=True) def register_app(self, app, metadata): # Register installed app in list of installed apps self.installed_packages['apps'][app] = metadata self.save_installed_packages() def unregister_app(self, app): # Remove app from list of installed apps if app in self.installed_packages['apps']: del self.installed_packages['apps'][app] self.save_installed_packages() def create_containers(self, app): # Create LXC containers from image and app metadata for container in self.online_packages['apps'][app]['containers']: image = self.online_packages['apps'][app]['containers'][container]['image'] image = self.online_packages['images'][image].copy() if 'mounts' in self.online_packages['apps'][app]['containers'][container]: image['mounts'] = self.online_packages['apps'][app]['containers'][container]['mounts'] lxcmgr.create_container(container, image) # TODO: Create services @flock.flock_ex(REPO_LOCK) def uninstall_app(self, app): # Main uninstallation function. Wrapper for uninstall script and filesystem purge if app not in self.installed_packages['apps']: return self.run_uninstall_script(app) self.destroy_containers(app) self.purge_scripts(app) self.purge_unused_layers() def destroy_containers(self, app): # Destroy LXC containers for container in self.installed_packages['apps'][app]['containers']: lxcmgr.destroy_container(container) def purge_unused_layers(self): # Remove layers which are no longer used by any installed application layers = set(os.list(LXC_STORAGE_DIR)) for app in self.installed_packages['apps']: for container in self.installed_packages['apps'][app]['containers']: image = self.installed_packages['apps'][app]['containers'][container]['image'] for layer in self.installed_packages['images'][image]['layers']: if layer in layers: del layers[layer] for layer in layers: self.purge_layer(layer) @flock.flock_ex(REPO_LOCK) def update_app(self, app, item): # Main update function. # TODO: Implement actual update uninstall_app(app) install_app(app, item) def has_update(self, app): # Check if online repository list a newer version of app if not self.online_packages: self.fetch_online_packages() if app not in self.online_packages['apps']: # Application has been removed from online repo return False # Compare version strings return parse_version(self.installed_packages['apps'][app]['version']) < parse_version(self.online_packages['apps'][app]['version'])