# -*- coding: utf-8 -*- import json import os import requests import shutil import subprocess from pkg_resources import parse_version from . import crypto from . import flock from . import lxcmgr from . import svcmgr from .paths import LXC_STORAGE_DIR, REPO_CACHE_DIR, REPO_CONF_FILE, REPO_LOCAL_FILE, REPO_LOCK, REPO_SIG_FILE class Stage: QUEUED = 1 DOWNLOAD = 2 UNPACK = 3 INSTALL = 4 UNINSTALL = 5 UPDATE = 6 DONE = 7 class RepoUnauthorized(Exception): pass class RepoFileNotFound(Exception): pass class RepoBadRequest(Exception): pass class App: def __init__(self, name): self.name = name self.stage = Stage.QUEUED self.bytes_total = 1 self.bytes_processed = 0 @property def percent_processed(self): # Limit the displayed percentage to 0 - 99 return min(99, round(self.bytes_processed / self.bytes_total * 100)) class PkgMgr: def __init__(self): self.repo_url = None self.repo_auth = None self._online_packages = None with open(REPO_LOCAL_FILE, 'r') as f: self.installed_packages = json.load(f) @property def online_packages(self): if not self._online_packages: # Fetches and verifies online packages. Can raise InvalidSignature packages = self.get_repo_resource('packages').content packages_sig = self.get_repo_resource('packages.sig').content crypto.verify_signature(REPO_SIG_FILE, packages, packages_sig) self._online_packages = json.loads(packages) return self._online_packages def save_installed_packages(self): with open(REPO_LOCAL_FILE, 'w') as f: json.dump(self.installed_packages, f, sort_keys=True, indent=4) def load_repo_conf(self): with open(REPO_CONF_FILE, 'r') as f: conf = json.load(f) self.repo_url = conf['url'] user = conf['user'] if 'user' in conf and conf['user'] else None pwd = conf['pwd'] if 'pwd' in conf and conf['pwd'] else None self.repo_auth = (user, pwd) if user else None def get_repo_resource(self, resource_url, stream=False): # Download requested repository resource if not self.repo_url: self.load_repo_conf() # Make a HTTP request r = requests.get('{}/{}'.format(self.repo_url, resource_url), auth=self.repo_auth, timeout=5, stream=stream) if r.status_code == 401: raise RepoUnauthorized(r.url) elif r.status_code == 404: raise RepoFileNotFound(r.url) elif r.status_code != 200: raise RepoBadRequest(r.url) return r @flock.flock_ex(REPO_LOCK) def install_app(self, app): # Main installation function. Wrapper for download, registration and install script if app.name in self.installed_packages['apps']: app.stage = Stage.DONE return self.download_and_unpack_deps(app) # Run setup scripts app.stage = Stage.INSTALL # Run uninstall script to clean previous failed installation self.run_script(app.name, 'uninstall') # Build containers and services self.create_containers(self.online_packages['apps'][app.name]['containers']) # Run install script and register the app self.run_script(app.name, 'install') self.register_app(app.name, self.online_packages['apps'][app.name]) app.stage = Stage.DONE def download_and_unpack_deps(self, app): # Common download and unpack function for install and update # Get all packages on which the app and its containers depend and which have not been installed yet images = [] image_deps = [container['image'] for container in self.online_packages['apps'][app.name]['containers'].values()] for image in image_deps: images.extend(self.online_packages['images'][image]['layers']) images = [image for image in set(images) if image not in self.installed_packages['images']] # Calculate bytes to download app.bytes_total = sum(self.online_packages['images'][image]['pkgsize'] for image in images) + self.online_packages['apps'][app.name]['pkgsize'] # Download layers and setup script files app.stage = Stage.DOWNLOAD for image in images: self.download_image(app, image) self.download_scripts(app) # Purge old data to clean previous failed installation and unpack downloaded archives app.stage = Stage.UNPACK self.destroy_containers(self.online_packages['apps'][app.name]['containers']) for image in images: self.purge_image(image) self.unpack_image(image) self.register_image(image, self.online_packages['images'][image]) self.purge_scripts(app.name) self.unpack_scripts(app.name) def download_image(self, app, image): # Download image archive and verify hash archive = 'images/{}.tar.xz'.format(image) self.download_archive(app, archive, self.online_packages['images'][image]['sha512']) def download_scripts(self, app): # Download scripts archive and verify hash archive = 'apps/{}.tar.xz'.format(app.name) self.download_archive(app, archive, self.online_packages['apps'][app.name]['sha512']) def download_archive(self, app, archive, hash): # Download the archive from online repository tmp_archive = os.path.join(REPO_CACHE_DIR, archive) res = self.get_repo_resource(archive, True) with open(tmp_archive, 'wb') as f: for chunk in res.iter_content(chunk_size=65536): if chunk: app.bytes_processed += f.write(chunk) crypto.verify_hash(tmp_archive, hash) def purge_image(self, image): # Delete layer files from storage directory try: shutil.rmtree(os.path.join(LXC_STORAGE_DIR, image)) except FileNotFoundError: pass def unpack_image(self, image): # Unpack layer archive tmp_archive = os.path.join(REPO_CACHE_DIR, 'images/{}.tar.xz'.format(image)) subprocess.run(['tar', 'xJf', tmp_archive], cwd=LXC_STORAGE_DIR, check=True) os.unlink(tmp_archive) def register_image(self, image, metadata): # Add installed layer to list of installed images self.installed_packages['images'][image] = metadata self.save_installed_packages() def unregister_image(self, image): # Remove image from list of installed images if image in self.installed_packages['images']: del self.installed_packages['images'][image] self.save_installed_packages() def purge_scripts(self, app): # Delete application setup scripts from storage directory try: shutil.rmtree(os.path.join(REPO_CACHE_DIR, 'apps', app)) except FileNotFoundError: pass def unpack_scripts(self, app): # Unpack setup scripts archive tmp_archive = os.path.join(REPO_CACHE_DIR, 'apps/{}.tar.xz'.format(app)) subprocess.run(['tar', 'xJf', tmp_archive], cwd=os.path.join(REPO_CACHE_DIR, 'apps'), check=True) os.unlink(tmp_archive) def run_script(self, app, action): # Runs script for an app, if the script is present cache_dir = os.path.join(REPO_CACHE_DIR, 'apps', app) script_dir = os.path.join(cache_dir, action) script_path = '{}.sh'.format(script_dir) if os.path.exists(script_path): # Run the script in its working directory, if there is one, so it doesn't have to figure out paths to packaged files cwd = script_dir if os.path.exists(script_dir) else cache_dir subprocess.run(script_path, cwd=cwd, check=True) def register_app(self, app, metadata): # Register installed app in list of installed apps self.installed_packages['apps'][app] = metadata self.save_installed_packages() def unregister_app(self, app): # Remove app from list of installed apps if app in self.installed_packages['apps']: del self.installed_packages['apps'][app] self.save_installed_packages() def create_containers(self, containers): # Create LXC containers from image and app metadata for container in containers: image = containers[container]['image'] image = self.online_packages['images'][image].copy() if 'mounts' in containers[container]: image['mounts'] = containers[container]['mounts'] if 'depends' in containers[container]: image['depends'] = containers[container]['depends'] lxcmgr.create_container(container, image) svcmgr.create_service(container, image) svcmgr.update_services() def destroy_containers(self, containers): # Destroy LXC containers for container in containers: svcmgr.delete_service(container) lxcmgr.destroy_container(container) svcmgr.update_services() @flock.flock_ex(REPO_LOCK) def uninstall_app(self, app): # Main uninstallation function. Wrapper for uninstall script and filesystem purge if app.name not in self.installed_packages['apps']: app.stage = Stage.DONE return app.stage = Stage.UNINSTALL self.run_script(app.name, 'uninstall') self.destroy_containers(self.installed_packages['apps'][app.name]['containers']) self.purge_scripts(app.name) self.unregister_app(app.name) self.purge_unused_layers() app.stage = Stage.DONE def purge_unused_layers(self): # Remove layers which are no longer used by any installed application layers = set(os.listdir(LXC_STORAGE_DIR)) for app in self.installed_packages['apps']: for container in self.installed_packages['apps'][app]['containers']: image = self.installed_packages['apps'][app]['containers'][container]['image'] for layer in self.installed_packages['images'][image]['layers']: if layer in layers: # Layer is still used, remove from set of layers to be purged layers.remove(layer) for layer in layers: self.purge_image(layer) @flock.flock_ex(REPO_LOCK) def update_app(self, app): # Main update function. self.download_and_unpack_deps(app) # Run setup scripts app.stage = Stage.UPDATE # Build containers and services self.create_containers(self.online_packages['apps'][app.name]['containers']) # Run update script and register the app self.run_script(app.name, 'update') self.register_app(app.name, self.online_packages['apps'][app.name]) app.stage = Stage.DONE def has_update(self, app): # Check if online repository list a newer version of app if app not in self.online_packages['apps']: # Application has been removed from online repo return False # Compare version strings return parse_version(self.installed_packages['apps'][app]['version']) < parse_version(self.online_packages['apps'][app]['version'])