# -*- coding: utf-8 -*- import json import os import requests import shutil import subprocess from enum import Enum from pkg_resources import parse_version from . import crypto from . import flock from . import lxcmgr from .paths import LXC_STORAGE_DIR, REPO_CACHE_DIR, REPO_LOCAL_FILE, REPO_LOCK class Stage(Enum): QUEUED = 1 DOWNLOAD = 2 UNPACK = 3 INSTALL = 4 UNINSTALL = 5 DONE = 6 class RepoUnauthorized(Exception): pass class RepoFileNotFound(Exception): pass class RepoBadRequest(Exception): pass class AppInstall: def __init__(self, name): self.name = name self.stage = Stage.QUEUED self.bytes_total = 1 self.bytes_downloaded = 0 @property def percent_downloaded(self): # Limit the displayed percentage to 0 - 99 return min(99, round(self.bytes_downloaded / self.bytes_total * 100)) class PkgMgr: def __init__(self, repo_url, repo_auth=None): self.repo_url = repo_url self.repo_auth = repo_auth self.installed_packages = None self.online_packages = None def load_installed_packages(self): with open(REPO_LOCAL_FILE, 'r') as f: self.installed_packages = json.load(f) def save_installed_packages(self, packages): with open(REPO_LOCAL_FILE, 'w') as f: json.dump(packages, f, sort_keys=True, indent=4) def get_repo_resource(self, resource_url, stream=False): # Download requested repository resource r = requests.get('{}/{}'.format(self.repo_url, resource_url), auth=self.repo_auth, timeout=5, stream=stream) if r.status_code == 401: raise RepoUnauthorized(r.url) elif r.status_code == 404: raise RepoFileNotFound(r.url) elif r.status_code != 200: raise RepoBadRequest(r.url) return r def fetch_online_packages(self): # Fetches and verifies online packages. Can raise InvalidSignature packages = self.get_repo_resource('packages').content packages_sig = self.get_repo_resource('packages.sig').content crypto.verify_signature(packages, packages_sig) self.online_packages = json.loads(packages) @flock.flock_ex(REPO_LOCK) def install_app(self, app): # Main installation function. Wrapper for download, registration and install script if not self.installed_packages: self.load_installed_packages() # Request for installation of already installed app immediately returns with success if app.name in self.installed_packages['apps']: app.stage = Stage.DONE return if not self.online_packages: self.fetch_online_packages() # Get all packages on which the app depends and which have not been installed yet layers = [] images = [container['image'] for container in self.online_packages['apps'][app]['containers'].values()] for image in images: layers.extend(self.online_packages['images'][image]['layers']) layers = [layer for layer in set(layers) if layer not in self.installed_packages['images']] # Calculate bytes to download app.bytes_total = sum(self.online_packages['images'][layer]['size'] for layer in layers) + self.online_packages['apps'][app.name]['size'] # Download layers and setup script files app.stage = Stage.DOWNLOAD for layer in layers: self.download_layer(app, layer) self.download_scripts(app) # Purge old data (to clean previous failed installation) and unpack app.stage = Stage.UNPACK for layer in layers: self.purge_layer(layer) self.unpack_layer(layer) self.purge_scripts(app.name) self.unpack_scripts(app.name) # Run setup scripts app.stage = Stage.INSTALL self.run_uninstall_script(app.name) # Build containers and services self.create_containers(app.name) # Run install script and finish the installation self.run_install_script(app.name) self.installed_packages['apps'][app.name] = self.online_packages['apps'][app.name] self.save_installed_packages() app.stage = Stage.DONE def download_layer(self, app, layer): pkg_archive = 'images/{}.tar.xz'.format(layer) self.download_archive(app, pkg_archive) # Verify hash crypto.verify_hash(tmp_archive, self.online_packages['images'][layer]['sha512']) def download_scripts(self, app): pkg_archive = 'apps/{}.tar.xz'.format(app.name) self.download_archive(app, pkg_archive) # Verify hash crypto.verify_hash(tmp_archive, self.online_packages['apps'][app.name]['sha512']) def download_archive(self, app, archive): # Download the archive tmp_archive = os.path.join(REPO_CACHE_DIR, pkg_archive) res = self.get_repo_resource('{}/{}'.format(type, pkg_archive), True) with open(tmp_archive, 'wb') as f: for chunk in res.iter_content(chunk_size=65536): if chunk: app.bytes_downloaded += f.write(chunk) def purge_layer(self, layer): # Delete layer files from storage directory shutil.rmtree(os.path.join(LXC_STORAGE_DIR, layer)) if layer in self.installed_packages['images']: del self.installed_packages['images'][layer] self.save_installed_packages() def unpack_layer(self, layer): # Unpack layer archive tmp_archive = os.path.join(REPO_CACHE_DIR, 'images/{}.tar.xz'.format(layer)) subprocess.run(['tar', 'xJf', tmp_archive], cwd=LXC_STORAGE_DIR, check=True) os.unlink(tmp_archive) self.installed_packages['images'][layer] = self.online_packages['images'][layer] self.save_installed_packages() def purge_scripts(self, app): # Delete application setup scripts from storage directory shutil.rmtree(os.path.join(REPO_CACHE_DIR, 'apps', app)) del self.installed_packages['apps'][app] self.save_installed_packages() def unpack_scripts(self, app): # Unpack setup scripts archive tmp_archive = os.path.join(REPO_CACHE_DIR, 'apps/{}.tar.xz'.format(app)) subprocess.run(['tar', 'xJf', tmp_archive], cwd=os.path.join(REPO_CACHE_DIR, 'apps'), check=True) os.unlink(tmp_archive) def run_uninstall_script(self, app): # Runs uninstall.sh for an app, if the script is present self.run_script(app, 'uninstall.sh') def run_install_script(self, app): # Runs install.sh for a package, if the script is present self.run_script(app, 'install.sh') def run_script(self, app, script): # Runs script for an app, if the script is present script_path = os.path.join(REPO_CACHE_DIR, app, script) if os.path.exists(script_path): subprocess.run(script_path, check=True) def create_containers(self, app): # Create LXC containers from image and app metadata for container in self.online_packages['apps'][app]['containers']: image = self.online_packages['apps'][app]['containers'][container]['image'] image = self.online_packages['images'][image]['containers'].copy() if 'mounts' in self.online_packages['apps'][app]['containers'][container]: image['mounts'] = self.online_packages['apps'][app]['containers'][container]['mounts'] lxcmgr.create_container(container, image) @flock.flock_ex(REPO_LOCK) def uninstall_app(self, app): # Main uninstallation function. Wrapper for uninstall script and filesystem purge self.run_uninstall_script(app) self.destroy_containers(app) self.purge_scripts(app) self.purge_unused_layers() def destroy_containers(self, app): # Destroy LXC containers for container in self.installed_packages['apps'][app]['containers']: lxcmgr.destroy_container(container) def purge_unused_layers(self): # Remove layers which are no longer used by any installed application layers = set(os.list(LXC_STORAGE_DIR)) for app in self.installed_packages['apps']: for container in self.installed_packages['apps'][app]['containers']: image = self.installed_packages['apps'][app]['containers'][container]['image'] for layer in self.installed_packages['images'][image]['layers']: if layer in layers: del layers[layer] for layer in layers: self.purge_layer(layer) @flock.flock_ex(REPO_LOCK) def update_app(self, app, item): # Main update function. # TODO: Implement actual update #uninstall_app(app) #install_app(app, item) def has_update(self, app): if not self.installed_packages: self.load_installed_packages() if not self.online_packages: self.fetch_online_packages() return parse_version(self.installed_packages['apps'][app]['version']) < parse_version(self.online_packages['apps'][app]['version'])