vmmgr/usr/lib/python3.6/lxcmgr/pkgmgr.py

# -*- coding: utf-8 -*-

import json
import os
import requests
import shutil
import subprocess

from pkg_resources import parse_version

from . import crypto
from . import flock
from . import lxcmgr
from . import svcmgr
from .paths import LXC_STORAGE_DIR, REPO_CACHE_DIR, REPO_CONF_FILE, REPO_LOCAL_FILE, REPO_LOCK, REPO_SIG_FILE

class Stage:
    QUEUED = 1
    DOWNLOAD = 2
    UNPACK = 3
    INSTALL = 4
    UNINSTALL = 5
    UPDATE = 6
    DONE = 7

class RepoUnauthorized(Exception):
    pass

class RepoFileNotFound(Exception):
    pass

class RepoBadRequest(Exception):
    pass

class App:
    def __init__(self, name):
        self.name = name
        self.stage = Stage.QUEUED
        self.bytes_total = 1
        self.bytes_processed = 0

    @property
    def percent_processed(self):
        # Limit the displayed percentage to 0 - 99
        return min(99, round(self.bytes_processed / self.bytes_total * 100))

class PkgMgr:
    def __init__(self):
        self.repo_url = None
        self.repo_auth = None
        self._online_packages = None
        with open(REPO_LOCAL_FILE, 'r') as f:
            self.installed_packages = json.load(f)

    @property
    def online_packages(self):
        if not self._online_packages:
            # Fetches and verifies online packages. Can raise InvalidSignature
            packages = self.get_repo_resource('packages').content
            packages_sig = self.get_repo_resource('packages.sig').content
            crypto.verify_signature(REPO_SIG_FILE, packages, packages_sig)
            self._online_packages = json.loads(packages)
        return self._online_packages

    def save_installed_packages(self):
        with open(REPO_LOCAL_FILE, 'w') as f:
            json.dump(self.installed_packages, f, sort_keys=True, indent=4)

    def load_repo_conf(self):
        with open(REPO_CONF_FILE, 'r') as f:
            conf = json.load(f)
        self.repo_url = conf['url']
        user = conf['user'] if 'user' in conf and conf['user'] else None
        pwd = conf['pwd'] if 'pwd' in conf and conf['pwd'] else None
        self.repo_auth = (user, pwd) if user else None

    def get_repo_resource(self, resource_url, stream=False):
        # Download requested repository resource
        if not self.repo_url:
            self.load_repo_conf()
        # Make a HTTP request
        r = requests.get('{}/{}'.format(self.repo_url, resource_url), auth=self.repo_auth, timeout=5, stream=stream)
        if r.status_code == 401:
            raise RepoUnauthorized(r.url)
        elif r.status_code == 404:
            raise RepoFileNotFound(r.url)
        elif r.status_code != 200:
            raise RepoBadRequest(r.url)
        return r

    @flock.flock_ex(REPO_LOCK)
    def install_app(self, app):
        # Main installation function. Wrapper for download, registration and install script
        if app.name in self.installed_packages['apps']:
            app.stage = Stage.DONE
            return
        self.download_and_unpack_deps(app)
        # Run setup scripts
        app.stage = Stage.INSTALL
        # Run uninstall script to clean previous failed installation
        self.run_script(app.name, 'uninstall')
        # Build containers and services
        self.create_containers(self.online_packages['apps'][app.name]['containers'])
        # Run install script and register the app
        self.run_script(app.name, 'install')
        self.register_app(app.name, self.online_packages['apps'][app.name])
        app.stage = Stage.DONE

    def download_and_unpack_deps(self, app):
        # Common download and unpack function for install and update
        # Get all packages on which the app and its containers depend and which have not been installed yet
        images = []
        image_deps = [container['image'] for container in self.online_packages['apps'][app.name]['containers'].values()]
        for image in image_deps:
            images.extend(self.online_packages['images'][image]['layers'])
        images = [image for image in set(images) if image not in self.installed_packages['images']]
        # Calculate bytes to download
        app.bytes_total = sum(self.online_packages['images'][image]['pkgsize'] for image in images) + self.online_packages['apps'][app.name]['pkgsize']
        # Download layers and setup script files
        app.stage = Stage.DOWNLOAD
        for image in images:
            self.download_image(app, image)
        self.download_scripts(app)
        # Purge old data to clean previous failed installation and unpack downloaded archives
        app.stage = Stage.UNPACK
        self.destroy_containers(self.online_packages['apps'][app.name]['containers'])
        for image in images:
            self.purge_image(image)
            self.unpack_image(image)
            self.register_image(image, self.online_packages['images'][image])
        self.purge_scripts(app.name)
        self.unpack_scripts(app.name)

    def download_image(self, app, image):
        # Download image archive and verify hash
        archive = 'images/{}.tar.xz'.format(image)
        self.download_archive(app, archive, self.online_packages['images'][image]['sha512'])

    def download_scripts(self, app):
        # Download scripts archive and verify hash
        archive = 'apps/{}.tar.xz'.format(app.name)
        self.download_archive(app, archive, self.online_packages['apps'][app.name]['sha512'])

    def download_archive(self, app, archive, hash):
        # Download the archive from online repository
        tmp_archive = os.path.join(REPO_CACHE_DIR, archive)
        res = self.get_repo_resource(archive, True)
        with open(tmp_archive, 'wb') as f:
            for chunk in res.iter_content(chunk_size=65536):
                if chunk:
                    app.bytes_processed += f.write(chunk)
        crypto.verify_hash(tmp_archive, hash)

    def purge_image(self, image):
        # Delete layer files from storage directory
        try:
            shutil.rmtree(os.path.join(LXC_STORAGE_DIR, image))
        except FileNotFoundError:
            pass

    def unpack_image(self, image):
        # Unpack layer archive
        tmp_archive = os.path.join(REPO_CACHE_DIR, 'images/{}.tar.xz'.format(image))
        subprocess.run(['tar', 'xJf', tmp_archive], cwd=LXC_STORAGE_DIR, check=True)
        os.unlink(tmp_archive)

    def register_image(self, image, metadata):
        # Add installed layer to list of installed images
        self.installed_packages['images'][image] = metadata
        self.save_installed_packages()

    def unregister_image(self, image):
        # Remove image from list of installed images
        if image in self.installed_packages['images']:
            del self.installed_packages['images'][image]
            self.save_installed_packages()

    def purge_scripts(self, app):
        # Delete application setup scripts from storage directory
        try:
            shutil.rmtree(os.path.join(REPO_CACHE_DIR, 'apps', app))
        except FileNotFoundError:
            pass

    def unpack_scripts(self, app):
        # Unpack setup scripts archive
        tmp_archive = os.path.join(REPO_CACHE_DIR, 'apps/{}.tar.xz'.format(app))
        subprocess.run(['tar', 'xJf', tmp_archive], cwd=os.path.join(REPO_CACHE_DIR, 'apps'), check=True)
        os.unlink(tmp_archive)

    def run_script(self, app, action):
        # Runs script for an app, if the script is present
        cache_dir = os.path.join(REPO_CACHE_DIR, 'apps', app)
        script_dir = os.path.join(cache_dir, action)
        script_path = '{}.sh'.format(script_dir)
        if os.path.exists(script_path):
            # Run the script in its working directory, if there is one, so it doesn't have to figure out paths to packaged files
            cwd = script_dir if os.path.exists(script_dir) else cache_dir
            subprocess.run(script_path, cwd=cwd, check=True)

    def register_app(self, app, metadata):
        # Register installed app in list of installed apps
        self.installed_packages['apps'][app] = metadata
        self.save_installed_packages()

    def unregister_app(self, app):
        # Remove app from list of installed apps
        if app in self.installed_packages['apps']:
            del self.installed_packages['apps'][app]
            self.save_installed_packages()

    def create_containers(self, containers):
        # Create LXC containers from image and app metadata
        for container in containers:
            image = containers[container]['image']
            image = self.online_packages['images'][image].copy()
            if 'mounts' in containers[container]:
                image['mounts'] = containers[container]['mounts']
            if 'depends' in containers[container]:
                image['depends'] = containers[container]['depends']
            lxcmgr.create_container(container, image)
            svcmgr.create_service(container, image)
        svcmgr.update_services()

    def destroy_containers(self, containers):
        # Destroy LXC containers
        for container in containers:
            svcmgr.delete_service(container)
            lxcmgr.destroy_container(container)
        svcmgr.update_services()

    @flock.flock_ex(REPO_LOCK)
    def uninstall_app(self, app):
        # Main uninstallation function. Wrapper for uninstall script and filesystem purge
        if app.name not in self.installed_packages['apps']:
            app.stage = Stage.DONE
            return
        app.stage = Stage.UNINSTALL
        self.run_script(app.name, 'uninstall')
        self.destroy_containers(self.installed_packages['apps'][app.name]['containers'])
        self.purge_scripts(app.name)
        self.unregister_app(app.name)
        self.purge_unused_layers()
        app.stage = Stage.DONE

    def purge_unused_layers(self):
        # Remove layers which are no longer used by any installed application
        layers = set(os.listdir(LXC_STORAGE_DIR))
        for app in self.installed_packages['apps']:
            for container in self.installed_packages['apps'][app]['containers']:
                image = self.installed_packages['apps'][app]['containers'][container]['image']
                for layer in self.installed_packages['images'][image]['layers']:
                    if layer in layers:
                        # Layer is still used, remove from set of layers to be purged
                        layers.remove(layer)
        for layer in layers:
            self.purge_image(layer)

    @flock.flock_ex(REPO_LOCK)
    def update_app(self, app):
        # Main update function.
        self.download_and_unpack_deps(app)
        # Run setup scripts
        app.stage = Stage.UPDATE
        # Build containers and services
        self.create_containers(self.online_packages['apps'][app.name]['containers'])
        # Run update script and register the app
        self.run_script(app.name, 'update')
        self.register_app(app.name, self.online_packages['apps'][app.name])
        app.stage = Stage.DONE

    def has_update(self, app):
        # Check if online repository list a newer version of app
        if app not in self.online_packages['apps']:
            # Application has been removed from online repo
            return False
        # Compare version strings
        return parse_version(self.installed_packages['apps'][app]['version']) < parse_version(self.online_packages['apps'][app]['version'])