Separate repo conf, build should now be almost complete

etc/lxcmgr/repo.json

@@ -0,0 +1,5 @@
+{
+    "url": "https://repo.spotter.cz/lxc",
+    "user": "",
+    "pwd": ""
+}

etc/vmmgr/config.default.json

@@ -8,11 +8,5 @@
         "adminpwd": "${ADMINPWD}",
         "domain": "spotter.vm",
         "port": "443"
-    },
-    "packages": {},
-    "repo": {
-        "pwd": "",
-        "url": "https://repo.spotter.cz/lxc",
-        "user": ""
     }
 }

usr/bin/lxcmgr

@@ -44,16 +44,6 @@ parser_container_cleanup.set_defaults(action='container-cleanup')
 parser_container_cleanup.add_argument('container', help='Container name')
 parser_container_cleanup.add_argument('lxc', nargs=argparse.REMAINDER)
 
-parser_container_create = subparsers_container.add_parser('create')
-parser_container_create.set_defaults(action='container-create')
-parser_container_create.add_argument('container', help='Container name')
-parser_container_create.add_argument('lxc', nargs=argparse.REMAINDER)
-
-parser_container_destroy = subparsers_container.add_parser('destroy')
-parser_container_destroy.set_defaults(action='container-destroy')
-parser_container_destroy.add_argument('container', help='Container name')
-parser_container_destroy.add_argument('lxc', nargs=argparse.REMAINDER)
-
 def print_apps(packages):
     for app, meta in packages.items():
         print('{} {}'.format(app, meta['version']))
@@ -110,9 +100,3 @@ elif args.action == 'container-prepare':
 elif args.action == 'container-cleanup':
     # Used with LXC hooks on container stop
     lxcmgr.cleanup_container(args.container)
-elif args.action == 'container-create':
-    # Used by package installer and builder
-    lxcmgr.register_container(args.container)
-elif args.action == 'container-destroy':
-    # Used by package installer and builder
-    lxcmgr.unregister_container(args.container)

usr/lib/python3.6/lxcmgr/crypto.py

@@ -9,20 +9,20 @@ from cryptography.hazmat.primitives.asymmetric import ec
 
 from .paths import REPO_SIG_FILE
 
-def verify_signature(file, signature):
+def verify_signature(public_key_path, input_data, signature_data):
     # Verifies ECDSA HMAC SHA512 signature of a file
-    with open(REPO_SIG_FILE, 'rb') as f:
+    with open(public_key_path, 'rb') as f:
         pub_key = serialization.load_pem_public_key(f.read(), default_backend())
-    pub_key.verify(signature, file, ec.ECDSA(hashes.SHA512()))
+    pub_key.verify(signature_data, input_data, ec.ECDSA(hashes.SHA512()))
 
-def verify_hash(file, expected_hash):
+def verify_hash(input_path, expected_hash):
     # Verifies SHA512 hash of a file against expected hash
     sha512 = hashlib.sha512()
-    with open(file, 'rb') as f:
+    with open(input_path, 'rb') as f:
         while True:
             data = f.read(65536)
             if not data:
                 break
             sha512.update(data)
     if sha512.hexdigest() != expected_hash:
-        raise InvalidSignature(file)
+        raise InvalidSignature(input_path)

usr/lib/python3.6/lxcmgr/lxcmgr.py

@@ -6,7 +6,7 @@ import shutil
 import subprocess
 
 from . import flock
-from .paths import HOSTS_FILE, HOSTS_LOCK, LXC_ROOT
+from .paths import HOSTS_FILE, HOSTS_LOCK, LXC_LOGS, LXC_ROOT, LXC_STORAGE_DIR
 from .templates import LXC_CONTAINER
 
 def prepare_container(container, layers):
@@ -67,6 +67,7 @@ def create_container(container, image):
 def destroy_container(container):
     # Remove container configuration and directories
     shutil.rmtree(os.path.join(LXC_ROOT, container))
+    os.unlink(os.path.join(LXC_LOGS, '{}.log'.format(container)))
     # Release the IP address
     update_hosts_lease(container, False)
 

usr/lib/python3.6/lxcmgr/paths.py

@@ -2,12 +2,14 @@
 
 # Package manager
 REPO_CACHE_DIR = '/var/lib/lxcmgr/cache'
+REPO_CONF_FILE = '/etc/lxcmgr/repo.json'
 REPO_LOCAL_FILE = '/var/lib/lxcmgr/packages'
 REPO_LOCK = '/var/lock/lxcmgr-repo.lock'
-REPO_SIG_FILE = '/var/lib/lxcmgr/packages.pub'
+REPO_SIG_FILE = '/etc/lxcmgr/packages.pub'
 
 # LXC
 HOSTS_FILE = '/etc/hosts'
 HOSTS_LOCK = '/var/lock/lxcmgr-hosts.lock'
+LXC_LOGS = '/var/log/lxc'
 LXC_ROOT = '/var/lib/lxc'
 LXC_STORAGE_DIR = '/var/lib/lxcmgr/storage'

usr/lib/python3.6/lxcmgr/pkgmgr.py

@@ -12,7 +12,7 @@ from pkg_resources import parse_version
 from . import crypto
 from . import flock
 from . import lxcmgr
-from .paths import LXC_STORAGE_DIR, REPO_CACHE_DIR, REPO_LOCAL_FILE, REPO_LOCK
+from .paths import LXC_STORAGE_DIR, REPO_CACHE_DIR, REPO_CONF_FILE, REPO_LOCAL_FILE, REPO_LOCK, REPO_SIG_FILE
 
 class Stage(Enum):
     QUEUED = 1
@@ -31,7 +31,7 @@ class RepoFileNotFound(Exception):
 class RepoBadRequest(Exception):
     pass
 
-class AppInstall:
+class App:
     def __init__(self, name):
         self.name = name
         self.stage = Stage.QUEUED
@@ -44,22 +44,30 @@ class AppInstall:
         return min(99, round(self.bytes_downloaded / self.bytes_total * 100))
 
 class PkgMgr:
-    def __init__(self, repo_url, repo_auth=None):
-        self.repo_url = repo_url
-        self.repo_auth = repo_auth
-        self.installed_packages = None
+    def __init__(self):
+        self.repo_url = None
+        self.repo_auth = None
         self.online_packages = None
-
-    def load_installed_packages(self):
         with open(REPO_LOCAL_FILE, 'r') as f:
             self.installed_packages = json.load(f)
 
-    def save_installed_packages(self, packages):
+    def save_installed_packages(self):
         with open(REPO_LOCAL_FILE, 'w') as f:
-            json.dump(packages, f, sort_keys=True, indent=4)
+            json.dump(self.installed_packages, f, sort_keys=True, indent=4)
+
+    def load_repo_conf(self):
+        with open(REPO_CONF_FILE, 'r') as f:
+            conf = json.load(f)
+        self.repo_url = conf['url']
+        user = conf['user'] if 'user' in conf and conf['user'] else None
+        pwd = conf['pwd'] if 'pwd' in conf and conf['pwd'] else None
+        self.repo_auth = (user, pwd) if user else None
 
     def get_repo_resource(self, resource_url, stream=False):
         # Download requested repository resource
+        if not self.repo_url:
+            self.load_repo_conf()
+        # Make a HTTP request
         r = requests.get('{}/{}'.format(self.repo_url, resource_url), auth=self.repo_auth, timeout=5, stream=stream)
         if r.status_code == 401:
             raise RepoUnauthorized(r.url)
@@ -73,65 +81,65 @@ class PkgMgr:
         # Fetches and verifies online packages. Can raise InvalidSignature
         packages = self.get_repo_resource('packages').content
         packages_sig = self.get_repo_resource('packages.sig').content
-        crypto.verify_signature(packages, packages_sig)
+        crypto.verify_signature(REPO_SIG_FILE, packages, packages_sig)
         self.online_packages = json.loads(packages)
 
     @flock.flock_ex(REPO_LOCK)
     def install_app(self, app):
         # Main installation function. Wrapper for download, registration and install script
-        if not self.installed_packages:
-            self.load_installed_packages()
-        # Request for installation of already installed app immediately returns with success
         if app.name in self.installed_packages['apps']:
             app.stage = Stage.DONE
             return
         if not self.online_packages:
             self.fetch_online_packages()
         # Get all packages on which the app depends and which have not been installed yet
+        #TODO: flatten and change name to "images"
         layers = []
         images = [container['image'] for container in self.online_packages['apps'][app]['containers'].values()]
         for image in images:
             layers.extend(self.online_packages['images'][image]['layers'])
         layers = [layer for layer in set(layers) if layer not in self.installed_packages['images']]
         # Calculate bytes to download
-        app.bytes_total = sum(self.online_packages['images'][layer]['size'] for layer in layers) + self.online_packages['apps'][app.name]['size']
+        app.bytes_total = sum(self.online_packages['images'][image]['size'] for image in layers) + self.online_packages['apps'][app.name]['size']
         # Download layers and setup script files
         app.stage = Stage.DOWNLOAD
-        for layer in layers:
-            self.download_layer(app, layer)
+        for image in layers:
+            self.download_image(app, image)
         self.download_scripts(app)
-        # Purge old data (to clean previous failed installation) and unpack
+        # Purge old data to clean previous failed installation and unpack downloaded archives
         app.stage = Stage.UNPACK
-        for layer in layers:
-            self.purge_layer(layer)
-            self.unpack_layer(layer)
+        for image in layers:
+            self.purge_image(image)
+            self.unpack_image(image)
+            self.register_image(image, self.online_packages['images'][image])
         self.purge_scripts(app.name)
         self.unpack_scripts(app.name)
         # Run setup scripts
         app.stage = Stage.INSTALL
+        # Run uninstall script to clean previous failed installation
         self.run_uninstall_script(app.name)
         # Build containers and services
         self.create_containers(app.name)
-        # Run install script and finish the installation
+        # TODO: Create services
+        # Run install script and register the app
         self.run_install_script(app.name)
-        self.installed_packages['apps'][app.name] = self.online_packages['apps'][app.name]
-        self.save_installed_packages()
+        self.register_app(app.name, self.online_packages['apps'][app.name])
         app.stage = Stage.DONE
 
-    def download_layer(self, app, layer):
-        pkg_archive = 'images/{}.tar.xz'.format(layer)
+    def download_image(self, app, image):
+        # Download image archive and verify hash
+        pkg_archive = 'images/{}.tar.xz'.format(image)
         self.download_archive(app, pkg_archive)
-        # Verify hash
-        crypto.verify_hash(tmp_archive, self.online_packages['images'][layer]['sha512'])
+        crypto.verify_hash(tmp_archive, self.online_packages['images'][image]['sha512'])
 
     def download_scripts(self, app):
+        # Download scripts archive and verify hash
         pkg_archive = 'apps/{}.tar.xz'.format(app.name)
         self.download_archive(app, pkg_archive)
-        # Verify hash
         crypto.verify_hash(tmp_archive, self.online_packages['apps'][app.name]['sha512'])
 
     def download_archive(self, app, archive):
-        # Download the archive
+        # Download the archive from online repository
         tmp_archive = os.path.join(REPO_CACHE_DIR, pkg_archive)
         res = self.get_repo_resource('{}/{}'.format(type, pkg_archive), True)
         with open(tmp_archive, 'wb') as f:
@@ -139,26 +147,30 @@ class PkgMgr:
                 if chunk:
                     app.bytes_downloaded += f.write(chunk)
 
-    def purge_layer(self, layer):
+    def purge_image(self, image):
         # Delete layer files from storage directory
-        shutil.rmtree(os.path.join(LXC_STORAGE_DIR, layer))
-        if layer in self.installed_packages['images']:
-            del self.installed_packages['images'][layer]
-            self.save_installed_packages()
+        shutil.rmtree(os.path.join(LXC_STORAGE_DIR, image))
 
-    def unpack_layer(self, layer):
+    def unpack_image(self, image):
         # Unpack layer archive
-        tmp_archive = os.path.join(REPO_CACHE_DIR, 'images/{}.tar.xz'.format(layer))
+        tmp_archive = os.path.join(REPO_CACHE_DIR, 'images/{}.tar.xz'.format(image))
         subprocess.run(['tar', 'xJf', tmp_archive], cwd=LXC_STORAGE_DIR, check=True)
         os.unlink(tmp_archive)
-        self.installed_packages['images'][layer] = self.online_packages['images'][layer]
+
+    def register_image(self, image, metadata):
+        # Add installed layer to list of installed images
+        self.installed_packages['images'][image] = metadata
         self.save_installed_packages()
 
+    def unregister_image(self, image):
+        # Remove image from list of installed images
+        if image in self.installed_packages['images']:
+            del self.installed_packages['images'][image]
+            self.save_installed_packages()
+
     def purge_scripts(self, app):
         # Delete application setup scripts from storage directory
         shutil.rmtree(os.path.join(REPO_CACHE_DIR, 'apps', app))
-        del self.installed_packages['apps'][app]
-        self.save_installed_packages()
 
     def unpack_scripts(self, app):
         # Unpack setup scripts archive
@@ -180,6 +192,17 @@ class PkgMgr:
         if os.path.exists(script_path):
             subprocess.run(script_path, check=True)
 
+    def register_app(self, app, metadata):
+        # Register installed app in list of installed apps
+        self.installed_packages['apps'][app] = metadata
+        self.save_installed_packages()
+
+    def unregister_app(self, app):
+        # Remove app from list of installed apps
+        if app in self.installed_packages['apps']:
+            del self.installed_packages['apps'][app]
+            self.save_installed_packages()
+
     def create_containers(self, app):
         # Create LXC containers from image and app metadata
         for container in self.online_packages['apps'][app]['containers']:
@@ -188,10 +211,13 @@ class PkgMgr:
             if 'mounts' in self.online_packages['apps'][app]['containers'][container]:
                 image['mounts'] = self.online_packages['apps'][app]['containers'][container]['mounts']
             lxcmgr.create_container(container, image)
+        # TODO: Create services
 
     @flock.flock_ex(REPO_LOCK)
     def uninstall_app(self, app):
         # Main uninstallation function. Wrapper for uninstall script and filesystem purge
+        if app not in self.installed_packages['apps']:
+            return
         self.run_uninstall_script(app)
         self.destroy_containers(app)
         self.purge_scripts(app)
@@ -218,8 +244,8 @@ class PkgMgr:
     def update_app(self, app, item):
         # Main update function.
         # TODO: Implement actual update
-        #uninstall_app(app)
-        #install_app(app, item)
+        uninstall_app(app)
+        install_app(app, item)
 
     def has_update(self, app):
         if not self.installed_packages:

usr/lib/python3.6/lxcmgr/templates.py

@@ -40,8 +40,8 @@ lxc.idmap = u 0 100000 65536
 lxc.idmap = g 0 100000 65536
 
 # Hooks
-lxc.hook.pre-start = /usr/bin/vmmgr prepare-container {layers}
-lxc.hook.post-stop = /usr/bin/vmmgr cleanup-container
+lxc.hook.pre-start = /usr/bin/lxcmgr container prepare {layers}
+lxc.hook.post-stop = /usr/bin/lxcmgr container cleanup
 
 # Other
 lxc.arch = linux64

var/lib/lxcmgr/packages

@@ -0,0 +1 @@
+{"apps": {}, "images": {}}