Prototype implementation of ActionItem + observers

usr/bin/spoc-image

@@ -8,7 +8,7 @@ from spoc import repo_online
 from spoc import repo_publish
 from spoc.image import Image
 from spoc.imagebuilder import ImageBuilder
-from spoc.utils import readable_size
+from spoc.utils import ActionItem, readable_size
 
 ACTION_LIST = 1
 ACTION_DOWNLOAD = 2
@@ -36,11 +36,15 @@ def listing(repo_type):
         print(image)
 
 def download(image_name):
+    plan = []
     local_images = repo_local.get_images()
     for layer in repo_online.get_image(image_name)['layers']:
         if layer not in local_images:
-            print(f'Downloading {layer}...')
+            image = Image(layer, False)
-            Image(layer, False).download()
+            plan.append(ActionItem(f'Downloading {image_name}', image.download))
+            plan.append(ActionItem(f'Unpacking {image_name}', image.unpack_downloaded))
+    for item in plan:
+        item.run()
 
 def delete(image_name):
     Image(image_name, False).delete()
@@ -50,9 +54,9 @@ def build(filename, force, do_publish):
     image_name = get_image_name(filename)
     if force or image_name not in repo_local.get_images():
         image = Image(image_name, False)
-        image.delete()
         print(f'Building image {image_name} from file {filename}')
-        ImageBuilder().build(image, filename)
+        image.delete()
+        image.create(ImageBuilder(), filename)
         print(f'Image {image_name} built successfully')
         # If publishing was requested, force publish after successful build
         force = True
@@ -119,6 +123,6 @@ elif args.action == ACTION_BUILD:
 elif args.action == ACTION_PUBLISH:
     publish(args.image, args.force)
 elif args.action == ACTION_UNPUBLISH:
-    publishreop.unpublish_image(args.image)
+    unpublish(args.image)
 else:
     parser.print_usage()

usr/lib/python3.8/spoc/config.py

@@ -14,6 +14,8 @@ APPS_DIR =          os.path.join(DATA_DIR, 'apps/')
 CONTAINERS_DIR =    os.path.join(DATA_DIR, 'containers/')
 LAYERS_DIR =        os.path.join(DATA_DIR, 'layers/')
 VOLUME_DIR =        os.path.join(DATA_DIR, 'volumes/')
+TMP_APPS_DIR =      os.path.join(DATA_DIR, 'tmp/apps/')
+TMP_LAYERS_DIR =    os.path.join(DATA_DIR, 'tmp/layers/')
 HOSTS_FILE =        os.path.join(DATA_DIR, 'hosts')
 REPO_FILE =         os.path.join(DATA_DIR, 'repository.json')
 LOG_DIR =           config.get('general', 'log-dir', fallback='/var/log/spoc')

usr/lib/python3.8/spoc/image.py

@@ -8,7 +8,7 @@ import urllib.parse
 from . import repo_local
 from . import repo_online
 from . import repo_publish
-from .config import LAYERS_DIR, PUB_LAYERS_DIR, ONLINE_LAYERS_URL
+from .config import LAYERS_DIR, PUB_LAYERS_DIR, ONLINE_LAYERS_URL, TMP_LAYERS_DIR
 
 DEFINITION_MEMBERS = {'layers', 'env', 'uid', 'gid', 'cmd', 'cwd', 'ready', 'halt', 'size', 'dlsize', 'hash'}
 
@@ -16,8 +16,6 @@ class Image:
     def __init__(self, name, load_from_repo=True):
         self.name = name
         self.layer_path = os.path.join(LAYERS_DIR, name)
-        self.archive_path = os.path.join(PUB_LAYERS_DIR, f'{name}.tar.xz')
-        self.online_path = urllib.parse.urljoin(ONLINE_LAYERS_URL, f'{name}.tar.xz')
         self.layers = [name]
         self.env = {}
         self.uid = None
@@ -44,12 +42,12 @@ class Image:
                 definition[key] = value
         return definition
 
-    def create(self, imagebuilder):
+    def create(self, imagebuilder, filename):
         # Build the container from image file and save to local repository
         # Chown is possible only when the process is running as root, for user namespaces, see https://linuxcontainers.org/lxc/manpages/man1/lxc-usernsexec.1.html
         os.makedirs(self.layer_path, 0o755, True)
         os.chown(self.layer_path, 100000, 100000)
-        imagebuilder.process_file()
+        imagebuilder.build(self, filename)
         repo_local.register_image(self.name, self.get_definition())
 
     def delete(self):
@@ -60,24 +58,36 @@ class Image:
             pass
 
     def publish(self):
-        os.makedirs(PUB_LAYERS_DIR, 0o755, True)
+        os.makedirs(PUB_LAYERS_DIR, 0o700, True)
         files = repo_publish.TarSizeCounter()
-        with tarfile.open(self.archive_path, 'w:xz') as tar:
+        archive_path = os.path.join(PUB_LAYERS_DIR, f'{self.name}.tar.xz')
+        with tarfile.open(archive_path, 'w:xz') as tar:
             tar.add(self.layer_path, self.name, filter=files.add_file)
         self.size = files.size
-        self.dlsize = os.path.getsize(self.archive_path)
+        self.dlsize = os.path.getsize(archive_path)
-        self.hash = repo_publish.sign_file(self.archive_path).hex()
+        self.hash = repo_publish.sign_file(archive_path).hex()
         repo_publish.register_image(self.name, self.get_definition())
 
     def unpublish(self):
         repo_publish.unregister_image(self.name)
+        archive_path = os.path.join(PUB_LAYERS_DIR, f'{self.name}.tar.xz')
         try:
-            os.unlink(self.archive_path)
+            os.unlink(archive_path)
         except FileNotFoundError:
             pass
 
-    def download(self):
+    def download(self, observer):
+        os.makedirs(TMP_LAYERS_DIR, 0o700, True)
         definition = repo_online.get_image(self.name)
-        repo_online.download_archive(self.online_path, LAYERS_DIR, definition['hash'])
+        observer.units_total = definition['dlsize']
+        archive_url = urllib.parse.urljoin(ONLINE_LAYERS_URL, f'{self.name}.tar.xz')
+        archive_path = os.path.join(TMP_LAYERS_DIR, f'{self.name}.tar.xz')
+        repo_online.download_archive(archive_url, archive_path, definition['hash'], observer)
+
+    def unpack_downloaded(self, observer):
+        definition = repo_online.get_image(self.name)
+        observer.units_total = definition['size']
+        archive_path = os.path.join(TMP_LAYERS_DIR, f'{self.name}.tar.xz')
+        repo_online.unpack_archive(archive_path, LAYERS_DIR, definition['hash'], observer)
         self.set_definition(definition)
         repo_local.register_image(self.name, definition)

usr/lib/python3.8/spoc/imagebuilder.py

@@ -10,19 +10,16 @@ import zipfile
 
 from .container import Container
 from .image import Image
-from .config import VOLUME_DIR
+from .config import LAYERS_DIR
 
 class ImageBuilder:
     def build(self, image, filename):
         # Reset internal state, read and process lines from filename
         self.image = image
-        self.filename = filename
+        self.builddir = os.path.dirname(filename)
         self.script_eof = None
         self.script_lines = []
-        self.image.create(self)
+        with open(filename, 'r') as f:
-
-    def process_file(self):
-        with open(self.filename, 'r') as f:
             for line in f:
                 self.process_line(line.strip())
 
@@ -71,7 +68,6 @@ class ImageBuilder:
     def run_script(self, script_lines):
         # Creates a temporary container, runs a script in its namespace, and stores the files modified by it as part of the layer
         # Note: If USER or WORKDIR directive has already been set, the command is run under that UID/GID or working directory
-        os.makedirs(VOLUME_DIR, 0o755, True)
         script_fd, script_path = tempfile.mkstemp(suffix='.sh', dir=self.image.layer_path, text=True)
         script_name = os.path.basename(script_path)
         script_lines = '\n'.join(script_lines)
@@ -116,7 +112,7 @@ class ImageBuilder:
         if src.startswith('http://') or src.startswith('https://'):
             unpack_http_archive(src, dst)
         else:
-            src = os.path.join(os.path.dirname(self.filename), src)
+            src = os.path.join(self.builddir, src)
             copy_tree(src, dst)
         # Shift UID/GID of the files to the unprivileged range
         shift_uid(dst, os.stat(dst, follow_symlinks=False))

usr/lib/python3.8/spoc/repo_online.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+import copy
 import hashlib
 import json
 import os
@@ -8,6 +9,7 @@ import tarfile
 import tempfile
 import time
 
+from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.asymmetric import ec, utils
@@ -27,24 +29,68 @@ PUBLIC_KEY = get_pubkey()
 
 # TODO: HTTP Error handling for all downloads (including imagebuilder)
 
-def download_archive(src, dst, expected_hash):
+def verify_fileobj(fileobj, expected_hash):
-    # Download archive via http(s), verify hash and decompress
+    hasher = hashes.Hash(hashes.SHA512(), default_backend())
-    with tempfile.TemporaryFile() as tmp_archive:
+    while True:
-        sha512 = hashes.SHA512()
+        data = fileobj.read(64*1024)
-        hasher = hashes.Hash(sha512, default_backend())
+        if not data:
-        # Download the file via http(s) and store as temporary file
+            break
-        with requests.Session() as session:
+        hasher.update(data)
-            resource = session.get(src, stream=True)
+    PUBLIC_KEY.verify(bytes.fromhex(expected_hash), hasher.finalize(), ec.ECDSA(utils.Prehashed(hashes.SHA512())))
+
+def download_archive(archive_url, archive_path, expected_hash, observer):
+    # Check if an archive needs to be downloaded via http(s)
+    do_download = True
+    # If the file already exists in the temporary directory, verify the signature
+    if os.path.exists(archive_path):
+        try:
+            with open(archive_path, 'rb') as f:
+                verify_fileobj(f, expected_hash)
+            # If the signature matches, skip download
+            observer.units_done = os.path.getsize(archive_path)
+            do_download = False
+        except InvalidSignature:
+            pass
+    if do_download:
+        # Download archive via http(s) and store in temporary directory
+        with open(archive_path, 'wb') as f, requests.Session() as session:
+            resource = session.get(archive_url, stream=True)
             for chunk in resource.iter_content(chunk_size=None):
                 if chunk:
-                    tmp_archive.write(chunk)
+                    observer.units_done += f.write(chunk)
-                    hasher.update(chunk)
+
-        # Verify hash
+def unpack_archive(archive_path, destination, expected_hash, observer):
-        PUBLIC_KEY.verify(bytes.fromhex(expected_hash), hasher.finalize(), ec.ECDSA(utils.Prehashed(sha512)))
+    with open(archive_path, 'rb') as f:
-        # Extract the tar.xz file
+        # Verify file object, then seek back and open it as tar without losing handle, thus preventing posible malicious race conditions
-        tmp_archive.seek(0)
+        verify_fileobj(f, expected_hash)
-        with tarfile.open(fileobj=tmp_archive) as tar:
+        f.seek(0)
-            tar.extractall(dst, numeric_owner=True)
+        tar = tarfile.open(fileobj=f)
+        # Extract the tar members while counting their size, taken from https://github.com/python/cpython/blob/master/Lib/tarfile.py
+        directories = []
+        for tarinfo in tar:
+            if tarinfo.isdir():
+                # Extract directories with a safe mode
+                directories.append(tarinfo)
+                tarinfo = copy.copy(tarinfo)
+                tarinfo.mode = 0o700
+            # Do not set_attrs directories, as we will do that further down
+            tar.extract(tarinfo, destination, set_attrs=not tarinfo.isdir(), numeric_owner=True)
+            observer.units_done += tarinfo.size
+        # Reverse sort directories
+        directories.sort(key=lambda a: a.name)
+        directories.reverse()
+        # Set correct owner, mtime and filemode on directories
+        for tarinfo in directories:
+            dirpath = os.path.join(destination, tarinfo.name)
+            try:
+                tar.chown(tarinfo, dirpath, numeric_owner=True)
+                tar.utime(tarinfo, dirpath)
+                tar.chmod(tarinfo, dirpath)
+            except ExtractError as e:
+                if tar.errorlevel > 1:
+                    raise
+    # Remove the archive
+    os.unlink(archive_path)
 
 def download_metadata():
     with requests.Session() as session:

usr/lib/python3.8/spoc/utils.py

@@ -2,6 +2,17 @@
 
 SIZE_PREFIXES = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
 
+class ActionItem:
+    def __init__(self, text, action):
+        self.text = text
+        self.action = action
+        self.units_total = 0
+        self.units_done = 0
+
+    def run(self):
+        print(self.text)
+        self.action(self)
+
 def readable_size(bytes):
     i = 0
     while bytes > 1024: