From dae215424eecd9b0cebad588543e7ad229e35232 Mon Sep 17 00:00:00 2001 From: Disassembler Date: Mon, 17 Feb 2020 01:05:00 +0100 Subject: [PATCH] Prototype implementation of ActionItem + observers --- usr/bin/spoc-image | 16 ++++-- usr/lib/python3.8/spoc/config.py | 2 + usr/lib/python3.8/spoc/image.py | 34 +++++++---- usr/lib/python3.8/spoc/imagebuilder.py | 12 ++-- usr/lib/python3.8/spoc/repo_online.py | 78 ++++++++++++++++++++------ usr/lib/python3.8/spoc/utils.py | 11 ++++ 6 files changed, 111 insertions(+), 42 deletions(-) diff --git a/usr/bin/spoc-image b/usr/bin/spoc-image index 428796d..d46f1bd 100644 --- a/usr/bin/spoc-image +++ b/usr/bin/spoc-image @@ -8,7 +8,7 @@ from spoc import repo_online from spoc import repo_publish from spoc.image import Image from spoc.imagebuilder import ImageBuilder -from spoc.utils import readable_size +from spoc.utils import ActionItem, readable_size ACTION_LIST = 1 ACTION_DOWNLOAD = 2 @@ -36,11 +36,15 @@ def listing(repo_type): print(image) def download(image_name): + plan = [] local_images = repo_local.get_images() for layer in repo_online.get_image(image_name)['layers']: if layer not in local_images: - print(f'Downloading {layer}...') - Image(layer, False).download() + image = Image(layer, False) + plan.append(ActionItem(f'Downloading {image_name}', image.download)) + plan.append(ActionItem(f'Unpacking {image_name}', image.unpack_downloaded)) + for item in plan: + item.run() def delete(image_name): Image(image_name, False).delete() @@ -50,9 +54,9 @@ def build(filename, force, do_publish): image_name = get_image_name(filename) if force or image_name not in repo_local.get_images(): image = Image(image_name, False) - image.delete() print(f'Building image {image_name} from file {filename}') - ImageBuilder().build(image, filename) + image.delete() + image.create(ImageBuilder(), filename) print(f'Image {image_name} built successfully') # If publishing was requested, force publish after successful build force = True @@ -119,6 +123,6 @@ elif args.action == ACTION_BUILD: elif args.action == ACTION_PUBLISH: publish(args.image, args.force) elif args.action == ACTION_UNPUBLISH: - publishreop.unpublish_image(args.image) + unpublish(args.image) else: parser.print_usage() diff --git a/usr/lib/python3.8/spoc/config.py b/usr/lib/python3.8/spoc/config.py index e749a28..6f42992 100644 --- a/usr/lib/python3.8/spoc/config.py +++ b/usr/lib/python3.8/spoc/config.py @@ -14,6 +14,8 @@ APPS_DIR = os.path.join(DATA_DIR, 'apps/') CONTAINERS_DIR = os.path.join(DATA_DIR, 'containers/') LAYERS_DIR = os.path.join(DATA_DIR, 'layers/') VOLUME_DIR = os.path.join(DATA_DIR, 'volumes/') +TMP_APPS_DIR = os.path.join(DATA_DIR, 'tmp/apps/') +TMP_LAYERS_DIR = os.path.join(DATA_DIR, 'tmp/layers/') HOSTS_FILE = os.path.join(DATA_DIR, 'hosts') REPO_FILE = os.path.join(DATA_DIR, 'repository.json') LOG_DIR = config.get('general', 'log-dir', fallback='/var/log/spoc') diff --git a/usr/lib/python3.8/spoc/image.py b/usr/lib/python3.8/spoc/image.py index ea39ffc..a1c8e7f 100644 --- a/usr/lib/python3.8/spoc/image.py +++ b/usr/lib/python3.8/spoc/image.py @@ -8,7 +8,7 @@ import urllib.parse from . import repo_local from . import repo_online from . import repo_publish -from .config import LAYERS_DIR, PUB_LAYERS_DIR, ONLINE_LAYERS_URL +from .config import LAYERS_DIR, PUB_LAYERS_DIR, ONLINE_LAYERS_URL, TMP_LAYERS_DIR DEFINITION_MEMBERS = {'layers', 'env', 'uid', 'gid', 'cmd', 'cwd', 'ready', 'halt', 'size', 'dlsize', 'hash'} @@ -16,8 +16,6 @@ class Image: def __init__(self, name, load_from_repo=True): self.name = name self.layer_path = os.path.join(LAYERS_DIR, name) - self.archive_path = os.path.join(PUB_LAYERS_DIR, f'{name}.tar.xz') - self.online_path = urllib.parse.urljoin(ONLINE_LAYERS_URL, f'{name}.tar.xz') self.layers = [name] self.env = {} self.uid = None @@ -44,12 +42,12 @@ class Image: definition[key] = value return definition - def create(self, imagebuilder): + def create(self, imagebuilder, filename): # Build the container from image file and save to local repository # Chown is possible only when the process is running as root, for user namespaces, see https://linuxcontainers.org/lxc/manpages/man1/lxc-usernsexec.1.html os.makedirs(self.layer_path, 0o755, True) os.chown(self.layer_path, 100000, 100000) - imagebuilder.process_file() + imagebuilder.build(self, filename) repo_local.register_image(self.name, self.get_definition()) def delete(self): @@ -60,24 +58,36 @@ class Image: pass def publish(self): - os.makedirs(PUB_LAYERS_DIR, 0o755, True) + os.makedirs(PUB_LAYERS_DIR, 0o700, True) files = repo_publish.TarSizeCounter() - with tarfile.open(self.archive_path, 'w:xz') as tar: + archive_path = os.path.join(PUB_LAYERS_DIR, f'{self.name}.tar.xz') + with tarfile.open(archive_path, 'w:xz') as tar: tar.add(self.layer_path, self.name, filter=files.add_file) self.size = files.size - self.dlsize = os.path.getsize(self.archive_path) - self.hash = repo_publish.sign_file(self.archive_path).hex() + self.dlsize = os.path.getsize(archive_path) + self.hash = repo_publish.sign_file(archive_path).hex() repo_publish.register_image(self.name, self.get_definition()) def unpublish(self): repo_publish.unregister_image(self.name) + archive_path = os.path.join(PUB_LAYERS_DIR, f'{self.name}.tar.xz') try: - os.unlink(self.archive_path) + os.unlink(archive_path) except FileNotFoundError: pass - def download(self): + def download(self, observer): + os.makedirs(TMP_LAYERS_DIR, 0o700, True) definition = repo_online.get_image(self.name) - repo_online.download_archive(self.online_path, LAYERS_DIR, definition['hash']) + observer.units_total = definition['dlsize'] + archive_url = urllib.parse.urljoin(ONLINE_LAYERS_URL, f'{self.name}.tar.xz') + archive_path = os.path.join(TMP_LAYERS_DIR, f'{self.name}.tar.xz') + repo_online.download_archive(archive_url, archive_path, definition['hash'], observer) + + def unpack_downloaded(self, observer): + definition = repo_online.get_image(self.name) + observer.units_total = definition['size'] + archive_path = os.path.join(TMP_LAYERS_DIR, f'{self.name}.tar.xz') + repo_online.unpack_archive(archive_path, LAYERS_DIR, definition['hash'], observer) self.set_definition(definition) repo_local.register_image(self.name, definition) diff --git a/usr/lib/python3.8/spoc/imagebuilder.py b/usr/lib/python3.8/spoc/imagebuilder.py index e444790..47d3b54 100644 --- a/usr/lib/python3.8/spoc/imagebuilder.py +++ b/usr/lib/python3.8/spoc/imagebuilder.py @@ -10,19 +10,16 @@ import zipfile from .container import Container from .image import Image -from .config import VOLUME_DIR +from .config import LAYERS_DIR class ImageBuilder: def build(self, image, filename): # Reset internal state, read and process lines from filename self.image = image - self.filename = filename + self.builddir = os.path.dirname(filename) self.script_eof = None self.script_lines = [] - self.image.create(self) - - def process_file(self): - with open(self.filename, 'r') as f: + with open(filename, 'r') as f: for line in f: self.process_line(line.strip()) @@ -71,7 +68,6 @@ class ImageBuilder: def run_script(self, script_lines): # Creates a temporary container, runs a script in its namespace, and stores the files modified by it as part of the layer # Note: If USER or WORKDIR directive has already been set, the command is run under that UID/GID or working directory - os.makedirs(VOLUME_DIR, 0o755, True) script_fd, script_path = tempfile.mkstemp(suffix='.sh', dir=self.image.layer_path, text=True) script_name = os.path.basename(script_path) script_lines = '\n'.join(script_lines) @@ -116,7 +112,7 @@ class ImageBuilder: if src.startswith('http://') or src.startswith('https://'): unpack_http_archive(src, dst) else: - src = os.path.join(os.path.dirname(self.filename), src) + src = os.path.join(self.builddir, src) copy_tree(src, dst) # Shift UID/GID of the files to the unprivileged range shift_uid(dst, os.stat(dst, follow_symlinks=False)) diff --git a/usr/lib/python3.8/spoc/repo_online.py b/usr/lib/python3.8/spoc/repo_online.py index 49e42cf..c531bae 100644 --- a/usr/lib/python3.8/spoc/repo_online.py +++ b/usr/lib/python3.8/spoc/repo_online.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import copy import hashlib import json import os @@ -8,6 +9,7 @@ import tarfile import tempfile import time +from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import ec, utils @@ -27,24 +29,68 @@ PUBLIC_KEY = get_pubkey() # TODO: HTTP Error handling for all downloads (including imagebuilder) -def download_archive(src, dst, expected_hash): - # Download archive via http(s), verify hash and decompress - with tempfile.TemporaryFile() as tmp_archive: - sha512 = hashes.SHA512() - hasher = hashes.Hash(sha512, default_backend()) - # Download the file via http(s) and store as temporary file - with requests.Session() as session: - resource = session.get(src, stream=True) +def verify_fileobj(fileobj, expected_hash): + hasher = hashes.Hash(hashes.SHA512(), default_backend()) + while True: + data = fileobj.read(64*1024) + if not data: + break + hasher.update(data) + PUBLIC_KEY.verify(bytes.fromhex(expected_hash), hasher.finalize(), ec.ECDSA(utils.Prehashed(hashes.SHA512()))) + +def download_archive(archive_url, archive_path, expected_hash, observer): + # Check if an archive needs to be downloaded via http(s) + do_download = True + # If the file already exists in the temporary directory, verify the signature + if os.path.exists(archive_path): + try: + with open(archive_path, 'rb') as f: + verify_fileobj(f, expected_hash) + # If the signature matches, skip download + observer.units_done = os.path.getsize(archive_path) + do_download = False + except InvalidSignature: + pass + if do_download: + # Download archive via http(s) and store in temporary directory + with open(archive_path, 'wb') as f, requests.Session() as session: + resource = session.get(archive_url, stream=True) for chunk in resource.iter_content(chunk_size=None): if chunk: - tmp_archive.write(chunk) - hasher.update(chunk) - # Verify hash - PUBLIC_KEY.verify(bytes.fromhex(expected_hash), hasher.finalize(), ec.ECDSA(utils.Prehashed(sha512))) - # Extract the tar.xz file - tmp_archive.seek(0) - with tarfile.open(fileobj=tmp_archive) as tar: - tar.extractall(dst, numeric_owner=True) + observer.units_done += f.write(chunk) + +def unpack_archive(archive_path, destination, expected_hash, observer): + with open(archive_path, 'rb') as f: + # Verify file object, then seek back and open it as tar without losing handle, thus preventing posible malicious race conditions + verify_fileobj(f, expected_hash) + f.seek(0) + tar = tarfile.open(fileobj=f) + # Extract the tar members while counting their size, taken from https://github.com/python/cpython/blob/master/Lib/tarfile.py + directories = [] + for tarinfo in tar: + if tarinfo.isdir(): + # Extract directories with a safe mode + directories.append(tarinfo) + tarinfo = copy.copy(tarinfo) + tarinfo.mode = 0o700 + # Do not set_attrs directories, as we will do that further down + tar.extract(tarinfo, destination, set_attrs=not tarinfo.isdir(), numeric_owner=True) + observer.units_done += tarinfo.size + # Reverse sort directories + directories.sort(key=lambda a: a.name) + directories.reverse() + # Set correct owner, mtime and filemode on directories + for tarinfo in directories: + dirpath = os.path.join(destination, tarinfo.name) + try: + tar.chown(tarinfo, dirpath, numeric_owner=True) + tar.utime(tarinfo, dirpath) + tar.chmod(tarinfo, dirpath) + except ExtractError as e: + if tar.errorlevel > 1: + raise + # Remove the archive + os.unlink(archive_path) def download_metadata(): with requests.Session() as session: diff --git a/usr/lib/python3.8/spoc/utils.py b/usr/lib/python3.8/spoc/utils.py index 0bd0acb..a2099eb 100644 --- a/usr/lib/python3.8/spoc/utils.py +++ b/usr/lib/python3.8/spoc/utils.py @@ -2,6 +2,17 @@ SIZE_PREFIXES = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') +class ActionItem: + def __init__(self, text, action): + self.text = text + self.action = action + self.units_total = 0 + self.units_done = 0 + + def run(self): + print(self.text) + self.action(self) + def readable_size(bytes): i = 0 while bytes > 1024: