diff --git a/etc/spoc/spoc.conf b/etc/spoc/spoc.conf new file mode 100644 index 0000000..9fab907 --- /dev/null +++ b/etc/spoc/spoc.conf @@ -0,0 +1,14 @@ +[general] +data-dir = /var/lib/spoc +log-dir = /var/log/spoc +network-interface = spocbr0 + +[publish] +publish-dir = /srv/build/spoc +signing-key = /etc/spoc/publish.key + +[repo] +url = https://repo.spotter.cz/spoc +username = +password = +public-key = MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEWJXH4Qm0kt2L86sntQH+C1zOJNQ0qMRt0vx4krTxRs9HQTQYAy//JC92ea2aKleA8OL0JF90b1NYXcQCWdAS+vE/ng9IEAii8C2+5nfuFeZ5YUjbQhfFblwHSM0c7hEG diff --git a/usr/bin/spoc-container b/usr/bin/spoc-container index 19c35eb..78ccb8d 100644 --- a/usr/bin/spoc-container +++ b/usr/bin/spoc-container @@ -8,7 +8,7 @@ import shlex from spoc import repo_local from spoc.container import Container, STATE_RUNNING, STATE_STOPPED from spoc.image import Image -from spoc.paths import VOLUME_DIR +from spoc.config import VOLUME_DIR ACTION_LIST = 1 ACTION_CREATE = 2 diff --git a/usr/bin/spoc-image b/usr/bin/spoc-image index 699b643..ab2a3d0 100644 --- a/usr/bin/spoc-image +++ b/usr/bin/spoc-image @@ -17,9 +17,9 @@ ACTION_BUILD = 4 ACTION_PUBLISH = 5 ACTION_UNPUBLISH = 6 -def get_image_name(filepath): +def get_image_name(file_path): # Read and return image name from image file - with open(filepath) as f: + with open(file_path) as f: for line in f: if line.startswith('IMAGE '): return line.split()[1] @@ -36,7 +36,7 @@ def listing(repo_type): print(image) def download(image_name): - raise NotImplementedException() # TODO + Image(image_name, False).download() def delete(image_name): Image(image_name, False).delete() diff --git a/usr/lib/python3.8/spoc/config.py b/usr/lib/python3.8/spoc/config.py new file mode 100644 index 0000000..bd39d0c --- /dev/null +++ b/usr/lib/python3.8/spoc/config.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +import configparser +import os +import urllib.parse + +config = configparser.ConfigParser() +config.read('/etc/spoc/spoc.conf') +print ('CONFIG LOADED') # TODO: Debug, remove + +def get_repo_auth(config): + username = config.get('repo', 'username', fallback='') + password = config.get('repo', 'password', fallback='') + if not username and not password: + return None + return (username, password) + +def get_repo_pubkey(config): + pubkey = config.get('repo', 'public-key', fallback='') + pubkey = f'-----BEGIN PUBLIC KEY-----\n{pubkey}\n-----END PUBLIC KEY-----' + return pubkey.encode() + +NETWORK_INTERFACE = config.get('general', 'network-interface', 'spocbr0') + +DATA_DIR = config.get('general', 'data-dir', fallback='/var/lib/spoc') +APPS_DIR = os.path.join(DATA_DIR, 'apps') +CONTAINERS_DIR = os.path.join(DATA_DIR, 'containers') +LAYERS_DIR = os.path.join(DATA_DIR, 'layers') +VOLUME_DIR = os.path.join(DATA_DIR, 'volumes') +HOSTS_FILE = os.path.join(DATA_DIR, 'hosts') +REPO_FILE = os.path.join(DATA_DIR, 'packages.json') +LOG_DIR = config.get('general', 'log-dir', fallback='/var/log/spoc') +LOCK_FILE = '/run/lock/spoc.lock' + +PUB_DIR = config.get('publish', 'publish-dir', fallback=os.path.join(DATA_DIR, 'publish')) +PUB_LAYERS_DIR = os.path.join(PUB_DIR, 'layers') +PUB_APPS_DIR = os.path.join(PUB_DIR, 'apps') +PUB_PACKAGES_FILE = os.path.join(PUB_DIR, 'packages.json') +PUB_SIG_FILE = os.path.join(PUB_DIR, 'packages.sig') +PUB_PRIVKEY_FILE = config.get('publish', 'signing-key', fallback='/etc/spoc/publish.key') + +REPO_URL = config.get('repo', 'url', fallback='https://localhost') +REPO_PACKAGES_URL = urllib.parse.urljoin(REPO_URL, 'packages.json') +REPO_SIG_URL = urllib.parse.urljoin(REPO_URL, 'packages.sig') +REPO_AUTH = get_repo_auth(config) +REPO_PUBKEY = get_repo_pubkey(config) diff --git a/usr/lib/python3.8/spoc/container.py b/usr/lib/python3.8/spoc/container.py index 81d22ee..6aa082d 100644 --- a/usr/lib/python3.8/spoc/container.py +++ b/usr/lib/python3.8/spoc/container.py @@ -12,7 +12,7 @@ from . import depsolver from . import network from . import repo_local from .exceptions import InvalidContainerStateError -from .paths import CONTAINERS_DIR, LAYERS_DIR, LOG_DIR, HOSTS_FILE, VOLUME_DIR +from .config import CONTAINERS_DIR, LAYERS_DIR, LOG_DIR, HOSTS_FILE, VOLUME_DIR from .templates import LXC_CONTAINER_TEMPLATE # States taken from https://github.com/lxc/lxc/blob/master/src/lxc/state.h diff --git a/usr/lib/python3.8/spoc/image.py b/usr/lib/python3.8/spoc/image.py index bcdb339..39e06b3 100644 --- a/usr/lib/python3.8/spoc/image.py +++ b/usr/lib/python3.8/spoc/image.py @@ -3,11 +3,12 @@ import os import shutil import tarfile +import urllib.parse from . import repo_local +from . import repo_online from . import repo_publish -from . import utils -from .paths import LAYERS_DIR, PUB_LAYERS_DIR +from .config import LAYERS_DIR, PUB_LAYERS_DIR, REPO_URL DEFINITION_MEMBERS = {'layers', 'env', 'uid', 'gid', 'cmd', 'cwd', 'ready', 'halt', 'size', 'dlsize', 'hash'} @@ -16,6 +17,7 @@ class Image: self.name = name self.layer_path = os.path.join(LAYERS_DIR, name) self.archive_path = os.path.join(PUB_LAYERS_DIR, f'{name}.tar.xz') + self.online_path = urllib.parse.urljoin(REPO_URL, 'images', f'{image_name}.tar.xz') self.layers = [name] self.env = {} self.uid = None @@ -58,13 +60,13 @@ class Image: pass def publish(self): - ctr = utils.TarSizeCounter() os.makedirs(PUB_LAYERS_DIR, 0o755, True) + files = repo_publish.TarSizeCounter() with tarfile.open(self.archive_path, 'w:xz') as tar: - tar.add(self.layer_path, self.name, filter=ctr.add_file) - self.size = ctr.size + tar.add(self.layer_path, self.name, filter=files.add_file) + self.size = files.size self.dlsize = os.path.getsize(self.archive_path) - self.hash = utils.hash_file(self.archive_path) + self.hash = repo_publish.sign_file(self.archive_path).hex() repo_publish.register_image(self.name, self.get_definition()) def unpublish(self): @@ -73,3 +75,9 @@ class Image: os.unlink(self.archive_path) except FileNotFoundError: pass + + def download(self): + definition = repo_online.get_image(self.name) + repo_online.download_archive(self.online_path, self.layer_path, definition['hash']) + self.set_definition(definition) + repo_local.register_image(self.name, definition) diff --git a/usr/lib/python3.8/spoc/imagebuilder.py b/usr/lib/python3.8/spoc/imagebuilder.py index 54ef1fa..e444790 100644 --- a/usr/lib/python3.8/spoc/imagebuilder.py +++ b/usr/lib/python3.8/spoc/imagebuilder.py @@ -10,7 +10,7 @@ import zipfile from .container import Container from .image import Image -from .paths import VOLUME_DIR +from .config import VOLUME_DIR class ImageBuilder: def build(self, image, filename): @@ -114,9 +114,60 @@ class ImageBuilder: # Copy files from the host or download them from a http(s) URL dst = os.path.join(self.image.layer_path, dst.lstrip('/')) if src.startswith('http://') or src.startswith('https://'): - utils.unpack_http_archive(src, dst) + unpack_http_archive(src, dst) else: src = os.path.join(os.path.dirname(self.filename), src) - utils.copy_tree(src, dst) + copy_tree(src, dst) # Shift UID/GID of the files to the unprivileged range - utils.shift_uid(dst, os.stat(dst, follow_symlinks=False)) + shift_uid(dst, os.stat(dst, follow_symlinks=False)) + +def unpack_http_archive(src, dst): + # Decompress an archive downloaded via http(s) + with tempfile.TemporaryFile() as tmp_archive: + # Download the file via http(s) and store as temporary file + with requests.Session() as session: + resource = session.get(src, stream=True) + for chunk in resource.iter_content(chunk_size=None): + if chunk: + tmp_archive.write(chunk) + # Check if the magic bytes and determine if the file is zip + tmp_archive.seek(0) + is_zip = zipfile.is_zipfile(tmp_archive) + # Extract the file. If it is not zip, assume tar (bzip2, gizp or xz) + tmp_archive.seek(0) + if is_zip: + with zipfile.ZipFile(tmp_archive) as zip: + zip.extractall(dst) + else: + with tarfile.open(fileobj=tmp_archive) as tar: + tar.extractall(dst, numeric_owner=True) + +def copy_tree(src, dst): + # TODO: shutil.copytree? + # Copies files from the host + if not os.path.isdir(src): + shutil.copy2(src, dst) + else: + os.makedirs(dst, exist_ok=True) + for name in os.listdir(src): + copy_tree(os.path.join(src, name), os.path.join(dst, name)) + shutil.copystat(src, dst) + +def shift_uid(path, path_stat): + # Shifts UID/GID of a file or a directory and its contents to the unprivileged range + # The function parameters could arguably be more friendly, but os.scandir() already calls stat() on the entires, + # so it would be wasteful to not reuse them for considerable performance gain + uid = path_stat.st_uid + gid = path_stat.st_gid + do_chown = False + if uid < 100000: + uid = uid + 100000 + do_chown = True + if gid < 100000: + gid = gid + 100000 + do_chown = True + if do_chown: + os.chown(path, uid, gid, follow_symlinks=False) + if stat.S_ISDIR(path_stat.st_mode): + for entry in os.scandir(path): + shift_uid(entry.path, entry.stat(follow_symlinks=False)) diff --git a/usr/lib/python3.8/spoc/network.py b/usr/lib/python3.8/spoc/network.py index 5c9ff58..1ca0c98 100644 --- a/usr/lib/python3.8/spoc/network.py +++ b/usr/lib/python3.8/spoc/network.py @@ -5,9 +5,8 @@ import ipaddress import socket import struct -from .paths import HOSTS_FILE +from .config import HOSTS_FILE, NETWORK_INTERFACE -INTERFACE_NAME = 'spocbr0' # ioctl magic constants taken from https://git.musl-libc.org/cgit/musl/tree/include/sys/ioctl.h (same as glibc) IOCTL_SIOCGIFADDR = 0x8915 IOCTL_SIOCGIFNETMASK = 0x891b @@ -32,7 +31,7 @@ def get_bridge_interface(): # Returns bridge interface's IP address and netmask with socket.socket(socket.AF_INET) as sock: # Get IPv4Interface for given interface name - packed_ifname = struct.pack('256s', INTERFACE_NAME.encode()) + packed_ifname = struct.pack('256s', NETWORK_INTERFACE.encode()) ip = socket.inet_ntoa(fcntl.ioctl(sock.fileno(), IOCTL_SIOCGIFADDR, packed_ifname)[20:24]) netmask = socket.inet_ntoa(fcntl.ioctl(sock.fileno(), IOCTL_SIOCGIFNETMASK, packed_ifname)[20:24]) return ipaddress.IPv4Interface(f'{ip}/{netmask}') diff --git a/usr/lib/python3.8/spoc/paths.py b/usr/lib/python3.8/spoc/paths.py deleted file mode 100644 index c111ca2..0000000 --- a/usr/lib/python3.8/spoc/paths.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- - -ROOT_DIR = '/var/lib/spoc' -CONTAINERS_DIR = '/var/lib/spoc/containers' -LAYERS_DIR = '/var/lib/spoc/layers' -VOLUME_DIR = '/var/lib/spoc/volumes' -HOSTS_FILE = '/var/lib/spoc/hosts' -REPO_FILE = '/var/lib/spoc/repository.json' -REPO_LOCK = '/run/lock/spoc-repository.lock' - -LOG_DIR = '/var/log/spoc' - -PUB_ROOT_DIR = '/srv/build/spoc' -PUB_LAYERS_DIR = '/srv/build/spoc/layers' -PUB_APPS_DIR = '/srv/build/spoc/apps' -PUB_REPO_FILE = '/srv/build/spoc/repository.json' -PUB_SIG_FILE = '/srv/build/spoc/repository.sig' -PUB_REPO_LOCK = '/run/lock/spoc-publish.lock' -PUB_PRIVATE_KEY = '/etc/spoc/publish.key' diff --git a/usr/lib/python3.8/spoc/repo_local.py b/usr/lib/python3.8/spoc/repo_local.py index 3f0d275..3bd3699 100644 --- a/usr/lib/python3.8/spoc/repo_local.py +++ b/usr/lib/python3.8/spoc/repo_local.py @@ -4,7 +4,7 @@ import json from .exceptions import AppNotFoundError, ContainerNotFoundError, ImageNotFoundError from .flock import lock_ex -from .paths import REPO_FILE, REPO_LOCK +from .config import REPO_FILE, LOCK_FILE TYPE_APP = 'apps' TYPE_CONTAINER = 'containers' @@ -22,7 +22,7 @@ def save(data): json.dump(data, f, sort_keys=True, indent=4) def get_entries(entry_type): - with lock_ex(REPO_LOCK): + with lock_ex(LOCK_FILE): data = load() return data[entry_type] @@ -30,13 +30,13 @@ def get_entry(entry_type, name): return get_entries(entry_type)[name] def add_entry(entry_type, name, definition): - with lock_ex(REPO_LOCK): + with lock_ex(LOCK_FILE): data = load() data[entry_type][name] = definition save(data) def delete_entry(entry_type, name): - with lock_ex(REPO_LOCK): + with lock_ex(LOCK_FILE): data = load() try: del data[entry_type][name] diff --git a/usr/lib/python3.8/spoc/repo_online.py b/usr/lib/python3.8/spoc/repo_online.py index 1896831..d6de9a5 100644 --- a/usr/lib/python3.8/spoc/repo_online.py +++ b/usr/lib/python3.8/spoc/repo_online.py @@ -1,16 +1,47 @@ # -*- coding: utf-8 -*- +import hashlib import json import requests +import tarfile +import tempfile + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec, utils +from cryptography.hazmat.primitives.serialization import load_pem_public_key -from . import utils from .exceptions import AppNotFoundError, ImageNotFoundError +from .config import REPO_AUTH, REPO_PUBKEY, REPO_PACKAGES_URL, REPO_SIG_URL TYPE_APP = 'apps' TYPE_IMAGE = 'images' +def download_archive(src, dst, expected_hash): + # Download archive via http(s), verify hash and decompress + with tempfile.TemporaryFile() as tmp_archive: + sha512 = hashes.SHA512() + hasher = hashes.Hash(sha512, default_backend()) + # Download the file via http(s) and store as temporary file + with requests.Session(auth=REPO_AUTH) as session: + resource = session.get(src, stream=True) + for chunk in resource.iter_content(chunk_size=None): + if chunk: + tmp_archive.write(chunk) + hasher.update(chunk) + # Verify hash + REPO_PUBKEY.verify(bytes.fromhex(expected_hash), hasher.finalize(), ec.ECDSA(utils.Prehashed(sha512))) + # Extract the tar.xz file + tmp_archive.seek(0) + with tarfile.open(fileobj=tmp_archive) as tar: + tar.extractall(dst, numeric_owner=True) + def load(): - raise NotImplementedError() + with requests.Session(auth=REPO_AUTH) as session: + packages = session.get(REPO_PACKAGES_URL, timout=5).content + packages_sig = bytes.fromhex(session.get(REPO_SIG_URL, timout=5).content) + REPO_PUBKEY.verify(packages_sig, packages, ec.ECDSA(hashes.SHA512())) + return json.loads(packages) def get_entries(entry_type): data = load() diff --git a/usr/lib/python3.8/spoc/repo_publish.py b/usr/lib/python3.8/spoc/repo_publish.py index ae324a0..c307d11 100644 --- a/usr/lib/python3.8/spoc/repo_publish.py +++ b/usr/lib/python3.8/spoc/repo_publish.py @@ -2,31 +2,57 @@ import json -from . import utils +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec, utils +from cryptography.hazmat.primitives.serialization import load_pem_private_key + from .exceptions import AppNotFoundError, ImageNotFoundError from .flock import lock_ex -from .paths import PUB_PRIVATE_KEY, PUB_REPO_FILE, PUB_REPO_LOCK, PUB_SIG_FILE +from .config import LOCK_FILE, PUB_PRIVKEY_FILE, PUB_PACKAGES_FILE, PUB_SIG_FILE TYPE_APP = 'apps' TYPE_IMAGE = 'images' +class TarSizeCounter: + def __init__(self): + self.size = 0 + + def add_file(self, tarinfo): + self.size += tarinfo.size + return tarinfo + +def sign_file(file_path): + # Generate ECDSA HMAC SHA512 signature of a file using EC private key + sha512 = hashes.SHA512() + hasher = hashes.Hash(sha512, default_backend()) + with open(file_path, 'rb') as f: + while True: + data = f.read(64*1024) + if not data: + break + hasher.update(data) + with open(PUB_PRIVKEY_FILE, 'rb') as f: + private_key = load_pem_private_key(f.read(), None, default_backend()) + return private_key.sign(hasher.finalize(), ec.ECDSA(utils.Prehashed(sha512))) + def load(): try: - with open(PUB_REPO_FILE) as f: + with open(PUB_PACKAGES_FILE) as f: return json.load(f) except FileNotFoundError: return {TYPE_IMAGE: {}, TYPE_APP: {}} def save(data): - with open(PUB_REPO_FILE, 'w') as f: + with open(PUB_PACKAGES_FILE, 'w') as f: json.dump(data, f, sort_keys=True, indent=4) # Cryptographically sign the repository file - signature = utils.sign_file(PUB_PRIVATE_KEY, PUB_REPO_FILE) + signature = sign_file(PUB_PACKAGES_FILE) with open(PUB_SIG_FILE, 'wb') as f: f.write(signature) def get_entries(entry_type): - with lock_ex(PUB_REPO_LOCK): + with lock_ex(LOCK_FILE): data = load() return data[entry_type] @@ -34,13 +60,13 @@ def get_entry(entry_type, name): return get_entries(entry_type)[name] def add_entry(entry_type, name, definition): - with lock_ex(PUB_REPO_LOCK): + with lock_ex(LOCK_FILE): data = load() data[entry_type][name] = definition save(data) def delete_entry(entry_type, name): - with lock_ex(PUB_REPO_LOCK): + with lock_ex(LOCK_FILE): data = load() try: del data[entry_type][name] diff --git a/usr/lib/python3.8/spoc/utils.py b/usr/lib/python3.8/spoc/utils.py index 2b42636..0bd0acb 100644 --- a/usr/lib/python3.8/spoc/utils.py +++ b/usr/lib/python3.8/spoc/utils.py @@ -1,24 +1,5 @@ # -*- coding: utf-8 -*- -import hashlib -import requests -import tarfile -import tempfile -import zipfile - -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.asymmetric import ec -from cryptography.hazmat.primitives.serialization import load_pem_private_key - -class TarSizeCounter: - def __init__(self): - self.size = 0 - - def add_file(self, tarinfo): - self.size += tarinfo.size - return tarinfo - SIZE_PREFIXES = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') def readable_size(bytes): @@ -27,81 +8,3 @@ def readable_size(bytes): i += 1 bytes /= 1024 return f'{bytes:.2f} {SIZE_PREFIXES[i]}B' - -def sign_file(private_key_path, input_path): - # Generate SHA512 signature of a file using EC private key - with open(private_key_path, 'rb') as private_key: - priv_key = load_pem_private_key(private_key.read(), None, default_backend()) - with open(input_path, 'rb') as input: - data = input.read() - return priv_key.sign(data, ec.ECDSA(hashes.SHA512())) - -def hash_file_fd(file): - # Calculate SHA512 hash of a file from file descriptor - sha512 = hashlib.sha512() - while True: - data = file.read(65536) - if not data: - break - sha512.update(data) - return sha512.hexdigest() - -def hash_file(file_path): - # Calculate SHA512 hash of a file - with open(file_path, 'rb') as file: - return hash_file_fd(file) - -def unpack_http_archive(src, dst, verify_hash=False): - # Decompress an archive downloaded via http(s) with optional hash verification - with tempfile.TemporaryFile() as tmp_archive: - # Download the file via http(s) and store as temporary file - with requests.Session() as session: - resource = session.get(src, stream=True) - for chunk in resource.iter_content(chunk_size=None): - if chunk: - tmp_archive.write(chunk) - if verify_hash: - # If a hash has been given, verify if - tmp_archive.seek(0) - if verify_hash != hash_file_fd(tmp_archive): - raise # TODO - # Check if the magic bytes and determine if the file is zip - tmp_archive.seek(0) - is_zip = zipfile.is_zipfile(tmp_archive) - # Extract the file. If it is not zip, assume tar (bzip2, gizp or xz) - tmp_archive.seek(0) - if is_zip: - with zipfile.ZipFile(tmp_archive) as zip: - zip.extractall(dst) - else: - with tarfile.open(fileobj=tmp_archive) as tar: - tar.extractall(dst, numeric_owner=True) - -def copy_tree(src, dst): - # Copies files from the host - if not os.path.isdir(src): - shutil.copy2(src, dst) - else: - os.makedirs(dst, exist_ok=True) - for name in os.listdir(src): - copy_tree(os.path.join(src, name), os.path.join(dst, name)) - shutil.copystat(src, dst) - -def shift_uid(path, path_stat): - # Shifts UID/GID of a file or a directory and its contents to the unprivileged range - # The function parameters could arguably be more friendly, but os.scandir() already calls stat() on the entires, - # so it would be wasteful to not reuse them for considerable performance gain - uid = path_stat.st_uid - gid = path_stat.st_gid - do_chown = False - if uid < 100000: - uid = uid + 100000 - do_chown = True - if gid < 100000: - gid = gid + 100000 - do_chown = True - if do_chown: - os.chown(path, uid, gid, follow_symlinks=False) - if stat.S_ISDIR(path_stat.st_mode): - for entry in os.scandir(path): - shift_uid(entry.path, entry.stat(follow_symlinks=False))