From 467cce9ac3544249e577c22ead926e8b1e2c79e7 Mon Sep 17 00:00:00 2001 From: Disassembler Date: Wed, 12 Feb 2020 16:03:32 +0100 Subject: [PATCH] Merge crypto + some imagebuilder functions into utils --- usr/bin/spoc-image | 7 +- usr/lib/python3.8/spoc/crypto.py | 27 -------- usr/lib/python3.8/spoc/image.py | 3 +- usr/lib/python3.8/spoc/imagebuilder.py | 53 +-------------- usr/lib/python3.8/spoc/repo_online.py | 2 +- usr/lib/python3.8/spoc/repo_publish.py | 4 +- usr/lib/python3.8/spoc/utils.py | 89 ++++++++++++++++++++++++++ 7 files changed, 99 insertions(+), 86 deletions(-) delete mode 100644 usr/lib/python3.8/spoc/crypto.py diff --git a/usr/bin/spoc-image b/usr/bin/spoc-image index bb41353..699b643 100644 --- a/usr/bin/spoc-image +++ b/usr/bin/spoc-image @@ -4,6 +4,7 @@ import argparse from spoc import repo_local +from spoc import repo_online from spoc import repo_publish from spoc.image import Image from spoc.imagebuilder import ImageBuilder @@ -38,8 +39,7 @@ def download(image_name): raise NotImplementedException() # TODO def delete(image_name): - image = Image(image_name, False) - image.delete() + Image(image_name, False).delete() def build(filename, force, do_publish): # Check if a build is needed and attempt to build the image from image file @@ -69,8 +69,7 @@ def publish(image_name, force): print(f'Image {image_name} already published, skipping publish task') def unpublish(image_name): - image = Image(image_name, False) - image.unpublish() + Image(image_name, False).unpublish() parser = argparse.ArgumentParser(description='SPOC image manager') parser.set_defaults(action=None) diff --git a/usr/lib/python3.8/spoc/crypto.py b/usr/lib/python3.8/spoc/crypto.py deleted file mode 100644 index de11c97..0000000 --- a/usr/lib/python3.8/spoc/crypto.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- - -import hashlib - -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.asymmetric import ec -from cryptography.hazmat.primitives.serialization import load_pem_private_key - -def sign_file(private_key_path, input_path): - # Generate SHA512 signature of a file using EC private key - with open(private_key_path, 'rb') as f: - priv_key = load_pem_private_key(f.read(), None, default_backend()) - with open(input_path, 'rb') as f: - data = f.read() - return priv_key.sign(data, ec.ECDSA(hashes.SHA512())) - -def hash_file(file_path): - # Calculate SHA512 hash of a file - sha512 = hashlib.sha512() - with open(file_path, 'rb') as f: - while True: - data = f.read(65536) - if not data: - break - sha512.update(data) - return sha512.hexdigest() diff --git a/usr/lib/python3.8/spoc/image.py b/usr/lib/python3.8/spoc/image.py index 28190f3..bcdb339 100644 --- a/usr/lib/python3.8/spoc/image.py +++ b/usr/lib/python3.8/spoc/image.py @@ -4,7 +4,6 @@ import os import shutil import tarfile -from . import crypto from . import repo_local from . import repo_publish from . import utils @@ -65,7 +64,7 @@ class Image: tar.add(self.layer_path, self.name, filter=ctr.add_file) self.size = ctr.size self.dlsize = os.path.getsize(self.archive_path) - self.hash = crypto.hash_file(self.archive_path) + self.hash = utils.hash_file(self.archive_path) repo_publish.register_image(self.name, self.get_definition()) def unpublish(self): diff --git a/usr/lib/python3.8/spoc/imagebuilder.py b/usr/lib/python3.8/spoc/imagebuilder.py index 6786186..54ef1fa 100644 --- a/usr/lib/python3.8/spoc/imagebuilder.py +++ b/usr/lib/python3.8/spoc/imagebuilder.py @@ -114,56 +114,9 @@ class ImageBuilder: # Copy files from the host or download them from a http(s) URL dst = os.path.join(self.image.layer_path, dst.lstrip('/')) if src.startswith('http://') or src.startswith('https://'): - unpack_http_archive(src, dst) + utils.unpack_http_archive(src, dst) else: src = os.path.join(os.path.dirname(self.filename), src) - copy_tree(src, dst) + utils.copy_tree(src, dst) # Shift UID/GID of the files to the unprivileged range - shift_uid(dst, os.stat(dst, follow_symlinks=False)) - -def unpack_http_archive(src, dst): - # Decompress an archive downloaded via http(s) - with tempfile.TemporaryFile() as tmp_archive: - with requests.Session() as session: - resource = session.get(src, stream=True) - for chunk in resource.iter_content(chunk_size=None): - if chunk: - tmp_archive.write(chunk) - tmp_archive.seek(0) - is_zip = zipfile.is_zipfile(tmp_archive) - tmp_archive.seek(0) - if is_zip: - with zipfile.ZipFile(tmp_archive) as zip: - zip.extractall(dst) - else: - with tarfile.open(fileobj=tmp_archive) as tar: - tar.extractall(dst, numeric_owner=True) - -def copy_tree(src, dst): - # Copies files from the host - if not os.path.isdir(src): - shutil.copy2(src, dst) - else: - os.makedirs(dst, exist_ok=True) - for name in os.listdir(src): - copy_tree(os.path.join(src, name), os.path.join(dst, name)) - shutil.copystat(src, dst) - -def shift_uid(path, path_stat): - # Shifts UID/GID of a file or a directory and its contents to the unprivileged range - # The function parameters could arguably be more friendly, but os.scandir() already calls stat() on the entires, - # so it would be wasteful to not reuse them for considerable performance gain - uid = path_stat.st_uid - gid = path_stat.st_gid - do_chown = False - if uid < 100000: - uid = uid + 100000 - do_chown = True - if gid < 100000: - gid = gid + 100000 - do_chown = True - if do_chown: - os.chown(path, uid, gid, follow_symlinks=False) - if stat.S_ISDIR(path_stat.st_mode): - for entry in os.scandir(path): - shift_uid(entry.path, entry.stat(follow_symlinks=False)) + utils.shift_uid(dst, os.stat(dst, follow_symlinks=False)) diff --git a/usr/lib/python3.8/spoc/repo_online.py b/usr/lib/python3.8/spoc/repo_online.py index 4a50ded..1896831 100644 --- a/usr/lib/python3.8/spoc/repo_online.py +++ b/usr/lib/python3.8/spoc/repo_online.py @@ -3,7 +3,7 @@ import json import requests -from . import crypto +from . import utils from .exceptions import AppNotFoundError, ImageNotFoundError TYPE_APP = 'apps' diff --git a/usr/lib/python3.8/spoc/repo_publish.py b/usr/lib/python3.8/spoc/repo_publish.py index da035ef..ae324a0 100644 --- a/usr/lib/python3.8/spoc/repo_publish.py +++ b/usr/lib/python3.8/spoc/repo_publish.py @@ -2,7 +2,7 @@ import json -from . import crypto +from . import utils from .exceptions import AppNotFoundError, ImageNotFoundError from .flock import lock_ex from .paths import PUB_PRIVATE_KEY, PUB_REPO_FILE, PUB_REPO_LOCK, PUB_SIG_FILE @@ -21,7 +21,7 @@ def save(data): with open(PUB_REPO_FILE, 'w') as f: json.dump(data, f, sort_keys=True, indent=4) # Cryptographically sign the repository file - signature = crypto.sign_file(PUB_PRIVATE_KEY, PUB_REPO_FILE) + signature = utils.sign_file(PUB_PRIVATE_KEY, PUB_REPO_FILE) with open(PUB_SIG_FILE, 'wb') as f: f.write(signature) diff --git a/usr/lib/python3.8/spoc/utils.py b/usr/lib/python3.8/spoc/utils.py index 043576c..2b42636 100644 --- a/usr/lib/python3.8/spoc/utils.py +++ b/usr/lib/python3.8/spoc/utils.py @@ -1,5 +1,16 @@ # -*- coding: utf-8 -*- +import hashlib +import requests +import tarfile +import tempfile +import zipfile + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.serialization import load_pem_private_key + class TarSizeCounter: def __init__(self): self.size = 0 @@ -16,3 +27,81 @@ def readable_size(bytes): i += 1 bytes /= 1024 return f'{bytes:.2f} {SIZE_PREFIXES[i]}B' + +def sign_file(private_key_path, input_path): + # Generate SHA512 signature of a file using EC private key + with open(private_key_path, 'rb') as private_key: + priv_key = load_pem_private_key(private_key.read(), None, default_backend()) + with open(input_path, 'rb') as input: + data = input.read() + return priv_key.sign(data, ec.ECDSA(hashes.SHA512())) + +def hash_file_fd(file): + # Calculate SHA512 hash of a file from file descriptor + sha512 = hashlib.sha512() + while True: + data = file.read(65536) + if not data: + break + sha512.update(data) + return sha512.hexdigest() + +def hash_file(file_path): + # Calculate SHA512 hash of a file + with open(file_path, 'rb') as file: + return hash_file_fd(file) + +def unpack_http_archive(src, dst, verify_hash=False): + # Decompress an archive downloaded via http(s) with optional hash verification + with tempfile.TemporaryFile() as tmp_archive: + # Download the file via http(s) and store as temporary file + with requests.Session() as session: + resource = session.get(src, stream=True) + for chunk in resource.iter_content(chunk_size=None): + if chunk: + tmp_archive.write(chunk) + if verify_hash: + # If a hash has been given, verify if + tmp_archive.seek(0) + if verify_hash != hash_file_fd(tmp_archive): + raise # TODO + # Check if the magic bytes and determine if the file is zip + tmp_archive.seek(0) + is_zip = zipfile.is_zipfile(tmp_archive) + # Extract the file. If it is not zip, assume tar (bzip2, gizp or xz) + tmp_archive.seek(0) + if is_zip: + with zipfile.ZipFile(tmp_archive) as zip: + zip.extractall(dst) + else: + with tarfile.open(fileobj=tmp_archive) as tar: + tar.extractall(dst, numeric_owner=True) + +def copy_tree(src, dst): + # Copies files from the host + if not os.path.isdir(src): + shutil.copy2(src, dst) + else: + os.makedirs(dst, exist_ok=True) + for name in os.listdir(src): + copy_tree(os.path.join(src, name), os.path.join(dst, name)) + shutil.copystat(src, dst) + +def shift_uid(path, path_stat): + # Shifts UID/GID of a file or a directory and its contents to the unprivileged range + # The function parameters could arguably be more friendly, but os.scandir() already calls stat() on the entires, + # so it would be wasteful to not reuse them for considerable performance gain + uid = path_stat.st_uid + gid = path_stat.st_gid + do_chown = False + if uid < 100000: + uid = uid + 100000 + do_chown = True + if gid < 100000: + gid = gid + 100000 + do_chown = True + if do_chown: + os.chown(path, uid, gid, follow_symlinks=False) + if stat.S_ISDIR(path_stat.st_mode): + for entry in os.scandir(path): + shift_uid(entry.path, entry.stat(follow_symlinks=False))