From 5e1b153c3d05d86a20a15b66f827ee24daf4a83a Mon Sep 17 00:00:00 2001 From: Disassembler Date: Tue, 18 Feb 2020 10:48:57 +0100 Subject: [PATCH] ActionQueue + DepSolver for image removal / cleanup --- usr/bin/spoc-container | 2 +- usr/bin/spoc-image | 52 ++++++++++++++++++++------ usr/lib/python3.8/spoc/container.py | 24 ++++++------ usr/lib/python3.8/spoc/depsolver.py | 43 ++++++++++++--------- usr/lib/python3.8/spoc/image.py | 6 ++- usr/lib/python3.8/spoc/imagebuilder.py | 4 +- 6 files changed, 86 insertions(+), 45 deletions(-) diff --git a/usr/bin/spoc-container b/usr/bin/spoc-container index 78ccb8d..da58d7c 100644 --- a/usr/bin/spoc-container +++ b/usr/bin/spoc-container @@ -80,7 +80,7 @@ def modify_container(container, depends, mounts, envs, uid, gid, cmd, cwd, ready def create(container_name, image_name, depends, mounts, env, uid, gid, cmd, cwd, ready, halt, autostart): # Create container based on image definition and extrea fields container = Container(container_name, False) - container.set_definition(Image(image_name).get_definition()) + container.set_definition(Image(image_name).get_definition(True)) modify_container(container, depends, mounts, env, uid, gid, cmd, cwd, ready, halt, autostart) container.create() diff --git a/usr/bin/spoc-image b/usr/bin/spoc-image index 39e5f1c..b092e73 100644 --- a/usr/bin/spoc-image +++ b/usr/bin/spoc-image @@ -2,10 +2,12 @@ # -*- coding: utf-8 -*- import argparse +import sys from spoc import repo_local from spoc import repo_online from spoc import repo_publish +from spoc.depsolver import DepSolver from spoc.exceptions import ImageNotFoundError from spoc.image import Image from spoc.imagebuilder import ImageBuilder @@ -14,9 +16,10 @@ from spoc.cli import ActionQueue, readable_size ACTION_LIST = 1 ACTION_DOWNLOAD = 2 ACTION_DELETE = 3 -ACTION_BUILD = 4 -ACTION_PUBLISH = 5 -ACTION_UNPUBLISH = 6 +ACTION_CLEAN = 4 +ACTION_BUILD = 5 +ACTION_PUBLISH = 6 +ACTION_UNPUBLISH = 7 def get_image_name(file_path): # Read and return image name from image file @@ -45,15 +48,37 @@ def download(image_name): queue.process() def delete(image_name): - # TODO: Check if any container or image doesn't depend - # TODO: Some kind of autoremove routine for images unused in containers - try: - image = Image(image_name) - except ImageNotFoundError: - return + # Remove the image including all images that have it as one of its parents + # Check if image is in use + used = [c for c,d in repo_local.get_containers().items() if image_name in d['layers']] + if used: + sys.exit(f'Error: Image {image_name} is used by container{"s" if len(used) > 1 else ""} {", ".join(used)}') + # Build dependency tree to safely remove the images in order of dependency + depsolver = DepSolver() + for image,definition in repo_local.get_images().items(): + if image_name in definition['layers']: + image = Image(image) + depsolver.add(image.name, image.layers, image) + # Enqueue and run the removal actions queue = ActionQueue() - for layer in reversed(image.layers): - queue.delete_image(Image(layer, False)) + for image in reversed(depsolver.solve()): + queue.delete_image(image) + queue.process() + +def clean(): + # Remove images which aren't used in any locally defined containers + used = set() + for definition in repo_local.get_containers().values(): + used.update(definition['layers']) + # Build dependency tree to safely remove the images in order of dependency + depsolver = DepSolver() + for image in set(repo_local.get_images()) - used: + image = Image(image) + depsolver.add(image.name, image.layers, image) + # Enqueue and run the removal actions + queue = ActionQueue() + for image in reversed(depsolver.solve()): + queue.delete_image(image) queue.process() def build(filename, force, do_publish): @@ -102,6 +127,9 @@ parser_delete = subparsers.add_parser('delete') parser_delete.set_defaults(action=ACTION_DELETE) parser_delete.add_argument('image') +parser_clean = subparsers.add_parser('clean') +parser_clean.set_defaults(action=ACTION_CLEAN) + parser_build = subparsers.add_parser('build') parser_build.set_defaults(action=ACTION_BUILD) parser_build.add_argument('-f', '--force', action='store_true', help='Force rebuild already existing image') @@ -125,6 +153,8 @@ elif args.action == ACTION_DOWNLOAD: download(args.image) elif args.action == ACTION_DELETE: delete(args.image) +elif args.action == ACTION_CLEAN: + clean() elif args.action == ACTION_BUILD: build(args.file, args.force, args.publish) elif args.action == ACTION_PUBLISH: diff --git a/usr/lib/python3.8/spoc/container.py b/usr/lib/python3.8/spoc/container.py index 6aa082d..e078b1a 100644 --- a/usr/lib/python3.8/spoc/container.py +++ b/usr/lib/python3.8/spoc/container.py @@ -8,9 +8,9 @@ import time from concurrent.futures import ThreadPoolExecutor -from . import depsolver from . import network from . import repo_local +from .depsolver import DepSolver from .exceptions import InvalidContainerStateError from .config import CONTAINERS_DIR, LAYERS_DIR, LOG_DIR, HOSTS_FILE, VOLUME_DIR from .templates import LXC_CONTAINER_TEMPLATE @@ -145,7 +145,9 @@ class Container: def start(self): # Start the container including its dependencies - for dependency in depsolver.solve(self.get_dependency_nodes()): + depsolver = DepSolver() + self.get_start_dependencies(depsolver) + for dependency in depsolver.solve(): if dependency.get_state() != STATE_RUNNING: dependency.do_start() @@ -175,7 +177,9 @@ class Container: def stop(self): # Stop the containers depending on the current cotnainer - for dependency in depsolver.solve(self.get_reverse_dependency_nodes()): + depsolver = DepSolver() + self.get_stop_dependencies(depsolver) + for dependency in depsolver.solve(): if dependency.get_state() != STATE_STOPPED: dependency.do_stop() @@ -218,18 +222,16 @@ class Container: gid = self.execute(['/usr/bin/getent', 'group', group], capture_output=True, check=True).stdout.decode().split(':')[2] return (uid,gid) - def get_dependency_nodes(self): - nodes = [depsolver.Node(self.name, self.depends, self)] + def get_start_dependencies(self, depsolver): + depsolver.add(self.name, self.depends, self) for dependency in self.depends: - nodes.extend(Container(dependency).get_dependency_nodes()) - return nodes + Container(dependency).get_start_dependencies(depsolver) - def get_reverse_dependency_nodes(self): + def get_stop_dependencies(self, depsolver): reverse_depends = [] for name, definition in repo_local.get_containers().items(): if 'depends' in definition and self.name in definition['depends']: reverse_depends.append(name) - nodes = [depsolver.Node(self.name, reverse_depends, self)] + depsolver.add(self.name, reverse_depends, self) for dependency in reverse_depends: - nodes.extend(Container(dependency).get_reverse_dependency_nodes()) - return nodes + Container(dependency).get_stop_dependencies(depsolver) diff --git a/usr/lib/python3.8/spoc/depsolver.py b/usr/lib/python3.8/spoc/depsolver.py index 7d07b90..a22ff38 100644 --- a/usr/lib/python3.8/spoc/depsolver.py +++ b/usr/lib/python3.8/spoc/depsolver.py @@ -8,21 +8,28 @@ class Node: self.depends = set(depends) self.instance = instance -def solve(nodes): - # Returns a list of instances ordered by dependency - deps = {node.name: node for node in nodes} - result = [] - while deps: - # Get a batch of nodes not depending on anything (or originally depending on already resolved nodes) - batch = {name for name, node in deps.items() if not node.depends} - if not batch: - # If there are no such nodes, we have found a circular dependency - raise CircularDependencyError(deps) - # Add instances tied to the resolved keys to the result and remove resolved keys from the dependecy map - for name in batch: - result.append(deps[name].instance) - del deps[name] - # Remove resolved keys from the dependencies of yet unresolved nodes - for node in deps.values(): - node.depends -= batch - return result +class DepSolver: + def __init__(self): + self.nodes = [] + + def add(self, name, depends, instance): + self.nodes.append(Node(name, depends, instance)) + + def solve(self): + # Returns a list of instances ordered by dependency + deps = {node.name: node for node in self.nodes} + result = [] + while deps: + # Get a batch of nodes not depending on anything (or originally depending on already resolved nodes) + batch = {name for name, node in deps.items() if not node.depends} + if not batch: + # If there are no such nodes, we have found a circular dependency + raise CircularDependencyError(deps) + # Add instances tied to the resolved keys to the result and remove resolved keys from the dependecy map + for name in batch: + result.append(deps[name].instance) + del deps[name] + # Remove resolved keys from the dependencies of yet unresolved nodes + for node in deps.values(): + node.depends -= batch + return result diff --git a/usr/lib/python3.8/spoc/image.py b/usr/lib/python3.8/spoc/image.py index a1c8e7f..629b6db 100644 --- a/usr/lib/python3.8/spoc/image.py +++ b/usr/lib/python3.8/spoc/image.py @@ -16,7 +16,7 @@ class Image: def __init__(self, name, load_from_repo=True): self.name = name self.layer_path = os.path.join(LAYERS_DIR, name) - self.layers = [name] + self.layers = [] self.env = {} self.uid = None self.gid = None @@ -34,12 +34,14 @@ class Image: for key in DEFINITION_MEMBERS.intersection(definition): setattr(self, key, definition[key]) - def get_definition(self): + def get_definition(self, including_self_layer=False): definition = {} for key in DEFINITION_MEMBERS: value = getattr(self, key) if value: definition[key] = value + if including_self_layer: + definition['layers'].append(self.name) return definition def create(self, imagebuilder, filename): diff --git a/usr/lib/python3.8/spoc/imagebuilder.py b/usr/lib/python3.8/spoc/imagebuilder.py index 47d3b54..9316bfc 100644 --- a/usr/lib/python3.8/spoc/imagebuilder.py +++ b/usr/lib/python3.8/spoc/imagebuilder.py @@ -41,7 +41,7 @@ class ImageBuilder: self.script_eof = args elif 'FROM' == directive: # Set the values of image from which this one inherits - self.image.set_definition(Image(args).get_definition()) + self.image.set_definition(Image(args).get_definition(True)) self.image.layers.append(self.image.name) elif 'COPY' == directive: srcdst = args.split() @@ -77,7 +77,7 @@ class ImageBuilder: os.chown(script_path, 100000, 100000) # Create a temporary container from the current image definition and execute the script within the container container = Container(self.image.name, False) - container.set_definition(self.image.get_definition()) + container.set_definition(self.image.get_definition(True)) container.build = True container.create() container.execute(['/bin/sh', '-lc', os.path.join('/', script_name)], check=True)