ActionQueue + DepSolver for image removal / cleanup

usr/bin/spoc-container

@@ -80,7 +80,7 @@ def modify_container(container, depends, mounts, envs, uid, gid, cmd, cwd, ready
 def create(container_name, image_name, depends, mounts, env, uid, gid, cmd, cwd, ready, halt, autostart):
     # Create container based on image definition and extrea fields
     container = Container(container_name, False)
-    container.set_definition(Image(image_name).get_definition())
+    container.set_definition(Image(image_name).get_definition(True))
     modify_container(container, depends, mounts, env, uid, gid, cmd, cwd, ready, halt, autostart)
     container.create()
 

usr/bin/spoc-image

@@ -2,10 +2,12 @@
 # -*- coding: utf-8 -*-
 
 import argparse
+import sys
 
 from spoc import repo_local
 from spoc import repo_online
 from spoc import repo_publish
+from spoc.depsolver import DepSolver
 from spoc.exceptions import ImageNotFoundError
 from spoc.image import Image
 from spoc.imagebuilder import ImageBuilder
@@ -14,9 +16,10 @@ from spoc.cli import ActionQueue, readable_size
 ACTION_LIST = 1
 ACTION_DOWNLOAD = 2
 ACTION_DELETE = 3
-ACTION_BUILD = 4
+ACTION_CLEAN = 4
-ACTION_PUBLISH = 5
+ACTION_BUILD = 5
-ACTION_UNPUBLISH = 6
+ACTION_PUBLISH = 6
+ACTION_UNPUBLISH = 7
 
 def get_image_name(file_path):
     # Read and return image name from image file
@@ -45,15 +48,37 @@ def download(image_name):
     queue.process()
 
 def delete(image_name):
-    # TODO: Check if any container or image doesn't depend
+    # Remove the image including all images that have it as one of its parents
-    # TODO: Some kind of autoremove routine for images unused in containers
+    # Check if image is in use
-    try:
+    used = [c for c,d in repo_local.get_containers().items() if image_name in d['layers']]
-        image = Image(image_name)
+    if used:
-    except ImageNotFoundError:
+        sys.exit(f'Error: Image {image_name} is used by container{"s" if len(used) > 1 else ""} {", ".join(used)}')
-        return
+    # Build dependency tree to safely remove the images in order of dependency
+    depsolver = DepSolver()
+    for image,definition in repo_local.get_images().items():
+        if image_name in definition['layers']:
+            image = Image(image)
+            depsolver.add(image.name, image.layers, image)
+    # Enqueue and run the removal actions
     queue = ActionQueue()
-    for layer in reversed(image.layers):
+    for image in reversed(depsolver.solve()):
-        queue.delete_image(Image(layer, False))
+        queue.delete_image(image)
+    queue.process()
+
+def clean():
+    # Remove images which aren't used in any locally defined containers
+    used = set()
+    for definition in repo_local.get_containers().values():
+        used.update(definition['layers'])
+    # Build dependency tree to safely remove the images in order of dependency
+    depsolver = DepSolver()
+    for image in set(repo_local.get_images()) - used:
+        image = Image(image)
+        depsolver.add(image.name, image.layers, image)
+    # Enqueue and run the removal actions
+    queue = ActionQueue()
+    for image in reversed(depsolver.solve()):
+        queue.delete_image(image)
     queue.process()
 
 def build(filename, force, do_publish):
@@ -102,6 +127,9 @@ parser_delete = subparsers.add_parser('delete')
 parser_delete.set_defaults(action=ACTION_DELETE)
 parser_delete.add_argument('image')
 
+parser_clean = subparsers.add_parser('clean')
+parser_clean.set_defaults(action=ACTION_CLEAN)
+
 parser_build = subparsers.add_parser('build')
 parser_build.set_defaults(action=ACTION_BUILD)
 parser_build.add_argument('-f', '--force', action='store_true', help='Force rebuild already existing image')
@@ -125,6 +153,8 @@ elif args.action == ACTION_DOWNLOAD:
     download(args.image)
 elif args.action == ACTION_DELETE:
     delete(args.image)
+elif args.action == ACTION_CLEAN:
+    clean()
 elif args.action == ACTION_BUILD:
     build(args.file, args.force, args.publish)
 elif args.action == ACTION_PUBLISH:

usr/lib/python3.8/spoc/container.py

@@ -8,9 +8,9 @@ import time
 
 from concurrent.futures import ThreadPoolExecutor
 
-from . import depsolver
 from . import network
 from . import repo_local
+from .depsolver import DepSolver
 from .exceptions import InvalidContainerStateError
 from .config import CONTAINERS_DIR, LAYERS_DIR, LOG_DIR, HOSTS_FILE, VOLUME_DIR
 from .templates import LXC_CONTAINER_TEMPLATE
@@ -145,7 +145,9 @@ class Container:
 
     def start(self):
         # Start the container including its dependencies
-        for dependency in depsolver.solve(self.get_dependency_nodes()):
+        depsolver = DepSolver()
+        self.get_start_dependencies(depsolver)
+        for dependency in depsolver.solve():
             if dependency.get_state() != STATE_RUNNING:
                 dependency.do_start()
 
@@ -175,7 +177,9 @@ class Container:
 
     def stop(self):
         # Stop the containers depending on the current cotnainer
-        for dependency in depsolver.solve(self.get_reverse_dependency_nodes()):
+        depsolver = DepSolver()
+        self.get_stop_dependencies(depsolver)
+        for dependency in depsolver.solve():
             if dependency.get_state() != STATE_STOPPED:
                 dependency.do_stop()
 
@@ -218,18 +222,16 @@ class Container:
             gid = self.execute(['/usr/bin/getent', 'group', group], capture_output=True, check=True).stdout.decode().split(':')[2]
         return (uid,gid)
 
-    def get_dependency_nodes(self):
+    def get_start_dependencies(self, depsolver):
-        nodes = [depsolver.Node(self.name, self.depends, self)]
+        depsolver.add(self.name, self.depends, self)
         for dependency in self.depends:
-            nodes.extend(Container(dependency).get_dependency_nodes())
+            Container(dependency).get_start_dependencies(depsolver)
-        return nodes
 
-    def get_reverse_dependency_nodes(self):
+    def get_stop_dependencies(self, depsolver):
         reverse_depends = []
         for name, definition in repo_local.get_containers().items():
             if 'depends' in definition and self.name in definition['depends']:
                 reverse_depends.append(name)
-        nodes = [depsolver.Node(self.name, reverse_depends, self)]
+        depsolver.add(self.name, reverse_depends, self)
         for dependency in reverse_depends:
-            nodes.extend(Container(dependency).get_reverse_dependency_nodes())
+            Container(dependency).get_stop_dependencies(depsolver)
-        return nodes

usr/lib/python3.8/spoc/depsolver.py

@@ -8,9 +8,16 @@ class Node:
         self.depends = set(depends)
         self.instance = instance
 
-def solve(nodes):
+class DepSolver:
+    def __init__(self):
+        self.nodes = []
+
+    def add(self, name, depends, instance):
+        self.nodes.append(Node(name, depends, instance))
+
+    def solve(self):
         # Returns a list of instances ordered by dependency
-    deps = {node.name: node for node in nodes}
+        deps = {node.name: node for node in self.nodes}
         result = []
         while deps:
             # Get a batch of nodes not depending on anything (or originally depending on already resolved nodes)

usr/lib/python3.8/spoc/image.py

@@ -16,7 +16,7 @@ class Image:
     def __init__(self, name, load_from_repo=True):
         self.name = name
         self.layer_path = os.path.join(LAYERS_DIR, name)
-        self.layers = [name]
+        self.layers = []
         self.env = {}
         self.uid = None
         self.gid = None
@@ -34,12 +34,14 @@ class Image:
         for key in DEFINITION_MEMBERS.intersection(definition):
             setattr(self, key, definition[key])
 
-    def get_definition(self):
+    def get_definition(self, including_self_layer=False):
         definition = {}
         for key in DEFINITION_MEMBERS:
             value = getattr(self, key)
             if value:
                 definition[key] = value
+        if including_self_layer:
+            definition['layers'].append(self.name)
         return definition
 
     def create(self, imagebuilder, filename):

usr/lib/python3.8/spoc/imagebuilder.py

@@ -41,7 +41,7 @@ class ImageBuilder:
             self.script_eof = args
         elif 'FROM' == directive:
             # Set the values of image from which this one inherits
-            self.image.set_definition(Image(args).get_definition())
+            self.image.set_definition(Image(args).get_definition(True))
             self.image.layers.append(self.image.name)
         elif 'COPY' == directive:
             srcdst = args.split()
@@ -77,7 +77,7 @@ class ImageBuilder:
         os.chown(script_path, 100000, 100000)
         # Create a temporary container from the current image definition and execute the script within the container
         container = Container(self.image.name, False)
-        container.set_definition(self.image.get_definition())
+        container.set_definition(self.image.get_definition(True))
         container.build = True
         container.create()
         container.execute(['/bin/sh', '-lc', os.path.join('/', script_name)], check=True)