Spotter-Cluster/spoc
2
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

ActionQueue + DepSolver for image removal / cleanup

Browse Source
This commit is contained in:
Disassembler 2020-02-18 10:48:57 +01:00
parent 67f994f190
commit 5e1b153c3d
No known key found for this signature in database
GPG Key ID: 524BD33A0EE29499
6 changed files with 86 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
usr/bin/spoc-container
View File

@ -80,7 +80,7 @@ def modify_container(container, depends, mounts, envs, uid, gid, cmd, cwd, ready
def create(container_name, image_name, depends, mounts, env, uid, gid, cmd, cwd, ready, halt, autostart):
# Create container based on image definition and extrea fields
container = Container(container_name, False)
container.set_definition(Image(image_name).get_definition())
container.set_definition(Image(image_name).get_definition(True))
modify_container(container, depends, mounts, env, uid, gid, cmd, cwd, ready, halt, autostart)
container.create()

52
usr/bin/spoc-image
View File

@ -2,10 +2,12 @@
# -*- coding: utf-8 -*-
import argparse
import sys
from spoc import repo_local
from spoc import repo_online
from spoc import repo_publish
from spoc.depsolver import DepSolver
from spoc.exceptions import ImageNotFoundError
from spoc.image import Image
from spoc.imagebuilder import ImageBuilder
@ -14,9 +16,10 @@ from spoc.cli import ActionQueue, readable_size
ACTION_LIST = 1
ACTION_DOWNLOAD = 2
ACTION_DELETE = 3
ACTION_BUILD = 4
ACTION_PUBLISH = 5
ACTION_UNPUBLISH = 6
ACTION_CLEAN = 4
ACTION_BUILD = 5
ACTION_PUBLISH = 6
ACTION_UNPUBLISH = 7
def get_image_name(file_path):
# Read and return image name from image file
@ -45,15 +48,37 @@ def download(image_name):
queue.process()
def delete(image_name):
# TODO: Check if any container or image doesn't depend
# TODO: Some kind of autoremove routine for images unused in containers
try:
image = Image(image_name)
except ImageNotFoundError:
return
# Remove the image including all images that have it as one of its parents
# Check if image is in use
used = [c for c,d in repo_local.get_containers().items() if image_name in d['layers']]
if used:
sys.exit(f'Error: Image {image_name} is used by container{"s" if len(used) > 1 else ""} {", ".join(used)}')
# Build dependency tree to safely remove the images in order of dependency
depsolver = DepSolver()
for image,definition in repo_local.get_images().items():
if image_name in definition['layers']:
image = Image(image)
depsolver.add(image.name, image.layers, image)
# Enqueue and run the removal actions
queue = ActionQueue()
for layer in reversed(image.layers):
queue.delete_image(Image(layer, False))
for image in reversed(depsolver.solve()):
queue.delete_image(image)
queue.process()
def clean():
# Remove images which aren't used in any locally defined containers
used = set()
for definition in repo_local.get_containers().values():
used.update(definition['layers'])
# Build dependency tree to safely remove the images in order of dependency
depsolver = DepSolver()
for image in set(repo_local.get_images()) - used:
image = Image(image)
depsolver.add(image.name, image.layers, image)
# Enqueue and run the removal actions
queue = ActionQueue()
for image in reversed(depsolver.solve()):
queue.delete_image(image)
queue.process()
def build(filename, force, do_publish):
@ -102,6 +127,9 @@ parser_delete = subparsers.add_parser('delete')
parser_delete.set_defaults(action=ACTION_DELETE)
parser_delete.add_argument('image')
parser_clean = subparsers.add_parser('clean')
parser_clean.set_defaults(action=ACTION_CLEAN)
parser_build = subparsers.add_parser('build')
parser_build.set_defaults(action=ACTION_BUILD)
parser_build.add_argument('-f', '--force', action='store_true', help='Force rebuild already existing image')
@ -125,6 +153,8 @@ elif args.action == ACTION_DOWNLOAD:
download(args.image)
elif args.action == ACTION_DELETE:
delete(args.image)
elif args.action == ACTION_CLEAN:
clean()
elif args.action == ACTION_BUILD:
build(args.file, args.force, args.publish)
elif args.action == ACTION_PUBLISH:

24
usr/lib/python3.8/spoc/container.py
View File

@ -8,9 +8,9 @@ import time
from concurrent.futures import ThreadPoolExecutor
from . import depsolver
from . import network
from . import repo_local
from .depsolver import DepSolver
from .exceptions import InvalidContainerStateError
from .config import CONTAINERS_DIR, LAYERS_DIR, LOG_DIR, HOSTS_FILE, VOLUME_DIR
from .templates import LXC_CONTAINER_TEMPLATE
@ -145,7 +145,9 @@ class Container:
def start(self):
# Start the container including its dependencies
for dependency in depsolver.solve(self.get_dependency_nodes()):
depsolver = DepSolver()
self.get_start_dependencies(depsolver)
for dependency in depsolver.solve():
if dependency.get_state() != STATE_RUNNING:
dependency.do_start()
@ -175,7 +177,9 @@ class Container:
def stop(self):
# Stop the containers depending on the current cotnainer
for dependency in depsolver.solve(self.get_reverse_dependency_nodes()):
depsolver = DepSolver()
self.get_stop_dependencies(depsolver)
for dependency in depsolver.solve():
if dependency.get_state() != STATE_STOPPED:
dependency.do_stop()
@ -218,18 +222,16 @@ class Container:
gid = self.execute(['/usr/bin/getent', 'group', group], capture_output=True, check=True).stdout.decode().split(':')[2]
return (uid,gid)
def get_dependency_nodes(self):
nodes = [depsolver.Node(self.name, self.depends, self)]
def get_start_dependencies(self, depsolver):
depsolver.add(self.name, self.depends, self)
for dependency in self.depends:
nodes.extend(Container(dependency).get_dependency_nodes())
return nodes
Container(dependency).get_start_dependencies(depsolver)
def get_reverse_dependency_nodes(self):
def get_stop_dependencies(self, depsolver):
reverse_depends = []
for name, definition in repo_local.get_containers().items():
if 'depends' in definition and self.name in definition['depends']:
reverse_depends.append(name)
nodes = [depsolver.Node(self.name, reverse_depends, self)]
depsolver.add(self.name, reverse_depends, self)
for dependency in reverse_depends:
nodes.extend(Container(dependency).get_reverse_dependency_nodes())
return nodes
Container(dependency).get_stop_dependencies(depsolver)

43
usr/lib/python3.8/spoc/depsolver.py
View File

@ -8,21 +8,28 @@ class Node:
self.depends = set(depends)
self.instance = instance
def solve(nodes):
# Returns a list of instances ordered by dependency
deps = {node.name: node for node in nodes}
result = []
while deps:
# Get a batch of nodes not depending on anything (or originally depending on already resolved nodes)
batch = {name for name, node in deps.items() if not node.depends}
if not batch:
# If there are no such nodes, we have found a circular dependency
raise CircularDependencyError(deps)
# Add instances tied to the resolved keys to the result and remove resolved keys from the dependecy map
for name in batch:
result.append(deps[name].instance)
del deps[name]
# Remove resolved keys from the dependencies of yet unresolved nodes
for node in deps.values():
node.depends -= batch
return result
class DepSolver:
def __init__(self):
self.nodes = []
def add(self, name, depends, instance):
self.nodes.append(Node(name, depends, instance))
def solve(self):
# Returns a list of instances ordered by dependency
deps = {node.name: node for node in self.nodes}
result = []
while deps:
# Get a batch of nodes not depending on anything (or originally depending on already resolved nodes)
batch = {name for name, node in deps.items() if not node.depends}
if not batch:
# If there are no such nodes, we have found a circular dependency
raise CircularDependencyError(deps)
# Add instances tied to the resolved keys to the result and remove resolved keys from the dependecy map
for name in batch:
result.append(deps[name].instance)
del deps[name]
# Remove resolved keys from the dependencies of yet unresolved nodes
for node in deps.values():
node.depends -= batch
return result

6
usr/lib/python3.8/spoc/image.py
View File

@ -16,7 +16,7 @@ class Image:
def __init__(self, name, load_from_repo=True):
self.name = name
self.layer_path = os.path.join(LAYERS_DIR, name)
self.layers = [name]
self.layers = []
self.env = {}
self.uid = None
self.gid = None
@ -34,12 +34,14 @@ class Image:
for key in DEFINITION_MEMBERS.intersection(definition):
setattr(self, key, definition[key])
def get_definition(self):
def get_definition(self, including_self_layer=False):
definition = {}
for key in DEFINITION_MEMBERS:
value = getattr(self, key)
if value:
definition[key] = value
if including_self_layer:
definition['layers'].append(self.name)
return definition
def create(self, imagebuilder, filename):

4
usr/lib/python3.8/spoc/imagebuilder.py
View File

@ -41,7 +41,7 @@ class ImageBuilder:
self.script_eof = args
elif 'FROM' == directive:
# Set the values of image from which this one inherits
self.image.set_definition(Image(args).get_definition())
self.image.set_definition(Image(args).get_definition(True))
self.image.layers.append(self.image.name)
elif 'COPY' == directive:
srcdst = args.split()
@ -77,7 +77,7 @@ class ImageBuilder:
os.chown(script_path, 100000, 100000)
# Create a temporary container from the current image definition and execute the script within the container
container = Container(self.image.name, False)
container.set_definition(self.image.get_definition())
container.set_definition(self.image.get_definition(True))
container.build = True
container.create()
container.execute(['/bin/sh', '-lc', os.path.join('/', script_name)], check=True)