Spotter-Cluster/Spotter-VM
2
0
Fork 0
Code Issues 78 Pull Requests Actions Packages Projects Releases Wiki Activity

Implement scratch containers and image/app removal

Browse Source
This commit is contained in:
Disassembler 2019-10-05 22:26:54 +02:00
parent f2016d1b71
commit 62a6612a79
No known key found for this signature in database
GPG Key ID: 524BD33A0EE29499
8 changed files with 243 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apk/vmmgr

@ -1 +1 @@
Subproject commit 6045349f9c3602d6ba9b081a62d4338b202521d6
Subproject commit b02fc3f42c65d8833451e41b550f7588c9de2cc2

77
build/usr/bin/lxcbuild
View File

@ -8,37 +8,62 @@ from lxcbuild.app import App
from lxcbuild.image import Image
parser = argparse.ArgumentParser(description='VM application builder and packager')
parser.add_argument('-f', '--force', action='store_true', help='Force rebuild already built package')
parser.add_argument('buildpath', help='Either specific "lxcfile" or "meta" file or a directory containing at least one')
group = parser.add_mutually_exclusive_group()
group.add_argument('-f', '--force', action='store_true', help='Force rebuild already built package')
group.add_argument('-s', '--scratch', action='store_true', help='Build container for testing purposes, i.e. without cleanup on failure and packaging')
group.add_argument('-r', '--remove-image', action='store_true', help='Delete image (including scratch) from build repository')
group.add_argument('-e', '--remove-app', action='store_true', help='Delete application from build repository')
parser.add_argument('buildarg', help='Either specific "lxcfile" or "meta" file or a directory containing at least one of them')
if len(sys.argv) < 2:
parser.print_usage()
sys.exit(1)
args = parser.parse_args()
buildpath = os.path.realpath(args.buildpath)
if os.path.isfile(buildpath):
basename = os.path.basename(buildpath)
if basename == 'lxcfile' or basename.endswith('.lxcfile'):
image = Image(buildpath)
image.build_and_pack(args.force)
elif basename == 'meta' or basename.endswith('.meta'):
app = App(buildpath)
app.pack()
else:
print('Unknown file {} given, expected "lxcfile" or "meta"'.format(buildpath))
sys.exit(1)
def build_and_pack_image(args, path):
image = Image()
image.force_build = args.force or args.scratch
image.scratch_build = args.scratch
image.build_and_pack(path)
def pack_app(path):
app = App()
app.pack(path)
if args.remove_image:
image = Image()
image.name = args.buildarg
image.remove()
elif args.remove_app:
app = App()
app.name = args.buildarg
app.remove()
else:
valid_dir = False
for entry in os.scandir(buildpath):
if entry.is_file() and (entry.name == 'lxcfile' or entry.name.endswith('.lxcfile')):
buildpath = os.path.realpath(args.buildarg)
# If the buildpath is a file, determine type from filename
if os.path.isfile(buildpath):
basename = os.path.basename(buildpath)
if basename == 'lxcfile' or basename.endswith('.lxcfile'):
build_and_pack_image(args, buildpath)
# Compose files needs to be ignored when performing scratch builds
elif not args.scratch and basename == 'meta':
pack_app(buildpath)
else:
print('Unknown file {} given, expected "lxcfile"{}'.format(buildpath, '' if args.scratch else ' or "meta"'))
sys.exit(1)
# If the buildpath is a directory, build as much as possible, unless scratch build was requested, in which case don't build anything
else:
if args.scratch:
print('Please specify an lxcfile for scratch build')
sys.exit(1)
valid_dir = False
for entry in os.scandir(buildpath):
if entry.is_file() and (entry.name == 'lxcfile' or entry.name.endswith('.lxcfile')):
valid_dir = True
build_and_pack_image(args, entry.path)
meta = os.path.join(buildpath, 'meta')
if os.path.exists(meta):
valid_dir = True
image = Image(entry.path)
image.build_and_pack(args.force)
meta = os.path.join(buildpath, 'meta')
if os.path.exists(meta):
valid_dir = True
app = App(meta)
app.pack()
if not valid_dir:
print('Directory {} doesn\'t contain anything to build, skipping'.format(buildpath))
pack_app(meta)
if not valid_dir:
print('Directory {} doesn\'t contain anything to build, skipping'.format(buildpath))

22
build/usr/lib/python3.6/lxcbuild/app.py
View File

@ -4,11 +4,16 @@ import json
import os
import sys
from .builder import ImageNotFoundError
from .packer import Packer
from .apppacker import AppPacker
from .imagebuilder import ImageNotFoundError
class App:
def __init__(self, metafile):
def __init__(self):
self.name = None
self.conf = {}
self.build_dir = None
def load_metafile(self, metafile):
self.build_dir = os.path.dirname(metafile)
if os.path.basename(metafile) == 'meta':
self.name = os.path.basename(self.build_dir)
@ -17,10 +22,15 @@ class App:
with open(metafile, 'r') as f:
self.conf = json.load(f)
def pack(self):
packer = Packer()
def pack(self, metafile):
self.load_metafile(metafile)
packer = AppPacker(self)
try:
packer.pack_app(self)
packer.pack()
except ImageNotFoundError as e:
print('Image {} not found, can\'t pack {}'.format(e, self.name))
sys.exit(1)
def remove(self):
packer = AppPacker(self)
packer.remove()

61
build/usr/lib/python3.6/lxcbuild/apppacker.py Normal file
View File

@ -0,0 +1,61 @@
# -*- coding: utf-8 -*-
import os
import subprocess
from . import crypto
from .imagebuilder import ImageNotFoundError
from .packer import Packer
from .paths import REPO_APPS_DIR
class AppPacker(Packer):
def __init__(self, app):
super().__init__()
self.app = app
# Prepare package file names
self.tar_path = os.path.join(REPO_APPS_DIR, '{}.tar'.format(self.app.name))
self.xz_path = '{}.xz'.format(self.tar_path)
def pack(self):
# Check if all images used by containers exist
for container in self.app.conf['containers']:
image = self.app.conf['containers'][container]['image']
if image not in self.packages['images']:
raise ImageNotFoundError(image)
try:
os.unlink(self.xz_path)
except FileNotFoundError:
pass
self.create_archive()
self.register()
self.sign_packages()
def remove(self):
self.unregister()
try:
os.unlink(self.xz_path)
except FileNotFoundError:
pass
def create_archive(self):
# Create archive with application setup scripts
print('Archiving setup scripts for', self.app.name)
scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh')
scripts = [s for s in scripts if os.path.exists(os.path.join(self.app.build_dir, s))]
subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, '--transform', 's,^,{}/,'.format(self.app.name)] + scripts, cwd=self.app.build_dir)
self.compress_archive()
def register(self):
# Register package in global repository metadata file
print('Registering package {}'.format(self.app.name))
self.packages['apps'][self.app.name] = self.app.conf.copy()
self.packages['apps'][self.app.name]['size'] = self.tar_size
self.packages['apps'][self.app.name]['pkgsize'] = self.xz_size
self.packages['apps'][self.app.name]['sha512'] = crypto.hash_file(self.xz_path)
self.save_repo_meta()
def unregister(self):
# Removes package from global repository metadata file
if self.app.name in self.packages['apps']:
del self.packages['apps'][self.app.name]
self.save_repo_meta()

47
build/usr/lib/python3.6/lxcbuild/image.py
View File

@ -3,24 +3,30 @@
import os
import sys
from .builder import Builder, ImageExistsError, ImageNotFoundError
from .packer import Packer, PackageExistsError
from lxcmgr import lxcmgr
from .imagebuilder import ImageBuilder, ImageExistsError, ImageNotFoundError
from .imagepacker import ImagePacker
from .packer import PackageExistsError
class Image:
def __init__(self, lxcfile):
def __init__(self):
self.name = None
self.path = None
self.conf = {}
self.lxcfile = None
self.build_dir = None
self.force_build = False
self.scratch_build = False
def build_and_pack(self, lxcfile):
self.lxcfile = lxcfile
self.build_dir = os.path.dirname(lxcfile)
def build_and_pack(self, force):
self.conf['build'] = True
try:
builder = Builder()
builder.build(self, force)
builder = ImageBuilder(self)
builder.build()
# In case of successful build, packaging needs to happen in all cases to prevent outdated packages
force = True
self.force_build = True
except ImageExistsError as e:
print('Image {} already exists, skipping build tasks'.format(e))
except ImageNotFoundError as e:
@ -28,11 +34,22 @@ class Image:
builder.clean()
sys.exit(1)
except:
builder.clean()
if not self.scratch_build:
builder.clean()
raise
del self.conf['build']
try:
packer = Packer()
packer.pack_image(self, force)
except PackageExistsError as e:
print('Package {} already exists, skipping packaging tasks'.format(e))
# If we're doing a scratch build, regenerate the final LXC container configuration including ephemeral layer
if self.scratch_build:
lxcmgr.create_container(self.name, self.conf)
else:
try:
packer = ImagePacker(self)
packer.pack()
except PackageExistsError as e:
print('Package {} already exists, skipping packaging tasks'.format(e))
def remove(self):
builder = ImageBuilder(self)
builder.clean()
packer = ImagePacker(self)
packer.remove()

32
build/usr/lib/python3.6/lxcbuild/builder.py → build/usr/lib/python3.6/lxcbuild/imagebuilder.py
View File

@ -14,16 +14,13 @@ class ImageExistsError(Exception):
class ImageNotFoundError(Exception):
pass
class Builder:
def __init__(self):
self.image = None
class ImageBuilder:
def __init__(self, image):
self.image = image
self.script = []
self.script_eof = None
self.force = False
def build(self, image, force=False):
self.image = image
self.force = force
def build(self):
with open(self.image.lxcfile, 'r') as f:
for line in f:
line = line.strip()
@ -67,26 +64,27 @@ class Builder:
def run_script(self, script):
lxcmgr.create_container(self.image.name, self.image.conf)
sh = os.path.join(self.image.path, 'run.sh')
sh = os.path.join(LXC_STORAGE_DIR, self.image.name, 'run.sh')
with open(sh, 'w') as f:
f.write('#!/bin/sh\nset -ev\n\n{}\n'.format('\n'.join(script)))
os.chmod(sh, 0o700)
os.chown(sh, 100000, 100000)
subprocess.run(['lxc-execute', self.image.name, '--', '/bin/sh', '-lc', '/run.sh'], check=True)
os.unlink(sh)
lxcmgr.destroy_container(self.image.name)
if not self.image.scratch_build:
lxcmgr.destroy_container(self.image.name)
def set_name(self, name):
self.image.name = name
self.image.path = self.get_layer_path(name)
self.image.conf['layers'] = [name]
if os.path.exists(self.image.path):
if self.force:
image_path = self.get_layer_path(name)
if os.path.exists(image_path):
if self.image.force_build:
self.clean()
else:
raise ImageExistsError(self.image.path)
os.makedirs(self.image.path, 0o755, True)
os.chown(self.image.path, 100000, 100000)
raise ImageExistsError(image_path)
os.makedirs(image_path, 0o755, True)
os.chown(image_path, 100000, 100000)
def add_layer(self, name):
layer_path = self.get_layer_path(name)
@ -99,7 +97,7 @@ class Builder:
subprocess.run(cmd + layers, check=True)
def copy_files(self, src, dst):
dst = os.path.join(self.image.path, dst)
dst = os.path.join(LXC_STORAGE_DIR, self.image.name, dst)
if src.startswith('http://') or src.startswith('https://'):
unpack_http_archive(src, dst)
else:
@ -128,8 +126,8 @@ class Builder:
self.image.conf['ready'] = cmd
def clean(self):
shutil.rmtree(self.image.path)
lxcmgr.destroy_container(self.image.name)
shutil.rmtree(self.get_layer_path(self.image.name))
def unpack_http_archive(src, dst):
xf = 'xzf'

66
build/usr/lib/python3.6/lxcbuild/imagepacker.py Normal file
View File

@ -0,0 +1,66 @@
# -*- coding: utf-8 -*-
import os
import subprocess
from lxcmgr.paths import LXC_STORAGE_DIR
from lxcmgr.pkgmgr import PkgMgr
from . import crypto
from .packer import Packer
from .paths import REPO_IMAGES_DIR
class ImagePacker(Packer):
def __init__(self, image):
super().__init__()
self.image = image
# Prepare package file names
self.tar_path = os.path.join(REPO_IMAGES_DIR, '{}.tar'.format(self.image.name))
self.xz_path = '{}.xz'.format(self.tar_path)
def pack(self):
if self.image.force_build:
self.unregister()
try:
os.unlink(self.xz_path)
except FileNotFoundError:
pass
elif os.path.exists(self.xz_path):
raise PackageExistsError(self.xz_path)
self.create_archive()
self.register()
self.sign_packages()
def remove(self):
self.unregister()
try:
os.unlink(self.xz_path)
except FileNotFoundError:
pass
def create_archive(self):
# Create archive
print('Archiving', self.image.path)
subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, self.image.name], cwd=LXC_STORAGE_DIR)
self.compress_archive()
def register(self):
# Register image in global repository metadata file
print('Registering package {}'.format(self.image.name))
self.packages['images'][self.image.name] = self.image.conf.copy()
self.packages['images'][self.image.name]['size'] = self.tar_size
self.packages['images'][self.image.name]['pkgsize'] = self.xz_size
self.packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path)
self.save_repo_meta()
# Register the image also to locally installed images for package manager
pm = PkgMgr()
pm.register_image(self.image.name, self.packages['images'][self.image.name])
def unregister(self):
# Removes package from global repository metadata file
if self.image.name in self.packages['images']:
del self.packages['images'][self.image.name]
self.save_repo_meta()
# Unregister the image also from locally installed images for package manager
pm = PkgMgr()
pm.unregister_image(self.image.name)

85
build/usr/lib/python3.6/lxcbuild/packer.py
View File

@ -3,22 +3,15 @@
import json
import os
import subprocess
import sys
from lxcmgr.paths import LXC_STORAGE_DIR
from lxcmgr.pkgmgr import PkgMgr
from . import crypto
from .builder import ImageNotFoundError
from .paths import PRIVATE_KEY, REPO_APPS_DIR, REPO_IMAGES_DIR, REPO_META_FILE, REPO_SIG_FILE
from .paths import PRIVATE_KEY, REPO_META_FILE, REPO_SIG_FILE
class PackageExistsError(Exception):
pass
class Packer:
def __init__(self):
self.app = None
self.image = None
self.tar_path = None
self.tar_size = 0
self.xz_path = None
@ -33,29 +26,6 @@ class Packer:
with open(REPO_META_FILE, 'w') as f:
json.dump(self.packages, f, sort_keys=True, indent=4)
def pack_image(self, image, force):
self.image = image
# Prepare package file names
self.tar_path = os.path.join(REPO_IMAGES_DIR, '{}.tar'.format(self.image.name))
self.xz_path = '{}.xz'.format(self.tar_path)
if force:
self.unregister_image()
try:
os.unlink(self.xz_path)
except FileNotFoundError:
pass
elif os.path.exists(self.xz_path):
raise PackageExistsError(self.xz_path)
self.create_image_archive()
self.register_image()
self.sign_packages()
def create_image_archive(self):
# Create archive
print('Archiving', self.image.path)
subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, self.image.name], cwd=LXC_STORAGE_DIR)
self.compress_archive()
def compress_archive(self):
# Compress the tarball with xz (LZMA2)
self.tar_size = os.path.getsize(self.tar_path)
@ -64,60 +34,7 @@ class Packer:
self.xz_size = os.path.getsize(self.xz_path)
print('Compressed ', self.xz_path, '({:.2f} MB)'.format(self.xz_size/1048576))
def register_image(self):
# Register image in global repository metadata file
print('Registering package {}'.format(self.image.name))
self.packages['images'][self.image.name] = self.image.conf.copy()
self.packages['images'][self.image.name]['size'] = self.tar_size
self.packages['images'][self.image.name]['pkgsize'] = self.xz_size
self.packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path)
self.save_repo_meta()
# Register the image also to locally installed images for package manager
pm = PkgMgr()
pm.register_image(self.image.name, self.packages['images'][self.image.name])
def sign_packages(self):
signature = crypto.sign_file(PRIVATE_KEY, REPO_META_FILE)
with open(REPO_SIG_FILE, 'wb') as f:
f.write(signature)
def unregister_image(self):
# Removes package from global repository metadata file
if self.image.name in self.packages['images']:
del self.packages['images'][self.image.name]
self.save_repo_meta()
def pack_app(self, app):
self.app = app
# Check if all images exist
for container in app.conf['containers']:
image = app.conf['containers'][container]['image']
if image not in self.packages['images']:
raise ImageNotFoundError(image)
# Prepare package file names
self.tar_path = os.path.join(REPO_APPS_DIR, '{}.tar'.format(self.app.name))
self.xz_path = '{}.xz'.format(self.tar_path)
try:
os.unlink(self.xz_path)
except FileNotFoundError:
pass
self.create_app_archive()
self.register_app()
self.sign_packages()
def create_app_archive(self):
# Create archive with application setup scripts
print('Archiving setup scripts for', self.app.name)
scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh')
scripts = [s for s in scripts if os.path.exists(os.path.join(self.app.build_dir, s))]
subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, '--transform', 's,^,{}/,'.format(self.app.name)] + scripts, cwd=self.app.build_dir)
self.compress_archive()
def register_app(self):
# Register package in global repository metadata file
print('Registering package {}'.format(self.app.name))
self.packages['apps'][self.app.name] = self.app.conf.copy()
self.packages['apps'][self.app.name]['size'] = self.tar_size
self.packages['apps'][self.app.name]['pkgsize'] = self.xz_size
self.packages['apps'][self.app.name]['sha512'] = crypto.hash_file(self.xz_path)
self.save_repo_meta()