From 972ca0b6967edd56af96a7de159950ac9fcbc4a6 Mon Sep 17 00:00:00 2001 From: Disassembler Date: Wed, 18 Sep 2019 11:27:49 +0200 Subject: [PATCH] Introduce LXC-composer --- usr/bin/vmmgr | 20 ++++-- usr/lib/python3.6/vmmgr/lxcmgr.py | 104 ++++++++++++++++++--------- usr/lib/python3.6/vmmgr/templates.py | 50 +++++++++++++ 3 files changed, 135 insertions(+), 39 deletions(-) diff --git a/usr/bin/vmmgr b/usr/bin/vmmgr index a8dc8b0..b668b33 100755 --- a/usr/bin/vmmgr +++ b/usr/bin/vmmgr @@ -24,16 +24,21 @@ parser_rebuild_issue = subparsers.add_parser('rebuild-issue') parser_rebuild_issue.set_defaults(action='rebuild-issue') parser_prepare_container = subparsers.add_parser('prepare-container') -parser_prepare_container.add_argument('lxc', nargs=argparse.REMAINDER) parser_prepare_container.set_defaults(action='prepare-container') +parser_prepare_container.add_argument('layers', help='OverlayFS LXC rootfs layers') +parser_prepare_container.add_argument('lxc', nargs=argparse.REMAINDER) + +parser_cleanup_container = subparsers.add_parser('cleanup-container') +parser_cleanup_container.set_defaults(action='cleanup-container') +parser_cleanup_container.add_argument('lxc', nargs=argparse.REMAINDER) parser_register_container = subparsers.add_parser('register-container') -parser_register_container.add_argument('lxc', nargs=argparse.REMAINDER) parser_register_container.set_defaults(action='register-container') +parser_register_container.add_argument('lxc', nargs=argparse.REMAINDER) parser_unregister_container = subparsers.add_parser('unregister-container') -parser_unregister_container.add_argument('lxc', nargs=argparse.REMAINDER) parser_unregister_container.set_defaults(action='unregister-container') +parser_unregister_container.add_argument('lxc', nargs=argparse.REMAINDER) parser_register_proxy = subparsers.add_parser('register-proxy') parser_register_proxy.set_defaults(action='register-proxy') @@ -56,12 +61,15 @@ elif args.action == 'rebuild-issue': vmmgr.rebuild_issue() elif args.action == 'prepare-container': # Used with LXC hooks on container startup - lxcmgr.prepare_container() + lxcmgr.prepare_container(args.layers) +elif args.action == 'cleanup-container': + # Used with LXC hooks on container stop + lxcmgr.cleanup_container() elif args.action == 'register-container': - # Used with LXC hooks on container startup + # Used by package installer and builder lxcmgr.register_container() elif args.action == 'unregister-container': - # Used with LXC hooks on container stop + # Used by package installer and builder lxcmgr.unregister_container() elif args.action == 'register-proxy': # Used in init scripts on application startup diff --git a/usr/lib/python3.6/vmmgr/lxcmgr.py b/usr/lib/python3.6/vmmgr/lxcmgr.py index 95a7e8d..de64c0f 100644 --- a/usr/lib/python3.6/vmmgr/lxcmgr.py +++ b/usr/lib/python3.6/vmmgr/lxcmgr.py @@ -5,44 +5,94 @@ import os import shutil import subprocess -from . import templates from .config import Config -from .paths import HOSTS_FILE, HOSTS_LOCK, LXC_ROOT, NGINX_DIR +from .paths import HOSTS_FILE, HOSTS_LOCK, LXC_ROOT +from .templates import LXC_CONTAINER -def prepare_container(): +def prepare_container(layers): # Extract the variables from values given via lxc.hook.pre-start hook app = os.environ['LXC_NAME'] # Remove ephemeral layer data clean_ephemeral_layer(app) + # Prepare and mount overlayfs + prepare_overlayfs(app, layers) # Configure host and common params used in the app configure_app(app) def clean_ephemeral_layer(app): - # Cleans containers ephemeral layer. - # This is done early in the container start process, so the inode of the delta0 directory must remain unchanged - layer = os.path.join(LXC_ROOT, app, 'delta0') - if os.path.exists(layer): - for item in os.scandir(layer): - shutil.rmtree(item.path) if item.is_dir() else os.unlink(item.path) + # Cleans containers ephemeral layer. Called in lxc.hook.post-stop and lxc.hook.pre-start in case of unclean shutdown + # This is done early in the container start process, so the inode of the ephemeral directory must remain unchanged + ephemeral = os.path.join(LXC_ROOT, app, 'ephemeral') + for item in os.scandir(ephemeral): + shutil.rmtree(item.path) if item.is_dir() else os.unlink(item.path) -def register_container(): - # Extract the variables from values given via lxc.hook.start-host hook - app = os.environ['LXC_NAME'] - pid = os.environ['LXC_PID'] - # Lease the first unused IP to the container - ip = update_hosts_lease(app, True) - # Set IP in container based on PID given via lxc.hook.start-host hook - cmd = 'ip addr add {}/16 broadcast 172.17.255.255 dev eth0 && ip route add default via 172.17.0.1'.format(ip) - subprocess.run(['nsenter', '-a', '-t', pid, '--', '/bin/sh', '-c', cmd]) +def prepare_overlayfs(app, layers): + # Prepare and mount overlayfs + rootfs = os.path.join(LXC_ROOT, app, 'rootfs') + # Unmount rootfs in case it remained mounted for whatever reason + subprocess.run(['umount', rootfs]) + layers = layers.split(',') + if len(layers) == 1: + # We have only single layer, no overlay needed + subprocess.run(['mount', '--bind', layers[0], rootfs]) + else: + olwork = os.path.join(LXC_ROOT, app, 'olwork') + subprocess.run(['mount', '-t', 'overlay', '-o', 'upperdir={},lowerdir={},workdir={}'.format(layers[-1], ','.join(layers[:-1]), olwork), 'none', rootfs]) -def unregister_container(): +def configure_app(app): + # Supply common configuration for the application. Done as part of container preparation during service startup + script = os.path.join('/srv', app, 'update-conf.sh') + if os.path.exists(script): + conf = Config() + setup_env = os.environ.copy() + setup_env['DOMAIN'] = conf['host']['domain'] + setup_env['PORT'] = conf['host']['port'] + setup_env['EMAIL'] = conf['common']['email'] + setup_env['GMAPS_API_KEY'] = conf['common']['gmaps-api-key'] + subprocess.run([script], env=setup_env, check=True) + +def cleanup_container(): # Extract the variables from values given via lxc.hook.post-stop hook app = os.environ['LXC_NAME'] - # Release the container IP - update_hosts_lease(app, False) + # Unmount rootfs + rootfs = os.path.join(LXC_ROOT, app, 'rootfs') + subprocess.run(['umount', rootfs]) # Remove ephemeral layer data clean_ephemeral_layer(app) +def register_container(app, image): + # Create directories after container installation + rootfs = os.path.join(LXC_ROOT, app, 'rootfs') + olwork = os.path.join(LXC_ROOT, app, 'olwork') + ephemeral = os.path.join(LXC_ROOT, app, 'ephemeral') + os.makedirs(rootfs, 0o755, True) + os.makedirs(olwork, 0o755, True) + os.makedirs(ephemeral, 0o755, True) + os.chown(ephemeral, 100000, 100000) + # Create container configuration file + layers = ','.join([os.path.join(LXC_ROOT, 'storage', layer) for layer in image['layers']]) + if 'build' not in image: + layers = '{},{}'.format(layer, ephemeral) + mounts = '\n'.join(['lxc.mount.entry = {} {} none bind,create={} 0 0'.format(m[1], m[2], m[0].lower()) for m in image['mounts']]) if 'mounts' in image else '' + env = '\n'.join(['lxc.environment = {}={}'.format(e[0], e[1]) for e in image['env']]) if 'env' in image else '' + uid = image['uid'] if 'uid' in image else '0' + gid = image['gid'] if 'gid' in image else '0' + cmd = image['cmd'] if 'cmd' in image else '/bin/sh' + cwd = image['cwd'] if 'cwd' in image else '/' + halt = image['halt'] if 'halt' in image else 'SIGINT' + # Lease the first unused IP to the container + ipv4 = update_hosts_lease(app, True) + # Create the config file + with open(os.path.join(LXC_ROOT, app, 'config'), 'w') as f: + f.write(LXC_CONTAINER.format(name=app, ipv4=ipv4, layers=layers, mounts=mounts, env=env, uid=uid, gid=gid, cmd=cmd, cwd=cwd, halt=halt)) + +def unregister_container(app): + # Remove container configuration and directories + # TODO: Duplicated with what pkgmgr does, ale zustane to tady, protoze unregister se pouziva pri buildu + shutil.rmtree(os.path.join(LXC_ROOT, app)) + # Release the IP address + update_hosts_lease(app, False) + def update_hosts_lease(app, is_request): # This is a poor man's DHCP server which uses /etc/hosts as lease database # Leases the first unused IP from range 172.17.0.0/16 @@ -69,15 +119,3 @@ def update_hosts_lease(app, is_request): for lease in leases: f.write('{} {}\n'.format(lease[0], lease[1])) return ip - -def configure_app(app): - # Supply common configuration for the application. Done as part of container preparation during service startup - script = os.path.join('/srv', app, 'update-conf.sh') - if os.path.exists(script): - conf = Config() - setup_env = os.environ.copy() - setup_env['DOMAIN'] = conf['host']['domain'] - setup_env['PORT'] = conf['host']['port'] - setup_env['EMAIL'] = conf['common']['email'] - setup_env['GMAPS_API_KEY'] = conf['common']['gmaps-api-key'] - subprocess.run([script], env=setup_env, check=True) diff --git a/usr/lib/python3.6/vmmgr/templates.py b/usr/lib/python3.6/vmmgr/templates.py index 9fa8e9c..4d81da1 100644 --- a/usr/lib/python3.6/vmmgr/templates.py +++ b/usr/lib/python3.6/vmmgr/templates.py @@ -112,3 +112,53 @@ ISSUE = ''' - \x1b[1m{url}\x1b[0m - \x1b[1m{ip}\x1b[0m\x1b[?1c ''' + +LXC_CONTAINER = '''# Image name +lxc.uts.name = {name} + +# Network +lxc.net.0.type = veth +lxc.net.0.link = lxcbr0 +lxc.net.0.flags = up +lxc.net.0.ipv4.address = {ipv4}/16 +lxc.net.0.ipv4.gateway = 172.17.0.1 + +# Volumes +lxc.rootfs.path = /var/lib/lxc/{name}/rootfs + +# Mounts +lxc.mount.entry = shm dev/shm tmpfs rw,nodev,noexec,nosuid,relatime,mode=1777,create=dir 0 0 +lxc.mount.entry = /etc/hosts etc/hosts none bind,create=file 0 0 +lxc.mount.entry = /etc/resolv.conf etc/resolv.conf none bind,create=file 0 0 +{mounts} + +# Init +lxc.init.uid = {uid} +lxc.init.gid = {gid} +lxc.init.cwd = {cwd} + +# Environment +lxc.environment = PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +{env} + +# Halt +lxc.signal.halt = {halt} + +# Log +lxc.console.size = 1MB +lxc.console.logfile = /var/log/lxc/{name}.log + +# ID map +lxc.idmap = u 0 100000 65536 +lxc.idmap = g 0 100000 65536 + +# Hooks +lxc.hook.pre-start = /usr/bin/vmmgr prepare-container {layers} +lxc.hook.post-stop = /usr/bin/vmmgr cleanup-container + +# Other +lxc.arch = linux64 +lxc.cap.drop = sys_admin +lxc.include = /usr/share/lxc/config/common.conf +lxc.include = /usr/share/lxc/config/userns.conf +'''