Introduce LXC-composer

usr/bin/vmmgr

@@ -24,16 +24,21 @@ parser_rebuild_issue = subparsers.add_parser('rebuild-issue')
 parser_rebuild_issue.set_defaults(action='rebuild-issue')
 
 parser_prepare_container = subparsers.add_parser('prepare-container')
-parser_prepare_container.add_argument('lxc', nargs=argparse.REMAINDER)
 parser_prepare_container.set_defaults(action='prepare-container')
+parser_prepare_container.add_argument('layers', help='OverlayFS LXC rootfs layers')
+parser_prepare_container.add_argument('lxc', nargs=argparse.REMAINDER)
+
+parser_cleanup_container = subparsers.add_parser('cleanup-container')
+parser_cleanup_container.set_defaults(action='cleanup-container')
+parser_cleanup_container.add_argument('lxc', nargs=argparse.REMAINDER)
 
 parser_register_container = subparsers.add_parser('register-container')
-parser_register_container.add_argument('lxc', nargs=argparse.REMAINDER)
 parser_register_container.set_defaults(action='register-container')
+parser_register_container.add_argument('lxc', nargs=argparse.REMAINDER)
 
 parser_unregister_container = subparsers.add_parser('unregister-container')
-parser_unregister_container.add_argument('lxc', nargs=argparse.REMAINDER)
 parser_unregister_container.set_defaults(action='unregister-container')
+parser_unregister_container.add_argument('lxc', nargs=argparse.REMAINDER)
 
 parser_register_proxy = subparsers.add_parser('register-proxy')
 parser_register_proxy.set_defaults(action='register-proxy')
@@ -56,12 +61,15 @@ elif args.action == 'rebuild-issue':
     vmmgr.rebuild_issue()
 elif args.action == 'prepare-container':
     # Used with LXC hooks on container startup
-    lxcmgr.prepare_container()
+    lxcmgr.prepare_container(args.layers)
+elif args.action == 'cleanup-container':
+    # Used with LXC hooks on container stop
+    lxcmgr.cleanup_container()
 elif args.action == 'register-container':
-    # Used with LXC hooks on container startup
+    # Used by package installer and builder
     lxcmgr.register_container()
 elif args.action == 'unregister-container':
-    # Used with LXC hooks on container stop
+    # Used by package installer and builder
     lxcmgr.unregister_container()
 elif args.action == 'register-proxy':
     # Used in init scripts on application startup

usr/lib/python3.6/vmmgr/lxcmgr.py

@@ -5,44 +5,94 @@ import os
 import shutil
 import subprocess
 
-from . import templates
 from .config import Config
-from .paths import HOSTS_FILE, HOSTS_LOCK, LXC_ROOT, NGINX_DIR
+from .paths import HOSTS_FILE, HOSTS_LOCK, LXC_ROOT
+from .templates import LXC_CONTAINER
 
-def prepare_container():
+def prepare_container(layers):
     # Extract the variables from values given via lxc.hook.pre-start hook
     app = os.environ['LXC_NAME']
     # Remove ephemeral layer data
     clean_ephemeral_layer(app)
+    # Prepare and mount overlayfs
+    prepare_overlayfs(app, layers)
     # Configure host and common params used in the app
     configure_app(app)
 
 def clean_ephemeral_layer(app):
-    # Cleans containers ephemeral layer.
+    # Cleans containers ephemeral layer. Called in lxc.hook.post-stop and lxc.hook.pre-start in case of unclean shutdown
-    # This is done early in the container start process, so the inode of the delta0 directory must remain unchanged
+    # This is done early in the container start process, so the inode of the ephemeral directory must remain unchanged
-    layer = os.path.join(LXC_ROOT, app, 'delta0')
+    ephemeral = os.path.join(LXC_ROOT, app, 'ephemeral')
-    if os.path.exists(layer):
+    for item in os.scandir(ephemeral):
-        for item in os.scandir(layer):
+        shutil.rmtree(item.path) if item.is_dir() else os.unlink(item.path)
-            shutil.rmtree(item.path) if item.is_dir() else os.unlink(item.path)
 
-def register_container():
+def prepare_overlayfs(app, layers):
-    # Extract the variables from values given via lxc.hook.start-host hook
+    # Prepare and mount overlayfs
-    app = os.environ['LXC_NAME']
+    rootfs = os.path.join(LXC_ROOT, app, 'rootfs')
-    pid = os.environ['LXC_PID']
+    # Unmount rootfs in case it remained mounted for whatever reason
-    # Lease the first unused IP to the container
+    subprocess.run(['umount', rootfs])
-    ip = update_hosts_lease(app, True)
+    layers = layers.split(',')
-    # Set IP in container based on PID given via lxc.hook.start-host hook
+    if len(layers) == 1:
-    cmd = 'ip addr add {}/16 broadcast 172.17.255.255 dev eth0 && ip route add default via 172.17.0.1'.format(ip)
+        # We have only single layer, no overlay needed
-    subprocess.run(['nsenter', '-a', '-t', pid, '--', '/bin/sh', '-c', cmd])
+        subprocess.run(['mount', '--bind', layers[0], rootfs])
+    else:
+        olwork = os.path.join(LXC_ROOT, app, 'olwork')
+        subprocess.run(['mount', '-t', 'overlay', '-o', 'upperdir={},lowerdir={},workdir={}'.format(layers[-1], ','.join(layers[:-1]), olwork), 'none', rootfs])
 
-def unregister_container():
+def configure_app(app):
+    # Supply common configuration for the application. Done as part of container preparation during service startup
+    script = os.path.join('/srv', app, 'update-conf.sh')
+    if os.path.exists(script):
+        conf = Config()
+        setup_env = os.environ.copy()
+        setup_env['DOMAIN'] = conf['host']['domain']
+        setup_env['PORT'] = conf['host']['port']
+        setup_env['EMAIL'] = conf['common']['email']
+        setup_env['GMAPS_API_KEY'] = conf['common']['gmaps-api-key']
+        subprocess.run([script], env=setup_env, check=True)
+
+def cleanup_container():
     # Extract the variables from values given via lxc.hook.post-stop hook
     app = os.environ['LXC_NAME']
-    # Release the container IP
+    # Unmount rootfs
-    update_hosts_lease(app, False)
+    rootfs = os.path.join(LXC_ROOT, app, 'rootfs')
+    subprocess.run(['umount', rootfs])
     # Remove ephemeral layer data
     clean_ephemeral_layer(app)
 
+def register_container(app, image):
+    # Create directories after container installation
+    rootfs = os.path.join(LXC_ROOT, app, 'rootfs')
+    olwork = os.path.join(LXC_ROOT, app, 'olwork')
+    ephemeral = os.path.join(LXC_ROOT, app, 'ephemeral')
+    os.makedirs(rootfs, 0o755, True)
+    os.makedirs(olwork, 0o755, True)
+    os.makedirs(ephemeral, 0o755, True)
+    os.chown(ephemeral, 100000, 100000)
+    # Create container configuration file
+    layers = ','.join([os.path.join(LXC_ROOT, 'storage', layer) for layer in image['layers']])
+    if 'build' not in image:
+        layers = '{},{}'.format(layer, ephemeral)
+    mounts = '\n'.join(['lxc.mount.entry = {} {} none bind,create={} 0 0'.format(m[1], m[2], m[0].lower()) for m in image['mounts']]) if 'mounts' in image else ''
+    env = '\n'.join(['lxc.environment = {}={}'.format(e[0], e[1]) for e in image['env']]) if 'env' in image else ''
+    uid = image['uid'] if 'uid' in image else '0'
+    gid = image['gid'] if 'gid' in image else '0'
+    cmd = image['cmd'] if 'cmd' in image else '/bin/sh'
+    cwd = image['cwd'] if 'cwd' in image else '/'
+    halt = image['halt'] if 'halt' in image else 'SIGINT'
+    # Lease the first unused IP to the container
+    ipv4 = update_hosts_lease(app, True)
+    # Create the config file
+    with open(os.path.join(LXC_ROOT, app, 'config'), 'w') as f:
+        f.write(LXC_CONTAINER.format(name=app, ipv4=ipv4, layers=layers, mounts=mounts, env=env, uid=uid, gid=gid, cmd=cmd, cwd=cwd, halt=halt))
+
+def unregister_container(app):
+    # Remove container configuration and directories
+    # TODO: Duplicated with what pkgmgr does, ale zustane to tady, protoze unregister se pouziva pri buildu
+    shutil.rmtree(os.path.join(LXC_ROOT, app))
+    # Release the IP address
+    update_hosts_lease(app, False)
+
 def update_hosts_lease(app, is_request):
     # This is a poor man's DHCP server which uses /etc/hosts as lease database
     # Leases the first unused IP from range 172.17.0.0/16
@@ -69,15 +119,3 @@ def update_hosts_lease(app, is_request):
             for lease in leases:
                 f.write('{} {}\n'.format(lease[0], lease[1]))
     return ip
-
-def configure_app(app):
-    # Supply common configuration for the application. Done as part of container preparation during service startup
-    script = os.path.join('/srv', app, 'update-conf.sh')
-    if os.path.exists(script):
-        conf = Config()
-        setup_env = os.environ.copy()
-        setup_env['DOMAIN'] = conf['host']['domain']
-        setup_env['PORT'] = conf['host']['port']
-        setup_env['EMAIL'] = conf['common']['email']
-        setup_env['GMAPS_API_KEY'] = conf['common']['gmaps-api-key']
-        subprocess.run([script], env=setup_env, check=True)

usr/lib/python3.6/vmmgr/templates.py

@@ -112,3 +112,53 @@ ISSUE = '''
    - \x1b[1m{url}\x1b[0m
    - \x1b[1m{ip}\x1b[0m\x1b[?1c
 '''
+
+LXC_CONTAINER = '''# Image name
+lxc.uts.name = {name}
+
+# Network
+lxc.net.0.type = veth
+lxc.net.0.link = lxcbr0
+lxc.net.0.flags = up
+lxc.net.0.ipv4.address = {ipv4}/16
+lxc.net.0.ipv4.gateway = 172.17.0.1
+
+# Volumes
+lxc.rootfs.path = /var/lib/lxc/{name}/rootfs
+
+# Mounts
+lxc.mount.entry = shm dev/shm tmpfs rw,nodev,noexec,nosuid,relatime,mode=1777,create=dir 0 0
+lxc.mount.entry = /etc/hosts etc/hosts none bind,create=file 0 0
+lxc.mount.entry = /etc/resolv.conf etc/resolv.conf none bind,create=file 0 0
+{mounts}
+
+# Init
+lxc.init.uid = {uid}
+lxc.init.gid = {gid}
+lxc.init.cwd = {cwd}
+
+# Environment
+lxc.environment = PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+{env}
+
+# Halt
+lxc.signal.halt = {halt}
+
+# Log
+lxc.console.size = 1MB
+lxc.console.logfile = /var/log/lxc/{name}.log
+
+# ID map
+lxc.idmap = u 0 100000 65536
+lxc.idmap = g 0 100000 65536
+
+# Hooks
+lxc.hook.pre-start = /usr/bin/vmmgr prepare-container {layers}
+lxc.hook.post-stop = /usr/bin/vmmgr cleanup-container
+
+# Other
+lxc.arch = linux64
+lxc.cap.drop = sys_admin
+lxc.include = /usr/share/lxc/config/common.conf
+lxc.include = /usr/share/lxc/config/userns.conf
+'''