Spotter-Cluster/Spotter-VM
2
0
Fork 0
Code Issues 78 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove MERGE capability, add FROM

Browse Source 
layer inheritance is now linear
This commit is contained in:
Disassembler 2019-11-30 09:59:11 +01:00
parent 4d579ef8c1
commit f8403c5f42
No known key found for this signature in database
GPG Key ID: 524BD33A0EE29499
6 changed files with 41 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apk/vmmgr

@ -1 +1 @@
Subproject commit 41156fe4243b15b4b233b618082aae8ce32e5a2b
Subproject commit 7c25d22d4146033cfb1e0775d06912b5c8f77e73

174
build/usr/bin/lxcmerge
View File

@ -1,174 +0,0 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import argparse
import os
import shutil
import sys
import tarfile
import tempfile
APK_WORLD = 'etc/apk/world'
APK_INSTALLED = 'lib/apk/db/installed'
APK_SCRIPTS = 'lib/apk/db/scripts.tar'
APK_TRIGGERS = 'lib/apk/db/triggers'
ETC_PASSWD = 'etc/passwd'
ETC_GROUP = 'etc/groups'
ETC_SHADOW = 'etc/shadow'
def makedirs(path, mode=0o755, uid=100000, gid=100000):
try:
os.mkdir(path, mode)
os.chown(path, uid, gid)
except FileNotFoundError:
makedirs(os.path.dirname(path), mode, uid, gid)
os.mkdir(path, mode)
os.chown(path, uid, gid)
except FileExistsError:
pass
def merge_apk_world(layers):
world = []
for layer in layers:
try:
with open(os.path.join(layer, APK_WORLD), 'r') as f:
for line in f:
if line not in world:
world.append(line)
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(APK_WORLD)))
with open(os.path.join(layers[-1], APK_WORLD), 'w') as f:
f.writelines(world)
os.chown(os.path.join(layers[-1], APK_WORLD), 100000, 100000)
def merge_apk_installed(layers):
installed = []
for layer in layers:
try:
with open(os.path.join(layer, APK_INSTALLED), 'r') as f:
buffer = []
for line in f:
if line.startswith('C:'):
buffer = ''.join(buffer)
if buffer not in installed:
installed.append(buffer)
buffer = []
buffer.append(line)
buffer = ''.join(buffer)
if buffer not in installed:
installed.append(buffer)
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(APK_INSTALLED)))
with open(os.path.join(layers[-1], APK_INSTALLED), 'w') as f:
f.writelines(installed)
os.chown(os.path.join(layers[-1], APK_INSTALLED), 100000, 100000)
def merge_apk_scripts(layers):
tmp_tar_path = tempfile.mkstemp()[1]
files_in_tar = []
with tarfile.open(tmp_tar_path, 'w:') as tmp_tar:
for layer in layers:
tar_path = os.path.join(layer, APK_SCRIPTS)
if os.path.exists(tar_path):
with tarfile.open(tar_path, 'r:') as tar:
for member in tar.getmembers():
if member.name not in files_in_tar:
buffer = tar.extractfile(member)
tmp_tar.addfile(member, buffer)
files_in_tar.append(member.name)
if files_in_tar:
makedirs(os.path.join(layers[-1], os.path.dirname(APK_SCRIPTS)))
shutil.move(tmp_tar_path, os.path.join(layers[-1], APK_SCRIPTS))
os.chown(os.path.join(layers[-1], APK_SCRIPTS), 100000, 100000)
else:
os.unlink(tmp_tar_path)
def merge_apk_triggers(layers):
triggers = []
for layer in layers:
try:
with open(os.path.join(layer, APK_TRIGGERS), 'r') as f:
for line in f:
if line not in triggers:
triggers.append(line)
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(APK_TRIGGERS)))
with open(os.path.join(layers[-1], APK_TRIGGERS), 'w') as f:
f.writelines(triggers)
os.chown(os.path.join(layers[-1], APK_TRIGGERS), 100000, 100000)
def merge_etc_passwd(layers):
passwd = {}
for layer in layers:
try:
with open(os.path.join(layer, ETC_PASSWD), 'r') as f:
for line in f:
passwd[line.split(':')[0]] = line
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(ETC_PASSWD)))
with open(os.path.join(layers[-1], ETC_PASSWD), 'w') as f:
f.writelines(passwd.values())
os.chown(os.path.join(layers[-1], ETC_PASSWD), 100000, 100000)
def merge_etc_group(layers):
groups = {}
for layer in layers:
try:
with open(os.path.join(layer, ETC_GROUP), 'r') as f:
for line in f:
name,pwd,gid,users = line.split(':')
name = splitline[0]
users = splitline[3].strip().split(',')
if name not in groups:
groups[name] = [name,pwd,gid,users]
else:
groups[name][1] = pwd
groups[name][2] = gid
for user in users:
if user not in groups[name][3]:
groups[name][3].append(user)
except:
continue
for group in groups.values():
group[3] = '{}\n'.format(','.join(group[3]))
makedirs(os.path.join(layers[-1], os.path.dirname(ETC_GROUP)))
with open(os.path.join(layers[-1], ETC_GROUP), 'w') as f:
f.writelines([':'.join(group) for group in groups.values()])
os.chown(os.path.join(layers[-1], ETC_GROUP), 100000, 100000)
def merge_etc_shadow(layers):
shadow = {}
for layer in layers:
try:
with open(os.path.join(layer, ETC_SHADOW), 'r') as f:
for line in f:
shadow[line.split(':')[0]] = line
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(ETC_SHADOW)))
with open(os.path.join(layers[-1], ETC_SHADOW), 'w') as f:
f.writelines(shadow.values())
os.chown(os.path.join(layers[-1], ETC_SHADOW), 100000, 100042)
parser = argparse.ArgumentParser(description='APK database merge script')
parser.add_argument('layers', help='Path to LXC layers to be merged', nargs=argparse.REMAINDER)
if len(sys.argv) < 3:
parser.print_usage()
sys.exit(1)
args = parser.parse_args()
merge_apk_world(args.layers)
merge_apk_installed(args.layers)
merge_apk_scripts(args.layers)
merge_apk_triggers(args.layers)
merge_etc_passwd(args.layers)
merge_etc_group(args.layers)
merge_etc_shadow(args.layers)

2
build/usr/lib/python3.6/lxcbuild/image.py
View File

@ -25,7 +25,7 @@ class Image:
try:
builder = ImageBuilder(self)
builder.build()
# In case of successful build, packaging needs to happen in all cases to prevent outdated packages
# Packaging needs to happen in any case after a successful build in order to prevent outdated packages
self.force_build = True
except ImageExistsError as e:
print('Image {} already exists, skipping build tasks'.format(e))

38
build/usr/lib/python3.6/lxcbuild/imagebuilder.py
View File

@ -7,6 +7,7 @@ import sys
from lxcmgr import lxcmgr
from lxcmgr.paths import LXC_STORAGE_DIR
from lxcmgr.pkgmgr import PkgMgr
class ImageExistsError(Exception):
pass
@ -21,6 +22,7 @@ class ImageBuilder:
self.script_eof = None
def build(self):
# Read and process lines from lxcfile
with open(self.image.lxcfile, 'r') as f:
for line in f:
line = line.strip()
@ -34,15 +36,14 @@ class ImageBuilder:
self.process_line(*line.split(None, 1))
def process_line(self, directive, args):
# Process directives from lxcfile
if 'RUN' == directive:
self.script = []
self.script_eof = args
elif 'IMAGE' == directive:
self.set_name(args)
elif 'LAYER' == directive:
self.add_layer(args)
elif 'MERGE' == directive:
self.merge_layers(args.split())
elif 'FROM' == directive:
self.set_layers(args)
elif 'COPY' == directive:
srcdst = args.split()
self.copy_files(srcdst[0], srcdst[1] if len(srcdst) == 2 else '')
@ -63,6 +64,7 @@ class ImageBuilder:
return os.path.join(LXC_STORAGE_DIR, layer)
def run_script(self, script):
# Creates a temporary container, runs a script in its namespace, and stores the modifications as part of the image
lxcmgr.create_container(self.image.name, self.image.conf)
sh = os.path.join(LXC_STORAGE_DIR, self.image.name, 'run.sh')
with open(sh, 'w') as f:
@ -75,6 +77,7 @@ class ImageBuilder:
lxcmgr.destroy_container(self.image.name)
def set_name(self, name):
# Set name and first (topmost) layer of the image
self.image.name = name
self.image.conf['layers'] = [name]
image_path = self.get_layer_path(name)
@ -86,43 +89,47 @@ class ImageBuilder:
os.makedirs(image_path, 0o755, True)
os.chown(image_path, 100000, 100000)
def add_layer(self, name):
layer_path = self.get_layer_path(name)
if not os.path.exists(layer_path):
raise ImageNotFoundError(layer_path)
self.image.conf['layers'].insert(1, name)
def merge_layers(self, cmd):
layers = [self.get_layer_path(layer) for layer in self.image.conf['layers']]
subprocess.run(cmd + layers[::-1], check=True)
def set_layers(self, image):
# Extend list of layers with the list of layers from parent image
# Raies an exception when IMAGE has no name
pkgmgr = PkgMgr()
self.image.conf['layers'].extend(pkgmgr.installed_packages[image]['layers'])
def copy_files(self, src, dst):
# Copy files from the host or download them from a http(s) URL
dst = os.path.join(LXC_STORAGE_DIR, self.image.name, dst)
if src.startswith('http://') or src.startswith('https://'):
unpack_http_archive(src, dst)
else:
copy_tree(os.path.join(self.image.build_dir, src), dst)
# Shift UID/GID of the files to the unprivileged range
shift_uid(dst)
def add_env(self, key, value):
# Sets lxc.environment records for the image
if 'env' not in self.image.conf:
self.image.conf['env'] = []
self.image.conf['env'].append([key, value])
def set_user(self, uid, gid):
# Sets lxc.init.uid/gid for the image
self.image.conf['uid'] = uid
self.image.conf['gid'] = gid
def set_cmd(self, cmd):
# Sets lxc.init.cmd for the image
self.image.conf['cmd'] = cmd
def set_cwd(self, cwd):
# Sets lxc.init.cwd for the image
self.image.conf['cwd'] = cwd
def set_halt(self, halt):
# Sets lxc.signal.halt for the image
self.image.conf['halt'] = halt
def set_ready(self, cmd):
# Sets a command performed in OpenRC start_post to check readiness of the container
self.image.conf['ready'] = cmd
def clean(self):
@ -130,6 +137,7 @@ class ImageBuilder:
shutil.rmtree(self.get_layer_path(self.image.name))
def unpack_http_archive(src, dst):
# Decompress an archive downloaded via http(s)
xf = 'xzf'
if src.endswith('.bz2'):
xf = 'xjf'
@ -141,6 +149,7 @@ def unpack_http_archive(src, dst):
tar.wait()
def copy_tree(src, dst):
# Copies files from the host
if not os.path.isdir(src):
shutil.copy2(src, dst)
else:
@ -150,16 +159,19 @@ def copy_tree(src, dst):
shutil.copystat(src, dst)
def shift_uid(dir):
# Shifts UID/GID of a file or a directory and its contents to the unprivileged range
shift_uid_entry(dir, os.stat(dir, follow_symlinks=True))
shift_uid_recursively(dir)
def shift_uid_recursively(dir):
# Shifts UID/GID of a directory and its contents to the unprivileged range
for entry in os.scandir(dir):
shift_uid_entry(entry.path, entry.stat(follow_symlinks=False))
if entry.is_dir():
shift_uid_recursively(entry.path)
def shift_uid_entry(path, stat):
# Shifts UID/GID of a file or a directory to the unprivileged range
uid = stat.st_uid
gid = stat.st_gid
do_chown = False

9
build/usr/lib/python3.6/lxcbuild/imagepacker.py
View File

@ -47,10 +47,11 @@ class ImagePacker(Packer):
def register(self):
# Register image in global repository metadata file
print('Registering image package', self.image.name)
self.packages['images'][self.image.name] = self.image.conf.copy()
self.packages['images'][self.image.name]['size'] = self.tar_size
self.packages['images'][self.image.name]['pkgsize'] = self.xz_size
self.packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path)
image_conf = self.image.conf.copy()
image_conf['size'] = self.tar_size
image_conf['pkgsize'] = self.xz_size
image_conf['sha512'] = crypto.hash_file(self.xz_path)
self.packages['images'][self.image.name] = image_conf
self.save_repo_meta()
# Register the image also to locally installed images for package manager
pm = PkgMgr()

7
doc/toolchain/lxc-build.md
View File

@ -29,13 +29,6 @@ The *lxcfile* syntax is designed to resemble *Dockerfile* syntax in order to eas
- **Docker equivalent:** `FROM`
- **Populates LXC field:** `lxc.rootfs.path`
### MERGE
- **Usage:** `MERGE <scriptname>`
- **Description:** Runs `<scriptname>` on LXC host and passes all layer paths as parameter to this script. This helps you to resolve the conflicts in cases where you mix multiple OverlayFS layers with overlapping files, ie. package manager cache. The idea is that all layers are read separately by the `<scriptname>` script and the fixed result is written back to the uppermost layer.
- **Docker equivalent:** None
- **Populates LXC field:** None
### RUN
- **Usage:**