Spotter-Cluster/Spotter-VM
2
0
Fork 0
Code Issues 78 Pull Requests Actions Packages Projects Releases Wiki Activity

Rename FIXLAYER to MERGE and update for merging passwd/group/shadow

Browse Source
This commit is contained in:
Disassembler 2019-11-17 15:04:34 +01:00
parent d3129d81ab
commit 5084dc1579
No known key found for this signature in database
GPG Key ID: 524BD33A0EE29499
14 changed files with 190 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build/install-toolchain.sh
View File

@ -22,8 +22,8 @@ adduser root abuild
cp etc/abuild.conf /etc/abuild.conf
# Prepare LXC build toolchain
cp usr/bin/fix-apk /usr/bin/fix-apk
cp usr/bin/lxc-build /usr/bin/lxc-build
cp usr/bin/lxcbuild /usr/bin/lxcbuild
cp usr/bin/lxcmerge /usr/bin/lxcmerge
mkdir -p /srv/build/lxc/apps /srv/build/lxc/images
# Prepare local APK repository

94
build/usr/bin/fix-apk
View File

@ -1,94 +0,0 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import argparse
import os
import shutil
import sys
import tarfile
import tempfile
parser = argparse.ArgumentParser(description='APK database merge script')
parser.add_argument('layers', help='Path to LXC layers to be merged', nargs=argparse.REMAINDER)
if len(sys.argv) < 2:
parser.print_usage()
sys.exit(1)
args = parser.parse_args()
layers = args.layers[::-1]
# /etc/apk/world
world = []
for layer in layers:
try:
with open(os.path.join(layer, 'etc/apk/world'), 'r') as f:
for line in f:
if line not in world:
world.append(line)
except:
continue
os.makedirs(os.path.join(layers[-1], 'etc/apk'), 0o755, True)
with open(os.path.join(layers[-1], 'etc/apk/world'), 'w') as f:
f.writelines(world)
os.chown(os.path.join(layers[-1], 'etc'), 100000, 100000)
os.chown(os.path.join(layers[-1], 'etc/apk'), 100000, 100000)
os.chown(os.path.join(layers[-1], 'etc/apk/world'), 100000, 100000)
# /lib/apk/db/installed
installed = []
for layer in layers:
try:
with open(os.path.join(layer, 'lib/apk/db/installed'), 'r') as f:
buffer = []
for line in f:
if line.startswith('C:'):
buffer = ''.join(buffer)
if buffer not in installed:
installed.append(buffer)
buffer = []
buffer.append(line)
buffer = ''.join(buffer)
if buffer not in installed:
installed.append(buffer)
except:
continue
os.makedirs(os.path.join(layers[-1], 'lib/apk/db'), 0o755, True)
with open(os.path.join(layers[-1], 'lib/apk/db/installed'), 'w') as f:
f.writelines(installed)
os.chown(os.path.join(layers[-1], 'lib'), 100000, 100000)
os.chown(os.path.join(layers[-1], 'lib/apk'), 100000, 100000)
os.chown(os.path.join(layers[-1], 'lib/apk/db'), 100000, 100000)
os.chown(os.path.join(layers[-1], 'lib/apk/db/installed'), 100000, 100000)
# /lib/apk/db/scripts.tar
tmp_tar_path = tempfile.mkstemp()[1]
files_in_tar = []
with tarfile.open(tmp_tar_path, 'w:') as tmp_tar:
for layer in layers:
tar_path = os.path.join(layer, 'lib/apk/db/scripts.tar')
if os.path.exists(tar_path):
with tarfile.open(tar_path, 'r:') as tar:
for member in tar.getmembers():
if member.name not in files_in_tar:
buffer = tar.extractfile(member)
tmp_tar.addfile(member, buffer)
files_in_tar.append(member.name)
if files_in_tar:
shutil.move(tmp_tar_path, os.path.join(layers[-1], 'lib/apk/db/scripts.tar'))
os.chown(os.path.join(layers[-1], 'lib/apk/db/scripts.tar'), 100000, 100000)
else:
os.unlink(tmp_tar_path)
# /lib/apk/db/triggers
triggers = []
for layer in layers:
try:
with open(os.path.join(layer, 'lib/apk/db/triggers'), 'r') as f:
for line in f:
if line not in triggers:
triggers.append(line)
except:
continue
with open(os.path.join(layers[-1], 'lib/apk/db/triggers'), 'w') as f:
f.writelines(triggers)
os.chown(os.path.join(layers[-1], 'lib/apk/db/triggers'), 100000, 100000)

175
build/usr/bin/lxcmerge Executable file
View File

@ -0,0 +1,175 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import argparse
import os
import shutil
import sys
import tarfile
import tempfile
APK_WORLD = 'etc/apk/world'
APK_INSTALLED = 'lib/apk/db/installed'
APK_SCRIPTS = 'lib/apk/db/scripts.tar'
APK_TRIGGERS = 'lib/apk/db/triggers'
ETC_PASSWD = 'etc/passwd'
ETC_GROUP = 'etc/groups'
ETC_SHADOW = 'etc/shadow'
def makedirs(path, mode=0o755, uid=100000, gid=100000):
try:
os.mkdir(path, mode)
os.chown(path, uid, gid)
except FileNotFoundError:
makedirs(os.path.dirname(path), mode, uid, gid)
os.mkdir(path, mode)
os.chown(path, uid, gid)
except FileExistsError:
pass
def merge_apk_world():
world = []
for layer in layers:
try:
with open(os.path.join(layer, APK_WORLD), 'r') as f:
for line in f:
if line not in world:
world.append(line)
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(APK_WORLD)))
with open(os.path.join(layers[-1], APK_WORLD), 'w') as f:
f.writelines(world)
os.chown(os.path.join(layers[-1], APK_WORLD), 100000, 100000)
def merge_apk_installed():
installed = []
for layer in layers:
try:
with open(os.path.join(layer, APK_INSTALLED), 'r') as f:
buffer = []
for line in f:
if line.startswith('C:'):
buffer = ''.join(buffer)
if buffer not in installed:
installed.append(buffer)
buffer = []
buffer.append(line)
buffer = ''.join(buffer)
if buffer not in installed:
installed.append(buffer)
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(APK_INSTALLED)))
with open(os.path.join(layers[-1], APK_INSTALLED), 'w') as f:
f.writelines(installed)
os.chown(os.path.join(layers[-1], APK_INSTALLED), 100000, 100000)
def merge_apk_scripts():
tmp_tar_path = tempfile.mkstemp()[1]
files_in_tar = []
with tarfile.open(tmp_tar_path, 'w:') as tmp_tar:
for layer in layers:
tar_path = os.path.join(layer, APK_SCRIPTS)
if os.path.exists(tar_path):
with tarfile.open(tar_path, 'r:') as tar:
for member in tar.getmembers():
if member.name not in files_in_tar:
buffer = tar.extractfile(member)
tmp_tar.addfile(member, buffer)
files_in_tar.append(member.name)
if files_in_tar:
makedirs(os.path.join(layers[-1], os.path.dirname(APK_SCRIPTS)))
shutil.move(tmp_tar_path, os.path.join(layers[-1], APK_SCRIPTS))
os.chown(os.path.join(layers[-1], APK_SCRIPTS), 100000, 100000)
else:
os.unlink(tmp_tar_path)
def merge_apk_triggers():
triggers = []
for layer in layers:
try:
with open(os.path.join(layer, APK_TRIGGERS), 'r') as f:
for line in f:
if line not in triggers:
triggers.append(line)
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(APK_TRIGGERS)))
with open(os.path.join(layers[-1], APK_TRIGGERS), 'w') as f:
f.writelines(triggers)
os.chown(os.path.join(layers[-1], APK_TRIGGERS), 100000, 100000)
def merge_etc_passwd():
passwd = {}
for layer in layers:
try:
with open(os.path.join(layer, ETC_PASSWD), 'r') as f:
for line in f:
passwd[line.split(':')[0]] = line
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(ETC_PASSWD)))
with open(os.path.join(layers[-1], ETC_PASSWD), 'w') as f:
f.writelines(passwd.values())
os.chown(os.path.join(layers[-1], ETC_PASSWD), 100000, 100000)
def merge_etc_group():
groups = {}
for layer in layers:
try:
with open(os.path.join(layer, ETC_GROUP), 'r') as f:
for line in f:
name,pwd,gid,users = line.split(':')
name = splitline[0]
users = splitline[3].strip().split(',')
if name not in groups:
groups[name] = [name,pwd,gid,users]
else:
groups[name][1] = pwd
groups[name][2] = gid
for user in users:
if user not in groups[name][3]:
groups[name][3].append(user)
except:
continue
for group in groups.values():
group[3] = '{}\n'.format(','.join(group[3]))
makedirs(os.path.join(layers[-1], os.path.dirname(ETC_GROUP)))
with open(os.path.join(layers[-1], ETC_GROUP), 'w') as f:
f.writelines([':'.join(group) for group in groups.values()])
os.chown(os.path.join(layers[-1], ETC_GROUP), 100000, 100000)
def merge_etc_shadow():
shadow = {}
for layer in layers:
try:
with open(os.path.join(layer, ETC_SHADOW), 'r') as f:
for line in f:
shadow[line.split(':')[0]] = line
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(ETC_SHADOW)))
with open(os.path.join(layers[-1], ETC_SHADOW), 'w') as f:
f.writelines(shadow.values())
os.chown(os.path.join(layers[-1], ETC_SHADOW), 100000, 100042)
parser = argparse.ArgumentParser(description='APK database merge script')
parser.add_argument('layers', help='Path to LXC layers to be merged', nargs=argparse.REMAINDER)
if len(sys.argv) < 2:
parser.print_usage()
sys.exit(1)
args = parser.parse_args()
layers = args.layers[::-1]
merge_apk_world()
merge_apk_installed()
merge_apk_scripts()
merge_apk_triggers()
merge_etc_passwd()
merge_etc_group()
merge_etc_shadow()

6
build/usr/lib/python3.6/lxcbuild/imagebuilder.py
View File

@ -41,8 +41,8 @@ class ImageBuilder:
self.set_name(args)
elif 'LAYER' == directive:
self.add_layer(args)
elif 'FIXLAYER' == directive:
self.fix_layer(args.split())
elif 'MERGE' == directive:
self.merge_layers(args.split())
elif 'COPY' == directive:
srcdst = args.split()
self.copy_files(srcdst[0], srcdst[1] if len(srcdst) == 2 else '')
@ -92,7 +92,7 @@ class ImageBuilder:
raise ImageNotFoundError(layer_path)
self.image.conf['layers'].insert(1, name)
def fix_layer(self, cmd):
def merge_layers(self, cmd):
layers = [self.get_layer_path(layer) for layer in self.image.conf['layers']]
subprocess.run(cmd + layers, check=True)

2
lxc-apps/crisiscleanup/lxcfile
View File

@ -4,7 +4,7 @@ LAYER alpine3.8_3.8.4-190620
LAYER alpine3.8-ruby2.4_2.4.5-190620
LAYER alpine3.8-nodejs8_8.14.0-190620
FIXLAYER /usr/bin/fix-apk
MERGE /usr/bin/lxcmerge
ENV RAILS_ENV production

2
lxc-apps/cts/lxcfile
View File

@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620
LAYER alpine3.9-python2.7_2.7.16-190620
LAYER alpine3.9-nginx_1.14.2-191115
FIXLAYER /usr/bin/fix-apk
MERGE /usr/bin/lxcmerge
RUN EOF
# Install runtime dependencies

2
lxc-apps/decidim/lxcfile
View File

@ -5,7 +5,7 @@ LAYER alpine3.9-ruby2.6_2.6.3-190620
LAYER alpine3.9-nodejs10_10.14.2-190620
LAYER alpine3.9-nginx_1.14.2-191115
FIXLAYER /usr/bin/fix-apk
MERGE /usr/bin/lxcmerge
ENV RAILS_ENV production

2
lxc-apps/gnuhealth/lxcfile
View File

@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620
LAYER alpine3.9-python3.6_3.6.8-190620
LAYER alpine3.9-nodejs10_10.14.2-190620
FIXLAYER /usr/bin/fix-apk
MERGE /usr/bin/lxcmerge
RUN EOF
# Install runtime dependencies

2
lxc-apps/odoo/lxcfile
View File

@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620
LAYER alpine3.9-python3.6_3.6.8-190620
LAYER alpine3.9-nodejs10_10.14.2-190620
FIXLAYER /usr/bin/fix-apk
MERGE /usr/bin/lxcmerge
RUN EOF
# Install runtime dependencies

2
lxc-apps/opendatakit/opendatakit-build.lxcfile
View File

@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620
LAYER alpine3.9-ruby2.4_2.4.5-190620
LAYER alpine3.9-nodejs10_10.14.2-190620
FIXLAYER /usr/bin/fix-apk
MERGE /usr/bin/lxcmerge
RUN EOF
# Install runtime dependencies

2
lxc-apps/openmapkit/lxcfile
View File

@ -5,7 +5,7 @@ LAYER alpine3.9-java8_8.212.04-190620
LAYER alpine3.9-python2.7_2.7.16-190620
LAYER alpine3.9-nodejs10_10.14.2-190620
FIXLAYER /usr/bin/fix-apk
MERGE /usr/bin/lxcmerge
RUN EOF
# Install build dependencies

2
lxc-apps/pandora/lxcfile
View File

@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620
LAYER alpine3.9-python3.6_3.6.8-190620
LAYER alpine3.9-nginx_1.14.2-191115
FIXLAYER /usr/bin/fix-apk
MERGE /usr/bin/lxcmerge
RUN EOF
# Install runtime dependencies

2
lxc-apps/sahana/lxcfile
View File

@ -4,7 +4,7 @@ LAYER alpine3.9_3.9.4-190620
LAYER alpine3.9-python2.7_2.7.16-190620
LAYER alpine3.9-nginx_1.14.2-191115
FIXLAYER /usr/bin/fix-apk
MERGE /usr/bin/lxcmerge
RUN EOF
# Install runtime dependencies

2
lxc-apps/seeddms/lxcfile
View File

@ -5,7 +5,7 @@ LAYER alpine3.9-nginx_1.14.2-191115
LAYER alpine3.9-php7.2_7.2.19-190620
LAYER alpine3.9-python3.6_3.6.8-190620
FIXLAYER /usr/bin/fix-apk
MERGE /usr/bin/lxcmerge
RUN EOF
# Install runtime dependencies