Revert to the pre-abuild way of package handling

This commit is contained in:
Disassembler 2019-02-21 00:10:38 +01:00
parent 9e928a4c58
commit 57db520dbb
No known key found for this signature in database
GPG Key ID: 524BD33A0EE29499
13 changed files with 298 additions and 164 deletions

View File

@ -8,5 +8,11 @@
"adminpwd": "${ADMINPWD}",
"domain": "spotter.vm",
"port": "443"
},
"packages": {},
"repo": {
"pwd": "",
"url": "https://dl.dasm.cz/spotter-repo",
"user": ""
}
}

5
etc/vmmgr/packages.pub Normal file
View File

@ -0,0 +1,5 @@
-----BEGIN PUBLIC KEY-----
MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEWJXH4Qm0kt2L86sntQH+C1zOJNQ0qMRt
0vx4krTxRs9HQTQYAy//JC92ea2aKleA8OL0JF90b1NYXcQCWdAS+vE/ng9IEAii
8C2+5nfuFeZ5YUjbQhfFblwHSM0c7hEG
-----END PUBLIC KEY-----

View File

@ -12,6 +12,7 @@ subparsers = parser.add_subparsers()
parser_register_app = subparsers.add_parser('register-app')
parser_register_app.set_defaults(action='register-app')
parser_register_app.add_argument('app', help='Application name')
parser_register_app.add_argument('host', help='Application subdomain')
parser_register_app.add_argument('login', nargs='?', help='Admin login')
parser_register_app.add_argument('password', nargs='?', help='Admin password')
@ -47,7 +48,7 @@ args = parser.parse_args()
vmmgr = VMMgr(Config())
if args.action == 'register-app':
# Used by package install.sh script
vmmgr.register_app(args.app, args.login, args.password)
vmmgr.register_app(args.app, args.host, args.login, args.password)
elif args.action == 'unregister-app':
# Used by package uninstall.sh script
vmmgr.unregister_app(args.app)

View File

@ -1,68 +1,68 @@
# -*- coding: utf-8 -*-
from collections import deque
from threading import Lock
class ActionItem:
def __init__(self, key, action):
self.key = key
self.action = action
self.started = False
self.data = None
class ActionQueue:
def __init__(self):
self.actions = {}
self.queue = deque()
self.lock = Lock()
self.is_running = False
def get_actions(self):
# Return copy of actions, so they can be traversed without state changes
with self.lock:
return self.actions.copy()
def enqueue_action(self, key, action):
# Enqueue action
with self.lock:
if key in self.actions:
# If the key alredy has a pending action, reject any other actions
return
item = ActionItem(key, action)
self.actions[key] = item
self.queue.append(item)
def process_actions(self):
# Main method for deferred queue processing called by WSGI close handler
with self.lock:
# If the queue is being processesd by another thread, allow this thread to be terminated
if self.is_running:
return
while True:
with self.lock:
# Try to get an item from queue
item = None
if self.queue:
item = self.queue.popleft()
# If there are no more queued items, unset the processing flag and allow the thread to be terminated
if not item:
self.is_running = False
return
# If there is an item to be processed, set processing flags and exit the lock
self.is_running = True
item.started = True
try:
# Call the method passed in item.action with the whole item as parameter
item.action(item)
# If the action finished without errors, restore nominal state by deleting the item from action list
self.clear_action(item.key)
except BaseException as e:
# If the action failed, store the exception and leave it in the list for manual clearance
with self.lock:
item.data = e
def clear_action(self, key):
# Restore nominal state by deleting the item from action list
with self.lock:
if key in self.actions:
del self.actions[key]
# -*- coding: utf-8 -*-
from collections import deque
from threading import Lock
class ActionItem:
def __init__(self, key, action):
self.key = key
self.action = action
self.started = False
self.data = None
class ActionQueue:
def __init__(self):
self.actions = {}
self.queue = deque()
self.lock = Lock()
self.is_running = False
def get_actions(self):
# Return copy of actions, so they can be traversed without state changes
with self.lock:
return self.actions.copy()
def enqueue_action(self, key, action):
# Enqueue action
with self.lock:
if key in self.actions:
# If the key alredy has a pending action, reject any other actions
return
item = ActionItem(key, action)
self.actions[key] = item
self.queue.append(item)
def process_actions(self):
# Main method for deferred queue processing called by WSGI close handler
with self.lock:
# If the queue is being processesd by another thread, allow this thread to be terminated
if self.is_running:
return
while True:
with self.lock:
# Try to get an item from queue
item = None
if self.queue:
item = self.queue.popleft()
# If there are no more queued items, unset the processing flag and allow the thread to be terminated
if not item:
self.is_running = False
return
# If there is an item to be processed, set processing flags and exit the lock
self.is_running = True
item.started = True
try:
# Call the method passed in item.action with the whole item as parameter
item.action(item)
# If the action finished without errors, restore nominal state by deleting the item from action list
self.clear_action(item.key)
except BaseException as e:
# If the action failed, store the exception and leave it in the list for manual clearance
with self.lock:
item.data = e
def clear_action(self, key):
# Restore nominal state by deleting the item from action list
with self.lock:
if key in self.actions:
del self.actions[key]

View File

@ -1,16 +1,14 @@
# -*- coding: utf-8 -*-
import json
import math
import os
import requests
import subprocess
import time
from .pkgmgr import Pkg, PkgMgr
class AppMgr:
def __init__(self, conf):
self.conf = conf
self.online_packages = {}
self.pkgmgr = PkgMgr(conf)
def start_app(self, item):
# Start the actual app service
@ -55,42 +53,17 @@ class AppMgr:
return os.path.exists(os.path.join('/etc/runlevels/default', app))
def install_app(self, item):
# Main installation function. Wrapper for installation via native package manager
item.data = 0
# Alpine apk provides machine-readable progress in bytes_downloaded/bytes_total format output to file descriptor of choice, so create a pipe for it
pipe_rfd, pipe_wfd = os.pipe()
with os.fdopen(pipe_rfd) as pipe_rf:
with subprocess.Popen(['apk', '--progress-fd', str(pipe_wfd), '--no-cache', 'add', 'vm-{}@vm'.format(item.key)], pass_fds=[pipe_wfd]) as p:
# Close write pipe for vmmgr to not block the pipe once apk finishes
os.close(pipe_wfd)
while p.poll() == None:
# Wait for line end or EOF in read pipe and process it
data = pipe_rf.readline()
if data:
progress = data.rstrip().split('/')
item.data = math.floor(int(progress[0]) / int(progress[1]) * 100)
# If the apk command didn't finish with returncode 0, raise an exception
if p.returncode:
raise subprocess.CalledProcessError(p.returncode, p.args)
# Main installation function. Wrapper for download, registration and install script
item.data = Pkg()
self.pkgmgr.install_app(item.key, item.data)
def uninstall_app(self, item):
# Main uninstallation function. Wrapper for uninstallation via native package manager
# Main uninstallation function. Wrapper for uninstall script, filesystem purge and unregistration
app = item.key
self.stop_app(item)
if self.is_service_autostarted(app):
self.update_app_autostart(app, False)
subprocess.run(['apk', '--no-cache', 'del', 'vm-{}'.format(app)], check=True)
def fetch_online_packages(self, repo_conf):
# Fetches list of online packages
auth = (repo_conf['user'], repo_conf['pwd']) if repo_conf['user'] else None
try:
packages = requests.get('{}/packages.json'.format(repo_conf['url']), auth=auth, timeout=5)
except:
return 0
if packages.status_code == 200:
self.online_packages = json.loads(packages.content)
return packages.status_code
self.pkgmgr.uninstall_app(app)
def get_services_deps(self):
# Fisrt, build a dictionary of {app: [needs]}
@ -105,10 +78,17 @@ class AppMgr:
return deps
def get_service_deps(self, app):
# Get "need" line from init script and split it to list
# Get "need" line from init script and split it to a list
try:
with open(os.path.join('/etc/init.d', app), 'r') as f:
return [l for l in f.readlines() if l.strip().startswith('need')][0].split()[1:]
except:
pass
return []
def update_repo_settings(self, url, user, pwd):
# Update lxc repository configuration
self.conf['repo']['url'] = url
self.conf['repo']['user'] = user
self.conf['repo']['pwd'] = pwd
self.conf.save()

View File

@ -5,16 +5,33 @@ import datetime
import os
from cryptography import x509
from cryptography.exceptions import InvalidSignature
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.oid import NameOID, ExtendedKeyUsageOID
from .paths import CERT_PUB_FILE, CERT_KEY_FILE, ACME_CRON
from .paths import ACME_CRON, CERT_PUB_FILE, CERT_KEY_FILE, PKG_SIG_FILE
# TODO: Use old method without cryptography module?
def verify_signature(file, signature):
# Verifies ECDSA HMAC SHA512 signature of a file
with open(PKG_SIG_FILE, 'rb') as f:
pub_key = serialization.load_pem_public_key(f.read(), default_backend())
pub_key.verify(signature, file, ec.ECDSA(hashes.SHA512()))
def create_cert(domain):
def verify_hash(file, hash):
# Verifies SHA512 hash of a file against expected hash
sha512 = hashlib.sha512()
with open(file, 'rb') as f:
while True:
data = f.read(65536)
if not data:
break
sha512.update(data)
if sha512.hexdigest() != expected_hash:
raise InvalidSignature(file)
def create_selfsigned_cert(domain):
# Create selfsigned certificate with wildcard alternative subject name
private_key = ec.generate_private_key(ec.SECP384R1(), default_backend())
public_key = private_key.public_key()

View File

@ -9,6 +9,7 @@ ACME_CRON = '/etc/periodic/daily/acme-sh'
ACME_DIR = '/etc/acme.sh.d'
CERT_KEY_FILE = '/etc/ssl/services.key'
CERT_PUB_FILE = '/etc/ssl/services.pem'
PKG_SIG_FILE = '/etc/vmmgr/packages.pub'
# LXC
HOSTS_FILE = '/etc/hosts'
@ -23,3 +24,4 @@ REPO_FILE = '/etc/apk/repositories'
# URLs
MYIP_URL = 'https://tools.dasm.cz/myip.php'
PING_URL = 'https://tools.dasm.cz/vm-ping.php'
RELOAD_URL = 'http://127.0.0.1:8080/reload-config'

View File

@ -0,0 +1,148 @@
# -*- coding: utf-8 -*-
import json
import os
import requests
import shutil
import subprocess
from werkzeug.exceptions import BadRequest, Unauthorized
from . import crypto
from .paths import LXC_ROOT
STAGE_DOWNLOAD = 0
STAGE_INSTALL_DEPS = 1
STAGE_INSTALL_APP = 2
class Pkg:
def __init__(self):
self.stage = STAGE_DOWNLOAD
self.bytes_total = 1
self.bytes_downloaded = 0
@property
def percent_downloaded(self):
# Limit the displayed percentage to 0 - 99
return min(99, round(self.bytes_downloaded / self.bytes_total * 100))
class PkgMgr:
def __init__(self, conf):
self.repo_url = repo_url
self.conf = conf
self.online_packages = {}
def get_repo_resource(self, resource_url, stream=False):
r = requests.get('{}/{}'.format(self.repo_url, resource_url), auth=self.repo_auth, timeout=5, stream=stream)
if r.status_code == 401:
raise Unauthorized(r.url)
elif r.status_code != 200:
raise BadRequest(r.url)
return r
def fetch_online_packages(self):
# Fetches and verifies online packages. Can raise InvalidSignature
packages = self.get_repo_resource('packages').content
packages_sig = self.get_repo_resource('packages.sig').content
crypto.verify_signature(packages, packages_sig)
return json.loads(packages)
def install_app(self, app, item):
# Main installation function. Wrapper for download, registration and install script
self.online_packages = self.fetch_online_packages()
# Get all packages on which the app depends and which have not been installed yet
deps = [d for d in self.get_install_deps(app) if d not in self.conf['packages']]
item.bytes_total = sum(self.online_packages[d]['size'] for d in deps)
for dep in deps:
self.download_package(dep, item)
for dep in deps:
# Set stage to INSTALLING_DEPS or INSTALLING based on which package in sequence is being installed
item.stage = STAGE_INSTALL_APP if dep == deps[-1] else STAGE_INSTALL_DEPS
# Purge old data before unpacking to clean previous failed installation
self.purge_package(dep)
self.unpack_package(dep)
# Run uninstall script before installation to clean previous failed installation
self.run_uninstall_script(dep)
self.run_install_script(dep)
self.register_package(dep)
def uninstall_app(self, app):
# Main uninstallation function. Wrapper for uninstall script, filesystem purge and unregistration
deps = self.get_install_deps(app, False)[::-1]
for dep in deps:
if dep not in self.get_uninstall_deps():
self.run_uninstall_script(dep)
self.purge_package(dep)
self.unregister_package(dep)
def download_package(self, name, item):
# Download tar.xz package and verify its hash. Can raise InvalidSignature
tmp_archive = '/tmp/{}.tar.xz'.format(name)
r = self.get_repo_resource('{}.tar.xz'.format(name), True)
with open(tmp_archive, 'wb') as f:
for chunk in r.iter_content(chunk_size=65536):
if chunk:
item.bytes_downloaded += f.write(chunk)
# Verify hash
crypto.verify_hash(tmp_archive, self.online_packages[name]['sha512'])
def unpack_package(self, name):
# Unpack archive
tmp_archive = '/tmp/{}.tar.xz'.format(name)
subprocess.run(['tar', 'xJf', tmp_archive], cwd='/', check=True)
os.unlink(tmp_archive)
def purge_package(self, name):
# Removes package and shared data from filesystem
lxcpath = self.conf['packages'][name]['lxcpath'] if name in self.conf['packages'] else self.online_packages[name]['lxcpath']
lxc_dir = os.path.join(LXC_ROOT, lxcpath)
if os.path.exists(lxc_dir):
shutil.rmtree(lxc_dir)
srv_dir = os.path.join('/srv/', name)
if os.path.exists(srv_dir):
shutil.rmtree(srv_dir)
lxc_log = '/var/log/lxc/{}.log'.format(name)
if os.path.exists(lxc_log):
os.unlink(lxc_log)
def register_package(self, name):
# Registers a package in installed packages
metadata = self.online_packages[name].copy()
del metadata['sha512']
del metadata['size']
self.conf['packages'][name] = metadata
self.conf.save()
def unregister_package(self, name):
# Removes a package from installed packages
del self.conf['packages'][name]
self.conf.save()
def run_install_script(self, name):
# Runs install.sh for a package, if the script is present
install_script = os.path.join('/srv/', name, 'install.sh')
if os.path.exists(install_script):
subprocess.run(install_script, check=True)
def run_uninstall_script(self, name):
# Runs uninstall.sh for a package, if the script is present
uninstall_script = os.path.join('/srv/', name, 'uninstall.sh')
if os.path.exists(uninstall_script):
subprocess.run(uninstall_script, check=True)
def get_install_deps(self, name, online=True):
# Flatten dependency tree for a package while preserving the dependency order
packages = self.online_packages if online else self.conf['packages']
deps = packages[name]['deps'].copy()
for dep in deps[::-1]:
deps[:0] = [d for d in self.get_install_deps(dep, online)]
deps = list(dict.fromkeys(deps + [name]))
return deps
def get_uninstall_deps(self):
# Create reverse dependency tree for all installed packages
deps = {}
for name in self.conf['packages'].copy():
for d in self.conf['packages'][name]['deps']:
deps.setdefault(d, []).append(name)
return deps

View File

@ -112,9 +112,3 @@ ISSUE = '''
- \x1b[1m{url}\x1b[0m
- \x1b[1m{ip}\x1b[0m\x1b[?1c
'''
REPOSITORIES = '''
http://dl-cdn.alpinelinux.org/alpine/v3.9/main
http://dl-cdn.alpinelinux.org/alpine/v3.9/community
@vm {url}
'''

View File

@ -10,7 +10,7 @@ import urllib
from . import crypto
from . import templates
from . import net
from .paths import ACME_CRON, ACME_DIR, ISSUE_FILE, NGINX_DIR, REPO_FILE
from .paths import ACME_CRON, ACME_DIR, ISSUE_FILE, NGINX_DIR, RELOAD_URL, REPO_FILE
class VMMgr:
def __init__(self, conf):
@ -19,11 +19,9 @@ class VMMgr:
self.domain = conf['host']['domain']
self.port = conf['host']['port']
def register_app(self, app, login, password):
# Register newly installed application, its metadata and credentials (called at the end of package install.sh)
with open('/var/lib/lxcpkgs/{}/meta'.format(app)) as f:
meta = json.load(f)
self.conf['apps'][app] = {**meta,
def register_app(self, app, host, login, password):
# Register newly installed application, its subdomain and credentials (called at the end of package install.sh)
self.conf['apps'][app] = {'host': host,
'login': login if login else 'N/A',
'password': password if password else 'N/A',
'visible': False}
@ -41,14 +39,14 @@ class VMMgr:
def reload_wsgi_config(self):
# Attempt to contact running vmmgr WSGI application to reload config
try:
requests.get('http://127.0.0.1:8080/reload-config', timeout=3)
requests.get(RELOAD_URL, timeout=3)
except:
pass
def register_proxy(self, app, host):
def register_proxy(self, app):
# Setup proxy configuration and reload nginx
with open(os.path.join(NGINX_DIR, '{}.conf'.format(app)), 'w') as f:
f.write(templates.NGINX.format(app=app, host=host, domain=self.conf['host']['domain'], port=self.conf['host']['port']))
f.write(templates.NGINX.format(app=app, host=self.conf['apps'][app]['host'], domain=self.conf['host']['domain'], port=self.conf['host']['port']))
self.reload_nginx()
def unregister_proxy(self, app):
@ -95,35 +93,11 @@ class VMMgr:
# Save config to file
self.conf.save()
def get_repo_conf(self):
# Read, parse and return current @vm repository configuration
with open(REPO_FILE) as f:
url = [l for l in f.read().splitlines() if l.startswith('@vm')][0].split(' ', 2)[1]
url = urllib.parse.urlparse(url)
return {'url': '{}://{}{}'.format(url.scheme, url.netloc, url.path),
'user': url.username if url.username else '' ,
'pwd': url.password if url.password else ''}
def set_repo_conf(self, url, user, pwd):
# Update @vm repository configuration
url = urllib.parse.urlparse(url)
# Create URL with username and password
repo_url = [url.scheme, '://']
if user:
repo_url.append(urllib.quote(user, safe=''))
if pwd:
repo_url.extend((':', urllib.quote(pwd, safe='')))
repo_url.append('@')
repo_url.extend((url.netloc, url.path))
# Update URL in repositories file
with open(REPO_FILE, 'w') as f:
f.write(templates.REPOSITORIES.format(url=''.join(repo_url)))
def create_selfsigned_cert(self):
# Disable acme.sh cronjob
os.chmod(ACME_CRON, 0o640)
# Create selfsigned certificate with wildcard alternative subject name
crypto.create_cert(self.domain)
crypto.create_selfsigned_cert(self.domain)
# Reload nginx
self.reload_nginx()

View File

@ -3,7 +3,7 @@
import json
import os
from werkzeug.exceptions import HTTPException, NotFound
from werkzeug.exceptions import HTTPException, NotFound, Unauthorized
from werkzeug.routing import Map, Rule
from werkzeug.utils import redirect
from werkzeug.wrappers import Request, Response
@ -163,24 +163,26 @@ class WSGIApp:
def setup_apps_view(self, request):
# Application manager view.
repo_error = None
repo_conf = self.vmmgr.get_repo_conf()
status = self.appmgr.fetch_online_packages(repo_conf)
if status == 401:
try:
online_packages = self.appmgr.pkgmgr.fetch_online_packages()
except InvalidSignature:
repo_error = request.session.lang.invalid_packages_signature()
except Unauthorized:
repo_error = request.session.lang.repo_invalid_credentials()
elif status != 200:
except:
repo_error = request.session.lang.repo_unavailable()
table = self.render_setup_apps_table(request)
table = self.render_setup_apps_table(request, online_packages)
message = self.get_session_message(request)
return self.render_html('setup-apps.html', request, repo_error=repo_error, repo_conf=repo_conf, table=table, message=message)
return self.render_html('setup-apps.html', request, repo_error=repo_error, table=table, message=message)
def render_setup_apps_table(self, request):
def render_setup_apps_table(self, request, online_packages):
lang = request.session.lang
pending_actions = self.queue.get_actions()
actionable_apps = sorted(set([k for k, v in self.appmgr.online_packages.items() if 'host' in v] + list(self.conf['apps'].keys())))
actionable_apps = sorted(set([k for k, v in online_packages.items() if 'host' in v] + list(self.conf['apps'].keys())))
app_data = {}
for app in actionable_apps:
installed = app in self.conf['apps']
title = self.conf['apps'][app]['title'] if installed else self.appmgr.online_packages[app]['title']
title = self.conf['packages'][app]['title'] if installed else online_packages[app]['title']
visible = self.conf['apps'][app]['visible'] if installed else False
autostarted = self.appmgr.is_service_autostarted(app) if installed else False
if app in pending_actions:
@ -204,13 +206,15 @@ class WSGIApp:
status = lang.status_stopping()
elif item.action == self.appmgr.install_app:
if not item.started:
status = '{} ({})'.format(lang.status_installing(), lang.status_queued())
status = '{} ({})'.format(lang.status_downloading(), lang.status_queued())
elif isinstance(item.data, BaseException):
status = '<span class="error">{}</span> <a href="#" class="app-clear-status">OK</a>'.format(lang.package_manager_error())
actions = None
else:
if item.data < 100:
status = '{} ({} %)'.format(lang.status_installing(), item.data)
if item.data.stage == 0:
status = '{} ({} %)'.format(lang.status_downloading(), item.data)
elif item.data.stage == 1:
status = lang.status_installing_deps()
else:
status = lang.status_installing()
elif item.action == self.appmgr.uninstall_app:
@ -318,7 +322,7 @@ class WSGIApp:
if not validator.is_valid_repo_url(url):
request.session['msg'] = 'repo:error:{}'.format(request.session.lang.invalid_url(request.form['repourl']))
else:
self.vmmgr.update_repo_conf(url, request.form['repousername'], request.form['repopassword'])
self.appmgr.update_repo_settings(url, request.form['repousername'], request.form['repopassword'])
request.session['msg'] = 'repo:info:{}'.format(request.session.lang.repo_updated())
return redirect('/setup-apps')

View File

@ -23,6 +23,7 @@ class WSGILang:
'stop_start_error': 'Došlo k chybě při spouštění/zastavování. Zkuste akci opakovat nebo restartuje virtuální stroj.',
'installation_in_progress': 'Probíhá instalace jiného balíku. Vyčkejte na její dokončení.',
'package_manager_error': 'Došlo k chybě při instalaci aplikace. Zkuste akci opakovat nebo restartuje virtuální stroj.',
'invalid_packages_signature': 'Digitální podpis seznamu balíků není platný. Kontaktujte správce distribučního serveru.',
'repo_invalid_credentials': 'Přístupové údaje k distribučnímu serveru nejsou správné.',
'repo_unavailable': 'Distribuční server není dostupný. Zkontroluje připojení k síti',
'bad_password': 'Nesprávné heslo',
@ -36,7 +37,9 @@ class WSGILang:
'status_started': 'Spuštěna',
'status_stopping': 'Zastavuje se',
'status_stopped': 'Zastavena',
'status_downloading': 'Stahuje se',
'status_installing': 'Instaluje se',
'status_installing_deps': 'Instalují se závislosti',
'status_uninstalling': 'Odinstalovává se',
'status_not_installed': 'Není nainstalována',
'action_start': 'Spustit',

View File

@ -26,11 +26,11 @@
<table>
<tr>
<td>URL serveru:</td>
<td><input type="text" name="repourl" value="{{ repo_conf['url'] }}"></td>
<td><input type="text" name="repourl" value="{{ conf['repo']['url'] }}"></td>
</tr>
<tr>
<td>Uživatelské jméno:</td>
<td><input type="text" name="repousername" value="{{ repo_conf['user'] }}"></td>
<td><input type="text" name="repousername" value="{{ conf['repo']['user'] }}"></td>
</tr>
<tr>
<td>Heslo:</td>