Spotter-Cluster/vmmgr
2
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Add start/stop queuing, Remove unused layers on update/uninstall

Browse Source
This commit is contained in:
Disassembler 2020-04-03 10:57:05 +02:00
parent 1105aea148
commit 31372ac3e1
No known key found for this signature in database
GPG Key ID: 524BD33A0EE29499
7 changed files with 95 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
usr/lib/python3.8/vmmgr/actionqueue.py
View File

@ -15,6 +15,8 @@ class ActionItemType(Enum):
APP_INSTALL = 6 APP_INSTALL = 6
APP_UPDATE = 7 APP_UPDATE = 7
APP_UNINSTALL = 8 APP_UNINSTALL = 8
APP_START = 9
APP_STOP = 10
class ActionItem: class ActionItem:
def __init__(self, action_type, key, action, show_progress=True): def __init__(self, action_type, key, action, show_progress=True):
@ -36,7 +38,6 @@ class ActionAppQueue:
def __init__(self, action): def __init__(self, action):
self.action = action self.action = action
self.queue = [] self.queue = []
self.started = False
self.exception = None self.exception = None
self.index = 0 self.index = 0
@ -53,13 +54,21 @@ class ActionAppQueue:
self.queue.append(ActionItem(ActionItemType.APP_INSTALL, app.name, app.install, False)) self.queue.append(ActionItem(ActionItemType.APP_INSTALL, app.name, app.install, False))
def update_app(self, app): def update_app(self, app):
self.queue.append(ActionItem(ActionItemType.APP_STOP, app.name, app.stop, False))
self.queue.append(ActionItem(ActionItemType.APP_DOWNLOAD, app.name, app.download)) self.queue.append(ActionItem(ActionItemType.APP_DOWNLOAD, app.name, app.download))
self.queue.append(ActionItem(ActionItemType.APP_UNPACK, app.name, app.unpack_downloaded)) self.queue.append(ActionItem(ActionItemType.APP_UNPACK, app.name, app.unpack_downloaded))
self.queue.append(ActionItem(ActionItemType.APP_UPDATE, app.name, app.update, False)) self.queue.append(ActionItem(ActionItemType.APP_UPDATE, app.name, app.update, False))
def uninstall_app(self, app): def uninstall_app(self, app):
self.queue.append(ActionItem(ActionItemType.APP_STOP, app.name, app.stop, False))
self.queue.append(ActionItem(ActionItemType.APP_UNINSTALL, app.name, app.uninstall, False)) self.queue.append(ActionItem(ActionItemType.APP_UNINSTALL, app.name, app.uninstall, False))
def start_app(self, app):
self.queue.append(ActionItem(ActionItemType.APP_START, app.name, app.start, False))
def stop_app(self, app):
self.queue.append(ActionItem(ActionItemType.APP_STOP, app.name, app.stop, False))
def process(self): def process(self):
for item in self.queue: for item in self.queue:
self.index += 1 self.index += 1

4
usr/lib/python3.8/vmmgr/config.py
View File

@ -50,13 +50,13 @@ def unregister_app(name):
pass pass
@locked(CONF_LOCK) @locked(CONF_LOCK)
def set_host_value(key, value): def set_host(key, value):
load() load()
data['host'][key] = value data['host'][key] = value
save() save()
@locked(CONF_LOCK) @locked(CONF_LOCK)
def set_app_value(name, key, value): def set_app(name, key, value):
load() load()
data['apps'][name][key] = value data['apps'][name][key] = value
save() save()

8
usr/lib/python3.8/vmmgr/crypto.py
View File

@ -9,12 +9,10 @@ from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.oid import NameOID, ExtendedKeyUsageOID from cryptography.x509.oid import NameOID, ExtendedKeyUsageOID
from . import config
from .paths import ACME_CRON, CERT_PUB_FILE, CERT_KEY_FILE from .paths import ACME_CRON, CERT_PUB_FILE, CERT_KEY_FILE
def create_selfsigned_cert(): def create_selfsigned_cert(domain):
# Create selfsigned certificate with wildcard alternative subject name # Create selfsigned certificate with wildcard alternative subject name
domain = config.get_host()['domain']
private_key = ec.generate_private_key(ec.SECP384R1(), default_backend()) private_key = ec.generate_private_key(ec.SECP384R1(), default_backend())
public_key = private_key.public_key() public_key = private_key.public_key()
subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, domain)]) subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, domain)])
@ -59,5 +57,5 @@ def get_cert_info():
def adminpwd_hash(password): def adminpwd_hash(password):
return bcrypt.hashpw(password.encode(), bcrypt.gensalt()).decode() return bcrypt.hashpw(password.encode(), bcrypt.gensalt()).decode()
def adminpwd_verify(password): def adminpwd_verify(password, hash):
return bcrypt.checkpw(password.encode(), config.get_host()['adminpwd'].encode()) return bcrypt.checkpw(password.encode(), hash.encode())

80
usr/lib/python3.8/vmmgr/vmmgr.py
View File

@ -8,8 +8,9 @@ import urllib
from spoc.app import App from spoc.app import App
from spoc.config import ONLINE_BASE_URL from spoc.config import ONLINE_BASE_URL
from spoc.container import Container, ContainerState from spoc.container import Container, ContainerState
from spoc.depsolver import DepSolver
from . import crypto, net, templates from . import config, crypto, net, templates
from .paths import ACME_CRON, ACME_DIR, ISSUE_FILE, MOTD_FILE, NGINX_DIR from .paths import ACME_CRON, ACME_DIR, ISSUE_FILE, MOTD_FILE, NGINX_DIR
def register_app(app, host, login, password): def register_app(app, host, login, password):
@ -68,7 +69,7 @@ def update_password(oldpassword, newpassword):
pwinput = f'{oldpassword}\n{newpassword}'.encode() pwinput = f'{oldpassword}\n{newpassword}'.encode()
partition_uuid = open('/etc/crypttab').read().split()[1][5:] partition_uuid = open('/etc/crypttab').read().split()[1][5:]
partition_name = subprocess.run(['/sbin/blkid', '-U', partition_uuid], check=True, stdout=subprocess.PIPE).stdout.decode().strip() partition_name = subprocess.run(['/sbin/blkid', '-U', partition_uuid], check=True, stdout=subprocess.PIPE).stdout.decode().strip()
subprocess.run(['cryptsetup', 'luksChangeKey', partition_name], input=pwinput, check=True) subprocess.run(['/sbin/cryptsetup', 'luksChangeKey', partition_name], input=pwinput, check=True)
# Update bcrypt-hashed password in config # Update bcrypt-hashed password in config
hash = crypto.adminpwd_hash(newpassword) hash = crypto.adminpwd_hash(newpassword)
config.set_host('adminpwd', hash) config.set_host('adminpwd', hash)
@ -126,28 +127,13 @@ def shutdown_vm():
def reboot_vm(): def reboot_vm():
subprocess.run(['/sbin/reboot']) subprocess.run(['/sbin/reboot'])
def start_app(item): def start_app(app_name, queue):
# Start the actual app service # Enqueue application start
app = item.key queue.start_app(App(app_name))
if app in config.get_apps() and not is_app_started(app):
start_service(app)
def start_service(service): def stop_app(app_name, queue):
subprocess.run(['/sbin/service', service, 'start'], check=True) # Enqueue application stop
queue.stop_app(App(app_name))
def stop_app(item):
# Stop the actual app service
app = item.key
if app in config.get_apps() and is_app_started(app):
stop_service(app)
# Stop the app service's dependencies if they are not used by another running app
deps = get_services_deps()
for dep in get_service_deps(app):
if not any([is_app_started(d) for d in deps[dep]]):
stop_service(dep)
def stop_service(service):
subprocess.run(['/sbin/service', service, 'stop'], check=True)
def update_app_visibility(app_name, visible): def update_app_visibility(app_name, visible):
# Update visibility for the app in the configuration # Update visibility for the app in the configuration
@ -155,7 +141,7 @@ def update_app_visibility(app_name, visible):
def update_app_autostart(app_name, enabled): def update_app_autostart(app_name, enabled):
# Add/remove the app to OpenRC default runlevel # Add/remove the app to OpenRC default runlevel
App(app_name).set_autostart(enabled) App(app_name, False).set_autostart(enabled)
def is_app_started(app_name): def is_app_started(app_name):
# Assume that the main container has always the same name as app # Assume that the main container has always the same name as app
@ -166,9 +152,10 @@ def is_app_autostarted(app_name):
return App(app_name, False).autostart return App(app_name, False).autostart
def install_app(app_name, queue): def install_app(app_name, queue):
# Main installation function. Wrapper for download, registration and install script # Enqueue layers and application download, registration and install script execution
required_images = [] required_images = []
for container in repo_online.get_app(app_name)['containers'].values(): new_containers = repo_online.get_app(app_name)['containers'].values()
for container in new_containers:
required_images.extend(repo_online.get_image(container['image'])['layers']) required_images.extend(repo_online.get_image(container['image'])['layers'])
local_images = repo_local.get_images() local_images = repo_local.get_images()
for layer in set(required_images): for layer in set(required_images):
@ -177,21 +164,46 @@ def install_app(app_name, queue):
queue.install_app(App(app_name, False, False)) queue.install_app(App(app_name, False, False))
def uninstall_app(app_name, queue): def uninstall_app(app_name, queue):
# Main uninstallation function. Wrapper for uninstall script, filesystem purge and unregistration # Enqueue application uninstall script execution and removal
queue.uninstall_app(App(app_name, False)) app = App(app_name, False)
queue.uninstall_app(app)
# Remove unused layers
removed_containers = [container.name for container in app.containers]
retained_containers = [definition for name,definition in repo_local.get_containers() if name not in removed_containers]
remove_unused_layers(retained_containers, queue)
def update_app(app_name, queue): def update_app(app_name, queue):
# Main update function. Wrapper for download and update script # Enqueue layers and application download, registration and update script execution
required_images = [] required_images = []
for container in repo_online.get_app(app_name)['containers'].values(): new_containers = repo_online.get_app(app_name)['containers'].values()
for container in new_containers:
required_images.extend(repo_online.get_image(container['image'])['layers']) required_images.extend(repo_online.get_image(container['image'])['layers'])
local_images = repo_local.get_images() local_images = repo_local.get_images()
for layer in set(required_images): for layer in set(required_images):
if layer not in local_images: if layer not in local_images:
queue.download_image(Image(layer, False)) queue.download_image(Image(layer, False))
queue.update_app(App(app_name, False)) app = App(app_name, False)
queue.update_app(app)
# Remove unused layers
removed_containers = [container.name for container in app.containers]
retained_containers = [definition for name,definition in repo_local.get_containers() if name not in removed_containers] + new_containers
remove_unused_layers(retained_containers, queue)
def update_repo_settings(url, username, password): def remove_unused_layers(retained_containers, queue):
# Enqueue removal of images which won't be used in any locally defined containers anymore
used_images = set()
for definition in retained_containers:
used_images.update(definition['layers'])
# Build dependency tree to safely remove the images in order of dependency
depsolver = DepSolver()
for image in set(repo_local.get_images()) - used_images:
image = Image(image)
depsolver.add(image.name, set(image.layers) - used_images, image)
# Enqueue the removal actions
for image in reversed(depsolver.solve()):
queue.delete_image(image)
def update_repo_conf(url, username, password):
# Include credentials in the repo URL and save to SPOC config # Include credentials in the repo URL and save to SPOC config
spoc_config = configparser.ConfigParser() spoc_config = configparser.ConfigParser()
spoc_config.read('/etc/spoc/spoc.conf') spoc_config.read('/etc/spoc/spoc.conf')
@ -202,9 +214,9 @@ def update_repo_settings(url, username, password):
with open('/etc/spoc/spoc.conf', 'w') as f: with open('/etc/spoc/spoc.conf', 'w') as f:
config.write(f) config.write(f)
def get_repo_settings(): def get_repo_conf():
# Parse the SPOC config repo URL and return as tuple # Parse the SPOC config repo URL and return as tuple
parts = urllib.parse.urlsplit(ONLINE_BASE_URL) parts = urllib.parse.urlsplit(ONLINE_BASE_URL)
netloc = parts.netloc.split('@', 1)[1] if parts.username or parts.password else parts.netloc netloc = parts.netloc.split('@', 1)[1] if parts.username or parts.password else parts.netloc
url = urllib.parse.urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment)) url = urllib.parse.urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
return (url, parts.username, parts.password) return {'url':url, 'username':parts.username}

56
usr/lib/python3.8/vmmgr/wsgiapp.py
View File

@ -125,7 +125,8 @@ class WSGIApp:
def login_action(self, request): def login_action(self, request):
password = request.form['password'] password = request.form['password']
redir = request.form['redir'] redir = request.form['redir']
if crypto.adminpwd_verify(password): hash = config.get_host()['adminpwd']
if crypto.adminpwd_verify(password, hash):
request.session['admin'] = True request.session['admin'] = True
return redirect(f'/{redir}') return redirect(f'/{redir}')
request.session['msg'] = f'login:error:{request.session.lang.bad_password()}' request.session['msg'] = f'login:error:{request.session.lang.bad_password()}'
@ -170,8 +171,8 @@ class WSGIApp:
repo_error = request.session.lang.repo_unavailable() repo_error = request.session.lang.repo_unavailable()
table = self.render_setup_apps_table(request) table = self.render_setup_apps_table(request)
message = self.get_session_message(request) message = self.get_session_message(request)
repo_url, repo_user, _ = vmmgr.get_repo_settings() repo_conf = vmmgr.get_repo_conf()
return self.render_html('setup-apps.html', request, repo_url=repo_url, repo_user=repo_user, repo_error=repo_error, table=table, message=message) return self.render_html('setup-apps.html', request, repo_conf=repo_conf, repo_error=repo_error, table=table, message=message)
def render_setup_apps_table(self, request): def render_setup_apps_table(self, request):
lang = request.session.lang lang = request.session.lang
@ -210,27 +211,28 @@ class WSGIApp:
actions = None actions = None
else: else:
# Display task/subtask progress # Display task/subtask progress
if app_queue.action == vmmgr.start_app: action_item = app_queue.queue[app_queue.index-1]
if action_item.type in (ActionItemType.IMAGE_DOWNLOAD, ActionItemType.APP_DOWNLOAD):
status = lang.status_downloading(action_item.key)
elif action_item.type in (ActionItemType.IMAGE_UNPACK, ActionItemType.APP_UNPACK):
status = lang.status_unpacking(action_item.key)
elif action_item.type == ActionItemType.IMAGE_DELETE:
status = lang.status_deleting(action_item.key)
elif action_item.type == ActionItemType.APP_START:
status = lang.status_starting() status = lang.status_starting()
elif app_queue.action == vmmgr.stop_app: elif action_item.type == ActionItemType.APP_STOP:
status = lang.status_stopping() status = lang.status_stopping()
else: elif action_item.type == ActionItemType.APP_INSTALL:
action_item = app_queue.queue[app_queue.index-1] status = lang.status_installing()
if action_item.type in (ActionItemType.IMAGE_DOWNLOAD, ActionItemType.APP_DOWNLOAD): elif action_item.type == ActionItemType.APP_UPDATE:
status = lang.status_downloading(action_item.key) status = lang.status_updating()
elif action_item.type in (ActionItemType.IMAGE_UNPACK, ActionItemType.APP_UNPACK): elif action_item.type == ActionItemType.APP_UNINSTALL:
status = lang.status_unpacking(action_item.key) status = lang.status_uninstalling()
elif action_item.type == ActionItemType.IMAGE_DELETE: if app_queue.action not in (vmmgr.start_app, vmmgr.stop_app):
status = lang.status_deleting(action_item.key) # For tasks other than start/stop which have only a single subtask, display also index of the subtask in queue
elif action_item.type == ActionItemType.APP_INSTALL:
status = lang.status_installing(action_item.key)
elif action_item.type == ActionItemType.APP_UPDATE:
status = lang.status_updating(action_item.key)
elif action_item.type == ActionItemType.APP_UNINSTALL:
status = lang.status_uninstalling(action_item.key)
status = f'[{app_queue.index}/{len(app_queue.queue)}] {status}' status = f'[{app_queue.index}/{len(app_queue.queue)}] {status}'
if action_item.show_progress: if action_item.show_progress:
status = f'{status} ({floor(current_action.units_done/current_action.units_total*100)} %)' status = f'{status} ({floor(current_action.units_done/current_action.units_total*100)} %)'
actions = '<div class="loader"></div>' actions = '<div class="loader"></div>'
else: else:
# Display queued (pending, not started) task # Display queued (pending, not started) task
@ -239,11 +241,11 @@ class WSGIApp:
elif app_queue.action == vmmgr.stop_app: elif app_queue.action == vmmgr.stop_app:
status = lang.status_stopping() status = lang.status_stopping()
elif app_queue.action == vmmgr.install_app: elif app_queue.action == vmmgr.install_app:
status = lang.status_installing('') status = lang.status_installing()
elif app_queue.action == vmmgr.uninstall_app:
status = lang.status_uninstalling('')
elif app_queue.action == vmmgr.update_app: elif app_queue.action == vmmgr.update_app:
status = lang.status_updating('') status = lang.status_updating()
elif app_queue.action == vmmgr.uninstall_app:
status = lang.status_uninstalling()
status = f'{status} ({lang.status_queued()})' status = f'{status} ({lang.status_queued()})'
actions = '<div class="loader"></div>' actions = '<div class="loader"></div>'
else: else:
@ -348,7 +350,7 @@ class WSGIApp:
if not validator.is_valid_repo_url(url): if not validator.is_valid_repo_url(url):
request.session['msg'] = f'repo:error:{request.session.lang.invalid_url(url)}' request.session['msg'] = f'repo:error:{request.session.lang.invalid_url(url)}'
else: else:
vmmgr.update_repo_settings(url, request.form['repousername'], request.form['repopassword']) vmmgr.update_repo_conf(url, request.form['repousername'], request.form['repopassword'])
request.session['msg'] = f'repo:info:{request.session.lang.repo_updated()}' request.session['msg'] = f'repo:info:{request.session.lang.repo_updated()}'
return redirect('/setup-apps') return redirect('/setup-apps')
@ -406,7 +408,7 @@ class WSGIApp:
return self.render_json({'error': request.session.lang.password_mismatch()}) return self.render_json({'error': request.session.lang.password_mismatch()})
if request.form['newpassword'] == '': if request.form['newpassword'] == '':
return self.render_json({'error': request.session.lang.password_empty()}) return self.render_json({'error': request.session.lang.password_empty()})
# No need to explicitly validate old password, update_luks_password will raise exception if it's wrong # No need to explicitly validate old password, vmmgr.update_password will raise exception if it's wrong
vmmgr.update_password(request.form['oldpassword'], request.form['newpassword']) vmmgr.update_password(request.form['oldpassword'], request.form['newpassword'])
except: except:
return self.render_json({'error': request.session.lang.bad_password()}) return self.render_json({'error': request.session.lang.bad_password()})

6
usr/lib/python3.8/vmmgr/wsgilang.py
View File

@ -41,9 +41,9 @@ class WSGILang:
'status_stopped': 'Zastavena', 'status_stopped': 'Zastavena',
'status_downloading': 'Stahuje se {}', 'status_downloading': 'Stahuje se {}',
'status_unpacking': 'Rozbaluje se {}', 'status_unpacking': 'Rozbaluje se {}',
'status_installing': 'Instaluje se {}', 'status_installing': 'Instaluje se',
'status_updating': 'Aktualizuje se {}', 'status_updating': 'Aktualizuje se',
'status_uninstalling': 'Odinstalovává se {}', 'status_uninstalling': 'Odinstalovává se',
'status_not_installed': 'Není nainstalována', 'status_not_installed': 'Není nainstalována',
'action_start': 'Spustit', 'action_start': 'Spustit',
'action_stop': 'Zastavit', 'action_stop': 'Zastavit',

4
usr/share/vmmgr/templates/setup-apps.html
View File

@ -26,11 +26,11 @@
<table> <table>
<tr> <tr>
<td>URL serveru:</td> <td>URL serveru:</td>
<td><input type="text" name="repourl" value="{{ repo_url }}"></td> <td><input type="text" name="repourl" value="{{ repo_conf['url'] }}"></td>
</tr> </tr>
<tr> <tr>
<td>Uživatelské jméno:</td> <td>Uživatelské jméno:</td>
<td><input type="text" name="repousername" value="{{ repo_user }}"></td> <td><input type="text" name="repousername" value="{{ repo_conf['username'] }}"></td>
</tr> </tr>
<tr> <tr>
<td>Heslo:</td> <td>Heslo:</td>