Transform PackageMgr into queue-backed AppMgr

basic/srv/vm/mgr/appmgr.py

@@ -6,23 +6,48 @@ import os
 import requests
 import shutil
 import subprocess
-import tempfile
+import time
+import uuid
 
 from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.asymmetric import ec
 from cryptography.hazmat.primitives.serialization import load_pem_public_key
+from threading import Lock
+
+from . import tools
 
 PUB_FILE = '/srv/vm/packages.pub'
 LXC_ROOT = '/var/lib/lxc'
 
-class PackageManager:
-    def __init__(self, conf):
-        # Load JSON configuration
-        self.conf = conf
+class ActionItem:
+    def __init__(self, action, app):
+        self.timestamp = int(time.time())
+        self.action = action
+        self.app = app
+        self.started = False
+        self.finished = False
+        self.data = None
+
+class InstallItem:
+    def __init__(self, total):
+        # Stage 0 = download, 1 = deps install, 2 = app install
+        self.stage = 0
+        self.total = total
+        self.downloaded = 0
+
+    def __str__(self):
+        # Limit the disaplyed percentage between 1 - 99 for aestethical and psychological reasons
+        return str(max(1, min(99, round(self.downloaded / self.total * 100))))
+
+class AppMgr:
+    def __init__(self, vmmgr):
+        self.vmmgr = vmmgr
+        self.conf = vmmgr.conf
         self.online_packages = {}
-        self.bytes_downloaded = 0
+        self.action_queue = {}
+        self.lock = Lock()
 
     def get_repo_resource(self, url, stream=False):
         return requests.get('{}/{}'.format(self.conf['repo']['url'], url), auth=(self.conf['repo']['user'], self.conf['repo']['pwd']), stream=stream)
@@ -37,44 +62,78 @@ class PackageManager:
         pub_key.verify(packages_sig, packages, ec.ECDSA(hashes.SHA512()))
         self.online_packages = json.loads(packages)
 
-    def register_pending_installation(self, name):
-        # Registers pending installation. Fetch online packages here instead of install_pacakges() to fail early if the repo isn't reachable
-        self.fetch_online_packages()
-        self.bytes_downloaded = 1
-        # Return total size for download
-        deps = [d for d in self.get_install_deps(name) if d not in self.conf['packages']]
-        return sum(self.online_packages[d]['size'] for d in deps)
+    def enqueue_action(self, action, app):
+        # Remove actions older than 1 day
+        for id,item in self.action_queue.items():
+            if item.timestamp < time.time() - 86400:
+                del self.item[id]
+        # Enqueue action
+        id = '{}:{}'.format(app, uuid.uuid4())
+        item = ActionItem(action, app)
+        self.action_queue[id] = item
+        return id,item
 
-    def install_package(self, name):
+    def get_actions(self, ids):
+        # Return list of requested actions
+        result = {}
+        for id in ids:
+            result[id] = self.action_queue[id] if id in self.action_queue else None
+        return result
+
+    def process_action(self, id):
+        # Main method for deferred queue processing called by WSGI close handler
+        item = self.action_queue[id]
+        with self.lock:
+            item.started = True
+            try:
+                # Call the action method inside exclusive lock
+                getattr(self, item.action)(item)
+            except BaseException as e:
+                item.data = e
+            finally:
+                item.finished = True
+
+    def start_app(self, item):
+        if not tools.is_service_started(item.app):
+            self.vmmgr.start_app(item.app)
+
+    def stop_app(self, app):
+        if tools.is_service_started(item.app):
+            self.vmmgr.stop_app(item.app)
+
+    def install_app(self, item):
         # Main installation function. Wrapper for download, registration and install script
-        deps = [d for d in self.get_install_deps(name) if d not in self.conf['packages']]
-        try:
-            for dep in deps:
-                self.download_package(dep)
-                self.register_package(dep)
-                self.run_install_script(dep)
-            self.bytes_downloaded = 0
-        except:
-            # Store exception state for retrieval via get_install_progress_action()
-            self.bytes_downloaded = -1
+        deps = [d for d in self.get_install_deps(item.app) if d not in self.conf['packages']]
+        item.data = InstallItem(sum(self.online_packages[d]['size'] for d in deps))
+        for dep in deps:
+            self.download_package(dep, item.data)
+        for dep in deps:
+            item.data.stage = 2 if dep == deps[-1] else 1
+            self.unpack_package(dep)
+            # Run uninstall script before installation to purge previous failed installation
+            self.run_uninstall_script(dep)
+            self.register_package(dep)
+            self.run_install_script(dep)
 
-    def uninstall_package(self, name):
+    def uninstall_app(self, item):
         # Main uninstallation function. Wrapper for uninstall script, filesystem purge and unregistration
-        deps = self.get_install_deps(name, False)[::-1]
+        deps = self.get_install_deps(item.app, False)[::-1]
         for dep in deps:
             if dep not in self.get_uninstall_deps():
                 self.run_uninstall_script(dep)
                 self.purge_package(dep)
                 self.unregister_package(dep)
 
-    def download_package(self, name):
-        # Downloads, verifies, unpacks and sets up a package
-        tmp_archive = tempfile.mkstemp('.tar.xz')[1]
+    def download_package(self, name, installitem):
+        tmp_archive = '/tmp/{}.tar.xz'.format(name)
         r = self.get_repo_resource('{}.tar.xz'.format(name), True)
         with open(tmp_archive, 'wb') as f:
             for chunk in r.iter_content(chunk_size=65536):
                 if chunk:
-                    self.bytes_downloaded += f.write(chunk)
+                    installitem.downloaded += f.write(chunk)
+
+    def unpack_package(self, name):
+        tmp_archive = '/tmp/{}.tar.xz'.format(name)
         # Verify hash
         if self.online_packages[name]['sha512'] != hash_file(tmp_archive):
             raise InvalidSignature(name)

basic/srv/vm/mgr/config.py

@@ -1,25 +1,22 @@
 # -*- coding: utf-8 -*-
 
-import fcntl
 import json
+from threading import Lock
 
 CONF_FILE = '/srv/vm/config.json'
-# Locking is needed in order to prevent race conditions in WSGI threads
-LOCK_FILE = '/srv/vm/config.lock'
 
 class Config:
     def __init__(self):
+        self.lock = Lock()
         self.load()
 
     def load(self):
-        with open(LOCK_FILE, 'w') as l:
-            fcntl.flock(l, fcntl.LOCK_EX)
+        with self.lock:
             with open(CONF_FILE, 'r') as f:
                 self.data = json.load(f)
 
     def save(self):
-        with open(LOCK_FILE, 'w') as l:
-            fcntl.flock(l, fcntl.LOCK_EX)
+        with self.lock:
             with open(CONF_FILE, 'w') as f:
                 json.dump(self.data, f, sort_keys=True, indent=4)
 

basic/srv/vm/mgr/wsgiapp.py

@@ -12,7 +12,7 @@ from jinja2 import Environment, FileSystemLoader
 
 from . import VMMgr, CERT_PUB_FILE
 from . import tools
-from .pkgmgr import PackageManager
+from .appmgr import AppMgr
 from .validator import InvalidValueException
 from .wsgilang import WSGILang
 from .wsgisession import WSGISession
@@ -22,8 +22,8 @@ SESSION_KEY = os.urandom(26)
 class WSGIApp(object):
     def __init__(self):
         self.vmmgr = VMMgr()
+        self.appmgr = AppMgr(self.vmmgr)
         self.conf = self.vmmgr.conf
-        self.pkgmgr = PackageManager(self.conf)
         self.jinja_env = Environment(loader=FileSystemLoader('/srv/vm/templates'), autoescape=True, lstrip_blocks=True, trim_blocks=True)
         self.jinja_env.globals.update(is_app_visible=self.is_app_visible)
         self.jinja_env.globals.update(is_service_autostarted=tools.is_service_autostarted)
@@ -81,7 +81,7 @@ class WSGIApp(object):
                 Rule('/start-app', endpoint='start_app_action'),
                 Rule('/stop-app', endpoint='stop_app_action'),
                 Rule('/install-app', endpoint='install_app_action'),
-                Rule('/get-install-progress', endpoint='get_install_progress_action'),
+                Rule('/get-progress', endpoint='get_progress_action'),
                 Rule('/uninstall-app', endpoint='uninstall_app_action'),
                 Rule('/update-password', endpoint='update_password_action'),
                 Rule('/shutdown-vm', endpoint='shutdown_vm_action'),
@@ -148,15 +148,62 @@ class WSGIApp(object):
     def setup_apps_view(self, request):
         # Application manager view.
         try:
-            self.pkgmgr.fetch_online_packages()
+            self.appmgr.fetch_online_packages()
         except:
             pass
-        all_apps = sorted(set([k for k,v in self.pkgmgr.online_packages.items() if 'host' in v] + list(self.conf['apps'].keys())))
-        return self.render_template('setup-apps.html', request, all_apps=all_apps, online_packages=self.pkgmgr.online_packages)
+        all_apps = sorted(set([k for k,v in self.appmgr.online_packages.items() if 'host' in v] + list(self.conf['apps'].keys())))
+        return self.render_template('setup-apps.html', request, all_apps=all_apps, online_packages=self.appmgr.online_packages)
 
-    def render_setup_apps_row(self, request, app, app_title, total_size=None, install_error=False):
+    def render_setup_apps_row(self, request, app, app_title, item):
+        lang = request.session.lang
+        actions = '<div class="loader"></div>'
+        if item.action == 'start_app':
+            if not item.started:
+                status = 'Spouští se (ve frontě)'
+            elif not item.finished:
+                status = 'Spouští se'
+            elif isinstance(item.data, BaseException):
+                status = '<span class="error">{}</span>'.format(lang.stop_start_error())
+            else:
+                status = '<span class="info">Spuštěna</span>'
+                actions = '<a href="#" class="app-stop">Zastavit</a>'
+        elif item.action == 'stop_app':
+            if not item.started:
+                status = 'Zastavuje se (ve frontě)'
+            elif not item.finished:
+                status = 'Zastavuje se'
+            elif isinstance(item.data, BaseException):
+                status = '<span class="error">{}</span>'.format(lang.stop_start_error())
+            else:
+                status = '<span class="error">Zastavena</span>'
+                actions = '<a href="#" class="app-start">Spustit</a>, <a href="#" class="app-uninstall">Odinstalovat</a>'
+        elif item.action == 'install_app':
+            if not item.started:
+                status = 'Stahuje se (ve frontě)'
+            elif not item.finished:
+                if item.data.stage == 0:
+                    status = 'Stahuje se ({} %)'.format(item.data)
+                elif item.data.stage == 1:
+                    status = 'Instalují se závislosti'
+                else:
+                    status = 'Instaluje se'
+            elif isinstance(item.data, BaseException):
+                status = '<span class="error">{}</span>'.format(lang.package_manager_error())
+            else:
+                status = '<span class="error">Zastavena</span>'
+                actions = '<a href="#" class="app-start">Spustit</a>, <a href="#" class="app-uninstall">Odinstalovat</a>'
+        elif item.action == 'uninstall_app':
+            if not item.started:
+                status = 'Odinstalovává se (ve frontě)'
+            elif not item.finished:
+                status = 'Odinstalovává se'
+            elif isinstance(item.data, BaseException):
+                status = '<span class="error">{}</span>'.format(lang.package_manager_error())
+            else:
+                status = 'Není nainstalována'
+                actions = '<a href="#" class="app-install">Instalovat</a>'
         t = self.jinja_env.get_template('setup-apps-row.html')
-        return t.render({'conf': self.conf, 'session': request.session, 'app': app, 'app_title': app_title, 'total_size': total_size, 'install_error': install_error})
+        return t.render({'conf': self.conf, 'session': request.session, 'app': app, 'app_title': app_title, 'status': status, 'actions': actions})
 
     def update_host_action(self, request):
         # Update domain and port, then restart nginx
@@ -277,70 +324,47 @@ class WSGIApp(object):
             return self.render_json({'error': request.session.lang.malformed_request()})
         return self.render_json({'ok': 'ok'})
 
-    def start_app_action(self, request):
-        # Starts application along with its dependencies
+    def enqueue_action(self, request, action):
         try:
             app = request.form['app']
-            self.vmmgr.start_app(app)
-        except (BadRequest, InvalidValueException):
+        except BadRequest:
             return self.render_json({'error': request.session.lang.malformed_request()})
-        except:
-            return self.render_json({'error': request.session.lang.stop_start_error()})
-        app_title = self.conf['apps'][app]['title']
-        return self.render_json({'ok': self.render_setup_apps_row(request, app, app_title)})
-
-    def stop_app_action(self, request):
-        # Stops application along with its dependencies
-        try:
-            app = request.form['app']
-            if tools.is_service_started(app):
-                self.vmmgr.stop_app(app)
-        except (BadRequest, InvalidValueException):
-            return self.render_json({'error': request.session.lang.malformed_request()})
-        except:
-            return self.render_json({'error': request.session.lang.stop_start_error()})
-        app_title = self.conf['apps'][app]['title']
-        return self.render_json({'ok': self.render_setup_apps_row(request, app, app_title)})
-
-    def install_app_action(self, request):
-        # Registers the application installation as pending
-        if self.pkgmgr.bytes_downloaded > 0:
-            return self.render_json({'error': request.session.lang.installation_in_progress()})
-        try:
-            app = request.form['app']
-            total_size = self.pkgmgr.register_pending_installation(app)
-        except (BadRequest, InvalidValueException):
-            return self.render_json({'error': request.session.lang.malformed_request()})
-        except:
-            return self.render_json({'error': request.session.lang.package_manager_error()})
-        app_title = self.pkgmgr.online_packages[app]['title']
-        response = self.render_json({'ok': self.render_setup_apps_row(request, app, app_title, total_size)})
-        response.call_on_close(lambda: self.pkgmgr.install_package(app))
+        app_title = self.conf['apps'][app]['title'] if app in self.conf['apps'] else self.appmgr.online_packages[app]['title']
+        id,item = self.appmgr.enqueue_action(action, app)
+        response = self.render_json({'html': self.render_setup_apps_row(request, app, app_title, item), 'id': id})
+        response.call_on_close(lambda: self.appmgr.process_action(id))
         return response
 
-    def get_install_progress_action(self, request):
-        # Gets pending installation status
-        if self.pkgmgr.bytes_downloaded > 0:
-            return self.render_json({'progress': self.pkgmgr.bytes_downloaded})
-        app = request.form['app']
-        # In case of installation error, we need to get the name from online_packages as the app is not yet registered
-        app_title = self.conf['apps'][app]['title'] if app in self.conf['apps'] else self.pkgmgr.online_packages[app]['title']
-        install_error = True if self.pkgmgr.bytes_downloaded == -1 else False
-        return self.render_json({'ok': self.render_setup_apps_row(request, app, app_title, None, install_error)})
+    def start_app_action(self, request):
+        # Queues application start along with its dependencies
+        return self.enqueue_action(request, 'start_app')
+
+    def stop_app_action(self, request):
+        # Queues application stop along with its dependencies
+        return self.enqueue_action(request, 'stop_app')
+
+    def install_app_action(self, request):
+        # Queues application installation
+        return self.enqueue_action(request, 'install_app')
 
     def uninstall_app_action(self, request):
-        # Uninstalls application
-        if self.pkgmgr.bytes_downloaded > 0:
-            return self.render_json({'error': request.session.lang.installation_in_progress()})
+        # Queues application uninstallation
+        return self.enqueue_action(request, 'uninstall_app')
+
+    def get_progress_action(self, request):
+        # Gets appmgr queue status for given ids
+        json = {}
         try:
-            app = request.form['app']
-            app_title = self.conf['apps'][app]['title']
-            self.pkgmgr.uninstall_package(app)
-        except (BadRequest, InvalidValueException):
+            ids = request.form.getlist('ids[]')
+        except BadRequest:
             return self.render_json({'error': request.session.lang.malformed_request()})
-        except:
-            return self.render_json({'error': request.session.lang.package_manager_error()})
-        return self.render_json({'ok': self.render_setup_apps_row(request, app, app_title)})
+        actions = self.appmgr.get_actions(ids)
+        for id,item in actions.items():
+            app = item.app
+            # In case of installation error, we need to get the name from online_packages as the app is not yet registered
+            app_title = self.conf['apps'][app]['title'] if app in self.conf['apps'] else self.appmgr.online_packages[app]['title']
+            json[id] = {'html': self.render_setup_apps_row(request, app, app_title, item), 'last': item.finished}
+        return self.render_json(json)
 
     def update_password_action(self, request):
         # Updates password for both HDD encryption (LUKS-on-LVM) and web interface admin account

basic/srv/vm/static/js/admin.js

@@ -1,3 +1,5 @@
+var action_queue = [];
+
 $(function() {
     $('#update-host').on('submit', update_host);
     $('#verify-dns').on('click', verify_dns);
@@ -16,7 +18,7 @@ $(function() {
     $('#update-password').on('submit', update_password);
     $('#reboot-vm').on('click', reboot_vm);
     $('#shutdown-vm').on('click', shutdown_vm);
-    window.setInterval(check_progress, 2000);
+    window.setInterval(check_progress, 1000);
 });
 
 function update_host() {
@@ -147,7 +149,8 @@ function _do_app(action, ev) {
         if (data.error) {
             td.attr('class','error').html(data.error);
         } else if (action) {
-            tr.replaceWith(data.ok);
+            tr.html(data.html);
+            action_queue.push(data.id);
         }
     });
     return false;
@@ -174,20 +177,16 @@ function uninstall_app(ev) {
 }
 
 function check_progress() {
-    var progress = $('#install-progress');
-    if (progress.length) {
-        var td = progress.closest('td');
-        var tr = progress.closest('tr');
-        $.post('/get-install-progress', {'app': tr.data('app')}, function(data) {
-            if (data.progress) {
-                var value = parseInt(Math.max(1, data.progress / progress.data('total') * 100));
-                if (value < 100) {
-                    progress.text(parseInt(value));
-                } else {
-                    td.html('<span id="install-progress">Instaluje se</span>')
+    if (action_queue.length) {
+        $.post('/get-progress', {'ids': action_queue}, function(data) {
+            for (id in data) {
+                var app = id.split(':')[0];
+                $('#app-manager tr[data-app="'+app+'"]').html(data[id].html);
+                if (data[id].last) {
+                    action_queue = action_queue.filter(function(item) { 
+                        return item !== id
+                    });
                 }
-            } else {
-                tr.replaceWith(data.ok);
             }
         });
     }

basic/srv/vm/templates/setup-apps-row.html

@@ -1,22 +1,19 @@
-<tr data-app="{{ app }}">
-    <td>{{ app_title }}</td>
-    {% set not_installed = app not in conf['apps'] %}
-    <td class="center"><input type="checkbox" class="app-visible"{% if not_installed %} disabled{% elif conf['apps'][app]['visible'] %} checked{% endif %}></td>
-    <td class="center"><input type="checkbox" class="app-autostart"{% if not_installed %} disabled{% elif is_service_autostarted(app) %} checked{% endif %}></td>
-    {% if install_error %}
-    <td>Není nainstalována</td>
-    <td><span class="error">{{ session.lang.package_manager_error() }}</span></td>
-    {% elif total_size %}
-    <td>Stahuje se (<span id="install-progress" data-total="{{ total_size }}">1</span> %)</td>
-    <td><div class="loader"></div></td>
-    {% elif not_installed %}
-    <td>Není nainstalována</td>
-    <td><a href="#" class="app-install">Instalovat</a></td>
-    {% elif is_service_started(app) %}
-    <td><span class="info">Spuštěna</span></td>
-    <td><a href="#" class="app-stop">Zastavit</a></td>
-    {% else %}
-    <td><span class="error">Zastavena</span></td>
-    <td><a href="#" class="app-start">Spustit</a>, <a href="#" class="app-uninstall">Odinstalovat</a></td>
+{% set not_installed = app not in conf['apps'] %}
+{% if not status %}
+    {% if not_installed: %}
+        {% set status = 'Není nainstalována' %}
+        {% set actions = '<a href="#" class="app-install">Instalovat</a>' %}
+    {% elif is_service_started(app): %}
+        {% set status = '<span class="info">Spuštěna</span>' %}
+        {% set actions = '<a href="#" class="app-stop">Zastavit</a>' %}
+    {% else: %}
+        {% set status = '<span class="error">Zastavena</span>' %}
+        {% set actions = '<a href="#" class="app-start">Spustit</a>, <a href="#" class="app-uninstall">Odinstalovat</a>' %}
     {% endif %}
-</tr>
+{% endif %}
+
+<td>{{ app_title }}</td>
+<td class="center"><input type="checkbox" class="app-visible"{% if not_installed %} disabled{% elif conf['apps'][app]['visible'] %} checked{% endif %}></td>
+<td class="center"><input type="checkbox" class="app-autostart"{% if not_installed %} disabled{% elif is_service_autostarted(app) %} checked{% endif %}></td>
+<td>{{ status|safe }}</td>
+<td>{{ actions|safe }}</td>

basic/srv/vm/templates/setup-apps.html

@@ -17,7 +17,9 @@
         <tbody>
             {% for app in all_apps %}
             {% set app_title = conf['apps'][app]['title'] if app in conf['apps'] else online_packages[app]['title'] %}
-            {% include 'setup-apps-row.html' %}
+            <tr data-app="{{ app }}">
+                {% include 'setup-apps-row.html' %}
+            </tr>
             {% endfor %}
         </tbody>
     </table>