Add SPOC as a submodule

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "app-vmmgr"]
 	path = apk/vmmgr
 	url = ssh://git@git.spotter.cz:2222/Spotter-Cluster/vmmgr.git
+[submodule "spoc"]
+	path = apk/spoc
+	url = ssh://git@git.spotter.cz:2222/Spotter-Cluster/spoc.git

apk/spoc

@@ -0,0 +1 @@
+Subproject commit 1e12d78efe730fcf35bc949d70c8db04e5917945

build/build-all.sh

@@ -45,50 +45,88 @@ cd ${ROOT}/apk/wireguard-tools
 apk add -U libmnl-dev
 abuild -F
 
-# Build apd pack runtimes
+# Build runtimes
 cd ${ROOT}/lxc-shared
-lxcbuild alpine3.8
+spoc-image build -p alpine3.8/image
-lxcbuild alpine3.8-php5.6
+spoc-image build -p alpine3.8-php5.6/image
-lxcbuild alpine3.8-ruby2.4
+spoc-image build -p alpine3.8-ruby2.4/image
-lxcbuild alpine3.9
+spoc-image build -p alpine3.9/image
-lxcbuild alpine3.9-java8
+spoc-image build -p alpine3.9-java8/image
-lxcbuild alpine3.9-php7.2
+spoc-image build -p alpine3.9-php7.2/image
-lxcbuild alpine3.9-python2.7
+spoc-image build -p alpine3.9-python2.7/image
-lxcbuild alpine3.9-python3.6
+spoc-image build -p alpine3.9-python3.6/image
-lxcbuild alpine3.9-ruby2.4
+spoc-image build -p alpine3.9-ruby2.4/image
-lxcbuild alpine3.9-ruby2.6
+spoc-image build -p alpine3.9-ruby2.6/image
-lxcbuild alpine3.9-tomcat7
+spoc-image build -p alpine3.9-tomcat7/image
-lxcbuild alpine3.9-tomcat8.5
+spoc-image build -p alpine3.9-tomcat8.5/image
 
 # Build services
 cd ${ROOT}/lxc-services
-lxcbuild activemq
+spoc-image build -p activemq/image
-lxcbuild mariadb
+spoc-image build -p mariadb/image
-lxcbuild postgres
+spoc-image build -p postgres/image
-lxcbuild postgis
+spoc-image build -p postgis/image
-lxcbuild rabbitmq
+spoc-image build -p rabbitmq/image
-lxcbuild redis
+spoc-image build -p redis/image
-lxcbuild solr6
+spoc-image build -p solr6/image
 
 # Build applications
 cd ${ROOT}/lxc-apps
-lxcbuild ckan
+
-lxcbuild crisiscleanup
+spoc-image build -p ckan/image
-lxcbuild cts
+spoc-image build -p ckan-datapusher/image
-lxcbuild decidim
+spoc-app publish ckan/app
-lxcbuild ecogis
+
-lxcbuild frontlinesms
+spoc-image build -p crisiscleanup/image
-lxcbuild gnuhealth
+spoc-app publish crisiscleanup/app
-lxcbuild kanboard
+
-lxcbuild mifosx
+spoc-image build -p cts/image
-lxcbuild motech
+spoc-app publish cts/app
-lxcbuild odoo
+
-lxcbuild opendatakit
+spoc-image build -p decidim/image
-lxcbuild openmapkit
+spoc-app publish decidim/app
-lxcbuild pandora
+
-lxcbuild sahana
+spoc-image build -p ecogis/image
-lxcbuild sahana-demo
+# spoc-app publish ecogis/app
-lxcbuild sambro
+
-lxcbuild seeddms
+spoc-image build -p frontlinesms/image
-lxcbuild sigmah
+spoc-app publish frontlinesms/app
-lxcbuild ushahidi
+
+spoc-image build -p gnuhealth/image
+spoc-app publish gnuhealth/app
+
+spoc-image build -p kanboard/image
+spoc-app publish kanboard/app
+
+spoc-image build -p mifosx/image
+spoc-app publish mifosx/app
+
+spoc-image build -p motech/image
+spoc-app publish motech/app
+
+spoc-image build -p odoo/image
+spoc-app publish odoo/app
+
+spoc-image build -p opendatakit/image
+spoc-image build -p opendatakit-build/image
+spoc-app publish opendatakit/app
+
+spoc-image build -p openmapkit/image
+spoc-app publish openmapkit/app
+
+spoc-image build -p pandora/image
+spoc-app publish pandora/app
+
+spoc-image build -p sahana/image
+spoc-app publish sahana/app
+spoc-app publish sahana-demo/app
+spoc-app publish sambro/app
+
+spoc-image build -p seeddms/image
+spoc-app publish seeddms/app
+
+spoc-image build -p sigmah/image
+spoc-app publish sigmah/app
+
+spoc-image build -p ushahidi/image
+spoc-app publish ushahidi/app

build/clean-all.sh

@@ -11,28 +11,7 @@ rm -f /srv/build/vm.tar
 rm -rf /srv/build/alpine/*
 
 # Clean built LXC packages
-rm -rf /srv/build/lxc/apps/*
+rm -rf /srv/build/spoc
-rm -rf /srv/build/lxc/images/*
-rm -f /srv/build/lxc/packages.sig
-echo '{"apps":{},"images":{}}' >/srv/build/lxc/packages
-
-# Stop running containers
-for SERVICE in $(find /run/openrc/started -name 'lxc-*'); do
-    service $(basename ${SERVICE}) stop
-done
-
-# Remove services
-rm -f /etc/init.d/lxc-*
-rc-update -u
-
-# Remove containers
-rm -rf /var/lib/lxc/*
-rm -f /var/log/lxc/*
-
-# Remove application data
-for DIR in $(find /srv ! -path /srv/build -maxdepth 1 -mindepth 1); do
-    rm -rf ${DIR}
-done
 
 # Remove nginx configs
 for CONF in $(find /etc/nginx/conf.d -name '*.conf' -a ! -name repo.conf -a ! -name default.conf); do
@@ -40,20 +19,11 @@ for CONF in $(find /etc/nginx/conf.d -name '*.conf' -a ! -name repo.conf -a ! -n
 done
 service nginx reload
 
-# Reset /etc/hosts
+# Stop running containers
-cat <<EOF >/etc/hosts
+for APP in $(spoc-container list); do
-127.0.0.1 localhost
+    spoc-container stop ${APP}
-::1 localhost
+done
-172.17.0.1 host
-172.17.0.1 repo.build.vm
-EOF
 
-# Reset vmmgr config
+# Remove data
-export ADMINPWD=$(python3 -c "import json; f = open('/etc/vmmgr/config.json'); j = json.load(f); print(j['host']['adminpwd'])")
+rm -rf /var/lib/spoc
-envsubst </etc/vmmgr/config.default.json >/etc/vmmgr/config.json
+rm -rf /var/log/spoc
-
-# Clean locally installed LXC packages
-rm -rf /var/lib/lxcmgr/storage/*
-rm -rf /var/lib/lxcmgr/cache/apps/*
-rm -rf /var/lib/lxcmgr/cache/images/*
-echo '{"apps":{},"images":{}}' >/var/lib/lxcmgr/packages

build/install-toolchain.sh

@@ -5,7 +5,7 @@ cd $(realpath $(dirname "${0}"))
 
 # Install basic build tools
 apk update
-apk add git file htop less openssh-client tar xz
+apk add git file htop less openssh-client
 # Install Alpine SDK
 apk add alpine-sdk
 # Install Sphinx support
@@ -13,7 +13,7 @@ apk add py3-sphinx
 pip3 install recommonmark sphinx-markdown-tables
 
 # Copy root profile files and settings
-mkdir -p /root/.config/htop /root/.ssh
+mkdir -p /root/.config/htop
 cp root/.profile /root/.profile
 cp root/.config/htop/htoprc /root/.config/htop/htoprc
 
@@ -21,11 +21,6 @@ cp root/.config/htop/htoprc /root/.config/htop/htoprc
 adduser root abuild
 cp etc/abuild.conf /etc/abuild.conf
 
-# Prepare LXC build toolchain
-cp usr/bin/lxcbuild /usr/bin/lxcbuild
-cp usr/bin/lxcmerge /usr/bin/lxcmerge
-mkdir -p /srv/build/lxc/apps /srv/build/lxc/images
-
 # Prepare local APK repository
 cp etc/nginx/conf.d/repo.conf /etc/nginx/conf.d/repo.conf
 echo "172.17.0.1 repo.build.vm" >>/etc/hosts
@@ -38,5 +33,5 @@ echo '{"url":"http://repo.build.vm/lxc","user":"","pwd":""}' >/etc/lxcmgr/repo.j
 # echo '/srv/build/repokey.rsa' | abuild-keygen
 
 # Supply LXC build key
-# openssl ecparam -genkey -name secp384r1 -out /srv/build/packages.key
+# openssl ecparam -genkey -name secp384r1 -out /etc/spoc/publish.key
-# openssl ec -in /srv/build/packages.key -pubout -out /srv/build/lxc/packages.pub
+# openssl ec -in /etc/spoc/publish.key -pubout -out /tmp/repository.pub

build/usr/bin/lxcbuild

@@ -1,79 +0,0 @@
-#!/usr/bin/python3
-# -*- coding: utf-8 -*-
-
-import argparse
-import os
-import sys
-from lxcbuild.app import App
-from lxcbuild.image import Image
-from lxcbuild.imagebuilder import BuildType
-
-def build_and_pack_image(path, args):
-    image = Image()
-    if args.scratch:
-        image.build_type = BuildType.SCRATCH
-    elif args.force:
-        image.build_type = BuildType.FORCE
-    image.build_and_pack(path)
-
-def pack_app(path):
-    app = App()
-    app.pack(path)
-
-def main(args):
-    if args.remove_image:
-        image = Image()
-        image.name = args.buildarg
-        image.remove()
-    elif args.remove_app:
-        app = App()
-        app.name = args.buildarg
-        app.remove()
-    else:
-        buildpath = os.path.realpath(args.buildarg)
-        # If the buildpath is a file, determine type from filename
-        if os.path.isfile(buildpath):
-            basename = os.path.basename(buildpath)
-            if basename == 'lxcfile' or basename.endswith('.lxcfile'):
-                build_and_pack_image(buildpath, args)
-            # Compose files needs to be ignored when performing scratch builds
-            elif not args.scratch and basename == 'meta':
-                pack_app(buildpath)
-            else:
-                print('Unknown file {} given, expected "lxcfile"{}'.format(buildpath, '' if args.scratch else ' or "meta"'))
-                sys.exit(1)
-        # If the buildpath is a directory, build as much as possible, unless scratch build was requested, in which case don't build anything
-        else:
-            if args.scratch:
-                lxcfile = os.path.join(buildpath, 'lxcfile')
-                if os.path.exists(lxcfile):
-                    build_and_pack_image(lxcfile, args)
-                else:
-                    print('Please specify an lxcfile for scratch build')
-                    sys.exit(1)
-            else:
-                valid_dir = False
-                for entry in os.scandir(buildpath):
-                    if entry.is_file() and (entry.name == 'lxcfile' or entry.name.endswith('.lxcfile')):
-                        valid_dir = True
-                        build_and_pack_image(entry.path, args)
-                meta = os.path.join(buildpath, 'meta')
-                if os.path.exists(meta):
-                    valid_dir = True
-                    pack_app(meta)
-                if not valid_dir:
-                    print('Directory {} doesn\'t contain anything to build, skipping'.format(buildpath))
-
-parser = argparse.ArgumentParser(description='VM application builder and packager')
-group = parser.add_mutually_exclusive_group()
-group.add_argument('-f', '--force', action='store_true', help='Force rebuild already built package')
-group.add_argument('-s', '--scratch', action='store_true', help='Build container for testing purposes, i.e. without cleanup on failure and packaging')
-group.add_argument('-r', '--remove-image', action='store_true', help='Delete image (including scratch) from build repository')
-group.add_argument('-e', '--remove-app', action='store_true', help='Delete application from build repository')
-parser.add_argument('buildarg', help='Either specific "lxcfile" or "meta" file or a directory containing at least one of them')
-
-args = parser.parse_args()
-if hasattr(args, 'buildarg'):
-    main(args)
-else:
-    parser.print_usage()

build/usr/lib/python3.6/lxcbuild/__init__.py

@@ -1 +0,0 @@
-# -*- coding: utf-8 -*-

build/usr/lib/python3.6/lxcbuild/app.py

@@ -1,36 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import json
-import os
-import sys
-
-from .apppacker import AppPacker
-from .imagebuilder import ImageNotFoundError
-
-class App:
-    def __init__(self):
-        self.name = None
-        self.conf = {}
-        self.build_dir = None
-
-    def load_metafile(self, metafile):
-        self.build_dir = os.path.dirname(metafile)
-        if os.path.basename(metafile) == 'meta':
-            self.name = os.path.basename(self.build_dir)
-        else:
-            self.name = os.path.splitext(metafile)[0]
-        with open(metafile, 'r') as f:
-            self.conf = json.load(f)
-
-    def pack(self, metafile):
-        self.load_metafile(metafile)
-        packer = AppPacker(self)
-        try:
-            packer.pack()
-        except ImageNotFoundError as e:
-            print('Image {} not found, can\'t pack {}'.format(e, self.name))
-            sys.exit(1)
-
-    def remove(self):
-        packer = AppPacker(self)
-        packer.remove()

build/usr/lib/python3.6/lxcbuild/apppacker.py

@@ -1,61 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-import subprocess
-
-from . import crypto
-from .imagebuilder import ImageNotFoundError
-from .packer import Packer
-from .paths import REPO_APPS_DIR
-
-class AppPacker(Packer):
-    def __init__(self, app):
-        super().__init__()
-        self.app = app
-        # Prepare package file names
-        self.tar_path = os.path.join(REPO_APPS_DIR, '{}.tar'.format(self.app.name))
-        self.xz_path = '{}.xz'.format(self.tar_path)
-
-    def pack(self):
-        # Check if all images used by containers exist
-        for container in self.app.conf['containers']:
-            image = self.app.conf['containers'][container]['image']
-            if image not in self.packages['images']:
-                raise ImageNotFoundError(image)
-        try:
-            os.unlink(self.xz_path)
-        except FileNotFoundError:
-            pass
-        self.create_archive()
-        self.register()
-        self.sign_packages()
-
-    def remove(self):
-        self.unregister()
-        try:
-            os.unlink(self.xz_path)
-        except FileNotFoundError:
-            pass
-
-    def create_archive(self):
-        # Create archive with application setup scripts
-        print('Archiving setup scripts for', self.app.name)
-        scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh')
-        scripts = [s for s in scripts if os.path.exists(os.path.join(self.app.build_dir, s))]
-        subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, '--transform', 's,^,{}/,'.format(self.app.name)] + scripts, cwd=self.app.build_dir)
-        self.compress_archive()
-
-    def register(self):
-        # Register package in global repository metadata file
-        print('Registering application package', self.app.name)
-        self.packages['apps'][self.app.name] = self.app.conf.copy()
-        self.packages['apps'][self.app.name]['size'] = self.tar_size
-        self.packages['apps'][self.app.name]['pkgsize'] = self.xz_size
-        self.packages['apps'][self.app.name]['sha512'] = crypto.hash_file(self.xz_path)
-        self.save_repo_meta()
-
-    def unregister(self):
-        # Removes package from global repository metadata file
-        if self.app.name in self.packages['apps']:
-            del self.packages['apps'][self.app.name]
-            self.save_repo_meta()

build/usr/lib/python3.6/lxcbuild/crypto.py

@@ -1,28 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import hashlib
-
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import hashes
-from cryptography.hazmat.primitives.asymmetric import ec
-from cryptography.hazmat.primitives.serialization import load_pem_private_key
-
-def sign_file(private_key_path, input_path):
-    # Generate SHA512 signature of a file using EC private key
-    print('Signing packages')
-    with open(private_key_path, 'rb') as f:
-        priv_key = load_pem_private_key(f.read(), None, default_backend())
-    with open(input_path, 'rb') as f:
-        data = f.read()
-    return priv_key.sign(data, ec.ECDSA(hashes.SHA512()))
-
-def hash_file(file_path):
-    # Calculate SHA512 hash of a file
-    sha512 = hashlib.sha512()
-    with open(file_path, 'rb') as f:
-        while True:
-            data = f.read(65536)
-            if not data:
-                break
-            sha512.update(data)
-    return sha512.hexdigest()

build/usr/lib/python3.6/lxcbuild/image.py

@@ -1,60 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-import sys
-
-from lxcmgr import lxcmgr
-
-from .imagebuilder import BuildType, ImageBuilder, ImageExistsError, ImageNotFoundError
-from .imagepacker import ImagePacker
-from .packer import PackageExistsError
-
-class Image:
-    def __init__(self):
-        self.name = None
-        self.conf = {}
-        self.lxcfile = None
-        self.build_dir = None
-        self.build_type = BuildType.NORMAL
-        self.pack = False
-
-    def build_and_pack(self, lxcfile):
-        self.lxcfile = lxcfile
-        self.build_dir = os.path.dirname(lxcfile)
-        self.conf['build'] = True
-        builder = ImageBuilder(self)
-        try:
-            builder.build()
-            # Packaging needs to happen in any case after a successful build in order to prevent outdated packages
-            self.pack = True
-        except ImageExistsError as e:
-            # If container already exists and build hasn't been forced, rerun the build just for metadata which are still needed for packaging
-            print('Image {} already exists, skipping build tasks'.format(e))
-            self.build_type = BuildType.METADATA
-            builder.build()
-        except ImageNotFoundError as e:
-            # If one of the layers is missing, cleanup and die
-            print('Image {} not found, can\'t build {}'.format(e, self.name))
-            builder.clean()
-            sys.exit(1)
-        except:
-            # If build fails with another exception, cleanup (unless we were doing scratch build) and re-raise
-            if not self.build_type == BuildType.SCRATCH:
-                builder.clean()
-            raise
-        del self.conf['build']
-        # If we're doing a scratch build, regenerate the final LXC container configuration including ephemeral layer
-        if self.build_type == BuildType.SCRATCH:
-            lxcmgr.create_container(self.name, self.conf)
-        else:
-            try:
-                packer = ImagePacker(self)
-                packer.pack()
-            except PackageExistsError as e:
-                print('Package {} already exists, skipping packaging tasks'.format(e))
-
-    def remove(self):
-        builder = ImageBuilder(self)
-        builder.clean()
-        packer = ImagePacker(self)
-        packer.remove()

build/usr/lib/python3.6/lxcbuild/imagebuilder.py

@@ -1,203 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-import shutil
-import subprocess
-import sys
-
-from lxcmgr import lxcmgr
-from lxcmgr.paths import LXC_STORAGE_DIR
-from lxcmgr.pkgmgr import PkgMgr
-
-class ImageExistsError(Exception):
-    pass
-
-class ImageNotFoundError(Exception):
-    pass
-
-class BuildType:
-    NORMAL = 1
-    FORCE = 2
-    SCRATCH = 3
-    METADATA = 4
-
-class ImageBuilder:
-    def __init__(self, image):
-        self.image = image
-        self.script = []
-        self.script_eof = None
-
-    def build(self):
-        # Read and process lines from lxcfile
-        with open(self.image.lxcfile, 'r') as f:
-            for line in f:
-                line = line.strip()
-                if self.script_eof:
-                    if line == self.script_eof:
-                        self.script_eof = None
-                        self.run_script(self.script)
-                    else:
-                        self.script.append(line)
-                elif line:
-                    self.process_line(*line.split(None, 1))
-
-    def process_line(self, directive, args):
-        # Process directives from lxcfile
-        if 'RUN' == directive:
-            self.script = []
-            self.script_eof = args
-        elif 'IMAGE' == directive:
-            self.set_name(args)
-        elif 'FROM' == directive:
-            self.set_layers(args)
-        elif 'COPY' == directive:
-            srcdst = args.split()
-            self.copy_files(srcdst[0], srcdst[1] if len(srcdst) == 2 else '')
-        elif 'ENV' == directive:
-            self.add_env(*args.split(None, 1))
-        elif 'USER' == directive:
-            self.set_user(*args.split())
-        elif 'CMD' == directive:
-            self.set_cmd(args)
-        elif 'WORKDIR' == directive:
-            self.set_cwd(args)
-        elif 'HALT' == directive:
-            self.set_halt(args)
-        elif 'READY' == directive:
-            self.set_ready(args)
-
-    def get_layer_path(self, layer):
-        return os.path.join(LXC_STORAGE_DIR, layer)
-
-    def run_script(self, script):
-        # Creates a temporary container, runs a script in its namespace, and stores the modifications as part of the image
-        if self.image.build_type == BuildType.METADATA:
-            # Don't run anything if we're building just metadata
-            return
-        lxcmgr.create_container(self.image.name, self.image.conf)
-        sh = os.path.join(LXC_STORAGE_DIR, self.image.name, 'run.sh')
-        with open(sh, 'w') as f:
-            f.write('#!/bin/sh\nset -ev\n\n{}\n'.format('\n'.join(script)))
-        os.chmod(sh, 0o700)
-        os.chown(sh, 100000, 100000)
-        subprocess.run(['lxc-execute', self.image.name, '--', '/bin/sh', '-lc', '/run.sh'], check=True)
-        os.unlink(sh)
-        if not self.image.build_type == BuildType.SCRATCH:
-            # Don't delete the temporary container if we're doing scratch build
-            lxcmgr.destroy_container(self.image.name)
-
-    def set_name(self, name):
-        # Set name and first (topmost) layer of the image
-        self.image.name = name
-        self.image.conf['layers'] = [name]
-        if self.image.build_type == BuildType.METADATA:
-            # Don't check or create any directories if we're building just metadata
-            return
-        image_path = self.get_layer_path(name)
-        if os.path.exists(image_path):
-            if self.image.build_type in (BuildType.FORCE, BuildType.SCRATCH):
-                self.clean()
-            else:
-                raise ImageExistsError(image_path)
-        os.makedirs(image_path, 0o755, True)
-        os.chown(image_path, 100000, 100000)
-
-    def set_layers(self, image):
-        # Prepend list of layers from parent image
-        pkgmgr = PkgMgr()
-        self.image.conf['layers'] = pkgmgr.installed_packages['images'][image]['layers'] + [self.image.name]
-
-    def copy_files(self, src, dst):
-        # Copy files from the host or download them from a http(s) URL
-        if self.image.build_type == BuildType.METADATA:
-            # Don't copy anything if we're building just metadata
-            return
-        dst = os.path.join(LXC_STORAGE_DIR, self.image.name, dst)
-        if src.startswith('http://') or src.startswith('https://'):
-            unpack_http_archive(src, dst)
-        else:
-            copy_tree(os.path.join(self.image.build_dir, src), dst)
-        # Shift UID/GID of the files to the unprivileged range
-        shift_uid(dst)
-
-    def add_env(self, key, value):
-        # Sets lxc.environment records for the image
-        if 'env' not in self.image.conf:
-            self.image.conf['env'] = []
-        self.image.conf['env'].append([key, value])
-
-    def set_user(self, uid, gid):
-        # Sets lxc.init.uid/gid for the image
-        self.image.conf['uid'] = uid
-        self.image.conf['gid'] = gid
-
-    def set_cmd(self, cmd):
-        # Sets lxc.init.cmd for the image
-        self.image.conf['cmd'] = cmd
-
-    def set_cwd(self, cwd):
-        # Sets lxc.init.cwd for the image
-        self.image.conf['cwd'] = cwd
-
-    def set_halt(self, halt):
-        # Sets lxc.signal.halt for the image
-        self.image.conf['halt'] = halt
-
-    def set_ready(self, cmd):
-        # Sets a command performed in OpenRC start_post to check readiness of the container
-        self.image.conf['ready'] = cmd
-
-    def clean(self):
-        lxcmgr.destroy_container(self.image.name)
-        try:
-            shutil.rmtree(self.get_layer_path(self.image.name))
-        except FileNotFoundError:
-            pass
-
-def unpack_http_archive(src, dst):
-    # Decompress an archive downloaded via http(s)
-    xf = 'xzf'
-    if src.endswith('.bz2'):
-        xf = 'xjf'
-    elif src.endswith('.xz'):
-        xf = 'xJf'
-    with subprocess.Popen(['wget', src, '-O', '-'], stdout=subprocess.PIPE) as wget:
-        with subprocess.Popen(['tar', xf, '-', '-C', dst], stdin=wget.stdout) as tar:
-            wget.stdout.close()
-            tar.wait()
-
-def copy_tree(src, dst):
-    # Copies files from the host
-    if not os.path.isdir(src):
-        shutil.copy2(src, dst)
-    else:
-        os.makedirs(dst, exist_ok=True)
-        for name in os.listdir(src):
-            copy_tree(os.path.join(src, name), os.path.join(dst, name))
-    shutil.copystat(src, dst)
-
-def shift_uid(dir):
-    # Shifts UID/GID of a file or a directory and its contents to the unprivileged range
-    shift_uid_entry(dir, os.stat(dir, follow_symlinks=True))
-    shift_uid_recursively(dir)
-
-def shift_uid_recursively(dir):
-    # Shifts UID/GID of a directory and its contents to the unprivileged range
-    for entry in os.scandir(dir):
-        shift_uid_entry(entry.path, entry.stat(follow_symlinks=False))
-        if entry.is_dir():
-            shift_uid_recursively(entry.path)
-
-def shift_uid_entry(path, stat):
-    # Shifts UID/GID of a file or a directory to the unprivileged range
-    uid = stat.st_uid
-    gid = stat.st_gid
-    do_chown = False
-    if uid < 100000:
-        uid = uid + 100000
-        do_chown = True
-    if gid < 100000:
-        gid = gid + 100000
-        do_chown = True
-    if do_chown:
-        os.lchown(path, uid, gid)

build/usr/lib/python3.6/lxcbuild/imagepacker.py

@@ -1,67 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-import subprocess
-
-from lxcmgr.paths import LXC_STORAGE_DIR
-from lxcmgr.pkgmgr import PkgMgr
-
-from . import crypto
-from .packer import PackageExistsError, Packer
-from .paths import REPO_IMAGES_DIR
-
-class ImagePacker(Packer):
-    def __init__(self, image):
-        super().__init__()
-        self.image = image
-        # Prepare package file names
-        self.tar_path = os.path.join(REPO_IMAGES_DIR, '{}.tar'.format(self.image.name))
-        self.xz_path = '{}.xz'.format(self.tar_path)
-
-    def pack(self):
-        if self.image.pack:
-            self.unregister()
-            try:
-                os.unlink(self.xz_path)
-            except FileNotFoundError:
-                pass
-        elif os.path.exists(self.xz_path):
-            raise PackageExistsError(self.xz_path)
-        self.create_archive()
-        self.register()
-        self.sign_packages()
-
-    def remove(self):
-        self.unregister()
-        try:
-            os.unlink(self.xz_path)
-        except FileNotFoundError:
-            pass
-
-    def create_archive(self):
-        # Create archive
-        print('Archiving image', self.image.name)
-        subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, self.image.name], cwd=LXC_STORAGE_DIR)
-        self.compress_archive()
-
-    def register(self):
-        # Register image in global repository metadata file
-        print('Registering image package', self.image.name)
-        image_conf = self.image.conf.copy()
-        image_conf['size'] = self.tar_size
-        image_conf['pkgsize'] = self.xz_size
-        image_conf['sha512'] = crypto.hash_file(self.xz_path)
-        self.packages['images'][self.image.name] = image_conf
-        self.save_repo_meta()
-        # Register the image also to locally installed images for package manager
-        pm = PkgMgr()
-        pm.register_image(self.image.name, self.packages['images'][self.image.name])
-
-    def unregister(self):
-        # Removes package from global repository metadata file
-        if self.image.name in self.packages['images']:
-            del self.packages['images'][self.image.name]
-            self.save_repo_meta()
-        # Unregister the image also from locally installed images for package manager
-        pm = PkgMgr()
-        pm.unregister_image(self.image.name)

build/usr/lib/python3.6/lxcbuild/packer.py

@@ -1,40 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import json
-import os
-import subprocess
-
-from . import crypto
-from .paths import PRIVATE_KEY, REPO_META_FILE, REPO_SIG_FILE
-
-class PackageExistsError(Exception):
-    pass
-
-class Packer:
-    def __init__(self):
-        self.tar_path = None
-        self.tar_size = 0
-        self.xz_path = None
-        self.xz_size = 0
-        if os.path.exists(REPO_META_FILE):
-            with open(REPO_META_FILE, 'r') as f:
-                self.packages = json.load(f)
-        else:
-            self.packages = {'apps': {}, 'images': {}}
-
-    def save_repo_meta(self):
-        with open(REPO_META_FILE, 'w') as f:
-            json.dump(self.packages, f, sort_keys=True, indent=4)
-
-    def compress_archive(self):
-        # Compress the tarball with xz (LZMA2)
-        self.tar_size = os.path.getsize(self.tar_path)
-        print('Compressing', self.tar_path, '({:.2f} MB)'.format(self.tar_size/1048576))
-        subprocess.run(['xz', '-9', self.tar_path])
-        self.xz_size = os.path.getsize(self.xz_path)
-        print('Compressed ', self.xz_path, '({:.2f} MB)'.format(self.xz_size/1048576))
-
-    def sign_packages(self):
-        signature = crypto.sign_file(PRIVATE_KEY, REPO_META_FILE)
-        with open(REPO_SIG_FILE, 'wb') as f:
-            f.write(signature)

build/usr/lib/python3.6/lxcbuild/paths.py

@@ -1,7 +0,0 @@
-# -*- coding: utf-8 -*-
-
-PRIVATE_KEY = '/srv/build/packages.key'
-REPO_APPS_DIR = '/srv/build/lxc/apps'
-REPO_IMAGES_DIR = '/srv/build/lxc/images'
-REPO_META_FILE = '/srv/build/lxc/packages'
-REPO_SIG_FILE = '/srv/build/lxc/packages.sig'