From 7cb420dc4c3ddb30c0e127864e240ad06295da51 Mon Sep 17 00:00:00 2001 From: Disassembler Date: Thu, 12 Mar 2020 22:56:40 +0100 Subject: [PATCH] Add SPOC as a submodule --- .gitmodules | 3 + apk/spoc | 1 + build/build-all.sh | 118 ++++++---- build/clean-all.sh | 46 +--- build/install-toolchain.sh | 13 +- build/usr/bin/lxcbuild | 79 ------- build/usr/lib/python3.6/lxcbuild/__init__.py | 1 - build/usr/lib/python3.6/lxcbuild/app.py | 36 ---- build/usr/lib/python3.6/lxcbuild/apppacker.py | 61 ------ build/usr/lib/python3.6/lxcbuild/crypto.py | 28 --- build/usr/lib/python3.6/lxcbuild/image.py | 60 ------ .../lib/python3.6/lxcbuild/imagebuilder.py | 203 ------------------ .../usr/lib/python3.6/lxcbuild/imagepacker.py | 67 ------ build/usr/lib/python3.6/lxcbuild/packer.py | 40 ---- build/usr/lib/python3.6/lxcbuild/paths.py | 7 - 15 files changed, 94 insertions(+), 669 deletions(-) create mode 160000 apk/spoc delete mode 100755 build/usr/bin/lxcbuild delete mode 100644 build/usr/lib/python3.6/lxcbuild/__init__.py delete mode 100644 build/usr/lib/python3.6/lxcbuild/app.py delete mode 100644 build/usr/lib/python3.6/lxcbuild/apppacker.py delete mode 100644 build/usr/lib/python3.6/lxcbuild/crypto.py delete mode 100644 build/usr/lib/python3.6/lxcbuild/image.py delete mode 100644 build/usr/lib/python3.6/lxcbuild/imagebuilder.py delete mode 100644 build/usr/lib/python3.6/lxcbuild/imagepacker.py delete mode 100644 build/usr/lib/python3.6/lxcbuild/packer.py delete mode 100644 build/usr/lib/python3.6/lxcbuild/paths.py diff --git a/.gitmodules b/.gitmodules index 3c43375..a28e022 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "app-vmmgr"] path = apk/vmmgr url = ssh://git@git.spotter.cz:2222/Spotter-Cluster/vmmgr.git +[submodule "spoc"] + path = apk/spoc + url = ssh://git@git.spotter.cz:2222/Spotter-Cluster/spoc.git diff --git a/apk/spoc b/apk/spoc new file mode 160000 index 0000000..1e12d78 --- /dev/null +++ b/apk/spoc @@ -0,0 +1 @@ +Subproject commit 1e12d78efe730fcf35bc949d70c8db04e5917945 diff --git a/build/build-all.sh b/build/build-all.sh index e2f9d15..d186e78 100755 --- a/build/build-all.sh +++ b/build/build-all.sh @@ -45,50 +45,88 @@ cd ${ROOT}/apk/wireguard-tools apk add -U libmnl-dev abuild -F -# Build apd pack runtimes +# Build runtimes cd ${ROOT}/lxc-shared -lxcbuild alpine3.8 -lxcbuild alpine3.8-php5.6 -lxcbuild alpine3.8-ruby2.4 -lxcbuild alpine3.9 -lxcbuild alpine3.9-java8 -lxcbuild alpine3.9-php7.2 -lxcbuild alpine3.9-python2.7 -lxcbuild alpine3.9-python3.6 -lxcbuild alpine3.9-ruby2.4 -lxcbuild alpine3.9-ruby2.6 -lxcbuild alpine3.9-tomcat7 -lxcbuild alpine3.9-tomcat8.5 +spoc-image build -p alpine3.8/image +spoc-image build -p alpine3.8-php5.6/image +spoc-image build -p alpine3.8-ruby2.4/image +spoc-image build -p alpine3.9/image +spoc-image build -p alpine3.9-java8/image +spoc-image build -p alpine3.9-php7.2/image +spoc-image build -p alpine3.9-python2.7/image +spoc-image build -p alpine3.9-python3.6/image +spoc-image build -p alpine3.9-ruby2.4/image +spoc-image build -p alpine3.9-ruby2.6/image +spoc-image build -p alpine3.9-tomcat7/image +spoc-image build -p alpine3.9-tomcat8.5/image # Build services cd ${ROOT}/lxc-services -lxcbuild activemq -lxcbuild mariadb -lxcbuild postgres -lxcbuild postgis -lxcbuild rabbitmq -lxcbuild redis -lxcbuild solr6 +spoc-image build -p activemq/image +spoc-image build -p mariadb/image +spoc-image build -p postgres/image +spoc-image build -p postgis/image +spoc-image build -p rabbitmq/image +spoc-image build -p redis/image +spoc-image build -p solr6/image # Build applications cd ${ROOT}/lxc-apps -lxcbuild ckan -lxcbuild crisiscleanup -lxcbuild cts -lxcbuild decidim -lxcbuild ecogis -lxcbuild frontlinesms -lxcbuild gnuhealth -lxcbuild kanboard -lxcbuild mifosx -lxcbuild motech -lxcbuild odoo -lxcbuild opendatakit -lxcbuild openmapkit -lxcbuild pandora -lxcbuild sahana -lxcbuild sahana-demo -lxcbuild sambro -lxcbuild seeddms -lxcbuild sigmah -lxcbuild ushahidi + +spoc-image build -p ckan/image +spoc-image build -p ckan-datapusher/image +spoc-app publish ckan/app + +spoc-image build -p crisiscleanup/image +spoc-app publish crisiscleanup/app + +spoc-image build -p cts/image +spoc-app publish cts/app + +spoc-image build -p decidim/image +spoc-app publish decidim/app + +spoc-image build -p ecogis/image +# spoc-app publish ecogis/app + +spoc-image build -p frontlinesms/image +spoc-app publish frontlinesms/app + +spoc-image build -p gnuhealth/image +spoc-app publish gnuhealth/app + +spoc-image build -p kanboard/image +spoc-app publish kanboard/app + +spoc-image build -p mifosx/image +spoc-app publish mifosx/app + +spoc-image build -p motech/image +spoc-app publish motech/app + +spoc-image build -p odoo/image +spoc-app publish odoo/app + +spoc-image build -p opendatakit/image +spoc-image build -p opendatakit-build/image +spoc-app publish opendatakit/app + +spoc-image build -p openmapkit/image +spoc-app publish openmapkit/app + +spoc-image build -p pandora/image +spoc-app publish pandora/app + +spoc-image build -p sahana/image +spoc-app publish sahana/app +spoc-app publish sahana-demo/app +spoc-app publish sambro/app + +spoc-image build -p seeddms/image +spoc-app publish seeddms/app + +spoc-image build -p sigmah/image +spoc-app publish sigmah/app + +spoc-image build -p ushahidi/image +spoc-app publish ushahidi/app diff --git a/build/clean-all.sh b/build/clean-all.sh index 6edf1d7..1d35898 100755 --- a/build/clean-all.sh +++ b/build/clean-all.sh @@ -11,28 +11,7 @@ rm -f /srv/build/vm.tar rm -rf /srv/build/alpine/* # Clean built LXC packages -rm -rf /srv/build/lxc/apps/* -rm -rf /srv/build/lxc/images/* -rm -f /srv/build/lxc/packages.sig -echo '{"apps":{},"images":{}}' >/srv/build/lxc/packages - -# Stop running containers -for SERVICE in $(find /run/openrc/started -name 'lxc-*'); do - service $(basename ${SERVICE}) stop -done - -# Remove services -rm -f /etc/init.d/lxc-* -rc-update -u - -# Remove containers -rm -rf /var/lib/lxc/* -rm -f /var/log/lxc/* - -# Remove application data -for DIR in $(find /srv ! -path /srv/build -maxdepth 1 -mindepth 1); do - rm -rf ${DIR} -done +rm -rf /srv/build/spoc # Remove nginx configs for CONF in $(find /etc/nginx/conf.d -name '*.conf' -a ! -name repo.conf -a ! -name default.conf); do @@ -40,20 +19,11 @@ for CONF in $(find /etc/nginx/conf.d -name '*.conf' -a ! -name repo.conf -a ! -n done service nginx reload -# Reset /etc/hosts -cat </etc/hosts -127.0.0.1 localhost -::1 localhost -172.17.0.1 host -172.17.0.1 repo.build.vm -EOF +# Stop running containers +for APP in $(spoc-container list); do + spoc-container stop ${APP} +done -# Reset vmmgr config -export ADMINPWD=$(python3 -c "import json; f = open('/etc/vmmgr/config.json'); j = json.load(f); print(j['host']['adminpwd'])") -envsubst /etc/vmmgr/config.json - -# Clean locally installed LXC packages -rm -rf /var/lib/lxcmgr/storage/* -rm -rf /var/lib/lxcmgr/cache/apps/* -rm -rf /var/lib/lxcmgr/cache/images/* -echo '{"apps":{},"images":{}}' >/var/lib/lxcmgr/packages +# Remove data +rm -rf /var/lib/spoc +rm -rf /var/log/spoc diff --git a/build/install-toolchain.sh b/build/install-toolchain.sh index 5877d29..576790a 100755 --- a/build/install-toolchain.sh +++ b/build/install-toolchain.sh @@ -5,7 +5,7 @@ cd $(realpath $(dirname "${0}")) # Install basic build tools apk update -apk add git file htop less openssh-client tar xz +apk add git file htop less openssh-client # Install Alpine SDK apk add alpine-sdk # Install Sphinx support @@ -13,7 +13,7 @@ apk add py3-sphinx pip3 install recommonmark sphinx-markdown-tables # Copy root profile files and settings -mkdir -p /root/.config/htop /root/.ssh +mkdir -p /root/.config/htop cp root/.profile /root/.profile cp root/.config/htop/htoprc /root/.config/htop/htoprc @@ -21,11 +21,6 @@ cp root/.config/htop/htoprc /root/.config/htop/htoprc adduser root abuild cp etc/abuild.conf /etc/abuild.conf -# Prepare LXC build toolchain -cp usr/bin/lxcbuild /usr/bin/lxcbuild -cp usr/bin/lxcmerge /usr/bin/lxcmerge -mkdir -p /srv/build/lxc/apps /srv/build/lxc/images - # Prepare local APK repository cp etc/nginx/conf.d/repo.conf /etc/nginx/conf.d/repo.conf echo "172.17.0.1 repo.build.vm" >>/etc/hosts @@ -38,5 +33,5 @@ echo '{"url":"http://repo.build.vm/lxc","user":"","pwd":""}' >/etc/lxcmgr/repo.j # echo '/srv/build/repokey.rsa' | abuild-keygen # Supply LXC build key -# openssl ecparam -genkey -name secp384r1 -out /srv/build/packages.key -# openssl ec -in /srv/build/packages.key -pubout -out /srv/build/lxc/packages.pub +# openssl ecparam -genkey -name secp384r1 -out /etc/spoc/publish.key +# openssl ec -in /etc/spoc/publish.key -pubout -out /tmp/repository.pub diff --git a/build/usr/bin/lxcbuild b/build/usr/bin/lxcbuild deleted file mode 100755 index 114ca1f..0000000 --- a/build/usr/bin/lxcbuild +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- - -import argparse -import os -import sys -from lxcbuild.app import App -from lxcbuild.image import Image -from lxcbuild.imagebuilder import BuildType - -def build_and_pack_image(path, args): - image = Image() - if args.scratch: - image.build_type = BuildType.SCRATCH - elif args.force: - image.build_type = BuildType.FORCE - image.build_and_pack(path) - -def pack_app(path): - app = App() - app.pack(path) - -def main(args): - if args.remove_image: - image = Image() - image.name = args.buildarg - image.remove() - elif args.remove_app: - app = App() - app.name = args.buildarg - app.remove() - else: - buildpath = os.path.realpath(args.buildarg) - # If the buildpath is a file, determine type from filename - if os.path.isfile(buildpath): - basename = os.path.basename(buildpath) - if basename == 'lxcfile' or basename.endswith('.lxcfile'): - build_and_pack_image(buildpath, args) - # Compose files needs to be ignored when performing scratch builds - elif not args.scratch and basename == 'meta': - pack_app(buildpath) - else: - print('Unknown file {} given, expected "lxcfile"{}'.format(buildpath, '' if args.scratch else ' or "meta"')) - sys.exit(1) - # If the buildpath is a directory, build as much as possible, unless scratch build was requested, in which case don't build anything - else: - if args.scratch: - lxcfile = os.path.join(buildpath, 'lxcfile') - if os.path.exists(lxcfile): - build_and_pack_image(lxcfile, args) - else: - print('Please specify an lxcfile for scratch build') - sys.exit(1) - else: - valid_dir = False - for entry in os.scandir(buildpath): - if entry.is_file() and (entry.name == 'lxcfile' or entry.name.endswith('.lxcfile')): - valid_dir = True - build_and_pack_image(entry.path, args) - meta = os.path.join(buildpath, 'meta') - if os.path.exists(meta): - valid_dir = True - pack_app(meta) - if not valid_dir: - print('Directory {} doesn\'t contain anything to build, skipping'.format(buildpath)) - -parser = argparse.ArgumentParser(description='VM application builder and packager') -group = parser.add_mutually_exclusive_group() -group.add_argument('-f', '--force', action='store_true', help='Force rebuild already built package') -group.add_argument('-s', '--scratch', action='store_true', help='Build container for testing purposes, i.e. without cleanup on failure and packaging') -group.add_argument('-r', '--remove-image', action='store_true', help='Delete image (including scratch) from build repository') -group.add_argument('-e', '--remove-app', action='store_true', help='Delete application from build repository') -parser.add_argument('buildarg', help='Either specific "lxcfile" or "meta" file or a directory containing at least one of them') - -args = parser.parse_args() -if hasattr(args, 'buildarg'): - main(args) -else: - parser.print_usage() diff --git a/build/usr/lib/python3.6/lxcbuild/__init__.py b/build/usr/lib/python3.6/lxcbuild/__init__.py deleted file mode 100644 index 40a96af..0000000 --- a/build/usr/lib/python3.6/lxcbuild/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- diff --git a/build/usr/lib/python3.6/lxcbuild/app.py b/build/usr/lib/python3.6/lxcbuild/app.py deleted file mode 100644 index 2b25e73..0000000 --- a/build/usr/lib/python3.6/lxcbuild/app.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- - -import json -import os -import sys - -from .apppacker import AppPacker -from .imagebuilder import ImageNotFoundError - -class App: - def __init__(self): - self.name = None - self.conf = {} - self.build_dir = None - - def load_metafile(self, metafile): - self.build_dir = os.path.dirname(metafile) - if os.path.basename(metafile) == 'meta': - self.name = os.path.basename(self.build_dir) - else: - self.name = os.path.splitext(metafile)[0] - with open(metafile, 'r') as f: - self.conf = json.load(f) - - def pack(self, metafile): - self.load_metafile(metafile) - packer = AppPacker(self) - try: - packer.pack() - except ImageNotFoundError as e: - print('Image {} not found, can\'t pack {}'.format(e, self.name)) - sys.exit(1) - - def remove(self): - packer = AppPacker(self) - packer.remove() diff --git a/build/usr/lib/python3.6/lxcbuild/apppacker.py b/build/usr/lib/python3.6/lxcbuild/apppacker.py deleted file mode 100644 index 24da80a..0000000 --- a/build/usr/lib/python3.6/lxcbuild/apppacker.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import subprocess - -from . import crypto -from .imagebuilder import ImageNotFoundError -from .packer import Packer -from .paths import REPO_APPS_DIR - -class AppPacker(Packer): - def __init__(self, app): - super().__init__() - self.app = app - # Prepare package file names - self.tar_path = os.path.join(REPO_APPS_DIR, '{}.tar'.format(self.app.name)) - self.xz_path = '{}.xz'.format(self.tar_path) - - def pack(self): - # Check if all images used by containers exist - for container in self.app.conf['containers']: - image = self.app.conf['containers'][container]['image'] - if image not in self.packages['images']: - raise ImageNotFoundError(image) - try: - os.unlink(self.xz_path) - except FileNotFoundError: - pass - self.create_archive() - self.register() - self.sign_packages() - - def remove(self): - self.unregister() - try: - os.unlink(self.xz_path) - except FileNotFoundError: - pass - - def create_archive(self): - # Create archive with application setup scripts - print('Archiving setup scripts for', self.app.name) - scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh') - scripts = [s for s in scripts if os.path.exists(os.path.join(self.app.build_dir, s))] - subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, '--transform', 's,^,{}/,'.format(self.app.name)] + scripts, cwd=self.app.build_dir) - self.compress_archive() - - def register(self): - # Register package in global repository metadata file - print('Registering application package', self.app.name) - self.packages['apps'][self.app.name] = self.app.conf.copy() - self.packages['apps'][self.app.name]['size'] = self.tar_size - self.packages['apps'][self.app.name]['pkgsize'] = self.xz_size - self.packages['apps'][self.app.name]['sha512'] = crypto.hash_file(self.xz_path) - self.save_repo_meta() - - def unregister(self): - # Removes package from global repository metadata file - if self.app.name in self.packages['apps']: - del self.packages['apps'][self.app.name] - self.save_repo_meta() diff --git a/build/usr/lib/python3.6/lxcbuild/crypto.py b/build/usr/lib/python3.6/lxcbuild/crypto.py deleted file mode 100644 index 44c27d2..0000000 --- a/build/usr/lib/python3.6/lxcbuild/crypto.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- - -import hashlib - -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.asymmetric import ec -from cryptography.hazmat.primitives.serialization import load_pem_private_key - -def sign_file(private_key_path, input_path): - # Generate SHA512 signature of a file using EC private key - print('Signing packages') - with open(private_key_path, 'rb') as f: - priv_key = load_pem_private_key(f.read(), None, default_backend()) - with open(input_path, 'rb') as f: - data = f.read() - return priv_key.sign(data, ec.ECDSA(hashes.SHA512())) - -def hash_file(file_path): - # Calculate SHA512 hash of a file - sha512 = hashlib.sha512() - with open(file_path, 'rb') as f: - while True: - data = f.read(65536) - if not data: - break - sha512.update(data) - return sha512.hexdigest() diff --git a/build/usr/lib/python3.6/lxcbuild/image.py b/build/usr/lib/python3.6/lxcbuild/image.py deleted file mode 100644 index c07e293..0000000 --- a/build/usr/lib/python3.6/lxcbuild/image.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import sys - -from lxcmgr import lxcmgr - -from .imagebuilder import BuildType, ImageBuilder, ImageExistsError, ImageNotFoundError -from .imagepacker import ImagePacker -from .packer import PackageExistsError - -class Image: - def __init__(self): - self.name = None - self.conf = {} - self.lxcfile = None - self.build_dir = None - self.build_type = BuildType.NORMAL - self.pack = False - - def build_and_pack(self, lxcfile): - self.lxcfile = lxcfile - self.build_dir = os.path.dirname(lxcfile) - self.conf['build'] = True - builder = ImageBuilder(self) - try: - builder.build() - # Packaging needs to happen in any case after a successful build in order to prevent outdated packages - self.pack = True - except ImageExistsError as e: - # If container already exists and build hasn't been forced, rerun the build just for metadata which are still needed for packaging - print('Image {} already exists, skipping build tasks'.format(e)) - self.build_type = BuildType.METADATA - builder.build() - except ImageNotFoundError as e: - # If one of the layers is missing, cleanup and die - print('Image {} not found, can\'t build {}'.format(e, self.name)) - builder.clean() - sys.exit(1) - except: - # If build fails with another exception, cleanup (unless we were doing scratch build) and re-raise - if not self.build_type == BuildType.SCRATCH: - builder.clean() - raise - del self.conf['build'] - # If we're doing a scratch build, regenerate the final LXC container configuration including ephemeral layer - if self.build_type == BuildType.SCRATCH: - lxcmgr.create_container(self.name, self.conf) - else: - try: - packer = ImagePacker(self) - packer.pack() - except PackageExistsError as e: - print('Package {} already exists, skipping packaging tasks'.format(e)) - - def remove(self): - builder = ImageBuilder(self) - builder.clean() - packer = ImagePacker(self) - packer.remove() diff --git a/build/usr/lib/python3.6/lxcbuild/imagebuilder.py b/build/usr/lib/python3.6/lxcbuild/imagebuilder.py deleted file mode 100644 index 36df528..0000000 --- a/build/usr/lib/python3.6/lxcbuild/imagebuilder.py +++ /dev/null @@ -1,203 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import shutil -import subprocess -import sys - -from lxcmgr import lxcmgr -from lxcmgr.paths import LXC_STORAGE_DIR -from lxcmgr.pkgmgr import PkgMgr - -class ImageExistsError(Exception): - pass - -class ImageNotFoundError(Exception): - pass - -class BuildType: - NORMAL = 1 - FORCE = 2 - SCRATCH = 3 - METADATA = 4 - -class ImageBuilder: - def __init__(self, image): - self.image = image - self.script = [] - self.script_eof = None - - def build(self): - # Read and process lines from lxcfile - with open(self.image.lxcfile, 'r') as f: - for line in f: - line = line.strip() - if self.script_eof: - if line == self.script_eof: - self.script_eof = None - self.run_script(self.script) - else: - self.script.append(line) - elif line: - self.process_line(*line.split(None, 1)) - - def process_line(self, directive, args): - # Process directives from lxcfile - if 'RUN' == directive: - self.script = [] - self.script_eof = args - elif 'IMAGE' == directive: - self.set_name(args) - elif 'FROM' == directive: - self.set_layers(args) - elif 'COPY' == directive: - srcdst = args.split() - self.copy_files(srcdst[0], srcdst[1] if len(srcdst) == 2 else '') - elif 'ENV' == directive: - self.add_env(*args.split(None, 1)) - elif 'USER' == directive: - self.set_user(*args.split()) - elif 'CMD' == directive: - self.set_cmd(args) - elif 'WORKDIR' == directive: - self.set_cwd(args) - elif 'HALT' == directive: - self.set_halt(args) - elif 'READY' == directive: - self.set_ready(args) - - def get_layer_path(self, layer): - return os.path.join(LXC_STORAGE_DIR, layer) - - def run_script(self, script): - # Creates a temporary container, runs a script in its namespace, and stores the modifications as part of the image - if self.image.build_type == BuildType.METADATA: - # Don't run anything if we're building just metadata - return - lxcmgr.create_container(self.image.name, self.image.conf) - sh = os.path.join(LXC_STORAGE_DIR, self.image.name, 'run.sh') - with open(sh, 'w') as f: - f.write('#!/bin/sh\nset -ev\n\n{}\n'.format('\n'.join(script))) - os.chmod(sh, 0o700) - os.chown(sh, 100000, 100000) - subprocess.run(['lxc-execute', self.image.name, '--', '/bin/sh', '-lc', '/run.sh'], check=True) - os.unlink(sh) - if not self.image.build_type == BuildType.SCRATCH: - # Don't delete the temporary container if we're doing scratch build - lxcmgr.destroy_container(self.image.name) - - def set_name(self, name): - # Set name and first (topmost) layer of the image - self.image.name = name - self.image.conf['layers'] = [name] - if self.image.build_type == BuildType.METADATA: - # Don't check or create any directories if we're building just metadata - return - image_path = self.get_layer_path(name) - if os.path.exists(image_path): - if self.image.build_type in (BuildType.FORCE, BuildType.SCRATCH): - self.clean() - else: - raise ImageExistsError(image_path) - os.makedirs(image_path, 0o755, True) - os.chown(image_path, 100000, 100000) - - def set_layers(self, image): - # Prepend list of layers from parent image - pkgmgr = PkgMgr() - self.image.conf['layers'] = pkgmgr.installed_packages['images'][image]['layers'] + [self.image.name] - - def copy_files(self, src, dst): - # Copy files from the host or download them from a http(s) URL - if self.image.build_type == BuildType.METADATA: - # Don't copy anything if we're building just metadata - return - dst = os.path.join(LXC_STORAGE_DIR, self.image.name, dst) - if src.startswith('http://') or src.startswith('https://'): - unpack_http_archive(src, dst) - else: - copy_tree(os.path.join(self.image.build_dir, src), dst) - # Shift UID/GID of the files to the unprivileged range - shift_uid(dst) - - def add_env(self, key, value): - # Sets lxc.environment records for the image - if 'env' not in self.image.conf: - self.image.conf['env'] = [] - self.image.conf['env'].append([key, value]) - - def set_user(self, uid, gid): - # Sets lxc.init.uid/gid for the image - self.image.conf['uid'] = uid - self.image.conf['gid'] = gid - - def set_cmd(self, cmd): - # Sets lxc.init.cmd for the image - self.image.conf['cmd'] = cmd - - def set_cwd(self, cwd): - # Sets lxc.init.cwd for the image - self.image.conf['cwd'] = cwd - - def set_halt(self, halt): - # Sets lxc.signal.halt for the image - self.image.conf['halt'] = halt - - def set_ready(self, cmd): - # Sets a command performed in OpenRC start_post to check readiness of the container - self.image.conf['ready'] = cmd - - def clean(self): - lxcmgr.destroy_container(self.image.name) - try: - shutil.rmtree(self.get_layer_path(self.image.name)) - except FileNotFoundError: - pass - -def unpack_http_archive(src, dst): - # Decompress an archive downloaded via http(s) - xf = 'xzf' - if src.endswith('.bz2'): - xf = 'xjf' - elif src.endswith('.xz'): - xf = 'xJf' - with subprocess.Popen(['wget', src, '-O', '-'], stdout=subprocess.PIPE) as wget: - with subprocess.Popen(['tar', xf, '-', '-C', dst], stdin=wget.stdout) as tar: - wget.stdout.close() - tar.wait() - -def copy_tree(src, dst): - # Copies files from the host - if not os.path.isdir(src): - shutil.copy2(src, dst) - else: - os.makedirs(dst, exist_ok=True) - for name in os.listdir(src): - copy_tree(os.path.join(src, name), os.path.join(dst, name)) - shutil.copystat(src, dst) - -def shift_uid(dir): - # Shifts UID/GID of a file or a directory and its contents to the unprivileged range - shift_uid_entry(dir, os.stat(dir, follow_symlinks=True)) - shift_uid_recursively(dir) - -def shift_uid_recursively(dir): - # Shifts UID/GID of a directory and its contents to the unprivileged range - for entry in os.scandir(dir): - shift_uid_entry(entry.path, entry.stat(follow_symlinks=False)) - if entry.is_dir(): - shift_uid_recursively(entry.path) - -def shift_uid_entry(path, stat): - # Shifts UID/GID of a file or a directory to the unprivileged range - uid = stat.st_uid - gid = stat.st_gid - do_chown = False - if uid < 100000: - uid = uid + 100000 - do_chown = True - if gid < 100000: - gid = gid + 100000 - do_chown = True - if do_chown: - os.lchown(path, uid, gid) diff --git a/build/usr/lib/python3.6/lxcbuild/imagepacker.py b/build/usr/lib/python3.6/lxcbuild/imagepacker.py deleted file mode 100644 index 4ba2655..0000000 --- a/build/usr/lib/python3.6/lxcbuild/imagepacker.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import subprocess - -from lxcmgr.paths import LXC_STORAGE_DIR -from lxcmgr.pkgmgr import PkgMgr - -from . import crypto -from .packer import PackageExistsError, Packer -from .paths import REPO_IMAGES_DIR - -class ImagePacker(Packer): - def __init__(self, image): - super().__init__() - self.image = image - # Prepare package file names - self.tar_path = os.path.join(REPO_IMAGES_DIR, '{}.tar'.format(self.image.name)) - self.xz_path = '{}.xz'.format(self.tar_path) - - def pack(self): - if self.image.pack: - self.unregister() - try: - os.unlink(self.xz_path) - except FileNotFoundError: - pass - elif os.path.exists(self.xz_path): - raise PackageExistsError(self.xz_path) - self.create_archive() - self.register() - self.sign_packages() - - def remove(self): - self.unregister() - try: - os.unlink(self.xz_path) - except FileNotFoundError: - pass - - def create_archive(self): - # Create archive - print('Archiving image', self.image.name) - subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, self.image.name], cwd=LXC_STORAGE_DIR) - self.compress_archive() - - def register(self): - # Register image in global repository metadata file - print('Registering image package', self.image.name) - image_conf = self.image.conf.copy() - image_conf['size'] = self.tar_size - image_conf['pkgsize'] = self.xz_size - image_conf['sha512'] = crypto.hash_file(self.xz_path) - self.packages['images'][self.image.name] = image_conf - self.save_repo_meta() - # Register the image also to locally installed images for package manager - pm = PkgMgr() - pm.register_image(self.image.name, self.packages['images'][self.image.name]) - - def unregister(self): - # Removes package from global repository metadata file - if self.image.name in self.packages['images']: - del self.packages['images'][self.image.name] - self.save_repo_meta() - # Unregister the image also from locally installed images for package manager - pm = PkgMgr() - pm.unregister_image(self.image.name) diff --git a/build/usr/lib/python3.6/lxcbuild/packer.py b/build/usr/lib/python3.6/lxcbuild/packer.py deleted file mode 100644 index 0a55efd..0000000 --- a/build/usr/lib/python3.6/lxcbuild/packer.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- - -import json -import os -import subprocess - -from . import crypto -from .paths import PRIVATE_KEY, REPO_META_FILE, REPO_SIG_FILE - -class PackageExistsError(Exception): - pass - -class Packer: - def __init__(self): - self.tar_path = None - self.tar_size = 0 - self.xz_path = None - self.xz_size = 0 - if os.path.exists(REPO_META_FILE): - with open(REPO_META_FILE, 'r') as f: - self.packages = json.load(f) - else: - self.packages = {'apps': {}, 'images': {}} - - def save_repo_meta(self): - with open(REPO_META_FILE, 'w') as f: - json.dump(self.packages, f, sort_keys=True, indent=4) - - def compress_archive(self): - # Compress the tarball with xz (LZMA2) - self.tar_size = os.path.getsize(self.tar_path) - print('Compressing', self.tar_path, '({:.2f} MB)'.format(self.tar_size/1048576)) - subprocess.run(['xz', '-9', self.tar_path]) - self.xz_size = os.path.getsize(self.xz_path) - print('Compressed ', self.xz_path, '({:.2f} MB)'.format(self.xz_size/1048576)) - - def sign_packages(self): - signature = crypto.sign_file(PRIVATE_KEY, REPO_META_FILE) - with open(REPO_SIG_FILE, 'wb') as f: - f.write(signature) diff --git a/build/usr/lib/python3.6/lxcbuild/paths.py b/build/usr/lib/python3.6/lxcbuild/paths.py deleted file mode 100644 index bf4e455..0000000 --- a/build/usr/lib/python3.6/lxcbuild/paths.py +++ /dev/null @@ -1,7 +0,0 @@ -# -*- coding: utf-8 -*- - -PRIVATE_KEY = '/srv/build/packages.key' -REPO_APPS_DIR = '/srv/build/lxc/apps' -REPO_IMAGES_DIR = '/srv/build/lxc/images' -REPO_META_FILE = '/srv/build/lxc/packages' -REPO_SIG_FILE = '/srv/build/lxc/packages.sig'