From 62a6612a7944f4e9661725788d7268d3ad838ad2 Mon Sep 17 00:00:00 2001 From: Disassembler Date: Sat, 5 Oct 2019 22:26:54 +0200 Subject: [PATCH] Implement scratch containers and image/app removal --- apk/vmmgr | 2 +- build/usr/bin/lxcbuild | 77 +++++++++++------ build/usr/lib/python3.6/lxcbuild/app.py | 22 +++-- build/usr/lib/python3.6/lxcbuild/apppacker.py | 61 +++++++++++++ build/usr/lib/python3.6/lxcbuild/image.py | 47 ++++++---- .../lxcbuild/{builder.py => imagebuilder.py} | 32 ++++--- .../usr/lib/python3.6/lxcbuild/imagepacker.py | 66 ++++++++++++++ build/usr/lib/python3.6/lxcbuild/packer.py | 85 +------------------ 8 files changed, 243 insertions(+), 149 deletions(-) create mode 100644 build/usr/lib/python3.6/lxcbuild/apppacker.py rename build/usr/lib/python3.6/lxcbuild/{builder.py => imagebuilder.py} (88%) create mode 100644 build/usr/lib/python3.6/lxcbuild/imagepacker.py diff --git a/apk/vmmgr b/apk/vmmgr index 6045349..b02fc3f 160000 --- a/apk/vmmgr +++ b/apk/vmmgr @@ -1 +1 @@ -Subproject commit 6045349f9c3602d6ba9b081a62d4338b202521d6 +Subproject commit b02fc3f42c65d8833451e41b550f7588c9de2cc2 diff --git a/build/usr/bin/lxcbuild b/build/usr/bin/lxcbuild index a2673b8..9d47df9 100755 --- a/build/usr/bin/lxcbuild +++ b/build/usr/bin/lxcbuild @@ -8,37 +8,62 @@ from lxcbuild.app import App from lxcbuild.image import Image parser = argparse.ArgumentParser(description='VM application builder and packager') -parser.add_argument('-f', '--force', action='store_true', help='Force rebuild already built package') -parser.add_argument('buildpath', help='Either specific "lxcfile" or "meta" file or a directory containing at least one') +group = parser.add_mutually_exclusive_group() +group.add_argument('-f', '--force', action='store_true', help='Force rebuild already built package') +group.add_argument('-s', '--scratch', action='store_true', help='Build container for testing purposes, i.e. without cleanup on failure and packaging') +group.add_argument('-r', '--remove-image', action='store_true', help='Delete image (including scratch) from build repository') +group.add_argument('-e', '--remove-app', action='store_true', help='Delete application from build repository') +parser.add_argument('buildarg', help='Either specific "lxcfile" or "meta" file or a directory containing at least one of them') if len(sys.argv) < 2: parser.print_usage() sys.exit(1) args = parser.parse_args() -buildpath = os.path.realpath(args.buildpath) -if os.path.isfile(buildpath): - basename = os.path.basename(buildpath) - if basename == 'lxcfile' or basename.endswith('.lxcfile'): - image = Image(buildpath) - image.build_and_pack(args.force) - elif basename == 'meta' or basename.endswith('.meta'): - app = App(buildpath) - app.pack() - else: - print('Unknown file {} given, expected "lxcfile" or "meta"'.format(buildpath)) - sys.exit(1) +def build_and_pack_image(args, path): + image = Image() + image.force_build = args.force or args.scratch + image.scratch_build = args.scratch + image.build_and_pack(path) + +def pack_app(path): + app = App() + app.pack(path) + +if args.remove_image: + image = Image() + image.name = args.buildarg + image.remove() +elif args.remove_app: + app = App() + app.name = args.buildarg + app.remove() else: - valid_dir = False - for entry in os.scandir(buildpath): - if entry.is_file() and (entry.name == 'lxcfile' or entry.name.endswith('.lxcfile')): + buildpath = os.path.realpath(args.buildarg) + # If the buildpath is a file, determine type from filename + if os.path.isfile(buildpath): + basename = os.path.basename(buildpath) + if basename == 'lxcfile' or basename.endswith('.lxcfile'): + build_and_pack_image(args, buildpath) + # Compose files needs to be ignored when performing scratch builds + elif not args.scratch and basename == 'meta': + pack_app(buildpath) + else: + print('Unknown file {} given, expected "lxcfile"{}'.format(buildpath, '' if args.scratch else ' or "meta"')) + sys.exit(1) + # If the buildpath is a directory, build as much as possible, unless scratch build was requested, in which case don't build anything + else: + if args.scratch: + print('Please specify an lxcfile for scratch build') + sys.exit(1) + valid_dir = False + for entry in os.scandir(buildpath): + if entry.is_file() and (entry.name == 'lxcfile' or entry.name.endswith('.lxcfile')): + valid_dir = True + build_and_pack_image(args, entry.path) + meta = os.path.join(buildpath, 'meta') + if os.path.exists(meta): valid_dir = True - image = Image(entry.path) - image.build_and_pack(args.force) - meta = os.path.join(buildpath, 'meta') - if os.path.exists(meta): - valid_dir = True - app = App(meta) - app.pack() - if not valid_dir: - print('Directory {} doesn\'t contain anything to build, skipping'.format(buildpath)) + pack_app(meta) + if not valid_dir: + print('Directory {} doesn\'t contain anything to build, skipping'.format(buildpath)) diff --git a/build/usr/lib/python3.6/lxcbuild/app.py b/build/usr/lib/python3.6/lxcbuild/app.py index b1dfe74..2b25e73 100644 --- a/build/usr/lib/python3.6/lxcbuild/app.py +++ b/build/usr/lib/python3.6/lxcbuild/app.py @@ -4,11 +4,16 @@ import json import os import sys -from .builder import ImageNotFoundError -from .packer import Packer +from .apppacker import AppPacker +from .imagebuilder import ImageNotFoundError class App: - def __init__(self, metafile): + def __init__(self): + self.name = None + self.conf = {} + self.build_dir = None + + def load_metafile(self, metafile): self.build_dir = os.path.dirname(metafile) if os.path.basename(metafile) == 'meta': self.name = os.path.basename(self.build_dir) @@ -17,10 +22,15 @@ class App: with open(metafile, 'r') as f: self.conf = json.load(f) - def pack(self): - packer = Packer() + def pack(self, metafile): + self.load_metafile(metafile) + packer = AppPacker(self) try: - packer.pack_app(self) + packer.pack() except ImageNotFoundError as e: print('Image {} not found, can\'t pack {}'.format(e, self.name)) sys.exit(1) + + def remove(self): + packer = AppPacker(self) + packer.remove() diff --git a/build/usr/lib/python3.6/lxcbuild/apppacker.py b/build/usr/lib/python3.6/lxcbuild/apppacker.py new file mode 100644 index 0000000..dac61cd --- /dev/null +++ b/build/usr/lib/python3.6/lxcbuild/apppacker.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +import os +import subprocess + +from . import crypto +from .imagebuilder import ImageNotFoundError +from .packer import Packer +from .paths import REPO_APPS_DIR + +class AppPacker(Packer): + def __init__(self, app): + super().__init__() + self.app = app + # Prepare package file names + self.tar_path = os.path.join(REPO_APPS_DIR, '{}.tar'.format(self.app.name)) + self.xz_path = '{}.xz'.format(self.tar_path) + + def pack(self): + # Check if all images used by containers exist + for container in self.app.conf['containers']: + image = self.app.conf['containers'][container]['image'] + if image not in self.packages['images']: + raise ImageNotFoundError(image) + try: + os.unlink(self.xz_path) + except FileNotFoundError: + pass + self.create_archive() + self.register() + self.sign_packages() + + def remove(self): + self.unregister() + try: + os.unlink(self.xz_path) + except FileNotFoundError: + pass + + def create_archive(self): + # Create archive with application setup scripts + print('Archiving setup scripts for', self.app.name) + scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh') + scripts = [s for s in scripts if os.path.exists(os.path.join(self.app.build_dir, s))] + subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, '--transform', 's,^,{}/,'.format(self.app.name)] + scripts, cwd=self.app.build_dir) + self.compress_archive() + + def register(self): + # Register package in global repository metadata file + print('Registering package {}'.format(self.app.name)) + self.packages['apps'][self.app.name] = self.app.conf.copy() + self.packages['apps'][self.app.name]['size'] = self.tar_size + self.packages['apps'][self.app.name]['pkgsize'] = self.xz_size + self.packages['apps'][self.app.name]['sha512'] = crypto.hash_file(self.xz_path) + self.save_repo_meta() + + def unregister(self): + # Removes package from global repository metadata file + if self.app.name in self.packages['apps']: + del self.packages['apps'][self.app.name] + self.save_repo_meta() diff --git a/build/usr/lib/python3.6/lxcbuild/image.py b/build/usr/lib/python3.6/lxcbuild/image.py index f77bff8..0bc3260 100644 --- a/build/usr/lib/python3.6/lxcbuild/image.py +++ b/build/usr/lib/python3.6/lxcbuild/image.py @@ -3,24 +3,30 @@ import os import sys -from .builder import Builder, ImageExistsError, ImageNotFoundError -from .packer import Packer, PackageExistsError +from lxcmgr import lxcmgr + +from .imagebuilder import ImageBuilder, ImageExistsError, ImageNotFoundError +from .imagepacker import ImagePacker +from .packer import PackageExistsError class Image: - def __init__(self, lxcfile): + def __init__(self): self.name = None - self.path = None self.conf = {} + self.lxcfile = None + self.build_dir = None + self.force_build = False + self.scratch_build = False + + def build_and_pack(self, lxcfile): self.lxcfile = lxcfile self.build_dir = os.path.dirname(lxcfile) - - def build_and_pack(self, force): self.conf['build'] = True try: - builder = Builder() - builder.build(self, force) + builder = ImageBuilder(self) + builder.build() # In case of successful build, packaging needs to happen in all cases to prevent outdated packages - force = True + self.force_build = True except ImageExistsError as e: print('Image {} already exists, skipping build tasks'.format(e)) except ImageNotFoundError as e: @@ -28,11 +34,22 @@ class Image: builder.clean() sys.exit(1) except: - builder.clean() + if not self.scratch_build: + builder.clean() raise del self.conf['build'] - try: - packer = Packer() - packer.pack_image(self, force) - except PackageExistsError as e: - print('Package {} already exists, skipping packaging tasks'.format(e)) + # If we're doing a scratch build, regenerate the final LXC container configuration including ephemeral layer + if self.scratch_build: + lxcmgr.create_container(self.name, self.conf) + else: + try: + packer = ImagePacker(self) + packer.pack() + except PackageExistsError as e: + print('Package {} already exists, skipping packaging tasks'.format(e)) + + def remove(self): + builder = ImageBuilder(self) + builder.clean() + packer = ImagePacker(self) + packer.remove() diff --git a/build/usr/lib/python3.6/lxcbuild/builder.py b/build/usr/lib/python3.6/lxcbuild/imagebuilder.py similarity index 88% rename from build/usr/lib/python3.6/lxcbuild/builder.py rename to build/usr/lib/python3.6/lxcbuild/imagebuilder.py index 3e20780..337a9c5 100644 --- a/build/usr/lib/python3.6/lxcbuild/builder.py +++ b/build/usr/lib/python3.6/lxcbuild/imagebuilder.py @@ -14,16 +14,13 @@ class ImageExistsError(Exception): class ImageNotFoundError(Exception): pass -class Builder: - def __init__(self): - self.image = None +class ImageBuilder: + def __init__(self, image): + self.image = image self.script = [] self.script_eof = None - self.force = False - def build(self, image, force=False): - self.image = image - self.force = force + def build(self): with open(self.image.lxcfile, 'r') as f: for line in f: line = line.strip() @@ -67,26 +64,27 @@ class Builder: def run_script(self, script): lxcmgr.create_container(self.image.name, self.image.conf) - sh = os.path.join(self.image.path, 'run.sh') + sh = os.path.join(LXC_STORAGE_DIR, self.image.name, 'run.sh') with open(sh, 'w') as f: f.write('#!/bin/sh\nset -ev\n\n{}\n'.format('\n'.join(script))) os.chmod(sh, 0o700) os.chown(sh, 100000, 100000) subprocess.run(['lxc-execute', self.image.name, '--', '/bin/sh', '-lc', '/run.sh'], check=True) os.unlink(sh) - lxcmgr.destroy_container(self.image.name) + if not self.image.scratch_build: + lxcmgr.destroy_container(self.image.name) def set_name(self, name): self.image.name = name - self.image.path = self.get_layer_path(name) self.image.conf['layers'] = [name] - if os.path.exists(self.image.path): - if self.force: + image_path = self.get_layer_path(name) + if os.path.exists(image_path): + if self.image.force_build: self.clean() else: - raise ImageExistsError(self.image.path) - os.makedirs(self.image.path, 0o755, True) - os.chown(self.image.path, 100000, 100000) + raise ImageExistsError(image_path) + os.makedirs(image_path, 0o755, True) + os.chown(image_path, 100000, 100000) def add_layer(self, name): layer_path = self.get_layer_path(name) @@ -99,7 +97,7 @@ class Builder: subprocess.run(cmd + layers, check=True) def copy_files(self, src, dst): - dst = os.path.join(self.image.path, dst) + dst = os.path.join(LXC_STORAGE_DIR, self.image.name, dst) if src.startswith('http://') or src.startswith('https://'): unpack_http_archive(src, dst) else: @@ -128,8 +126,8 @@ class Builder: self.image.conf['ready'] = cmd def clean(self): - shutil.rmtree(self.image.path) lxcmgr.destroy_container(self.image.name) + shutil.rmtree(self.get_layer_path(self.image.name)) def unpack_http_archive(src, dst): xf = 'xzf' diff --git a/build/usr/lib/python3.6/lxcbuild/imagepacker.py b/build/usr/lib/python3.6/lxcbuild/imagepacker.py new file mode 100644 index 0000000..9c3b869 --- /dev/null +++ b/build/usr/lib/python3.6/lxcbuild/imagepacker.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +import os +import subprocess + +from lxcmgr.paths import LXC_STORAGE_DIR +from lxcmgr.pkgmgr import PkgMgr + +from . import crypto +from .packer import Packer +from .paths import REPO_IMAGES_DIR + +class ImagePacker(Packer): + def __init__(self, image): + super().__init__() + self.image = image + # Prepare package file names + self.tar_path = os.path.join(REPO_IMAGES_DIR, '{}.tar'.format(self.image.name)) + self.xz_path = '{}.xz'.format(self.tar_path) + + def pack(self): + if self.image.force_build: + self.unregister() + try: + os.unlink(self.xz_path) + except FileNotFoundError: + pass + elif os.path.exists(self.xz_path): + raise PackageExistsError(self.xz_path) + self.create_archive() + self.register() + self.sign_packages() + + def remove(self): + self.unregister() + try: + os.unlink(self.xz_path) + except FileNotFoundError: + pass + + def create_archive(self): + # Create archive + print('Archiving', self.image.path) + subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, self.image.name], cwd=LXC_STORAGE_DIR) + self.compress_archive() + + def register(self): + # Register image in global repository metadata file + print('Registering package {}'.format(self.image.name)) + self.packages['images'][self.image.name] = self.image.conf.copy() + self.packages['images'][self.image.name]['size'] = self.tar_size + self.packages['images'][self.image.name]['pkgsize'] = self.xz_size + self.packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path) + self.save_repo_meta() + # Register the image also to locally installed images for package manager + pm = PkgMgr() + pm.register_image(self.image.name, self.packages['images'][self.image.name]) + + def unregister(self): + # Removes package from global repository metadata file + if self.image.name in self.packages['images']: + del self.packages['images'][self.image.name] + self.save_repo_meta() + # Unregister the image also from locally installed images for package manager + pm = PkgMgr() + pm.unregister_image(self.image.name) diff --git a/build/usr/lib/python3.6/lxcbuild/packer.py b/build/usr/lib/python3.6/lxcbuild/packer.py index 11aab79..0a55efd 100644 --- a/build/usr/lib/python3.6/lxcbuild/packer.py +++ b/build/usr/lib/python3.6/lxcbuild/packer.py @@ -3,22 +3,15 @@ import json import os import subprocess -import sys - -from lxcmgr.paths import LXC_STORAGE_DIR -from lxcmgr.pkgmgr import PkgMgr from . import crypto -from .builder import ImageNotFoundError -from .paths import PRIVATE_KEY, REPO_APPS_DIR, REPO_IMAGES_DIR, REPO_META_FILE, REPO_SIG_FILE +from .paths import PRIVATE_KEY, REPO_META_FILE, REPO_SIG_FILE class PackageExistsError(Exception): pass class Packer: def __init__(self): - self.app = None - self.image = None self.tar_path = None self.tar_size = 0 self.xz_path = None @@ -33,29 +26,6 @@ class Packer: with open(REPO_META_FILE, 'w') as f: json.dump(self.packages, f, sort_keys=True, indent=4) - def pack_image(self, image, force): - self.image = image - # Prepare package file names - self.tar_path = os.path.join(REPO_IMAGES_DIR, '{}.tar'.format(self.image.name)) - self.xz_path = '{}.xz'.format(self.tar_path) - if force: - self.unregister_image() - try: - os.unlink(self.xz_path) - except FileNotFoundError: - pass - elif os.path.exists(self.xz_path): - raise PackageExistsError(self.xz_path) - self.create_image_archive() - self.register_image() - self.sign_packages() - - def create_image_archive(self): - # Create archive - print('Archiving', self.image.path) - subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, self.image.name], cwd=LXC_STORAGE_DIR) - self.compress_archive() - def compress_archive(self): # Compress the tarball with xz (LZMA2) self.tar_size = os.path.getsize(self.tar_path) @@ -64,60 +34,7 @@ class Packer: self.xz_size = os.path.getsize(self.xz_path) print('Compressed ', self.xz_path, '({:.2f} MB)'.format(self.xz_size/1048576)) - def register_image(self): - # Register image in global repository metadata file - print('Registering package {}'.format(self.image.name)) - self.packages['images'][self.image.name] = self.image.conf.copy() - self.packages['images'][self.image.name]['size'] = self.tar_size - self.packages['images'][self.image.name]['pkgsize'] = self.xz_size - self.packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path) - self.save_repo_meta() - # Register the image also to locally installed images for package manager - pm = PkgMgr() - pm.register_image(self.image.name, self.packages['images'][self.image.name]) - def sign_packages(self): signature = crypto.sign_file(PRIVATE_KEY, REPO_META_FILE) with open(REPO_SIG_FILE, 'wb') as f: f.write(signature) - - def unregister_image(self): - # Removes package from global repository metadata file - if self.image.name in self.packages['images']: - del self.packages['images'][self.image.name] - self.save_repo_meta() - - def pack_app(self, app): - self.app = app - # Check if all images exist - for container in app.conf['containers']: - image = app.conf['containers'][container]['image'] - if image not in self.packages['images']: - raise ImageNotFoundError(image) - # Prepare package file names - self.tar_path = os.path.join(REPO_APPS_DIR, '{}.tar'.format(self.app.name)) - self.xz_path = '{}.xz'.format(self.tar_path) - try: - os.unlink(self.xz_path) - except FileNotFoundError: - pass - self.create_app_archive() - self.register_app() - self.sign_packages() - - def create_app_archive(self): - # Create archive with application setup scripts - print('Archiving setup scripts for', self.app.name) - scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh') - scripts = [s for s in scripts if os.path.exists(os.path.join(self.app.build_dir, s))] - subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, '--transform', 's,^,{}/,'.format(self.app.name)] + scripts, cwd=self.app.build_dir) - self.compress_archive() - - def register_app(self): - # Register package in global repository metadata file - print('Registering package {}'.format(self.app.name)) - self.packages['apps'][self.app.name] = self.app.conf.copy() - self.packages['apps'][self.app.name]['size'] = self.tar_size - self.packages['apps'][self.app.name]['pkgsize'] = self.xz_size - self.packages['apps'][self.app.name]['sha512'] = crypto.hash_file(self.xz_path) - self.save_repo_meta()