From 7116566519f5d05fb495dc80e302bfb2adda8a69 Mon Sep 17 00:00:00 2001 From: Disassembler Date: Fri, 20 Sep 2019 10:13:41 +0200 Subject: [PATCH] Allow lxcbuilder to pack meta files --- apk/vmmgr | 2 +- build/build-all.sh | 3 +- build/install-toolchain.sh | 4 +- build/usr/bin/lxc-build | 12 -- build/usr/bin/lxcbuild | 43 +++++++ build/usr/lib/python3.6/lxcbuild/app.py | 16 +++ .../lxcbuild/{lxcbuilder.py => builder.py} | 60 ++++++---- build/usr/lib/python3.6/lxcbuild/crypto.py | 28 +++++ build/usr/lib/python3.6/lxcbuild/image.py | 38 ++++++ build/usr/lib/python3.6/lxcbuild/lxcimage.py | 24 ---- build/usr/lib/python3.6/lxcbuild/lxcpacker.py | 83 ------------- build/usr/lib/python3.6/lxcbuild/packer.py | 109 ++++++++++++++++++ build/usr/lib/python3.6/lxcbuild/paths.py | 8 ++ 13 files changed, 283 insertions(+), 147 deletions(-) delete mode 100755 build/usr/bin/lxc-build create mode 100644 build/usr/bin/lxcbuild create mode 100644 build/usr/lib/python3.6/lxcbuild/app.py rename build/usr/lib/python3.6/lxcbuild/{lxcbuilder.py => builder.py} (78%) create mode 100644 build/usr/lib/python3.6/lxcbuild/crypto.py create mode 100644 build/usr/lib/python3.6/lxcbuild/image.py delete mode 100644 build/usr/lib/python3.6/lxcbuild/lxcimage.py delete mode 100644 build/usr/lib/python3.6/lxcbuild/lxcpacker.py create mode 100644 build/usr/lib/python3.6/lxcbuild/packer.py create mode 100644 build/usr/lib/python3.6/lxcbuild/paths.py diff --git a/apk/vmmgr b/apk/vmmgr index 972ca0b..c3b7118 160000 --- a/apk/vmmgr +++ b/apk/vmmgr @@ -1 +1 @@ -Subproject commit 972ca0b6967edd56af96a7de159950ac9fcbc4a6 +Subproject commit c3b711850e02a6e228c4eb64ed82a4d1bc889ae9 diff --git a/build/build-all.sh b/build/build-all.sh index 9662bb6..986de40 100755 --- a/build/build-all.sh +++ b/build/build-all.sh @@ -66,14 +66,15 @@ cd ${ROOT}/lxc-services lxc-build activemq lxc-build mariadb lxc-build postgres +lxc-build postgis lxc-build rabbitmq lxc-build redis lxc-build solr # Build applications cd ${ROOT}/lxc-apps -lxc-build ckan-datapusher lxc-build ckan +lxc-build ckan-datapusher lxc-build crisiscleanup lxc-build cts lxc-build ecogis diff --git a/build/install-toolchain.sh b/build/install-toolchain.sh index 35b1a22..1530203 100755 --- a/build/install-toolchain.sh +++ b/build/install-toolchain.sh @@ -24,7 +24,7 @@ cp etc/abuild.conf /etc/abuild.conf # Prepare LXC build toolchain cp usr/bin/fix-apk /usr/bin/fix-apk cp usr/bin/lxc-build /usr/bin/lxc-build -cp usr/bin/lxc-pack /usr/bin/lxc-pack +mkdir -p /srv/build/lxc/apps /srv/build/lxc/images # Prepare local APK repository cp etc/nginx/conf.d/apkrepo.conf /etc/nginx/conf.d/apkrepo.conf @@ -36,4 +36,4 @@ service nginx reload # Supply LXC build key # openssl ecparam -genkey -name secp384r1 -out /srv/build/packages.key -# openssl ec -in /srv/build/packages.key -pubout -out /srv/build/packages.pub +# openssl ec -in /srv/build/packages.key -pubout -out /srv/build/lxc/packages.pub diff --git a/build/usr/bin/lxc-build b/build/usr/bin/lxc-build deleted file mode 100755 index 263f6f0..0000000 --- a/build/usr/bin/lxc-build +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- - -import sys -from lxcbuild.lxcimage import LXCImage - -if __name__ == '__main__': - if len(sys.argv) != 2 or sys.argv[1] in ('-h', '--help'): - print('Usage: lxc-build \n where the buildpath can be either specific lxcfile or a directory containing one') - else: - image = LXCImage(sys.argv[1]) - image.build_and_pack() diff --git a/build/usr/bin/lxcbuild b/build/usr/bin/lxcbuild new file mode 100644 index 0000000..6da06f8 --- /dev/null +++ b/build/usr/bin/lxcbuild @@ -0,0 +1,43 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +import argparse +import sys +from lxcbuild.app import App +from lxcbuild.image import Image + +parser = argparse.ArgumentParser(description='VM application builder and packager') +parser.add_argument('-f', '--force', action='store_true', help='Force rebuild already built package') +parser.add_argument('buildpath', help='Either specific "lxcfile" or "meta" file or a directory containing one') + +if len(sys.argv) < 2: + parser.print_usage() + sys.exit(1) +args = parser.parse_args() + +buildpath = os.path.realpath(args.buildpath) +if os.path.isfile(buildpath): + basename = os.path.basename(buildpath) + if basename == 'lxcfile' or basename.endswith('.lxcfile'): + image = Image(buildpath) + image.build_and_pack(args.force) + elif basename == 'meta' or basename.endswith('.meta'): + app = App(buildpath) + app.build_and_pack() + else: + print('Unknown file {} given, expected "lxcfile" or "meta"'.format(buildpath)) + sys.exit(1) +else: + valid_dir = False + lxcfile = os.path.join(buildpath, 'lxcfile') + meta = os.path.join(buildpath, 'meta') + if os.path.exists(lxcfile): + valid_dir = True + image = Image(lxcfile) + image.build_and_pack(args.force) + if os.path.exists(meta): + valid_dir = True + app = App(buildpath) + app.pack() + if not valid_dir: + print('Directory {} doesn\'t contain anything to build, skipping'.format(buildpath)) diff --git a/build/usr/lib/python3.6/lxcbuild/app.py b/build/usr/lib/python3.6/lxcbuild/app.py new file mode 100644 index 0000000..d7ae2b4 --- /dev/null +++ b/build/usr/lib/python3.6/lxcbuild/app.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import os + +from .packer import Packer + +class App: + def __init__(self, metafile): + self.build_dir = os.path.dirname(metafile) + self.name = os.path.basename(self.build_dir) + with open(metafile, 'r') as f: + self.conf = json.load(f) + + def pack(self): + packer = Packer() + packer.pack_app(self) diff --git a/build/usr/lib/python3.6/lxcbuild/lxcbuilder.py b/build/usr/lib/python3.6/lxcbuild/builder.py similarity index 78% rename from build/usr/lib/python3.6/lxcbuild/lxcbuilder.py rename to build/usr/lib/python3.6/lxcbuild/builder.py index 8605ee0..652d38b 100644 --- a/build/usr/lib/python3.6/lxcbuild/lxcbuilder.py +++ b/build/usr/lib/python3.6/lxcbuild/builder.py @@ -4,25 +4,26 @@ import os import shutil import subprocess import sys -from vmmgr import lxcmgr -LXC_ROOT = '/var/lib/lxc' +from lxcmgr import lxcmgr +from lxcmgr.paths import PKG_STORAGE_DIR -class LXCBuilder: - def __init__(self, image): - self.image = image +class ImageExistsError(Exception): + pass + +class ImageNotFoundError(Exception): + pass + +class Builder: + def __init__(self): + self.image = None self.script = [] self.script_eof = None + self.force = False - def build(self): - try: - self.image.conf['build'] = True - self.process_file() - except FileExistsError as e: - print(e) - del self.image.conf['build'] - - def process_file(self): + def build(self, image, force=False): + self.image = image + self.force = force with open(self.image.lxcfile, 'r') as f: for line in f: line = line.strip() @@ -62,11 +63,11 @@ class LXCBuilder: self.set_ready(args) def get_layer_path(self, layer): - return os.path.join(LXC_ROOT, 'storage', layer) + return os.path.join(PKG_STORAGE_DIR, layer) def run_script(self, script): lxcmgr.register_container(self.image.name, self.image.conf) - sh = os.path.join(self.get_layer_path(self.image.name), 'run.sh') + sh = os.path.join(self.image.path, 'run.sh') with open(sh, 'w') as f: f.write('#!/bin/sh\nset -ev\n\n{}\n'.format('\n'.join(script))) os.chmod(sh, 0o700) @@ -77,12 +78,20 @@ class LXCBuilder: def set_name(self, name): self.image.name = name - self.image.conf['layers'] = [self.image.name] - image_path = self.get_layer_path(self.image.name) - os.makedirs(image_path, 0o755, True) - os.chown(image_path, 100000, 100000) + self.image.path = self.get_layer_path(name) + self.image.conf['layers'] = [name] + if os.path.exists(self.image.path): + if self.force: + self.clean() + else: + raise ImageExistsError(self.image.path) + os.makedirs(self.image.path, 0o755, True) + os.chown(self.image.path, 100000, 100000) def add_layer(self, name): + layer_path = self.get_layer_path(name) + if not os.path.exists(layer_path): + raise ImageNotFoundError(layer_path) self.image.conf['layers'].insert(0, name) def fix_layer(self, cmd): @@ -90,17 +99,17 @@ class LXCBuilder: subprocess.run([cmd] + layers, check=True) def copy_files(self, src, dst): - dst = os.path.join(self.get_layer_path(self.image.name), dst) + dst = os.path.join(self.image.path, dst) if src.startswith('http://') or src.startswith('https://'): unpack_http_archive(src, dst) else: - copy_tree(os.path.join(self.build_dir, src), dst) + copy_tree(os.path.join(self.image.build_dir, src), dst) shift_uid(dst) - def add_env(self, args): + def add_env(self, key, value): if 'env' not in self.image.conf: self.image.conf['env'] = [] - self.image.conf['env'].append(args) + self.image.conf['env'].append('{}={}'.format(key, value)) def set_user(self, uid, gid): self.image.conf['uid'] = uid @@ -118,6 +127,9 @@ class LXCBuilder: def set_ready(self, cmd): self.image.conf['ready'] = cmd + def clean(self): + shutil.rmtree(self.image.path) + def unpack_http_archive(src, dst): xf = 'xzf' if src.endswith('.bz2'): diff --git a/build/usr/lib/python3.6/lxcbuild/crypto.py b/build/usr/lib/python3.6/lxcbuild/crypto.py new file mode 100644 index 0000000..fa94bde --- /dev/null +++ b/build/usr/lib/python3.6/lxcbuild/crypto.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +import hashlib + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.serialization import load_pem_private_key + +def sign_file(private_key, input_path): + # Generate SHA512 signature of a file using EC private key + print('Signing packages') + with open(private_key, 'rb') as f: + priv_key = load_pem_private_key(f.read(), None, default_backend()) + with open(input_path, 'rb') as f: + data = f.read() + return priv_key.sign(data, ec.ECDSA(hashes.SHA512())) + +def hash_file(file_path): + # Calculate SHA512 hash of a file + sha512 = hashlib.sha512() + with open(file_path, 'rb') as f: + while True: + data = f.read(65536) + if not data: + break + sha512.update(data) + return sha512.hexdigest() diff --git a/build/usr/lib/python3.6/lxcbuild/image.py b/build/usr/lib/python3.6/lxcbuild/image.py new file mode 100644 index 0000000..d026843 --- /dev/null +++ b/build/usr/lib/python3.6/lxcbuild/image.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +import os +import sys + +from .builder import Builder, ImageExistsError, ImageNotFoundError +from .packer import Packer, PackageExistsError + +class Image: + def __init__(self, lxcfile): + self.name = None + self.path = None + self.conf = {} + self.lxcfile = lxcfile + self.build_dir = os.path.dirname(lxcfile) + + def build_and_pack(self, force=False): + self.conf['build'] = True + try: + builder = Builder() + builder.build(self, force) + # In case of successful build, packaging needs to be forced to prevent outdated packages + force = True + except ImageExistsError as e: + print('Image {} already exists, skipping build tasks'.format(e)) + except ImageNotFoundError as e: + print('Image {} not found, can\'t build {}'.format(e, self.name)) + builder.clean() + sys.exit(1) + except: + builder.clean() + raise + try: + packer = Packer() + packer.pack_image(self, force) + except PackageExistsError as e: + print('Package {} already exists, skipping packaging tasks'.format(e)) + del self.conf['build'] diff --git a/build/usr/lib/python3.6/lxcbuild/lxcimage.py b/build/usr/lib/python3.6/lxcbuild/lxcimage.py deleted file mode 100644 index a5515e3..0000000 --- a/build/usr/lib/python3.6/lxcbuild/lxcimage.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- - -import os - -from .lxcbuilder import LXCBuilder -from .lxcpacker import LXCPacker - -class LXCImage: - def __init__(self, build_path): - self.name = None - self.conf = {} - - if os.path.isfile(build_path): - self.lxcfile = os.path.realpath(build_path) - self.build_dir = os.path.dirname(self.lxcfile) - else: - self.build_dir = os.path.realpath(build_path) - self.lxcfile = os.path.join(self.build_dir, 'lxcfile') - - def build_and_pack(self): - builder = LXCBuilder(self) - builder.build() - packer = LXCPacker(self) - packer.pack() diff --git a/build/usr/lib/python3.6/lxcbuild/lxcpacker.py b/build/usr/lib/python3.6/lxcbuild/lxcpacker.py deleted file mode 100644 index 8010460..0000000 --- a/build/usr/lib/python3.6/lxcbuild/lxcpacker.py +++ /dev/null @@ -1,83 +0,0 @@ -# -*- coding: utf-8 -*- - -import hashlib -import json -import os -import subprocess -import sys - -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.asymmetric import ec -from cryptography.hazmat.primitives.serialization import load_pem_private_key - -PKG_ROOT = '/srv/build/lxc' -PRIVATE_KEY = '/srv/build/packages.key' -LXC_STORAGE = '/var/lib/lxc/storage' - -class LXCPacker: - def __init__(self, image): - self.image = image - self.tar_path = None - self.xz_path = None - - def pack(self): - # Prepare package file names - self.tar_path = os.path.join(PKG_ROOT, '{}.tar'.format(self.image.name)) - self.xz_path = '{}.xz'.format(self.tar_path) - if os.path.exists(self.xz_path): - print('Package {} already exists, skipping packaging tasks'.format(self.xz_path)) - return - os.makedirs(PKG_ROOT, 0o755, True) - self.create_archive() - self.register_package() - self.sign_packages() - - def create_archive(self): - # Create archive - print('Archiving', self.image.name) - subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, os.path.join(LXC_STORAGE, self.image.name)], cwd='/') - # Add install/upgrade/uninstall scripts - # TODO: skripty balit jen s aplikacemi, ne s imagi - scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh') - scripts = [s for s in scripts if os.path.exists(os.path.join(self.image.build_dir, s))] - subprocess.run(['tar', '--transform', 's|^|srv/{}/|'.format(self.image.name), '-rpf', self.tar_path] + scripts, cwd=self.image.build_dir) - # Compress the tarball with xz (LZMA2) - print('Compressing', self.tar_path, '({:.2f} MB)'.format(os.path.getsize(self.tar_path)/1048576)) - subprocess.run(['xz', '-9', self.tar_path]) - print('Compressed ', self.xz_path, '({:.2f} MB)'.format(os.path.getsize(self.xz_path)/1048576)) - - def register_package(self): - # Register package - print('Registering package') - packages_file = os.path.join(PKG_ROOT, 'packages') - if os.path.exists(packages_file): - with open(packages_file, 'r') as f: - packages = json.load(f) - else: - packages = {'apps': {}, 'images': {}} - packages['images'][self.image.name] = self.image.conf.copy() - packages['images'][self.image.name]['size'] = os.path.getsize(self.xz_path) - packages['images'][self.image.name]['sha512'] = hash_file(self.xz_path) - with open(packages_file, 'w') as f: - json.dump(packages, f, sort_keys=True, indent=4) - - def sign_packages(self): - # Sign packages file - print('Signing packages') - with open(PRIVATE_KEY, 'rb') as f: - priv_key = load_pem_private_key(f.read(), None, default_backend()) - with open(os.path.join(PKG_ROOT, 'packages'), 'rb') as f: - data = f.read() - with open(os.path.join(PKG_ROOT, 'packages.sig'), 'wb') as f: - f.write(priv_key.sign(data, ec.ECDSA(hashes.SHA512()))) - -def hash_file(file_path): - sha512 = hashlib.sha512() - with open(file_path, 'rb') as f: - while True: - data = f.read(65536) - if not data: - break - sha512.update(data) - return sha512.hexdigest() diff --git a/build/usr/lib/python3.6/lxcbuild/packer.py b/build/usr/lib/python3.6/lxcbuild/packer.py new file mode 100644 index 0000000..737afe4 --- /dev/null +++ b/build/usr/lib/python3.6/lxcbuild/packer.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- + +import json +import os +import subprocess +import sys + +from lxcmgr.paths import LXC_STORAGE_DIR + +from . import crypto +from .paths import APP_DIR, IMAGE_DIR, META_FILE, PRIVATE_KEY, ROOT_DIR, SIGNATURE_FILE + +class PackageExistsError(Exception): + pass + +class Packer: + def __init__(self): + self.app = None + self.image = None + self.tar_path = None + self.xz_path = None + + def load_packages_meta(self): + if os.path.exists(PKG_META): + with open(PKG_META, 'r') as f: + return json.load(f) + else: + return {'apps': {}, 'images': {}} + + def save_packages_meta(self, packages): + with open(PKG_META, 'w') as f: + json.dump(packages, f, sort_keys=True, indent=4) + + + def pack_image(self, image, force): + self.image = image + # Prepare package file names + self.tar_path = os.path.join(IMAGE_DIR, '{}.tar'.format(self.image.name)) + self.xz_path = '{}.xz'.format(self.tar_path) + if os.path.exists(self.xz_path): + if force: + self.unregister_image() + os.unlink(self.xz_path) + else: + raise PackageExistsError(self.xz_path) + self.create_image_archive() + self.register_image() + self.sign_packages() + + def create_image_archive(self): + # Create archive + print('Archiving', self.image.path) + subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, self.image.name], cwd=LXC_STORAGE_DIR) + self.compress_archive() + + def compress_archive(self): + # Compress the tarball with xz (LZMA2) + print('Compressing', self.tar_path, '({:.2f} MB)'.format(os.path.getsize(self.tar_path)/1048576)) + subprocess.run(['xz', '-9', self.tar_path]) + print('Compressed ', self.xz_path, '({:.2f} MB)'.format(os.path.getsize(self.xz_path)/1048576)) + + def register_image(self): + # Register package in global repository metadata file + print('Registering package {}'.format(self.image.name)) + packages = self.load_packages_meta() + packages['images'][self.image.name] = self.image.conf.copy() + packages['images'][self.image.name]['size'] = os.path.getsize(self.xz_path) + packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path) + self.save_packages_meta(packages) + + def sign_packages(self): + signature = crypto.sign_file(PRIVATE_KEY, META_FILE) + with open(SIGNATURE_FILE, 'wb') as f: + f.write(signature) + + def unregister_image(self): + # Removes package from global repository metadata file + packages = self.load_packages_meta() + if self.image.name in packages['images']: + del packages['images'][self.image.name] + self.save_packages_meta(packages) + + def pack_app(self, app): + self.app = app + # Prepare package file names + self.tar_path = os.path.join(APP_DIR, '{}.tar'.format(self.image.name)) + self.xz_path = '{}.xz'.format(self.tar_path) + if os.path.exists(self.xz_path): + os.unlink(self.xz_path) + self.create_app_archive() + self.register_app() + self.sign_packages() + + def create_app_archive(self): + # Create archive with application setup scripts + print('Archiving setup scripts for', self.app.name) + scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh') + scripts = [s for s in scripts if os.path.exists(os.path.join(self.app.build_dir, s))] + subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path] + scripts, cwd=self.app.build_dir) + self.compress_archive() + + def register_app(self): + # Register package in global repository metadata file + print('Registering package {}'.format(self.app.name)) + packages = self.load_packages_meta() + packages['apps'][self.image.name] = self.app.conf.copy() + packages['apps'][self.image.name]['size'] = os.path.getsize(self.xz_path) + packages['apps'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path) + self.save_packages_meta(packages) diff --git a/build/usr/lib/python3.6/lxcbuild/paths.py b/build/usr/lib/python3.6/lxcbuild/paths.py new file mode 100644 index 0000000..bc1b8f9 --- /dev/null +++ b/build/usr/lib/python3.6/lxcbuild/paths.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +ROOT_DIR = '/srv/build/lxc' +IMAGE_DIR = os.path.join(ROOT_DIR, 'images') +APP_DIR = os.path.join(ROOT_DIR, 'apps') +META_FILE = os.path.join(ROOT_DIR, 'packages') +SIGNATURE_FILE = os.path.join(ROOT_DIR, 'packages.sig') +PRIVATE_KEY = '/srv/build/packages.key'