From f8403c5f42d6f65c3d42defb7b382ed7018af87e Mon Sep 17 00:00:00 2001 From: Disassembler Date: Sat, 30 Nov 2019 09:59:11 +0100 Subject: [PATCH] Remove MERGE capability, add FROM layer inheritance is now linear --- apk/vmmgr | 2 +- build/usr/bin/lxcmerge | 174 ------------------ build/usr/lib/python3.6/lxcbuild/image.py | 2 +- .../lib/python3.6/lxcbuild/imagebuilder.py | 56 +++--- .../usr/lib/python3.6/lxcbuild/imagepacker.py | 9 +- doc/toolchain/lxc-build.md | 7 - 6 files changed, 41 insertions(+), 209 deletions(-) delete mode 100755 build/usr/bin/lxcmerge diff --git a/apk/vmmgr b/apk/vmmgr index 41156fe..7c25d22 160000 --- a/apk/vmmgr +++ b/apk/vmmgr @@ -1 +1 @@ -Subproject commit 41156fe4243b15b4b233b618082aae8ce32e5a2b +Subproject commit 7c25d22d4146033cfb1e0775d06912b5c8f77e73 diff --git a/build/usr/bin/lxcmerge b/build/usr/bin/lxcmerge deleted file mode 100755 index e3bdb29..0000000 --- a/build/usr/bin/lxcmerge +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- - -import argparse -import os -import shutil -import sys -import tarfile -import tempfile - -APK_WORLD = 'etc/apk/world' -APK_INSTALLED = 'lib/apk/db/installed' -APK_SCRIPTS = 'lib/apk/db/scripts.tar' -APK_TRIGGERS = 'lib/apk/db/triggers' - -ETC_PASSWD = 'etc/passwd' -ETC_GROUP = 'etc/groups' -ETC_SHADOW = 'etc/shadow' - -def makedirs(path, mode=0o755, uid=100000, gid=100000): - try: - os.mkdir(path, mode) - os.chown(path, uid, gid) - except FileNotFoundError: - makedirs(os.path.dirname(path), mode, uid, gid) - os.mkdir(path, mode) - os.chown(path, uid, gid) - except FileExistsError: - pass - -def merge_apk_world(layers): - world = [] - for layer in layers: - try: - with open(os.path.join(layer, APK_WORLD), 'r') as f: - for line in f: - if line not in world: - world.append(line) - except: - continue - makedirs(os.path.join(layers[-1], os.path.dirname(APK_WORLD))) - with open(os.path.join(layers[-1], APK_WORLD), 'w') as f: - f.writelines(world) - os.chown(os.path.join(layers[-1], APK_WORLD), 100000, 100000) - -def merge_apk_installed(layers): - installed = [] - for layer in layers: - try: - with open(os.path.join(layer, APK_INSTALLED), 'r') as f: - buffer = [] - for line in f: - if line.startswith('C:'): - buffer = ''.join(buffer) - if buffer not in installed: - installed.append(buffer) - buffer = [] - buffer.append(line) - buffer = ''.join(buffer) - if buffer not in installed: - installed.append(buffer) - except: - continue - makedirs(os.path.join(layers[-1], os.path.dirname(APK_INSTALLED))) - with open(os.path.join(layers[-1], APK_INSTALLED), 'w') as f: - f.writelines(installed) - os.chown(os.path.join(layers[-1], APK_INSTALLED), 100000, 100000) - -def merge_apk_scripts(layers): - tmp_tar_path = tempfile.mkstemp()[1] - files_in_tar = [] - with tarfile.open(tmp_tar_path, 'w:') as tmp_tar: - for layer in layers: - tar_path = os.path.join(layer, APK_SCRIPTS) - if os.path.exists(tar_path): - with tarfile.open(tar_path, 'r:') as tar: - for member in tar.getmembers(): - if member.name not in files_in_tar: - buffer = tar.extractfile(member) - tmp_tar.addfile(member, buffer) - files_in_tar.append(member.name) - if files_in_tar: - makedirs(os.path.join(layers[-1], os.path.dirname(APK_SCRIPTS))) - shutil.move(tmp_tar_path, os.path.join(layers[-1], APK_SCRIPTS)) - os.chown(os.path.join(layers[-1], APK_SCRIPTS), 100000, 100000) - else: - os.unlink(tmp_tar_path) - -def merge_apk_triggers(layers): - triggers = [] - for layer in layers: - try: - with open(os.path.join(layer, APK_TRIGGERS), 'r') as f: - for line in f: - if line not in triggers: - triggers.append(line) - except: - continue - makedirs(os.path.join(layers[-1], os.path.dirname(APK_TRIGGERS))) - with open(os.path.join(layers[-1], APK_TRIGGERS), 'w') as f: - f.writelines(triggers) - os.chown(os.path.join(layers[-1], APK_TRIGGERS), 100000, 100000) - -def merge_etc_passwd(layers): - passwd = {} - for layer in layers: - try: - with open(os.path.join(layer, ETC_PASSWD), 'r') as f: - for line in f: - passwd[line.split(':')[0]] = line - except: - continue - makedirs(os.path.join(layers[-1], os.path.dirname(ETC_PASSWD))) - with open(os.path.join(layers[-1], ETC_PASSWD), 'w') as f: - f.writelines(passwd.values()) - os.chown(os.path.join(layers[-1], ETC_PASSWD), 100000, 100000) - -def merge_etc_group(layers): - groups = {} - for layer in layers: - try: - with open(os.path.join(layer, ETC_GROUP), 'r') as f: - for line in f: - name,pwd,gid,users = line.split(':') - name = splitline[0] - users = splitline[3].strip().split(',') - if name not in groups: - groups[name] = [name,pwd,gid,users] - else: - groups[name][1] = pwd - groups[name][2] = gid - for user in users: - if user not in groups[name][3]: - groups[name][3].append(user) - except: - continue - for group in groups.values(): - group[3] = '{}\n'.format(','.join(group[3])) - makedirs(os.path.join(layers[-1], os.path.dirname(ETC_GROUP))) - with open(os.path.join(layers[-1], ETC_GROUP), 'w') as f: - f.writelines([':'.join(group) for group in groups.values()]) - os.chown(os.path.join(layers[-1], ETC_GROUP), 100000, 100000) - -def merge_etc_shadow(layers): - shadow = {} - for layer in layers: - try: - with open(os.path.join(layer, ETC_SHADOW), 'r') as f: - for line in f: - shadow[line.split(':')[0]] = line - except: - continue - makedirs(os.path.join(layers[-1], os.path.dirname(ETC_SHADOW))) - with open(os.path.join(layers[-1], ETC_SHADOW), 'w') as f: - f.writelines(shadow.values()) - os.chown(os.path.join(layers[-1], ETC_SHADOW), 100000, 100042) - - -parser = argparse.ArgumentParser(description='APK database merge script') -parser.add_argument('layers', help='Path to LXC layers to be merged', nargs=argparse.REMAINDER) - -if len(sys.argv) < 3: - parser.print_usage() - sys.exit(1) -args = parser.parse_args() - -merge_apk_world(args.layers) -merge_apk_installed(args.layers) -merge_apk_scripts(args.layers) -merge_apk_triggers(args.layers) - -merge_etc_passwd(args.layers) -merge_etc_group(args.layers) -merge_etc_shadow(args.layers) diff --git a/build/usr/lib/python3.6/lxcbuild/image.py b/build/usr/lib/python3.6/lxcbuild/image.py index 0bc3260..838914a 100644 --- a/build/usr/lib/python3.6/lxcbuild/image.py +++ b/build/usr/lib/python3.6/lxcbuild/image.py @@ -25,7 +25,7 @@ class Image: try: builder = ImageBuilder(self) builder.build() - # In case of successful build, packaging needs to happen in all cases to prevent outdated packages + # Packaging needs to happen in any case after a successful build in order to prevent outdated packages self.force_build = True except ImageExistsError as e: print('Image {} already exists, skipping build tasks'.format(e)) diff --git a/build/usr/lib/python3.6/lxcbuild/imagebuilder.py b/build/usr/lib/python3.6/lxcbuild/imagebuilder.py index e77c782..b2a20e0 100644 --- a/build/usr/lib/python3.6/lxcbuild/imagebuilder.py +++ b/build/usr/lib/python3.6/lxcbuild/imagebuilder.py @@ -7,6 +7,7 @@ import sys from lxcmgr import lxcmgr from lxcmgr.paths import LXC_STORAGE_DIR +from lxcmgr.pkgmgr import PkgMgr class ImageExistsError(Exception): pass @@ -21,6 +22,7 @@ class ImageBuilder: self.script_eof = None def build(self): + # Read and process lines from lxcfile with open(self.image.lxcfile, 'r') as f: for line in f: line = line.strip() @@ -34,15 +36,14 @@ class ImageBuilder: self.process_line(*line.split(None, 1)) def process_line(self, directive, args): + # Process directives from lxcfile if 'RUN' == directive: self.script = [] self.script_eof = args elif 'IMAGE' == directive: self.set_name(args) - elif 'LAYER' == directive: - self.add_layer(args) - elif 'MERGE' == directive: - self.merge_layers(args.split()) + elif 'FROM' == directive: + self.set_layers(args) elif 'COPY' == directive: srcdst = args.split() self.copy_files(srcdst[0], srcdst[1] if len(srcdst) == 2 else '') @@ -63,6 +64,7 @@ class ImageBuilder: return os.path.join(LXC_STORAGE_DIR, layer) def run_script(self, script): + # Creates a temporary container, runs a script in its namespace, and stores the modifications as part of the image lxcmgr.create_container(self.image.name, self.image.conf) sh = os.path.join(LXC_STORAGE_DIR, self.image.name, 'run.sh') with open(sh, 'w') as f: @@ -75,6 +77,7 @@ class ImageBuilder: lxcmgr.destroy_container(self.image.name) def set_name(self, name): + # Set name and first (topmost) layer of the image self.image.name = name self.image.conf['layers'] = [name] image_path = self.get_layer_path(name) @@ -86,43 +89,47 @@ class ImageBuilder: os.makedirs(image_path, 0o755, True) os.chown(image_path, 100000, 100000) - def add_layer(self, name): - layer_path = self.get_layer_path(name) - if not os.path.exists(layer_path): - raise ImageNotFoundError(layer_path) - self.image.conf['layers'].insert(1, name) - - def merge_layers(self, cmd): - layers = [self.get_layer_path(layer) for layer in self.image.conf['layers']] - subprocess.run(cmd + layers[::-1], check=True) + def set_layers(self, image): + # Extend list of layers with the list of layers from parent image + # Raies an exception when IMAGE has no name + pkgmgr = PkgMgr() + self.image.conf['layers'].extend(pkgmgr.installed_packages[image]['layers']) def copy_files(self, src, dst): + # Copy files from the host or download them from a http(s) URL dst = os.path.join(LXC_STORAGE_DIR, self.image.name, dst) if src.startswith('http://') or src.startswith('https://'): unpack_http_archive(src, dst) else: copy_tree(os.path.join(self.image.build_dir, src), dst) + # Shift UID/GID of the files to the unprivileged range shift_uid(dst) def add_env(self, key, value): + # Sets lxc.environment records for the image if 'env' not in self.image.conf: self.image.conf['env'] = [] self.image.conf['env'].append([key, value]) def set_user(self, uid, gid): + # Sets lxc.init.uid/gid for the image self.image.conf['uid'] = uid self.image.conf['gid'] = gid def set_cmd(self, cmd): + # Sets lxc.init.cmd for the image self.image.conf['cmd'] = cmd def set_cwd(self, cwd): + # Sets lxc.init.cwd for the image self.image.conf['cwd'] = cwd def set_halt(self, halt): + # Sets lxc.signal.halt for the image self.image.conf['halt'] = halt def set_ready(self, cmd): + # Sets a command performed in OpenRC start_post to check readiness of the container self.image.conf['ready'] = cmd def clean(self): @@ -130,17 +137,19 @@ class ImageBuilder: shutil.rmtree(self.get_layer_path(self.image.name)) def unpack_http_archive(src, dst): - xf = 'xzf' - if src.endswith('.bz2'): - xf = 'xjf' - elif src.endswith('.xz'): - xf = 'xJf' - with subprocess.Popen(['wget', src, '-O', '-'], stdout=subprocess.PIPE) as wget: - with subprocess.Popen(['tar', xf, '-', '-C', dst], stdin=wget.stdout) as tar: - wget.stdout.close() - tar.wait() + # Decompress an archive downloaded via http(s) + xf = 'xzf' + if src.endswith('.bz2'): + xf = 'xjf' + elif src.endswith('.xz'): + xf = 'xJf' + with subprocess.Popen(['wget', src, '-O', '-'], stdout=subprocess.PIPE) as wget: + with subprocess.Popen(['tar', xf, '-', '-C', dst], stdin=wget.stdout) as tar: + wget.stdout.close() + tar.wait() def copy_tree(src, dst): + # Copies files from the host if not os.path.isdir(src): shutil.copy2(src, dst) else: @@ -150,16 +159,19 @@ def copy_tree(src, dst): shutil.copystat(src, dst) def shift_uid(dir): + # Shifts UID/GID of a file or a directory and its contents to the unprivileged range shift_uid_entry(dir, os.stat(dir, follow_symlinks=True)) shift_uid_recursively(dir) def shift_uid_recursively(dir): + # Shifts UID/GID of a directory and its contents to the unprivileged range for entry in os.scandir(dir): shift_uid_entry(entry.path, entry.stat(follow_symlinks=False)) if entry.is_dir(): shift_uid_recursively(entry.path) def shift_uid_entry(path, stat): + # Shifts UID/GID of a file or a directory to the unprivileged range uid = stat.st_uid gid = stat.st_gid do_chown = False diff --git a/build/usr/lib/python3.6/lxcbuild/imagepacker.py b/build/usr/lib/python3.6/lxcbuild/imagepacker.py index f423dfa..072c707 100644 --- a/build/usr/lib/python3.6/lxcbuild/imagepacker.py +++ b/build/usr/lib/python3.6/lxcbuild/imagepacker.py @@ -47,10 +47,11 @@ class ImagePacker(Packer): def register(self): # Register image in global repository metadata file print('Registering image package', self.image.name) - self.packages['images'][self.image.name] = self.image.conf.copy() - self.packages['images'][self.image.name]['size'] = self.tar_size - self.packages['images'][self.image.name]['pkgsize'] = self.xz_size - self.packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path) + image_conf = self.image.conf.copy() + image_conf['size'] = self.tar_size + image_conf['pkgsize'] = self.xz_size + image_conf['sha512'] = crypto.hash_file(self.xz_path) + self.packages['images'][self.image.name] = image_conf self.save_repo_meta() # Register the image also to locally installed images for package manager pm = PkgMgr() diff --git a/doc/toolchain/lxc-build.md b/doc/toolchain/lxc-build.md index 37b64c7..29d0172 100644 --- a/doc/toolchain/lxc-build.md +++ b/doc/toolchain/lxc-build.md @@ -29,13 +29,6 @@ The *lxcfile* syntax is designed to resemble *Dockerfile* syntax in order to eas - **Docker equivalent:** `FROM` - **Populates LXC field:** `lxc.rootfs.path` -### MERGE - -- **Usage:** `MERGE ` -- **Description:** Runs `` on LXC host and passes all layer paths as parameter to this script. This helps you to resolve the conflicts in cases where you mix multiple OverlayFS layers with overlapping files, ie. package manager cache. The idea is that all layers are read separately by the `` script and the fixed result is written back to the uppermost layer. -- **Docker equivalent:** None -- **Populates LXC field:** None - ### RUN - **Usage:**