Remove MERGE capability, add FROM

layer inheritance is now linear

apk/vmmgr

@@ -1 +1 @@
-Subproject commit 41156fe4243b15b4b233b618082aae8ce32e5a2b
+Subproject commit 7c25d22d4146033cfb1e0775d06912b5c8f77e73

build/usr/bin/lxcmerge

@@ -1,174 +0,0 @@
-#!/usr/bin/python3
-# -*- coding: utf-8 -*-
-
-import argparse
-import os
-import shutil
-import sys
-import tarfile
-import tempfile
-
-APK_WORLD = 'etc/apk/world'
-APK_INSTALLED = 'lib/apk/db/installed'
-APK_SCRIPTS = 'lib/apk/db/scripts.tar'
-APK_TRIGGERS = 'lib/apk/db/triggers'
-
-ETC_PASSWD = 'etc/passwd'
-ETC_GROUP = 'etc/groups'
-ETC_SHADOW = 'etc/shadow'
-
-def makedirs(path, mode=0o755, uid=100000, gid=100000):
-    try:
-        os.mkdir(path, mode)
-        os.chown(path, uid, gid)
-    except FileNotFoundError:
-        makedirs(os.path.dirname(path), mode, uid, gid)
-        os.mkdir(path, mode)
-        os.chown(path, uid, gid)
-    except FileExistsError:
-        pass
-
-def merge_apk_world(layers):
-    world = []
-    for layer in layers:
-        try:
-            with open(os.path.join(layer, APK_WORLD), 'r') as f:
-                for line in f:
-                    if line not in world:
-                        world.append(line)
-        except:
-            continue
-    makedirs(os.path.join(layers[-1], os.path.dirname(APK_WORLD)))
-    with open(os.path.join(layers[-1], APK_WORLD), 'w') as f:
-        f.writelines(world)
-    os.chown(os.path.join(layers[-1], APK_WORLD), 100000, 100000)
-
-def merge_apk_installed(layers):
-    installed = []
-    for layer in layers:
-        try:
-            with open(os.path.join(layer, APK_INSTALLED), 'r') as f:
-                buffer = []
-                for line in f:
-                    if line.startswith('C:'):
-                        buffer = ''.join(buffer)
-                        if buffer not in installed:
-                            installed.append(buffer)
-                        buffer = []
-                    buffer.append(line)
-                buffer = ''.join(buffer)
-                if buffer not in installed:
-                    installed.append(buffer)
-        except:
-            continue
-    makedirs(os.path.join(layers[-1], os.path.dirname(APK_INSTALLED)))
-    with open(os.path.join(layers[-1], APK_INSTALLED), 'w') as f:
-        f.writelines(installed)
-    os.chown(os.path.join(layers[-1], APK_INSTALLED), 100000, 100000)
-
-def merge_apk_scripts(layers):
-    tmp_tar_path = tempfile.mkstemp()[1]
-    files_in_tar = []
-    with tarfile.open(tmp_tar_path, 'w:') as tmp_tar:
-        for layer in layers:
-            tar_path = os.path.join(layer, APK_SCRIPTS)
-            if os.path.exists(tar_path):
-                with tarfile.open(tar_path, 'r:') as tar:
-                    for member in tar.getmembers():
-                        if member.name not in files_in_tar:
-                            buffer = tar.extractfile(member)
-                            tmp_tar.addfile(member, buffer)
-                            files_in_tar.append(member.name)
-    if files_in_tar:
-        makedirs(os.path.join(layers[-1], os.path.dirname(APK_SCRIPTS)))
-        shutil.move(tmp_tar_path, os.path.join(layers[-1], APK_SCRIPTS))
-        os.chown(os.path.join(layers[-1], APK_SCRIPTS), 100000, 100000)
-    else:
-        os.unlink(tmp_tar_path)
-
-def merge_apk_triggers(layers):
-    triggers = []
-    for layer in layers:
-        try:
-            with open(os.path.join(layer, APK_TRIGGERS), 'r') as f:
-                for line in f:
-                    if line not in triggers:
-                        triggers.append(line)
-        except:
-            continue
-    makedirs(os.path.join(layers[-1], os.path.dirname(APK_TRIGGERS)))
-    with open(os.path.join(layers[-1], APK_TRIGGERS), 'w') as f:
-        f.writelines(triggers)
-    os.chown(os.path.join(layers[-1], APK_TRIGGERS), 100000, 100000)
-
-def merge_etc_passwd(layers):
-    passwd = {}
-    for layer in layers:
-        try:
-            with open(os.path.join(layer, ETC_PASSWD), 'r') as f:
-                for line in f:
-                    passwd[line.split(':')[0]] = line
-        except:
-            continue
-    makedirs(os.path.join(layers[-1], os.path.dirname(ETC_PASSWD)))
-    with open(os.path.join(layers[-1], ETC_PASSWD), 'w') as f:
-        f.writelines(passwd.values())
-    os.chown(os.path.join(layers[-1], ETC_PASSWD), 100000, 100000)
-
-def merge_etc_group(layers):
-    groups = {}
-    for layer in layers:
-        try:
-            with open(os.path.join(layer, ETC_GROUP), 'r') as f:
-                for line in f:
-                    name,pwd,gid,users = line.split(':')
-                    name = splitline[0]
-                    users = splitline[3].strip().split(',')
-                    if name not in groups:
-                        groups[name] = [name,pwd,gid,users]
-                    else:
-                        groups[name][1] = pwd
-                        groups[name][2] = gid
-                        for user in users:
-                            if user not in groups[name][3]:
-                                groups[name][3].append(user)
-        except:
-            continue
-    for group in groups.values():
-        group[3] = '{}\n'.format(','.join(group[3]))
-    makedirs(os.path.join(layers[-1], os.path.dirname(ETC_GROUP)))
-    with open(os.path.join(layers[-1], ETC_GROUP), 'w') as f:
-        f.writelines([':'.join(group) for group in groups.values()])
-    os.chown(os.path.join(layers[-1], ETC_GROUP), 100000, 100000)
-
-def merge_etc_shadow(layers):
-    shadow = {}
-    for layer in layers:
-        try:
-            with open(os.path.join(layer, ETC_SHADOW), 'r') as f:
-                for line in f:
-                    shadow[line.split(':')[0]] = line
-        except:
-            continue
-    makedirs(os.path.join(layers[-1], os.path.dirname(ETC_SHADOW)))
-    with open(os.path.join(layers[-1], ETC_SHADOW), 'w') as f:
-        f.writelines(shadow.values())
-    os.chown(os.path.join(layers[-1], ETC_SHADOW), 100000, 100042)
-
-
-parser = argparse.ArgumentParser(description='APK database merge script')
-parser.add_argument('layers', help='Path to LXC layers to be merged', nargs=argparse.REMAINDER)
-
-if len(sys.argv) < 3:
-    parser.print_usage()
-    sys.exit(1)
-args = parser.parse_args()
-
-merge_apk_world(args.layers)
-merge_apk_installed(args.layers)
-merge_apk_scripts(args.layers)
-merge_apk_triggers(args.layers)
-
-merge_etc_passwd(args.layers)
-merge_etc_group(args.layers)
-merge_etc_shadow(args.layers)

build/usr/lib/python3.6/lxcbuild/image.py

@@ -25,7 +25,7 @@ class Image:
         try:
             builder = ImageBuilder(self)
             builder.build()
-            # In case of successful build, packaging needs to happen in all cases to prevent outdated packages
+            # Packaging needs to happen in any case after a successful build in order to prevent outdated packages
             self.force_build = True
         except ImageExistsError as e:
             print('Image {} already exists, skipping build tasks'.format(e))

build/usr/lib/python3.6/lxcbuild/imagebuilder.py

@@ -7,6 +7,7 @@ import sys
 
 from lxcmgr import lxcmgr
 from lxcmgr.paths import LXC_STORAGE_DIR
+from lxcmgr.pkgmgr import PkgMgr
 
 class ImageExistsError(Exception):
     pass
@@ -21,6 +22,7 @@ class ImageBuilder:
         self.script_eof = None
 
     def build(self):
+        # Read and process lines from lxcfile
         with open(self.image.lxcfile, 'r') as f:
             for line in f:
                 line = line.strip()
@@ -34,15 +36,14 @@ class ImageBuilder:
                     self.process_line(*line.split(None, 1))
 
     def process_line(self, directive, args):
+        # Process directives from lxcfile
         if 'RUN' == directive:
             self.script = []
             self.script_eof = args
         elif 'IMAGE' == directive:
             self.set_name(args)
-        elif 'LAYER' == directive:
+        elif 'FROM' == directive:
-            self.add_layer(args)
+            self.set_layers(args)
-        elif 'MERGE' == directive:
-            self.merge_layers(args.split())
         elif 'COPY' == directive:
             srcdst = args.split()
             self.copy_files(srcdst[0], srcdst[1] if len(srcdst) == 2 else '')
@@ -63,6 +64,7 @@ class ImageBuilder:
         return os.path.join(LXC_STORAGE_DIR, layer)
 
     def run_script(self, script):
+        # Creates a temporary container, runs a script in its namespace, and stores the modifications as part of the image
         lxcmgr.create_container(self.image.name, self.image.conf)
         sh = os.path.join(LXC_STORAGE_DIR, self.image.name, 'run.sh')
         with open(sh, 'w') as f:
@@ -75,6 +77,7 @@ class ImageBuilder:
             lxcmgr.destroy_container(self.image.name)
 
     def set_name(self, name):
+        # Set name and first (topmost) layer of the image
         self.image.name = name
         self.image.conf['layers'] = [name]
         image_path = self.get_layer_path(name)
@@ -86,43 +89,47 @@ class ImageBuilder:
         os.makedirs(image_path, 0o755, True)
         os.chown(image_path, 100000, 100000)
 
-    def add_layer(self, name):
+    def set_layers(self, image):
-        layer_path = self.get_layer_path(name)
+        # Extend list of layers with the list of layers from parent image
-        if not os.path.exists(layer_path):
+        # Raies an exception when IMAGE has no name
-            raise ImageNotFoundError(layer_path)
+        pkgmgr = PkgMgr()
-        self.image.conf['layers'].insert(1, name)
+        self.image.conf['layers'].extend(pkgmgr.installed_packages[image]['layers'])
-
-    def merge_layers(self, cmd):
-        layers = [self.get_layer_path(layer) for layer in self.image.conf['layers']]
-        subprocess.run(cmd + layers[::-1], check=True)
 
     def copy_files(self, src, dst):
+        # Copy files from the host or download them from a http(s) URL
         dst = os.path.join(LXC_STORAGE_DIR, self.image.name, dst)
         if src.startswith('http://') or src.startswith('https://'):
             unpack_http_archive(src, dst)
         else:
             copy_tree(os.path.join(self.image.build_dir, src), dst)
+        # Shift UID/GID of the files to the unprivileged range
         shift_uid(dst)
 
     def add_env(self, key, value):
+        # Sets lxc.environment records for the image
         if 'env' not in self.image.conf:
             self.image.conf['env'] = []
         self.image.conf['env'].append([key, value])
 
     def set_user(self, uid, gid):
+        # Sets lxc.init.uid/gid for the image
         self.image.conf['uid'] = uid
         self.image.conf['gid'] = gid
 
     def set_cmd(self, cmd):
+        # Sets lxc.init.cmd for the image
         self.image.conf['cmd'] = cmd
 
     def set_cwd(self, cwd):
+        # Sets lxc.init.cwd for the image
         self.image.conf['cwd'] = cwd
 
     def set_halt(self, halt):
+        # Sets lxc.signal.halt for the image
         self.image.conf['halt'] = halt
 
     def set_ready(self, cmd):
+        # Sets a command performed in OpenRC start_post to check readiness of the container
         self.image.conf['ready'] = cmd
 
     def clean(self):
@@ -130,17 +137,19 @@ class ImageBuilder:
         shutil.rmtree(self.get_layer_path(self.image.name))
 
 def unpack_http_archive(src, dst):
-        xf = 'xzf'
+    # Decompress an archive downloaded via http(s)
-        if src.endswith('.bz2'):
+    xf = 'xzf'
-            xf = 'xjf'
+    if src.endswith('.bz2'):
-        elif src.endswith('.xz'):
+        xf = 'xjf'
-            xf = 'xJf'
+    elif src.endswith('.xz'):
-        with subprocess.Popen(['wget', src, '-O', '-'], stdout=subprocess.PIPE) as wget:
+        xf = 'xJf'
-            with subprocess.Popen(['tar', xf, '-', '-C', dst], stdin=wget.stdout) as tar:
+    with subprocess.Popen(['wget', src, '-O', '-'], stdout=subprocess.PIPE) as wget:
-                wget.stdout.close()
+        with subprocess.Popen(['tar', xf, '-', '-C', dst], stdin=wget.stdout) as tar:
-                tar.wait()
+            wget.stdout.close()
+            tar.wait()
 
 def copy_tree(src, dst):
+    # Copies files from the host
     if not os.path.isdir(src):
         shutil.copy2(src, dst)
     else:
@@ -150,16 +159,19 @@ def copy_tree(src, dst):
     shutil.copystat(src, dst)
 
 def shift_uid(dir):
+    # Shifts UID/GID of a file or a directory and its contents to the unprivileged range
     shift_uid_entry(dir, os.stat(dir, follow_symlinks=True))
     shift_uid_recursively(dir)
 
 def shift_uid_recursively(dir):
+    # Shifts UID/GID of a directory and its contents to the unprivileged range
     for entry in os.scandir(dir):
         shift_uid_entry(entry.path, entry.stat(follow_symlinks=False))
         if entry.is_dir():
             shift_uid_recursively(entry.path)
 
 def shift_uid_entry(path, stat):
+    # Shifts UID/GID of a file or a directory to the unprivileged range
     uid = stat.st_uid
     gid = stat.st_gid
     do_chown = False

build/usr/lib/python3.6/lxcbuild/imagepacker.py

@@ -47,10 +47,11 @@ class ImagePacker(Packer):
     def register(self):
         # Register image in global repository metadata file
         print('Registering image package', self.image.name)
-        self.packages['images'][self.image.name] = self.image.conf.copy()
+        image_conf = self.image.conf.copy()
-        self.packages['images'][self.image.name]['size'] = self.tar_size
+        image_conf['size'] = self.tar_size
-        self.packages['images'][self.image.name]['pkgsize'] = self.xz_size
+        image_conf['pkgsize'] = self.xz_size
-        self.packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path)
+        image_conf['sha512'] = crypto.hash_file(self.xz_path)
+        self.packages['images'][self.image.name] = image_conf
         self.save_repo_meta()
         # Register the image also to locally installed images for package manager
         pm = PkgMgr()

doc/toolchain/lxc-build.md

@@ -29,13 +29,6 @@ The *lxcfile* syntax is designed to resemble *Dockerfile* syntax in order to eas
 - **Docker equivalent:** `FROM`
 - **Populates LXC field:** `lxc.rootfs.path`
 
-### MERGE
-
-- **Usage:** `MERGE <scriptname>`
-- **Description:** Runs `<scriptname>` on LXC host and passes all layer paths as parameter to this script. This helps you to resolve the conflicts in cases where you mix multiple OverlayFS layers with overlapping files, ie. package manager cache. The idea is that all layers are read separately by the `<scriptname>` script and the fixed result is written back to the uppermost layer.
-- **Docker equivalent:** None
-- **Populates LXC field:** None
-
 ### RUN
 
 - **Usage:**