Remove MERGE capability, add FROM

layer inheritance is now linear
This commit is contained in:
Disassembler 2019-11-30 09:59:11 +01:00
parent 4d579ef8c1
commit f8403c5f42
Signed by: Disassembler
GPG Key ID: 524BD33A0EE29499
6 changed files with 41 additions and 209 deletions

@ -1 +1 @@
Subproject commit 41156fe4243b15b4b233b618082aae8ce32e5a2b Subproject commit 7c25d22d4146033cfb1e0775d06912b5c8f77e73

View File

@ -1,174 +0,0 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import argparse
import os
import shutil
import sys
import tarfile
import tempfile
APK_WORLD = 'etc/apk/world'
APK_INSTALLED = 'lib/apk/db/installed'
APK_SCRIPTS = 'lib/apk/db/scripts.tar'
APK_TRIGGERS = 'lib/apk/db/triggers'
ETC_PASSWD = 'etc/passwd'
ETC_GROUP = 'etc/groups'
ETC_SHADOW = 'etc/shadow'
def makedirs(path, mode=0o755, uid=100000, gid=100000):
try:
os.mkdir(path, mode)
os.chown(path, uid, gid)
except FileNotFoundError:
makedirs(os.path.dirname(path), mode, uid, gid)
os.mkdir(path, mode)
os.chown(path, uid, gid)
except FileExistsError:
pass
def merge_apk_world(layers):
world = []
for layer in layers:
try:
with open(os.path.join(layer, APK_WORLD), 'r') as f:
for line in f:
if line not in world:
world.append(line)
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(APK_WORLD)))
with open(os.path.join(layers[-1], APK_WORLD), 'w') as f:
f.writelines(world)
os.chown(os.path.join(layers[-1], APK_WORLD), 100000, 100000)
def merge_apk_installed(layers):
installed = []
for layer in layers:
try:
with open(os.path.join(layer, APK_INSTALLED), 'r') as f:
buffer = []
for line in f:
if line.startswith('C:'):
buffer = ''.join(buffer)
if buffer not in installed:
installed.append(buffer)
buffer = []
buffer.append(line)
buffer = ''.join(buffer)
if buffer not in installed:
installed.append(buffer)
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(APK_INSTALLED)))
with open(os.path.join(layers[-1], APK_INSTALLED), 'w') as f:
f.writelines(installed)
os.chown(os.path.join(layers[-1], APK_INSTALLED), 100000, 100000)
def merge_apk_scripts(layers):
tmp_tar_path = tempfile.mkstemp()[1]
files_in_tar = []
with tarfile.open(tmp_tar_path, 'w:') as tmp_tar:
for layer in layers:
tar_path = os.path.join(layer, APK_SCRIPTS)
if os.path.exists(tar_path):
with tarfile.open(tar_path, 'r:') as tar:
for member in tar.getmembers():
if member.name not in files_in_tar:
buffer = tar.extractfile(member)
tmp_tar.addfile(member, buffer)
files_in_tar.append(member.name)
if files_in_tar:
makedirs(os.path.join(layers[-1], os.path.dirname(APK_SCRIPTS)))
shutil.move(tmp_tar_path, os.path.join(layers[-1], APK_SCRIPTS))
os.chown(os.path.join(layers[-1], APK_SCRIPTS), 100000, 100000)
else:
os.unlink(tmp_tar_path)
def merge_apk_triggers(layers):
triggers = []
for layer in layers:
try:
with open(os.path.join(layer, APK_TRIGGERS), 'r') as f:
for line in f:
if line not in triggers:
triggers.append(line)
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(APK_TRIGGERS)))
with open(os.path.join(layers[-1], APK_TRIGGERS), 'w') as f:
f.writelines(triggers)
os.chown(os.path.join(layers[-1], APK_TRIGGERS), 100000, 100000)
def merge_etc_passwd(layers):
passwd = {}
for layer in layers:
try:
with open(os.path.join(layer, ETC_PASSWD), 'r') as f:
for line in f:
passwd[line.split(':')[0]] = line
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(ETC_PASSWD)))
with open(os.path.join(layers[-1], ETC_PASSWD), 'w') as f:
f.writelines(passwd.values())
os.chown(os.path.join(layers[-1], ETC_PASSWD), 100000, 100000)
def merge_etc_group(layers):
groups = {}
for layer in layers:
try:
with open(os.path.join(layer, ETC_GROUP), 'r') as f:
for line in f:
name,pwd,gid,users = line.split(':')
name = splitline[0]
users = splitline[3].strip().split(',')
if name not in groups:
groups[name] = [name,pwd,gid,users]
else:
groups[name][1] = pwd
groups[name][2] = gid
for user in users:
if user not in groups[name][3]:
groups[name][3].append(user)
except:
continue
for group in groups.values():
group[3] = '{}\n'.format(','.join(group[3]))
makedirs(os.path.join(layers[-1], os.path.dirname(ETC_GROUP)))
with open(os.path.join(layers[-1], ETC_GROUP), 'w') as f:
f.writelines([':'.join(group) for group in groups.values()])
os.chown(os.path.join(layers[-1], ETC_GROUP), 100000, 100000)
def merge_etc_shadow(layers):
shadow = {}
for layer in layers:
try:
with open(os.path.join(layer, ETC_SHADOW), 'r') as f:
for line in f:
shadow[line.split(':')[0]] = line
except:
continue
makedirs(os.path.join(layers[-1], os.path.dirname(ETC_SHADOW)))
with open(os.path.join(layers[-1], ETC_SHADOW), 'w') as f:
f.writelines(shadow.values())
os.chown(os.path.join(layers[-1], ETC_SHADOW), 100000, 100042)
parser = argparse.ArgumentParser(description='APK database merge script')
parser.add_argument('layers', help='Path to LXC layers to be merged', nargs=argparse.REMAINDER)
if len(sys.argv) < 3:
parser.print_usage()
sys.exit(1)
args = parser.parse_args()
merge_apk_world(args.layers)
merge_apk_installed(args.layers)
merge_apk_scripts(args.layers)
merge_apk_triggers(args.layers)
merge_etc_passwd(args.layers)
merge_etc_group(args.layers)
merge_etc_shadow(args.layers)

View File

@ -25,7 +25,7 @@ class Image:
try: try:
builder = ImageBuilder(self) builder = ImageBuilder(self)
builder.build() builder.build()
# In case of successful build, packaging needs to happen in all cases to prevent outdated packages # Packaging needs to happen in any case after a successful build in order to prevent outdated packages
self.force_build = True self.force_build = True
except ImageExistsError as e: except ImageExistsError as e:
print('Image {} already exists, skipping build tasks'.format(e)) print('Image {} already exists, skipping build tasks'.format(e))

View File

@ -7,6 +7,7 @@ import sys
from lxcmgr import lxcmgr from lxcmgr import lxcmgr
from lxcmgr.paths import LXC_STORAGE_DIR from lxcmgr.paths import LXC_STORAGE_DIR
from lxcmgr.pkgmgr import PkgMgr
class ImageExistsError(Exception): class ImageExistsError(Exception):
pass pass
@ -21,6 +22,7 @@ class ImageBuilder:
self.script_eof = None self.script_eof = None
def build(self): def build(self):
# Read and process lines from lxcfile
with open(self.image.lxcfile, 'r') as f: with open(self.image.lxcfile, 'r') as f:
for line in f: for line in f:
line = line.strip() line = line.strip()
@ -34,15 +36,14 @@ class ImageBuilder:
self.process_line(*line.split(None, 1)) self.process_line(*line.split(None, 1))
def process_line(self, directive, args): def process_line(self, directive, args):
# Process directives from lxcfile
if 'RUN' == directive: if 'RUN' == directive:
self.script = [] self.script = []
self.script_eof = args self.script_eof = args
elif 'IMAGE' == directive: elif 'IMAGE' == directive:
self.set_name(args) self.set_name(args)
elif 'LAYER' == directive: elif 'FROM' == directive:
self.add_layer(args) self.set_layers(args)
elif 'MERGE' == directive:
self.merge_layers(args.split())
elif 'COPY' == directive: elif 'COPY' == directive:
srcdst = args.split() srcdst = args.split()
self.copy_files(srcdst[0], srcdst[1] if len(srcdst) == 2 else '') self.copy_files(srcdst[0], srcdst[1] if len(srcdst) == 2 else '')
@ -63,6 +64,7 @@ class ImageBuilder:
return os.path.join(LXC_STORAGE_DIR, layer) return os.path.join(LXC_STORAGE_DIR, layer)
def run_script(self, script): def run_script(self, script):
# Creates a temporary container, runs a script in its namespace, and stores the modifications as part of the image
lxcmgr.create_container(self.image.name, self.image.conf) lxcmgr.create_container(self.image.name, self.image.conf)
sh = os.path.join(LXC_STORAGE_DIR, self.image.name, 'run.sh') sh = os.path.join(LXC_STORAGE_DIR, self.image.name, 'run.sh')
with open(sh, 'w') as f: with open(sh, 'w') as f:
@ -75,6 +77,7 @@ class ImageBuilder:
lxcmgr.destroy_container(self.image.name) lxcmgr.destroy_container(self.image.name)
def set_name(self, name): def set_name(self, name):
# Set name and first (topmost) layer of the image
self.image.name = name self.image.name = name
self.image.conf['layers'] = [name] self.image.conf['layers'] = [name]
image_path = self.get_layer_path(name) image_path = self.get_layer_path(name)
@ -86,43 +89,47 @@ class ImageBuilder:
os.makedirs(image_path, 0o755, True) os.makedirs(image_path, 0o755, True)
os.chown(image_path, 100000, 100000) os.chown(image_path, 100000, 100000)
def add_layer(self, name): def set_layers(self, image):
layer_path = self.get_layer_path(name) # Extend list of layers with the list of layers from parent image
if not os.path.exists(layer_path): # Raies an exception when IMAGE has no name
raise ImageNotFoundError(layer_path) pkgmgr = PkgMgr()
self.image.conf['layers'].insert(1, name) self.image.conf['layers'].extend(pkgmgr.installed_packages[image]['layers'])
def merge_layers(self, cmd):
layers = [self.get_layer_path(layer) for layer in self.image.conf['layers']]
subprocess.run(cmd + layers[::-1], check=True)
def copy_files(self, src, dst): def copy_files(self, src, dst):
# Copy files from the host or download them from a http(s) URL
dst = os.path.join(LXC_STORAGE_DIR, self.image.name, dst) dst = os.path.join(LXC_STORAGE_DIR, self.image.name, dst)
if src.startswith('http://') or src.startswith('https://'): if src.startswith('http://') or src.startswith('https://'):
unpack_http_archive(src, dst) unpack_http_archive(src, dst)
else: else:
copy_tree(os.path.join(self.image.build_dir, src), dst) copy_tree(os.path.join(self.image.build_dir, src), dst)
# Shift UID/GID of the files to the unprivileged range
shift_uid(dst) shift_uid(dst)
def add_env(self, key, value): def add_env(self, key, value):
# Sets lxc.environment records for the image
if 'env' not in self.image.conf: if 'env' not in self.image.conf:
self.image.conf['env'] = [] self.image.conf['env'] = []
self.image.conf['env'].append([key, value]) self.image.conf['env'].append([key, value])
def set_user(self, uid, gid): def set_user(self, uid, gid):
# Sets lxc.init.uid/gid for the image
self.image.conf['uid'] = uid self.image.conf['uid'] = uid
self.image.conf['gid'] = gid self.image.conf['gid'] = gid
def set_cmd(self, cmd): def set_cmd(self, cmd):
# Sets lxc.init.cmd for the image
self.image.conf['cmd'] = cmd self.image.conf['cmd'] = cmd
def set_cwd(self, cwd): def set_cwd(self, cwd):
# Sets lxc.init.cwd for the image
self.image.conf['cwd'] = cwd self.image.conf['cwd'] = cwd
def set_halt(self, halt): def set_halt(self, halt):
# Sets lxc.signal.halt for the image
self.image.conf['halt'] = halt self.image.conf['halt'] = halt
def set_ready(self, cmd): def set_ready(self, cmd):
# Sets a command performed in OpenRC start_post to check readiness of the container
self.image.conf['ready'] = cmd self.image.conf['ready'] = cmd
def clean(self): def clean(self):
@ -130,17 +137,19 @@ class ImageBuilder:
shutil.rmtree(self.get_layer_path(self.image.name)) shutil.rmtree(self.get_layer_path(self.image.name))
def unpack_http_archive(src, dst): def unpack_http_archive(src, dst):
xf = 'xzf' # Decompress an archive downloaded via http(s)
if src.endswith('.bz2'): xf = 'xzf'
xf = 'xjf' if src.endswith('.bz2'):
elif src.endswith('.xz'): xf = 'xjf'
xf = 'xJf' elif src.endswith('.xz'):
with subprocess.Popen(['wget', src, '-O', '-'], stdout=subprocess.PIPE) as wget: xf = 'xJf'
with subprocess.Popen(['tar', xf, '-', '-C', dst], stdin=wget.stdout) as tar: with subprocess.Popen(['wget', src, '-O', '-'], stdout=subprocess.PIPE) as wget:
wget.stdout.close() with subprocess.Popen(['tar', xf, '-', '-C', dst], stdin=wget.stdout) as tar:
tar.wait() wget.stdout.close()
tar.wait()
def copy_tree(src, dst): def copy_tree(src, dst):
# Copies files from the host
if not os.path.isdir(src): if not os.path.isdir(src):
shutil.copy2(src, dst) shutil.copy2(src, dst)
else: else:
@ -150,16 +159,19 @@ def copy_tree(src, dst):
shutil.copystat(src, dst) shutil.copystat(src, dst)
def shift_uid(dir): def shift_uid(dir):
# Shifts UID/GID of a file or a directory and its contents to the unprivileged range
shift_uid_entry(dir, os.stat(dir, follow_symlinks=True)) shift_uid_entry(dir, os.stat(dir, follow_symlinks=True))
shift_uid_recursively(dir) shift_uid_recursively(dir)
def shift_uid_recursively(dir): def shift_uid_recursively(dir):
# Shifts UID/GID of a directory and its contents to the unprivileged range
for entry in os.scandir(dir): for entry in os.scandir(dir):
shift_uid_entry(entry.path, entry.stat(follow_symlinks=False)) shift_uid_entry(entry.path, entry.stat(follow_symlinks=False))
if entry.is_dir(): if entry.is_dir():
shift_uid_recursively(entry.path) shift_uid_recursively(entry.path)
def shift_uid_entry(path, stat): def shift_uid_entry(path, stat):
# Shifts UID/GID of a file or a directory to the unprivileged range
uid = stat.st_uid uid = stat.st_uid
gid = stat.st_gid gid = stat.st_gid
do_chown = False do_chown = False

View File

@ -47,10 +47,11 @@ class ImagePacker(Packer):
def register(self): def register(self):
# Register image in global repository metadata file # Register image in global repository metadata file
print('Registering image package', self.image.name) print('Registering image package', self.image.name)
self.packages['images'][self.image.name] = self.image.conf.copy() image_conf = self.image.conf.copy()
self.packages['images'][self.image.name]['size'] = self.tar_size image_conf['size'] = self.tar_size
self.packages['images'][self.image.name]['pkgsize'] = self.xz_size image_conf['pkgsize'] = self.xz_size
self.packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path) image_conf['sha512'] = crypto.hash_file(self.xz_path)
self.packages['images'][self.image.name] = image_conf
self.save_repo_meta() self.save_repo_meta()
# Register the image also to locally installed images for package manager # Register the image also to locally installed images for package manager
pm = PkgMgr() pm = PkgMgr()

View File

@ -29,13 +29,6 @@ The *lxcfile* syntax is designed to resemble *Dockerfile* syntax in order to eas
- **Docker equivalent:** `FROM` - **Docker equivalent:** `FROM`
- **Populates LXC field:** `lxc.rootfs.path` - **Populates LXC field:** `lxc.rootfs.path`
### MERGE
- **Usage:** `MERGE <scriptname>`
- **Description:** Runs `<scriptname>` on LXC host and passes all layer paths as parameter to this script. This helps you to resolve the conflicts in cases where you mix multiple OverlayFS layers with overlapping files, ie. package manager cache. The idea is that all layers are read separately by the `<scriptname>` script and the fixed result is written back to the uppermost layer.
- **Docker equivalent:** None
- **Populates LXC field:** None
### RUN ### RUN
- **Usage:** - **Usage:**