Remove MERGE capability, add FROM
layer inheritance is now linear
This commit is contained in:
parent
4d579ef8c1
commit
f8403c5f42
GPG Key ID:
524BD33A0EE29499
@ -1 +1 @@
|
||||
Subproject commit 41156fe4243b15b4b233b618082aae8ce32e5a2b
|
||||
Subproject commit 7c25d22d4146033cfb1e0775d06912b5c8f77e73
|
||||
@ -1,174 +0,0 @@
|
||||
#!/usr/bin/python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
APK_WORLD = 'etc/apk/world'
|
||||
APK_INSTALLED = 'lib/apk/db/installed'
|
||||
APK_SCRIPTS = 'lib/apk/db/scripts.tar'
|
||||
APK_TRIGGERS = 'lib/apk/db/triggers'
|
||||
|
||||
ETC_PASSWD = 'etc/passwd'
|
||||
ETC_GROUP = 'etc/groups'
|
||||
ETC_SHADOW = 'etc/shadow'
|
||||
|
||||
def makedirs(path, mode=0o755, uid=100000, gid=100000):
|
||||
try:
|
||||
os.mkdir(path, mode)
|
||||
os.chown(path, uid, gid)
|
||||
except FileNotFoundError:
|
||||
makedirs(os.path.dirname(path), mode, uid, gid)
|
||||
os.mkdir(path, mode)
|
||||
os.chown(path, uid, gid)
|
||||
except FileExistsError:
|
||||
pass
|
||||
|
||||
def merge_apk_world(layers):
|
||||
world = []
|
||||
for layer in layers:
|
||||
try:
|
||||
with open(os.path.join(layer, APK_WORLD), 'r') as f:
|
||||
for line in f:
|
||||
if line not in world:
|
||||
world.append(line)
|
||||
except:
|
||||
continue
|
||||
makedirs(os.path.join(layers[-1], os.path.dirname(APK_WORLD)))
|
||||
with open(os.path.join(layers[-1], APK_WORLD), 'w') as f:
|
||||
f.writelines(world)
|
||||
os.chown(os.path.join(layers[-1], APK_WORLD), 100000, 100000)
|
||||
|
||||
def merge_apk_installed(layers):
|
||||
installed = []
|
||||
for layer in layers:
|
||||
try:
|
||||
with open(os.path.join(layer, APK_INSTALLED), 'r') as f:
|
||||
buffer = []
|
||||
for line in f:
|
||||
if line.startswith('C:'):
|
||||
buffer = ''.join(buffer)
|
||||
if buffer not in installed:
|
||||
installed.append(buffer)
|
||||
buffer = []
|
||||
buffer.append(line)
|
||||
buffer = ''.join(buffer)
|
||||
if buffer not in installed:
|
||||
installed.append(buffer)
|
||||
except:
|
||||
continue
|
||||
makedirs(os.path.join(layers[-1], os.path.dirname(APK_INSTALLED)))
|
||||
with open(os.path.join(layers[-1], APK_INSTALLED), 'w') as f:
|
||||
f.writelines(installed)
|
||||
os.chown(os.path.join(layers[-1], APK_INSTALLED), 100000, 100000)
|
||||
|
||||
def merge_apk_scripts(layers):
|
||||
tmp_tar_path = tempfile.mkstemp()[1]
|
||||
files_in_tar = []
|
||||
with tarfile.open(tmp_tar_path, 'w:') as tmp_tar:
|
||||
for layer in layers:
|
||||
tar_path = os.path.join(layer, APK_SCRIPTS)
|
||||
if os.path.exists(tar_path):
|
||||
with tarfile.open(tar_path, 'r:') as tar:
|
||||
for member in tar.getmembers():
|
||||
if member.name not in files_in_tar:
|
||||
buffer = tar.extractfile(member)
|
||||
tmp_tar.addfile(member, buffer)
|
||||
files_in_tar.append(member.name)
|
||||
if files_in_tar:
|
||||
makedirs(os.path.join(layers[-1], os.path.dirname(APK_SCRIPTS)))
|
||||
shutil.move(tmp_tar_path, os.path.join(layers[-1], APK_SCRIPTS))
|
||||
os.chown(os.path.join(layers[-1], APK_SCRIPTS), 100000, 100000)
|
||||
else:
|
||||
os.unlink(tmp_tar_path)
|
||||
|
||||
def merge_apk_triggers(layers):
|
||||
triggers = []
|
||||
for layer in layers:
|
||||
try:
|
||||
with open(os.path.join(layer, APK_TRIGGERS), 'r') as f:
|
||||
for line in f:
|
||||
if line not in triggers:
|
||||
triggers.append(line)
|
||||
except:
|
||||
continue
|
||||
makedirs(os.path.join(layers[-1], os.path.dirname(APK_TRIGGERS)))
|
||||
with open(os.path.join(layers[-1], APK_TRIGGERS), 'w') as f:
|
||||
f.writelines(triggers)
|
||||
os.chown(os.path.join(layers[-1], APK_TRIGGERS), 100000, 100000)
|
||||
|
||||
def merge_etc_passwd(layers):
|
||||
passwd = {}
|
||||
for layer in layers:
|
||||
try:
|
||||
with open(os.path.join(layer, ETC_PASSWD), 'r') as f:
|
||||
for line in f:
|
||||
passwd[line.split(':')[0]] = line
|
||||
except:
|
||||
continue
|
||||
makedirs(os.path.join(layers[-1], os.path.dirname(ETC_PASSWD)))
|
||||
with open(os.path.join(layers[-1], ETC_PASSWD), 'w') as f:
|
||||
f.writelines(passwd.values())
|
||||
os.chown(os.path.join(layers[-1], ETC_PASSWD), 100000, 100000)
|
||||
|
||||
def merge_etc_group(layers):
|
||||
groups = {}
|
||||
for layer in layers:
|
||||
try:
|
||||
with open(os.path.join(layer, ETC_GROUP), 'r') as f:
|
||||
for line in f:
|
||||
name,pwd,gid,users = line.split(':')
|
||||
name = splitline[0]
|
||||
users = splitline[3].strip().split(',')
|
||||
if name not in groups:
|
||||
groups[name] = [name,pwd,gid,users]
|
||||
else:
|
||||
groups[name][1] = pwd
|
||||
groups[name][2] = gid
|
||||
for user in users:
|
||||
if user not in groups[name][3]:
|
||||
groups[name][3].append(user)
|
||||
except:
|
||||
continue
|
||||
for group in groups.values():
|
||||
group[3] = '{}\n'.format(','.join(group[3]))
|
||||
makedirs(os.path.join(layers[-1], os.path.dirname(ETC_GROUP)))
|
||||
with open(os.path.join(layers[-1], ETC_GROUP), 'w') as f:
|
||||
f.writelines([':'.join(group) for group in groups.values()])
|
||||
os.chown(os.path.join(layers[-1], ETC_GROUP), 100000, 100000)
|
||||
|
||||
def merge_etc_shadow(layers):
|
||||
shadow = {}
|
||||
for layer in layers:
|
||||
try:
|
||||
with open(os.path.join(layer, ETC_SHADOW), 'r') as f:
|
||||
for line in f:
|
||||
shadow[line.split(':')[0]] = line
|
||||
except:
|
||||
continue
|
||||
makedirs(os.path.join(layers[-1], os.path.dirname(ETC_SHADOW)))
|
||||
with open(os.path.join(layers[-1], ETC_SHADOW), 'w') as f:
|
||||
f.writelines(shadow.values())
|
||||
os.chown(os.path.join(layers[-1], ETC_SHADOW), 100000, 100042)
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(description='APK database merge script')
|
||||
parser.add_argument('layers', help='Path to LXC layers to be merged', nargs=argparse.REMAINDER)
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
parser.print_usage()
|
||||
sys.exit(1)
|
||||
args = parser.parse_args()
|
||||
|
||||
merge_apk_world(args.layers)
|
||||
merge_apk_installed(args.layers)
|
||||
merge_apk_scripts(args.layers)
|
||||
merge_apk_triggers(args.layers)
|
||||
|
||||
merge_etc_passwd(args.layers)
|
||||
merge_etc_group(args.layers)
|
||||
merge_etc_shadow(args.layers)
|
||||
@ -25,7 +25,7 @@ class Image:
|
||||
try:
|
||||
builder = ImageBuilder(self)
|
||||
builder.build()
|
||||
# In case of successful build, packaging needs to happen in all cases to prevent outdated packages
|
||||
# Packaging needs to happen in any case after a successful build in order to prevent outdated packages
|
||||
self.force_build = True
|
||||
except ImageExistsError as e:
|
||||
print('Image {} already exists, skipping build tasks'.format(e))
|
||||
|
||||
@ -7,6 +7,7 @@ import sys
|
||||
|
||||
from lxcmgr import lxcmgr
|
||||
from lxcmgr.paths import LXC_STORAGE_DIR
|
||||
from lxcmgr.pkgmgr import PkgMgr
|
||||
|
||||
class ImageExistsError(Exception):
|
||||
pass
|
||||
@ -21,6 +22,7 @@ class ImageBuilder:
|
||||
self.script_eof = None
|
||||
|
||||
def build(self):
|
||||
# Read and process lines from lxcfile
|
||||
with open(self.image.lxcfile, 'r') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
@ -34,15 +36,14 @@ class ImageBuilder:
|
||||
self.process_line(*line.split(None, 1))
|
||||
|
||||
def process_line(self, directive, args):
|
||||
# Process directives from lxcfile
|
||||
if 'RUN' == directive:
|
||||
self.script = []
|
||||
self.script_eof = args
|
||||
elif 'IMAGE' == directive:
|
||||
self.set_name(args)
|
||||
elif 'LAYER' == directive:
|
||||
self.add_layer(args)
|
||||
elif 'MERGE' == directive:
|
||||
self.merge_layers(args.split())
|
||||
elif 'FROM' == directive:
|
||||
self.set_layers(args)
|
||||
elif 'COPY' == directive:
|
||||
srcdst = args.split()
|
||||
self.copy_files(srcdst[0], srcdst[1] if len(srcdst) == 2 else '')
|
||||
@ -63,6 +64,7 @@ class ImageBuilder:
|
||||
return os.path.join(LXC_STORAGE_DIR, layer)
|
||||
|
||||
def run_script(self, script):
|
||||
# Creates a temporary container, runs a script in its namespace, and stores the modifications as part of the image
|
||||
lxcmgr.create_container(self.image.name, self.image.conf)
|
||||
sh = os.path.join(LXC_STORAGE_DIR, self.image.name, 'run.sh')
|
||||
with open(sh, 'w') as f:
|
||||
@ -75,6 +77,7 @@ class ImageBuilder:
|
||||
lxcmgr.destroy_container(self.image.name)
|
||||
|
||||
def set_name(self, name):
|
||||
# Set name and first (topmost) layer of the image
|
||||
self.image.name = name
|
||||
self.image.conf['layers'] = [name]
|
||||
image_path = self.get_layer_path(name)
|
||||
@ -86,43 +89,47 @@ class ImageBuilder:
|
||||
os.makedirs(image_path, 0o755, True)
|
||||
os.chown(image_path, 100000, 100000)
|
||||
|
||||
def add_layer(self, name):
|
||||
layer_path = self.get_layer_path(name)
|
||||
if not os.path.exists(layer_path):
|
||||
raise ImageNotFoundError(layer_path)
|
||||
self.image.conf['layers'].insert(1, name)
|
||||
|
||||
def merge_layers(self, cmd):
|
||||
layers = [self.get_layer_path(layer) for layer in self.image.conf['layers']]
|
||||
subprocess.run(cmd + layers[::-1], check=True)
|
||||
def set_layers(self, image):
|
||||
# Extend list of layers with the list of layers from parent image
|
||||
# Raies an exception when IMAGE has no name
|
||||
pkgmgr = PkgMgr()
|
||||
self.image.conf['layers'].extend(pkgmgr.installed_packages[image]['layers'])
|
||||
|
||||
def copy_files(self, src, dst):
|
||||
# Copy files from the host or download them from a http(s) URL
|
||||
dst = os.path.join(LXC_STORAGE_DIR, self.image.name, dst)
|
||||
if src.startswith('http://') or src.startswith('https://'):
|
||||
unpack_http_archive(src, dst)
|
||||
else:
|
||||
copy_tree(os.path.join(self.image.build_dir, src), dst)
|
||||
# Shift UID/GID of the files to the unprivileged range
|
||||
shift_uid(dst)
|
||||
|
||||
def add_env(self, key, value):
|
||||
# Sets lxc.environment records for the image
|
||||
if 'env' not in self.image.conf:
|
||||
self.image.conf['env'] = []
|
||||
self.image.conf['env'].append([key, value])
|
||||
|
||||
def set_user(self, uid, gid):
|
||||
# Sets lxc.init.uid/gid for the image
|
||||
self.image.conf['uid'] = uid
|
||||
self.image.conf['gid'] = gid
|
||||
|
||||
def set_cmd(self, cmd):
|
||||
# Sets lxc.init.cmd for the image
|
||||
self.image.conf['cmd'] = cmd
|
||||
|
||||
def set_cwd(self, cwd):
|
||||
# Sets lxc.init.cwd for the image
|
||||
self.image.conf['cwd'] = cwd
|
||||
|
||||
def set_halt(self, halt):
|
||||
# Sets lxc.signal.halt for the image
|
||||
self.image.conf['halt'] = halt
|
||||
|
||||
def set_ready(self, cmd):
|
||||
# Sets a command performed in OpenRC start_post to check readiness of the container
|
||||
self.image.conf['ready'] = cmd
|
||||
|
||||
def clean(self):
|
||||
@ -130,17 +137,19 @@ class ImageBuilder:
|
||||
shutil.rmtree(self.get_layer_path(self.image.name))
|
||||
|
||||
def unpack_http_archive(src, dst):
|
||||
xf = 'xzf'
|
||||
if src.endswith('.bz2'):
|
||||
xf = 'xjf'
|
||||
elif src.endswith('.xz'):
|
||||
xf = 'xJf'
|
||||
with subprocess.Popen(['wget', src, '-O', '-'], stdout=subprocess.PIPE) as wget:
|
||||
with subprocess.Popen(['tar', xf, '-', '-C', dst], stdin=wget.stdout) as tar:
|
||||
wget.stdout.close()
|
||||
tar.wait()
|
||||
# Decompress an archive downloaded via http(s)
|
||||
xf = 'xzf'
|
||||
if src.endswith('.bz2'):
|
||||
xf = 'xjf'
|
||||
elif src.endswith('.xz'):
|
||||
xf = 'xJf'
|
||||
with subprocess.Popen(['wget', src, '-O', '-'], stdout=subprocess.PIPE) as wget:
|
||||
with subprocess.Popen(['tar', xf, '-', '-C', dst], stdin=wget.stdout) as tar:
|
||||
wget.stdout.close()
|
||||
tar.wait()
|
||||
|
||||
def copy_tree(src, dst):
|
||||
# Copies files from the host
|
||||
if not os.path.isdir(src):
|
||||
shutil.copy2(src, dst)
|
||||
else:
|
||||
@ -150,16 +159,19 @@ def copy_tree(src, dst):
|
||||
shutil.copystat(src, dst)
|
||||
|
||||
def shift_uid(dir):
|
||||
# Shifts UID/GID of a file or a directory and its contents to the unprivileged range
|
||||
shift_uid_entry(dir, os.stat(dir, follow_symlinks=True))
|
||||
shift_uid_recursively(dir)
|
||||
|
||||
def shift_uid_recursively(dir):
|
||||
# Shifts UID/GID of a directory and its contents to the unprivileged range
|
||||
for entry in os.scandir(dir):
|
||||
shift_uid_entry(entry.path, entry.stat(follow_symlinks=False))
|
||||
if entry.is_dir():
|
||||
shift_uid_recursively(entry.path)
|
||||
|
||||
def shift_uid_entry(path, stat):
|
||||
# Shifts UID/GID of a file or a directory to the unprivileged range
|
||||
uid = stat.st_uid
|
||||
gid = stat.st_gid
|
||||
do_chown = False
|
||||
|
||||
@ -47,10 +47,11 @@ class ImagePacker(Packer):
|
||||
def register(self):
|
||||
# Register image in global repository metadata file
|
||||
print('Registering image package', self.image.name)
|
||||
self.packages['images'][self.image.name] = self.image.conf.copy()
|
||||
self.packages['images'][self.image.name]['size'] = self.tar_size
|
||||
self.packages['images'][self.image.name]['pkgsize'] = self.xz_size
|
||||
self.packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path)
|
||||
image_conf = self.image.conf.copy()
|
||||
image_conf['size'] = self.tar_size
|
||||
image_conf['pkgsize'] = self.xz_size
|
||||
image_conf['sha512'] = crypto.hash_file(self.xz_path)
|
||||
self.packages['images'][self.image.name] = image_conf
|
||||
self.save_repo_meta()
|
||||
# Register the image also to locally installed images for package manager
|
||||
pm = PkgMgr()
|
||||
|
||||
@ -29,13 +29,6 @@ The *lxcfile* syntax is designed to resemble *Dockerfile* syntax in order to eas
|
||||
- **Docker equivalent:** `FROM`
|
||||
- **Populates LXC field:** `lxc.rootfs.path`
|
||||
|
||||
### MERGE
|
||||
|
||||
- **Usage:** `MERGE <scriptname>`
|
||||
- **Description:** Runs `<scriptname>` on LXC host and passes all layer paths as parameter to this script. This helps you to resolve the conflicts in cases where you mix multiple OverlayFS layers with overlapping files, ie. package manager cache. The idea is that all layers are read separately by the `<scriptname>` script and the fixed result is written back to the uppermost layer.
|
||||
- **Docker equivalent:** None
|
||||
- **Populates LXC field:** None
|
||||
|
||||
### RUN
|
||||
|
||||
- **Usage:**
|
||||
|
||||
Loading…
Reference in New Issue
Block a user