Disassembler/Spotter-VM
1
0
Fork 0
Code Issues 78 Pull Requests Actions Packages Projects Releases Wiki Activity

Allow lxcbuilder to pack meta files

Browse Source
This commit is contained in:
Disassembler 2019-09-20 10:13:41 +02:00
parent 2ea88cabce
commit 7116566519
Signed by: Disassembler
GPG Key ID: 524BD33A0EE29499
13 changed files with 283 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apk/vmmgr

@ -1 +1 @@
Subproject commit 972ca0b6967edd56af96a7de159950ac9fcbc4a6
Subproject commit c3b711850e02a6e228c4eb64ed82a4d1bc889ae9

3
build/build-all.sh
View File

@ -66,14 +66,15 @@ cd ${ROOT}/lxc-services
lxc-build activemq
lxc-build mariadb
lxc-build postgres
lxc-build postgis
lxc-build rabbitmq
lxc-build redis
lxc-build solr
# Build applications
cd ${ROOT}/lxc-apps
lxc-build ckan-datapusher
lxc-build ckan
lxc-build ckan-datapusher
lxc-build crisiscleanup
lxc-build cts
lxc-build ecogis

4
build/install-toolchain.sh
View File

@ -24,7 +24,7 @@ cp etc/abuild.conf /etc/abuild.conf
# Prepare LXC build toolchain
cp usr/bin/fix-apk /usr/bin/fix-apk
cp usr/bin/lxc-build /usr/bin/lxc-build
cp usr/bin/lxc-pack /usr/bin/lxc-pack
mkdir -p /srv/build/lxc/apps /srv/build/lxc/images
# Prepare local APK repository
cp etc/nginx/conf.d/apkrepo.conf /etc/nginx/conf.d/apkrepo.conf
@ -36,4 +36,4 @@ service nginx reload
# Supply LXC build key
# openssl ecparam -genkey -name secp384r1 -out /srv/build/packages.key
# openssl ec -in /srv/build/packages.key -pubout -out /srv/build/packages.pub
# openssl ec -in /srv/build/packages.key -pubout -out /srv/build/lxc/packages.pub

12
build/usr/bin/lxc-build
View File

@ -1,12 +0,0 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import sys
from lxcbuild.lxcimage import LXCImage
if __name__ == '__main__':
if len(sys.argv) != 2 or sys.argv[1] in ('-h', '--help'):
print('Usage: lxc-build <buildpath>\n where the buildpath can be either specific lxcfile or a directory containing one')
else:
image = LXCImage(sys.argv[1])
image.build_and_pack()

43
build/usr/bin/lxcbuild Normal file
View File

@ -0,0 +1,43 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import argparse
import sys
from lxcbuild.app import App
from lxcbuild.image import Image
parser = argparse.ArgumentParser(description='VM application builder and packager')
parser.add_argument('-f', '--force', action='store_true', help='Force rebuild already built package')
parser.add_argument('buildpath', help='Either specific "lxcfile" or "meta" file or a directory containing one')
if len(sys.argv) < 2:
parser.print_usage()
sys.exit(1)
args = parser.parse_args()
buildpath = os.path.realpath(args.buildpath)
if os.path.isfile(buildpath):
basename = os.path.basename(buildpath)
if basename == 'lxcfile' or basename.endswith('.lxcfile'):
image = Image(buildpath)
image.build_and_pack(args.force)
elif basename == 'meta' or basename.endswith('.meta'):
app = App(buildpath)
app.build_and_pack()
else:
print('Unknown file {} given, expected "lxcfile" or "meta"'.format(buildpath))
sys.exit(1)
else:
valid_dir = False
lxcfile = os.path.join(buildpath, 'lxcfile')
meta = os.path.join(buildpath, 'meta')
if os.path.exists(lxcfile):
valid_dir = True
image = Image(lxcfile)
image.build_and_pack(args.force)
if os.path.exists(meta):
valid_dir = True
app = App(buildpath)
app.pack()
if not valid_dir:
print('Directory {} doesn\'t contain anything to build, skipping'.format(buildpath))

16
build/usr/lib/python3.6/lxcbuild/app.py Normal file
View File

@ -0,0 +1,16 @@
# -*- coding: utf-8 -*-
import os
from .packer import Packer
class App:
def __init__(self, metafile):
self.build_dir = os.path.dirname(metafile)
self.name = os.path.basename(self.build_dir)
with open(metafile, 'r') as f:
self.conf = json.load(f)
def pack(self):
packer = Packer()
packer.pack_app(self)

60
build/usr/lib/python3.6/lxcbuild/lxcbuilder.py → build/usr/lib/python3.6/lxcbuild/builder.py
View File

@ -4,25 +4,26 @@ import os
import shutil
import subprocess
import sys
from vmmgr import lxcmgr
LXC_ROOT = '/var/lib/lxc'
from lxcmgr import lxcmgr
from lxcmgr.paths import PKG_STORAGE_DIR
class LXCBuilder:
def __init__(self, image):
self.image = image
class ImageExistsError(Exception):
pass
class ImageNotFoundError(Exception):
pass
class Builder:
def __init__(self):
self.image = None
self.script = []
self.script_eof = None
self.force = False
def build(self):
try:
self.image.conf['build'] = True
self.process_file()
except FileExistsError as e:
print(e)
del self.image.conf['build']
def process_file(self):
def build(self, image, force=False):
self.image = image
self.force = force
with open(self.image.lxcfile, 'r') as f:
for line in f:
line = line.strip()
@ -62,11 +63,11 @@ class LXCBuilder:
self.set_ready(args)
def get_layer_path(self, layer):
return os.path.join(LXC_ROOT, 'storage', layer)
return os.path.join(PKG_STORAGE_DIR, layer)
def run_script(self, script):
lxcmgr.register_container(self.image.name, self.image.conf)
sh = os.path.join(self.get_layer_path(self.image.name), 'run.sh')
sh = os.path.join(self.image.path, 'run.sh')
with open(sh, 'w') as f:
f.write('#!/bin/sh\nset -ev\n\n{}\n'.format('\n'.join(script)))
os.chmod(sh, 0o700)
@ -77,12 +78,20 @@ class LXCBuilder:
def set_name(self, name):
self.image.name = name
self.image.conf['layers'] = [self.image.name]
image_path = self.get_layer_path(self.image.name)
os.makedirs(image_path, 0o755, True)
os.chown(image_path, 100000, 100000)
self.image.path = self.get_layer_path(name)
self.image.conf['layers'] = [name]
if os.path.exists(self.image.path):
if self.force:
self.clean()
else:
raise ImageExistsError(self.image.path)
os.makedirs(self.image.path, 0o755, True)
os.chown(self.image.path, 100000, 100000)
def add_layer(self, name):
layer_path = self.get_layer_path(name)
if not os.path.exists(layer_path):
raise ImageNotFoundError(layer_path)
self.image.conf['layers'].insert(0, name)
def fix_layer(self, cmd):
@ -90,17 +99,17 @@ class LXCBuilder:
subprocess.run([cmd] + layers, check=True)
def copy_files(self, src, dst):
dst = os.path.join(self.get_layer_path(self.image.name), dst)
dst = os.path.join(self.image.path, dst)
if src.startswith('http://') or src.startswith('https://'):
unpack_http_archive(src, dst)
else:
copy_tree(os.path.join(self.build_dir, src), dst)
copy_tree(os.path.join(self.image.build_dir, src), dst)
shift_uid(dst)
def add_env(self, args):
def add_env(self, key, value):
if 'env' not in self.image.conf:
self.image.conf['env'] = []
self.image.conf['env'].append(args)
self.image.conf['env'].append('{}={}'.format(key, value))
def set_user(self, uid, gid):
self.image.conf['uid'] = uid
@ -118,6 +127,9 @@ class LXCBuilder:
def set_ready(self, cmd):
self.image.conf['ready'] = cmd
def clean(self):
shutil.rmtree(self.image.path)
def unpack_http_archive(src, dst):
xf = 'xzf'
if src.endswith('.bz2'):

28
build/usr/lib/python3.6/lxcbuild/crypto.py Normal file
View File

@ -0,0 +1,28 @@
# -*- coding: utf-8 -*-
import hashlib
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.hazmat.primitives.serialization import load_pem_private_key
def sign_file(private_key, input_path):
# Generate SHA512 signature of a file using EC private key
print('Signing packages')
with open(private_key, 'rb') as f:
priv_key = load_pem_private_key(f.read(), None, default_backend())
with open(input_path, 'rb') as f:
data = f.read()
return priv_key.sign(data, ec.ECDSA(hashes.SHA512()))
def hash_file(file_path):
# Calculate SHA512 hash of a file
sha512 = hashlib.sha512()
with open(file_path, 'rb') as f:
while True:
data = f.read(65536)
if not data:
break
sha512.update(data)
return sha512.hexdigest()

38
build/usr/lib/python3.6/lxcbuild/image.py Normal file
View File

@ -0,0 +1,38 @@
# -*- coding: utf-8 -*-
import os
import sys
from .builder import Builder, ImageExistsError, ImageNotFoundError
from .packer import Packer, PackageExistsError
class Image:
def __init__(self, lxcfile):
self.name = None
self.path = None
self.conf = {}
self.lxcfile = lxcfile
self.build_dir = os.path.dirname(lxcfile)
def build_and_pack(self, force=False):
self.conf['build'] = True
try:
builder = Builder()
builder.build(self, force)
# In case of successful build, packaging needs to be forced to prevent outdated packages
force = True
except ImageExistsError as e:
print('Image {} already exists, skipping build tasks'.format(e))
except ImageNotFoundError as e:
print('Image {} not found, can\'t build {}'.format(e, self.name))
builder.clean()
sys.exit(1)
except:
builder.clean()
raise
try:
packer = Packer()
packer.pack_image(self, force)
except PackageExistsError as e:
print('Package {} already exists, skipping packaging tasks'.format(e))
del self.conf['build']

24
build/usr/lib/python3.6/lxcbuild/lxcimage.py
View File

@ -1,24 +0,0 @@
# -*- coding: utf-8 -*-
import os
from .lxcbuilder import LXCBuilder
from .lxcpacker import LXCPacker
class LXCImage:
def __init__(self, build_path):
self.name = None
self.conf = {}
if os.path.isfile(build_path):
self.lxcfile = os.path.realpath(build_path)
self.build_dir = os.path.dirname(self.lxcfile)
else:
self.build_dir = os.path.realpath(build_path)
self.lxcfile = os.path.join(self.build_dir, 'lxcfile')
def build_and_pack(self):
builder = LXCBuilder(self)
builder.build()
packer = LXCPacker(self)
packer.pack()

83
build/usr/lib/python3.6/lxcbuild/lxcpacker.py
View File

@ -1,83 +0,0 @@
# -*- coding: utf-8 -*-
import hashlib
import json
import os
import subprocess
import sys
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.hazmat.primitives.serialization import load_pem_private_key
PKG_ROOT = '/srv/build/lxc'
PRIVATE_KEY = '/srv/build/packages.key'
LXC_STORAGE = '/var/lib/lxc/storage'
class LXCPacker:
def __init__(self, image):
self.image = image
self.tar_path = None
self.xz_path = None
def pack(self):
# Prepare package file names
self.tar_path = os.path.join(PKG_ROOT, '{}.tar'.format(self.image.name))
self.xz_path = '{}.xz'.format(self.tar_path)
if os.path.exists(self.xz_path):
print('Package {} already exists, skipping packaging tasks'.format(self.xz_path))
return
os.makedirs(PKG_ROOT, 0o755, True)
self.create_archive()
self.register_package()
self.sign_packages()
def create_archive(self):
# Create archive
print('Archiving', self.image.name)
subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, os.path.join(LXC_STORAGE, self.image.name)], cwd='/')
# Add install/upgrade/uninstall scripts
# TODO: skripty balit jen s aplikacemi, ne s imagi
scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh')
scripts = [s for s in scripts if os.path.exists(os.path.join(self.image.build_dir, s))]
subprocess.run(['tar', '--transform', 's|^|srv/{}/|'.format(self.image.name), '-rpf', self.tar_path] + scripts, cwd=self.image.build_dir)
# Compress the tarball with xz (LZMA2)
print('Compressing', self.tar_path, '({:.2f} MB)'.format(os.path.getsize(self.tar_path)/1048576))
subprocess.run(['xz', '-9', self.tar_path])
print('Compressed ', self.xz_path, '({:.2f} MB)'.format(os.path.getsize(self.xz_path)/1048576))
def register_package(self):
# Register package
print('Registering package')
packages_file = os.path.join(PKG_ROOT, 'packages')
if os.path.exists(packages_file):
with open(packages_file, 'r') as f:
packages = json.load(f)
else:
packages = {'apps': {}, 'images': {}}
packages['images'][self.image.name] = self.image.conf.copy()
packages['images'][self.image.name]['size'] = os.path.getsize(self.xz_path)
packages['images'][self.image.name]['sha512'] = hash_file(self.xz_path)
with open(packages_file, 'w') as f:
json.dump(packages, f, sort_keys=True, indent=4)
def sign_packages(self):
# Sign packages file
print('Signing packages')
with open(PRIVATE_KEY, 'rb') as f:
priv_key = load_pem_private_key(f.read(), None, default_backend())
with open(os.path.join(PKG_ROOT, 'packages'), 'rb') as f:
data = f.read()
with open(os.path.join(PKG_ROOT, 'packages.sig'), 'wb') as f:
f.write(priv_key.sign(data, ec.ECDSA(hashes.SHA512())))
def hash_file(file_path):
sha512 = hashlib.sha512()
with open(file_path, 'rb') as f:
while True:
data = f.read(65536)
if not data:
break
sha512.update(data)
return sha512.hexdigest()

109
build/usr/lib/python3.6/lxcbuild/packer.py Normal file
View File

@ -0,0 +1,109 @@
# -*- coding: utf-8 -*-
import json
import os
import subprocess
import sys
from lxcmgr.paths import LXC_STORAGE_DIR
from . import crypto
from .paths import APP_DIR, IMAGE_DIR, META_FILE, PRIVATE_KEY, ROOT_DIR, SIGNATURE_FILE
class PackageExistsError(Exception):
pass
class Packer:
def __init__(self):
self.app = None
self.image = None
self.tar_path = None
self.xz_path = None
def load_packages_meta(self):
if os.path.exists(PKG_META):
with open(PKG_META, 'r') as f:
return json.load(f)
else:
return {'apps': {}, 'images': {}}
def save_packages_meta(self, packages):
with open(PKG_META, 'w') as f:
json.dump(packages, f, sort_keys=True, indent=4)
def pack_image(self, image, force):
self.image = image
# Prepare package file names
self.tar_path = os.path.join(IMAGE_DIR, '{}.tar'.format(self.image.name))
self.xz_path = '{}.xz'.format(self.tar_path)
if os.path.exists(self.xz_path):
if force:
self.unregister_image()
os.unlink(self.xz_path)
else:
raise PackageExistsError(self.xz_path)
self.create_image_archive()
self.register_image()
self.sign_packages()
def create_image_archive(self):
# Create archive
print('Archiving', self.image.path)
subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, self.image.name], cwd=LXC_STORAGE_DIR)
self.compress_archive()
def compress_archive(self):
# Compress the tarball with xz (LZMA2)
print('Compressing', self.tar_path, '({:.2f} MB)'.format(os.path.getsize(self.tar_path)/1048576))
subprocess.run(['xz', '-9', self.tar_path])
print('Compressed ', self.xz_path, '({:.2f} MB)'.format(os.path.getsize(self.xz_path)/1048576))
def register_image(self):
# Register package in global repository metadata file
print('Registering package {}'.format(self.image.name))
packages = self.load_packages_meta()
packages['images'][self.image.name] = self.image.conf.copy()
packages['images'][self.image.name]['size'] = os.path.getsize(self.xz_path)
packages['images'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path)
self.save_packages_meta(packages)
def sign_packages(self):
signature = crypto.sign_file(PRIVATE_KEY, META_FILE)
with open(SIGNATURE_FILE, 'wb') as f:
f.write(signature)
def unregister_image(self):
# Removes package from global repository metadata file
packages = self.load_packages_meta()
if self.image.name in packages['images']:
del packages['images'][self.image.name]
self.save_packages_meta(packages)
def pack_app(self, app):
self.app = app
# Prepare package file names
self.tar_path = os.path.join(APP_DIR, '{}.tar'.format(self.image.name))
self.xz_path = '{}.xz'.format(self.tar_path)
if os.path.exists(self.xz_path):
os.unlink(self.xz_path)
self.create_app_archive()
self.register_app()
self.sign_packages()
def create_app_archive(self):
# Create archive with application setup scripts
print('Archiving setup scripts for', self.app.name)
scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh')
scripts = [s for s in scripts if os.path.exists(os.path.join(self.app.build_dir, s))]
subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path] + scripts, cwd=self.app.build_dir)
self.compress_archive()
def register_app(self):
# Register package in global repository metadata file
print('Registering package {}'.format(self.app.name))
packages = self.load_packages_meta()
packages['apps'][self.image.name] = self.app.conf.copy()
packages['apps'][self.image.name]['size'] = os.path.getsize(self.xz_path)
packages['apps'][self.image.name]['sha512'] = crypto.hash_file(self.xz_path)
self.save_packages_meta(packages)

8
build/usr/lib/python3.6/lxcbuild/paths.py Normal file
View File

@ -0,0 +1,8 @@
# -*- coding: utf-8 -*-
ROOT_DIR = '/srv/build/lxc'
IMAGE_DIR = os.path.join(ROOT_DIR, 'images')
APP_DIR = os.path.join(ROOT_DIR, 'apps')
META_FILE = os.path.join(ROOT_DIR, 'packages')
SIGNATURE_FILE = os.path.join(ROOT_DIR, 'packages.sig')
PRIVATE_KEY = '/srv/build/packages.key'