Add SPOC as a submodule
This commit is contained in:
parent
99c39d5ee9
commit
7cb420dc4c
GPG Key ID:
524BD33A0EE29499
3
.gitmodules
vendored
3
.gitmodules
vendored
@ -1,3 +1,6 @@
|
||||
[submodule "app-vmmgr"]
|
||||
path = apk/vmmgr
|
||||
url = ssh://git@git.spotter.cz:2222/Spotter-Cluster/vmmgr.git
|
||||
[submodule "spoc"]
|
||||
path = apk/spoc
|
||||
url = ssh://git@git.spotter.cz:2222/Spotter-Cluster/spoc.git
|
||||
|
||||
1
apk/spoc
Submodule
1
apk/spoc
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 1e12d78efe730fcf35bc949d70c8db04e5917945
|
||||
@ -45,50 +45,88 @@ cd ${ROOT}/apk/wireguard-tools
|
||||
apk add -U libmnl-dev
|
||||
abuild -F
|
||||
|
||||
# Build apd pack runtimes
|
||||
# Build runtimes
|
||||
cd ${ROOT}/lxc-shared
|
||||
lxcbuild alpine3.8
|
||||
lxcbuild alpine3.8-php5.6
|
||||
lxcbuild alpine3.8-ruby2.4
|
||||
lxcbuild alpine3.9
|
||||
lxcbuild alpine3.9-java8
|
||||
lxcbuild alpine3.9-php7.2
|
||||
lxcbuild alpine3.9-python2.7
|
||||
lxcbuild alpine3.9-python3.6
|
||||
lxcbuild alpine3.9-ruby2.4
|
||||
lxcbuild alpine3.9-ruby2.6
|
||||
lxcbuild alpine3.9-tomcat7
|
||||
lxcbuild alpine3.9-tomcat8.5
|
||||
spoc-image build -p alpine3.8/image
|
||||
spoc-image build -p alpine3.8-php5.6/image
|
||||
spoc-image build -p alpine3.8-ruby2.4/image
|
||||
spoc-image build -p alpine3.9/image
|
||||
spoc-image build -p alpine3.9-java8/image
|
||||
spoc-image build -p alpine3.9-php7.2/image
|
||||
spoc-image build -p alpine3.9-python2.7/image
|
||||
spoc-image build -p alpine3.9-python3.6/image
|
||||
spoc-image build -p alpine3.9-ruby2.4/image
|
||||
spoc-image build -p alpine3.9-ruby2.6/image
|
||||
spoc-image build -p alpine3.9-tomcat7/image
|
||||
spoc-image build -p alpine3.9-tomcat8.5/image
|
||||
|
||||
# Build services
|
||||
cd ${ROOT}/lxc-services
|
||||
lxcbuild activemq
|
||||
lxcbuild mariadb
|
||||
lxcbuild postgres
|
||||
lxcbuild postgis
|
||||
lxcbuild rabbitmq
|
||||
lxcbuild redis
|
||||
lxcbuild solr6
|
||||
spoc-image build -p activemq/image
|
||||
spoc-image build -p mariadb/image
|
||||
spoc-image build -p postgres/image
|
||||
spoc-image build -p postgis/image
|
||||
spoc-image build -p rabbitmq/image
|
||||
spoc-image build -p redis/image
|
||||
spoc-image build -p solr6/image
|
||||
|
||||
# Build applications
|
||||
cd ${ROOT}/lxc-apps
|
||||
lxcbuild ckan
|
||||
lxcbuild crisiscleanup
|
||||
lxcbuild cts
|
||||
lxcbuild decidim
|
||||
lxcbuild ecogis
|
||||
lxcbuild frontlinesms
|
||||
lxcbuild gnuhealth
|
||||
lxcbuild kanboard
|
||||
lxcbuild mifosx
|
||||
lxcbuild motech
|
||||
lxcbuild odoo
|
||||
lxcbuild opendatakit
|
||||
lxcbuild openmapkit
|
||||
lxcbuild pandora
|
||||
lxcbuild sahana
|
||||
lxcbuild sahana-demo
|
||||
lxcbuild sambro
|
||||
lxcbuild seeddms
|
||||
lxcbuild sigmah
|
||||
lxcbuild ushahidi
|
||||
|
||||
spoc-image build -p ckan/image
|
||||
spoc-image build -p ckan-datapusher/image
|
||||
spoc-app publish ckan/app
|
||||
|
||||
spoc-image build -p crisiscleanup/image
|
||||
spoc-app publish crisiscleanup/app
|
||||
|
||||
spoc-image build -p cts/image
|
||||
spoc-app publish cts/app
|
||||
|
||||
spoc-image build -p decidim/image
|
||||
spoc-app publish decidim/app
|
||||
|
||||
spoc-image build -p ecogis/image
|
||||
# spoc-app publish ecogis/app
|
||||
|
||||
spoc-image build -p frontlinesms/image
|
||||
spoc-app publish frontlinesms/app
|
||||
|
||||
spoc-image build -p gnuhealth/image
|
||||
spoc-app publish gnuhealth/app
|
||||
|
||||
spoc-image build -p kanboard/image
|
||||
spoc-app publish kanboard/app
|
||||
|
||||
spoc-image build -p mifosx/image
|
||||
spoc-app publish mifosx/app
|
||||
|
||||
spoc-image build -p motech/image
|
||||
spoc-app publish motech/app
|
||||
|
||||
spoc-image build -p odoo/image
|
||||
spoc-app publish odoo/app
|
||||
|
||||
spoc-image build -p opendatakit/image
|
||||
spoc-image build -p opendatakit-build/image
|
||||
spoc-app publish opendatakit/app
|
||||
|
||||
spoc-image build -p openmapkit/image
|
||||
spoc-app publish openmapkit/app
|
||||
|
||||
spoc-image build -p pandora/image
|
||||
spoc-app publish pandora/app
|
||||
|
||||
spoc-image build -p sahana/image
|
||||
spoc-app publish sahana/app
|
||||
spoc-app publish sahana-demo/app
|
||||
spoc-app publish sambro/app
|
||||
|
||||
spoc-image build -p seeddms/image
|
||||
spoc-app publish seeddms/app
|
||||
|
||||
spoc-image build -p sigmah/image
|
||||
spoc-app publish sigmah/app
|
||||
|
||||
spoc-image build -p ushahidi/image
|
||||
spoc-app publish ushahidi/app
|
||||
|
||||
@ -11,28 +11,7 @@ rm -f /srv/build/vm.tar
|
||||
rm -rf /srv/build/alpine/*
|
||||
|
||||
# Clean built LXC packages
|
||||
rm -rf /srv/build/lxc/apps/*
|
||||
rm -rf /srv/build/lxc/images/*
|
||||
rm -f /srv/build/lxc/packages.sig
|
||||
echo '{"apps":{},"images":{}}' >/srv/build/lxc/packages
|
||||
|
||||
# Stop running containers
|
||||
for SERVICE in $(find /run/openrc/started -name 'lxc-*'); do
|
||||
service $(basename ${SERVICE}) stop
|
||||
done
|
||||
|
||||
# Remove services
|
||||
rm -f /etc/init.d/lxc-*
|
||||
rc-update -u
|
||||
|
||||
# Remove containers
|
||||
rm -rf /var/lib/lxc/*
|
||||
rm -f /var/log/lxc/*
|
||||
|
||||
# Remove application data
|
||||
for DIR in $(find /srv ! -path /srv/build -maxdepth 1 -mindepth 1); do
|
||||
rm -rf ${DIR}
|
||||
done
|
||||
rm -rf /srv/build/spoc
|
||||
|
||||
# Remove nginx configs
|
||||
for CONF in $(find /etc/nginx/conf.d -name '*.conf' -a ! -name repo.conf -a ! -name default.conf); do
|
||||
@ -40,20 +19,11 @@ for CONF in $(find /etc/nginx/conf.d -name '*.conf' -a ! -name repo.conf -a ! -n
|
||||
done
|
||||
service nginx reload
|
||||
|
||||
# Reset /etc/hosts
|
||||
cat <<EOF >/etc/hosts
|
||||
127.0.0.1 localhost
|
||||
::1 localhost
|
||||
172.17.0.1 host
|
||||
172.17.0.1 repo.build.vm
|
||||
EOF
|
||||
# Stop running containers
|
||||
for APP in $(spoc-container list); do
|
||||
spoc-container stop ${APP}
|
||||
done
|
||||
|
||||
# Reset vmmgr config
|
||||
export ADMINPWD=$(python3 -c "import json; f = open('/etc/vmmgr/config.json'); j = json.load(f); print(j['host']['adminpwd'])")
|
||||
envsubst </etc/vmmgr/config.default.json >/etc/vmmgr/config.json
|
||||
|
||||
# Clean locally installed LXC packages
|
||||
rm -rf /var/lib/lxcmgr/storage/*
|
||||
rm -rf /var/lib/lxcmgr/cache/apps/*
|
||||
rm -rf /var/lib/lxcmgr/cache/images/*
|
||||
echo '{"apps":{},"images":{}}' >/var/lib/lxcmgr/packages
|
||||
# Remove data
|
||||
rm -rf /var/lib/spoc
|
||||
rm -rf /var/log/spoc
|
||||
|
||||
@ -5,7 +5,7 @@ cd $(realpath $(dirname "${0}"))
|
||||
|
||||
# Install basic build tools
|
||||
apk update
|
||||
apk add git file htop less openssh-client tar xz
|
||||
apk add git file htop less openssh-client
|
||||
# Install Alpine SDK
|
||||
apk add alpine-sdk
|
||||
# Install Sphinx support
|
||||
@ -13,7 +13,7 @@ apk add py3-sphinx
|
||||
pip3 install recommonmark sphinx-markdown-tables
|
||||
|
||||
# Copy root profile files and settings
|
||||
mkdir -p /root/.config/htop /root/.ssh
|
||||
mkdir -p /root/.config/htop
|
||||
cp root/.profile /root/.profile
|
||||
cp root/.config/htop/htoprc /root/.config/htop/htoprc
|
||||
|
||||
@ -21,11 +21,6 @@ cp root/.config/htop/htoprc /root/.config/htop/htoprc
|
||||
adduser root abuild
|
||||
cp etc/abuild.conf /etc/abuild.conf
|
||||
|
||||
# Prepare LXC build toolchain
|
||||
cp usr/bin/lxcbuild /usr/bin/lxcbuild
|
||||
cp usr/bin/lxcmerge /usr/bin/lxcmerge
|
||||
mkdir -p /srv/build/lxc/apps /srv/build/lxc/images
|
||||
|
||||
# Prepare local APK repository
|
||||
cp etc/nginx/conf.d/repo.conf /etc/nginx/conf.d/repo.conf
|
||||
echo "172.17.0.1 repo.build.vm" >>/etc/hosts
|
||||
@ -38,5 +33,5 @@ echo '{"url":"http://repo.build.vm/lxc","user":"","pwd":""}' >/etc/lxcmgr/repo.j
|
||||
# echo '/srv/build/repokey.rsa' | abuild-keygen
|
||||
|
||||
# Supply LXC build key
|
||||
# openssl ecparam -genkey -name secp384r1 -out /srv/build/packages.key
|
||||
# openssl ec -in /srv/build/packages.key -pubout -out /srv/build/lxc/packages.pub
|
||||
# openssl ecparam -genkey -name secp384r1 -out /etc/spoc/publish.key
|
||||
# openssl ec -in /etc/spoc/publish.key -pubout -out /tmp/repository.pub
|
||||
|
||||
@ -1,79 +0,0 @@
|
||||
#!/usr/bin/python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from lxcbuild.app import App
|
||||
from lxcbuild.image import Image
|
||||
from lxcbuild.imagebuilder import BuildType
|
||||
|
||||
def build_and_pack_image(path, args):
|
||||
image = Image()
|
||||
if args.scratch:
|
||||
image.build_type = BuildType.SCRATCH
|
||||
elif args.force:
|
||||
image.build_type = BuildType.FORCE
|
||||
image.build_and_pack(path)
|
||||
|
||||
def pack_app(path):
|
||||
app = App()
|
||||
app.pack(path)
|
||||
|
||||
def main(args):
|
||||
if args.remove_image:
|
||||
image = Image()
|
||||
image.name = args.buildarg
|
||||
image.remove()
|
||||
elif args.remove_app:
|
||||
app = App()
|
||||
app.name = args.buildarg
|
||||
app.remove()
|
||||
else:
|
||||
buildpath = os.path.realpath(args.buildarg)
|
||||
# If the buildpath is a file, determine type from filename
|
||||
if os.path.isfile(buildpath):
|
||||
basename = os.path.basename(buildpath)
|
||||
if basename == 'lxcfile' or basename.endswith('.lxcfile'):
|
||||
build_and_pack_image(buildpath, args)
|
||||
# Compose files needs to be ignored when performing scratch builds
|
||||
elif not args.scratch and basename == 'meta':
|
||||
pack_app(buildpath)
|
||||
else:
|
||||
print('Unknown file {} given, expected "lxcfile"{}'.format(buildpath, '' if args.scratch else ' or "meta"'))
|
||||
sys.exit(1)
|
||||
# If the buildpath is a directory, build as much as possible, unless scratch build was requested, in which case don't build anything
|
||||
else:
|
||||
if args.scratch:
|
||||
lxcfile = os.path.join(buildpath, 'lxcfile')
|
||||
if os.path.exists(lxcfile):
|
||||
build_and_pack_image(lxcfile, args)
|
||||
else:
|
||||
print('Please specify an lxcfile for scratch build')
|
||||
sys.exit(1)
|
||||
else:
|
||||
valid_dir = False
|
||||
for entry in os.scandir(buildpath):
|
||||
if entry.is_file() and (entry.name == 'lxcfile' or entry.name.endswith('.lxcfile')):
|
||||
valid_dir = True
|
||||
build_and_pack_image(entry.path, args)
|
||||
meta = os.path.join(buildpath, 'meta')
|
||||
if os.path.exists(meta):
|
||||
valid_dir = True
|
||||
pack_app(meta)
|
||||
if not valid_dir:
|
||||
print('Directory {} doesn\'t contain anything to build, skipping'.format(buildpath))
|
||||
|
||||
parser = argparse.ArgumentParser(description='VM application builder and packager')
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
group.add_argument('-f', '--force', action='store_true', help='Force rebuild already built package')
|
||||
group.add_argument('-s', '--scratch', action='store_true', help='Build container for testing purposes, i.e. without cleanup on failure and packaging')
|
||||
group.add_argument('-r', '--remove-image', action='store_true', help='Delete image (including scratch) from build repository')
|
||||
group.add_argument('-e', '--remove-app', action='store_true', help='Delete application from build repository')
|
||||
parser.add_argument('buildarg', help='Either specific "lxcfile" or "meta" file or a directory containing at least one of them')
|
||||
|
||||
args = parser.parse_args()
|
||||
if hasattr(args, 'buildarg'):
|
||||
main(args)
|
||||
else:
|
||||
parser.print_usage()
|
||||
@ -1 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
@ -1,36 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
from .apppacker import AppPacker
|
||||
from .imagebuilder import ImageNotFoundError
|
||||
|
||||
class App:
|
||||
def __init__(self):
|
||||
self.name = None
|
||||
self.conf = {}
|
||||
self.build_dir = None
|
||||
|
||||
def load_metafile(self, metafile):
|
||||
self.build_dir = os.path.dirname(metafile)
|
||||
if os.path.basename(metafile) == 'meta':
|
||||
self.name = os.path.basename(self.build_dir)
|
||||
else:
|
||||
self.name = os.path.splitext(metafile)[0]
|
||||
with open(metafile, 'r') as f:
|
||||
self.conf = json.load(f)
|
||||
|
||||
def pack(self, metafile):
|
||||
self.load_metafile(metafile)
|
||||
packer = AppPacker(self)
|
||||
try:
|
||||
packer.pack()
|
||||
except ImageNotFoundError as e:
|
||||
print('Image {} not found, can\'t pack {}'.format(e, self.name))
|
||||
sys.exit(1)
|
||||
|
||||
def remove(self):
|
||||
packer = AppPacker(self)
|
||||
packer.remove()
|
||||
@ -1,61 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from . import crypto
|
||||
from .imagebuilder import ImageNotFoundError
|
||||
from .packer import Packer
|
||||
from .paths import REPO_APPS_DIR
|
||||
|
||||
class AppPacker(Packer):
|
||||
def __init__(self, app):
|
||||
super().__init__()
|
||||
self.app = app
|
||||
# Prepare package file names
|
||||
self.tar_path = os.path.join(REPO_APPS_DIR, '{}.tar'.format(self.app.name))
|
||||
self.xz_path = '{}.xz'.format(self.tar_path)
|
||||
|
||||
def pack(self):
|
||||
# Check if all images used by containers exist
|
||||
for container in self.app.conf['containers']:
|
||||
image = self.app.conf['containers'][container]['image']
|
||||
if image not in self.packages['images']:
|
||||
raise ImageNotFoundError(image)
|
||||
try:
|
||||
os.unlink(self.xz_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
self.create_archive()
|
||||
self.register()
|
||||
self.sign_packages()
|
||||
|
||||
def remove(self):
|
||||
self.unregister()
|
||||
try:
|
||||
os.unlink(self.xz_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
def create_archive(self):
|
||||
# Create archive with application setup scripts
|
||||
print('Archiving setup scripts for', self.app.name)
|
||||
scripts = ('install', 'install.sh', 'upgrade', 'upgrade.sh', 'uninstall', 'uninstall.sh')
|
||||
scripts = [s for s in scripts if os.path.exists(os.path.join(self.app.build_dir, s))]
|
||||
subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, '--transform', 's,^,{}/,'.format(self.app.name)] + scripts, cwd=self.app.build_dir)
|
||||
self.compress_archive()
|
||||
|
||||
def register(self):
|
||||
# Register package in global repository metadata file
|
||||
print('Registering application package', self.app.name)
|
||||
self.packages['apps'][self.app.name] = self.app.conf.copy()
|
||||
self.packages['apps'][self.app.name]['size'] = self.tar_size
|
||||
self.packages['apps'][self.app.name]['pkgsize'] = self.xz_size
|
||||
self.packages['apps'][self.app.name]['sha512'] = crypto.hash_file(self.xz_path)
|
||||
self.save_repo_meta()
|
||||
|
||||
def unregister(self):
|
||||
# Removes package from global repository metadata file
|
||||
if self.app.name in self.packages['apps']:
|
||||
del self.packages['apps'][self.app.name]
|
||||
self.save_repo_meta()
|
||||
@ -1,28 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import hashlib
|
||||
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.hazmat.primitives.serialization import load_pem_private_key
|
||||
|
||||
def sign_file(private_key_path, input_path):
|
||||
# Generate SHA512 signature of a file using EC private key
|
||||
print('Signing packages')
|
||||
with open(private_key_path, 'rb') as f:
|
||||
priv_key = load_pem_private_key(f.read(), None, default_backend())
|
||||
with open(input_path, 'rb') as f:
|
||||
data = f.read()
|
||||
return priv_key.sign(data, ec.ECDSA(hashes.SHA512()))
|
||||
|
||||
def hash_file(file_path):
|
||||
# Calculate SHA512 hash of a file
|
||||
sha512 = hashlib.sha512()
|
||||
with open(file_path, 'rb') as f:
|
||||
while True:
|
||||
data = f.read(65536)
|
||||
if not data:
|
||||
break
|
||||
sha512.update(data)
|
||||
return sha512.hexdigest()
|
||||
@ -1,60 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from lxcmgr import lxcmgr
|
||||
|
||||
from .imagebuilder import BuildType, ImageBuilder, ImageExistsError, ImageNotFoundError
|
||||
from .imagepacker import ImagePacker
|
||||
from .packer import PackageExistsError
|
||||
|
||||
class Image:
|
||||
def __init__(self):
|
||||
self.name = None
|
||||
self.conf = {}
|
||||
self.lxcfile = None
|
||||
self.build_dir = None
|
||||
self.build_type = BuildType.NORMAL
|
||||
self.pack = False
|
||||
|
||||
def build_and_pack(self, lxcfile):
|
||||
self.lxcfile = lxcfile
|
||||
self.build_dir = os.path.dirname(lxcfile)
|
||||
self.conf['build'] = True
|
||||
builder = ImageBuilder(self)
|
||||
try:
|
||||
builder.build()
|
||||
# Packaging needs to happen in any case after a successful build in order to prevent outdated packages
|
||||
self.pack = True
|
||||
except ImageExistsError as e:
|
||||
# If container already exists and build hasn't been forced, rerun the build just for metadata which are still needed for packaging
|
||||
print('Image {} already exists, skipping build tasks'.format(e))
|
||||
self.build_type = BuildType.METADATA
|
||||
builder.build()
|
||||
except ImageNotFoundError as e:
|
||||
# If one of the layers is missing, cleanup and die
|
||||
print('Image {} not found, can\'t build {}'.format(e, self.name))
|
||||
builder.clean()
|
||||
sys.exit(1)
|
||||
except:
|
||||
# If build fails with another exception, cleanup (unless we were doing scratch build) and re-raise
|
||||
if not self.build_type == BuildType.SCRATCH:
|
||||
builder.clean()
|
||||
raise
|
||||
del self.conf['build']
|
||||
# If we're doing a scratch build, regenerate the final LXC container configuration including ephemeral layer
|
||||
if self.build_type == BuildType.SCRATCH:
|
||||
lxcmgr.create_container(self.name, self.conf)
|
||||
else:
|
||||
try:
|
||||
packer = ImagePacker(self)
|
||||
packer.pack()
|
||||
except PackageExistsError as e:
|
||||
print('Package {} already exists, skipping packaging tasks'.format(e))
|
||||
|
||||
def remove(self):
|
||||
builder = ImageBuilder(self)
|
||||
builder.clean()
|
||||
packer = ImagePacker(self)
|
||||
packer.remove()
|
||||
@ -1,203 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from lxcmgr import lxcmgr
|
||||
from lxcmgr.paths import LXC_STORAGE_DIR
|
||||
from lxcmgr.pkgmgr import PkgMgr
|
||||
|
||||
class ImageExistsError(Exception):
|
||||
pass
|
||||
|
||||
class ImageNotFoundError(Exception):
|
||||
pass
|
||||
|
||||
class BuildType:
|
||||
NORMAL = 1
|
||||
FORCE = 2
|
||||
SCRATCH = 3
|
||||
METADATA = 4
|
||||
|
||||
class ImageBuilder:
|
||||
def __init__(self, image):
|
||||
self.image = image
|
||||
self.script = []
|
||||
self.script_eof = None
|
||||
|
||||
def build(self):
|
||||
# Read and process lines from lxcfile
|
||||
with open(self.image.lxcfile, 'r') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if self.script_eof:
|
||||
if line == self.script_eof:
|
||||
self.script_eof = None
|
||||
self.run_script(self.script)
|
||||
else:
|
||||
self.script.append(line)
|
||||
elif line:
|
||||
self.process_line(*line.split(None, 1))
|
||||
|
||||
def process_line(self, directive, args):
|
||||
# Process directives from lxcfile
|
||||
if 'RUN' == directive:
|
||||
self.script = []
|
||||
self.script_eof = args
|
||||
elif 'IMAGE' == directive:
|
||||
self.set_name(args)
|
||||
elif 'FROM' == directive:
|
||||
self.set_layers(args)
|
||||
elif 'COPY' == directive:
|
||||
srcdst = args.split()
|
||||
self.copy_files(srcdst[0], srcdst[1] if len(srcdst) == 2 else '')
|
||||
elif 'ENV' == directive:
|
||||
self.add_env(*args.split(None, 1))
|
||||
elif 'USER' == directive:
|
||||
self.set_user(*args.split())
|
||||
elif 'CMD' == directive:
|
||||
self.set_cmd(args)
|
||||
elif 'WORKDIR' == directive:
|
||||
self.set_cwd(args)
|
||||
elif 'HALT' == directive:
|
||||
self.set_halt(args)
|
||||
elif 'READY' == directive:
|
||||
self.set_ready(args)
|
||||
|
||||
def get_layer_path(self, layer):
|
||||
return os.path.join(LXC_STORAGE_DIR, layer)
|
||||
|
||||
def run_script(self, script):
|
||||
# Creates a temporary container, runs a script in its namespace, and stores the modifications as part of the image
|
||||
if self.image.build_type == BuildType.METADATA:
|
||||
# Don't run anything if we're building just metadata
|
||||
return
|
||||
lxcmgr.create_container(self.image.name, self.image.conf)
|
||||
sh = os.path.join(LXC_STORAGE_DIR, self.image.name, 'run.sh')
|
||||
with open(sh, 'w') as f:
|
||||
f.write('#!/bin/sh\nset -ev\n\n{}\n'.format('\n'.join(script)))
|
||||
os.chmod(sh, 0o700)
|
||||
os.chown(sh, 100000, 100000)
|
||||
subprocess.run(['lxc-execute', self.image.name, '--', '/bin/sh', '-lc', '/run.sh'], check=True)
|
||||
os.unlink(sh)
|
||||
if not self.image.build_type == BuildType.SCRATCH:
|
||||
# Don't delete the temporary container if we're doing scratch build
|
||||
lxcmgr.destroy_container(self.image.name)
|
||||
|
||||
def set_name(self, name):
|
||||
# Set name and first (topmost) layer of the image
|
||||
self.image.name = name
|
||||
self.image.conf['layers'] = [name]
|
||||
if self.image.build_type == BuildType.METADATA:
|
||||
# Don't check or create any directories if we're building just metadata
|
||||
return
|
||||
image_path = self.get_layer_path(name)
|
||||
if os.path.exists(image_path):
|
||||
if self.image.build_type in (BuildType.FORCE, BuildType.SCRATCH):
|
||||
self.clean()
|
||||
else:
|
||||
raise ImageExistsError(image_path)
|
||||
os.makedirs(image_path, 0o755, True)
|
||||
os.chown(image_path, 100000, 100000)
|
||||
|
||||
def set_layers(self, image):
|
||||
# Prepend list of layers from parent image
|
||||
pkgmgr = PkgMgr()
|
||||
self.image.conf['layers'] = pkgmgr.installed_packages['images'][image]['layers'] + [self.image.name]
|
||||
|
||||
def copy_files(self, src, dst):
|
||||
# Copy files from the host or download them from a http(s) URL
|
||||
if self.image.build_type == BuildType.METADATA:
|
||||
# Don't copy anything if we're building just metadata
|
||||
return
|
||||
dst = os.path.join(LXC_STORAGE_DIR, self.image.name, dst)
|
||||
if src.startswith('http://') or src.startswith('https://'):
|
||||
unpack_http_archive(src, dst)
|
||||
else:
|
||||
copy_tree(os.path.join(self.image.build_dir, src), dst)
|
||||
# Shift UID/GID of the files to the unprivileged range
|
||||
shift_uid(dst)
|
||||
|
||||
def add_env(self, key, value):
|
||||
# Sets lxc.environment records for the image
|
||||
if 'env' not in self.image.conf:
|
||||
self.image.conf['env'] = []
|
||||
self.image.conf['env'].append([key, value])
|
||||
|
||||
def set_user(self, uid, gid):
|
||||
# Sets lxc.init.uid/gid for the image
|
||||
self.image.conf['uid'] = uid
|
||||
self.image.conf['gid'] = gid
|
||||
|
||||
def set_cmd(self, cmd):
|
||||
# Sets lxc.init.cmd for the image
|
||||
self.image.conf['cmd'] = cmd
|
||||
|
||||
def set_cwd(self, cwd):
|
||||
# Sets lxc.init.cwd for the image
|
||||
self.image.conf['cwd'] = cwd
|
||||
|
||||
def set_halt(self, halt):
|
||||
# Sets lxc.signal.halt for the image
|
||||
self.image.conf['halt'] = halt
|
||||
|
||||
def set_ready(self, cmd):
|
||||
# Sets a command performed in OpenRC start_post to check readiness of the container
|
||||
self.image.conf['ready'] = cmd
|
||||
|
||||
def clean(self):
|
||||
lxcmgr.destroy_container(self.image.name)
|
||||
try:
|
||||
shutil.rmtree(self.get_layer_path(self.image.name))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
def unpack_http_archive(src, dst):
|
||||
# Decompress an archive downloaded via http(s)
|
||||
xf = 'xzf'
|
||||
if src.endswith('.bz2'):
|
||||
xf = 'xjf'
|
||||
elif src.endswith('.xz'):
|
||||
xf = 'xJf'
|
||||
with subprocess.Popen(['wget', src, '-O', '-'], stdout=subprocess.PIPE) as wget:
|
||||
with subprocess.Popen(['tar', xf, '-', '-C', dst], stdin=wget.stdout) as tar:
|
||||
wget.stdout.close()
|
||||
tar.wait()
|
||||
|
||||
def copy_tree(src, dst):
|
||||
# Copies files from the host
|
||||
if not os.path.isdir(src):
|
||||
shutil.copy2(src, dst)
|
||||
else:
|
||||
os.makedirs(dst, exist_ok=True)
|
||||
for name in os.listdir(src):
|
||||
copy_tree(os.path.join(src, name), os.path.join(dst, name))
|
||||
shutil.copystat(src, dst)
|
||||
|
||||
def shift_uid(dir):
|
||||
# Shifts UID/GID of a file or a directory and its contents to the unprivileged range
|
||||
shift_uid_entry(dir, os.stat(dir, follow_symlinks=True))
|
||||
shift_uid_recursively(dir)
|
||||
|
||||
def shift_uid_recursively(dir):
|
||||
# Shifts UID/GID of a directory and its contents to the unprivileged range
|
||||
for entry in os.scandir(dir):
|
||||
shift_uid_entry(entry.path, entry.stat(follow_symlinks=False))
|
||||
if entry.is_dir():
|
||||
shift_uid_recursively(entry.path)
|
||||
|
||||
def shift_uid_entry(path, stat):
|
||||
# Shifts UID/GID of a file or a directory to the unprivileged range
|
||||
uid = stat.st_uid
|
||||
gid = stat.st_gid
|
||||
do_chown = False
|
||||
if uid < 100000:
|
||||
uid = uid + 100000
|
||||
do_chown = True
|
||||
if gid < 100000:
|
||||
gid = gid + 100000
|
||||
do_chown = True
|
||||
if do_chown:
|
||||
os.lchown(path, uid, gid)
|
||||
@ -1,67 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from lxcmgr.paths import LXC_STORAGE_DIR
|
||||
from lxcmgr.pkgmgr import PkgMgr
|
||||
|
||||
from . import crypto
|
||||
from .packer import PackageExistsError, Packer
|
||||
from .paths import REPO_IMAGES_DIR
|
||||
|
||||
class ImagePacker(Packer):
|
||||
def __init__(self, image):
|
||||
super().__init__()
|
||||
self.image = image
|
||||
# Prepare package file names
|
||||
self.tar_path = os.path.join(REPO_IMAGES_DIR, '{}.tar'.format(self.image.name))
|
||||
self.xz_path = '{}.xz'.format(self.tar_path)
|
||||
|
||||
def pack(self):
|
||||
if self.image.pack:
|
||||
self.unregister()
|
||||
try:
|
||||
os.unlink(self.xz_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
elif os.path.exists(self.xz_path):
|
||||
raise PackageExistsError(self.xz_path)
|
||||
self.create_archive()
|
||||
self.register()
|
||||
self.sign_packages()
|
||||
|
||||
def remove(self):
|
||||
self.unregister()
|
||||
try:
|
||||
os.unlink(self.xz_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
def create_archive(self):
|
||||
# Create archive
|
||||
print('Archiving image', self.image.name)
|
||||
subprocess.run(['tar', '--xattrs', '-cpf', self.tar_path, self.image.name], cwd=LXC_STORAGE_DIR)
|
||||
self.compress_archive()
|
||||
|
||||
def register(self):
|
||||
# Register image in global repository metadata file
|
||||
print('Registering image package', self.image.name)
|
||||
image_conf = self.image.conf.copy()
|
||||
image_conf['size'] = self.tar_size
|
||||
image_conf['pkgsize'] = self.xz_size
|
||||
image_conf['sha512'] = crypto.hash_file(self.xz_path)
|
||||
self.packages['images'][self.image.name] = image_conf
|
||||
self.save_repo_meta()
|
||||
# Register the image also to locally installed images for package manager
|
||||
pm = PkgMgr()
|
||||
pm.register_image(self.image.name, self.packages['images'][self.image.name])
|
||||
|
||||
def unregister(self):
|
||||
# Removes package from global repository metadata file
|
||||
if self.image.name in self.packages['images']:
|
||||
del self.packages['images'][self.image.name]
|
||||
self.save_repo_meta()
|
||||
# Unregister the image also from locally installed images for package manager
|
||||
pm = PkgMgr()
|
||||
pm.unregister_image(self.image.name)
|
||||
@ -1,40 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from . import crypto
|
||||
from .paths import PRIVATE_KEY, REPO_META_FILE, REPO_SIG_FILE
|
||||
|
||||
class PackageExistsError(Exception):
|
||||
pass
|
||||
|
||||
class Packer:
|
||||
def __init__(self):
|
||||
self.tar_path = None
|
||||
self.tar_size = 0
|
||||
self.xz_path = None
|
||||
self.xz_size = 0
|
||||
if os.path.exists(REPO_META_FILE):
|
||||
with open(REPO_META_FILE, 'r') as f:
|
||||
self.packages = json.load(f)
|
||||
else:
|
||||
self.packages = {'apps': {}, 'images': {}}
|
||||
|
||||
def save_repo_meta(self):
|
||||
with open(REPO_META_FILE, 'w') as f:
|
||||
json.dump(self.packages, f, sort_keys=True, indent=4)
|
||||
|
||||
def compress_archive(self):
|
||||
# Compress the tarball with xz (LZMA2)
|
||||
self.tar_size = os.path.getsize(self.tar_path)
|
||||
print('Compressing', self.tar_path, '({:.2f} MB)'.format(self.tar_size/1048576))
|
||||
subprocess.run(['xz', '-9', self.tar_path])
|
||||
self.xz_size = os.path.getsize(self.xz_path)
|
||||
print('Compressed ', self.xz_path, '({:.2f} MB)'.format(self.xz_size/1048576))
|
||||
|
||||
def sign_packages(self):
|
||||
signature = crypto.sign_file(PRIVATE_KEY, REPO_META_FILE)
|
||||
with open(REPO_SIG_FILE, 'wb') as f:
|
||||
f.write(signature)
|
||||
@ -1,7 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
PRIVATE_KEY = '/srv/build/packages.key'
|
||||
REPO_APPS_DIR = '/srv/build/lxc/apps'
|
||||
REPO_IMAGES_DIR = '/srv/build/lxc/images'
|
||||
REPO_META_FILE = '/srv/build/lxc/packages'
|
||||
REPO_SIG_FILE = '/srv/build/lxc/packages.sig'
|
||||
Loading…
Reference in New Issue
Block a user