From a25186b5eadbd2781f6c756c9b0e391d0516460c Mon Sep 17 00:00:00 2001 From: Disassembler Date: Wed, 12 Sep 2018 16:08:10 +0200 Subject: [PATCH] Dockerize the lxc-build a bit more --- activemq/lxcfile | 4 ++-- basic-runtimes/alpine.lxcfile | 4 ++-- basic-runtimes/java.lxcfile | 4 ++-- basic-runtimes/libxml.lxcfile | 4 ++-- basic-runtimes/nodejs.lxcfile | 4 ++-- basic-runtimes/php.lxcfile | 4 ++-- basic-runtimes/python2.lxcfile | 4 ++-- basic-runtimes/python3.lxcfile | 4 ++-- basic-runtimes/ruby.lxcfile | 4 ++-- basic-runtimes/tomcat.lxcfile | 4 ++-- build-all.sh | 5 +++-- ckan-datapusher/lxcfile | 4 ++-- ckan/lxcfile | 4 ++-- postgres/lxcfile | 4 ++-- redis/lxcfile | 4 ++-- solr/lxcfile | 4 ++-- lxc-build => zz-extra/lxc-build | 40 ++++++++++++++++----------------- 17 files changed, 53 insertions(+), 52 deletions(-) rename lxc-build => zz-extra/lxc-build (86%) diff --git a/activemq/lxcfile b/activemq/lxcfile index 52e45b5..a20ce2f 100644 --- a/activemq/lxcfile +++ b/activemq/lxcfile @@ -3,7 +3,7 @@ LAYER shared/alpine LAYER shared/java LAYER activemq/activemq -SCRIPT +RUN EOF # Download and install ActiveMQ wget http://archive.apache.org/dist/activemq/5.15.5/apache-activemq-5.15.5-bin.tar.gz -O - | tar xzf - -C /srv mv /srv/apache-activemq-5.15.5 /srv/activemq @@ -16,7 +16,7 @@ SCRIPT # Configure Java heap size sed -i "s/-Xms64M -Xmx1G/-Xms32M -Xmx256M/" /srv/activemq/bin/env -RUN +EOF COPY lxc diff --git a/basic-runtimes/alpine.lxcfile b/basic-runtimes/alpine.lxcfile index 5aba891..b8dcc5d 100644 --- a/basic-runtimes/alpine.lxcfile +++ b/basic-runtimes/alpine.lxcfile @@ -1,6 +1,6 @@ IMAGE build LAYER shared/alpine -SCRIPT +RUN EOF apk --no-cache add s6 -RUN +EOF diff --git a/basic-runtimes/java.lxcfile b/basic-runtimes/java.lxcfile index bd83c79..3c49efe 100644 --- a/basic-runtimes/java.lxcfile +++ b/basic-runtimes/java.lxcfile @@ -2,6 +2,6 @@ IMAGE build LAYER shared/alpine LAYER shared/java -SCRIPT +RUN EOF apk --no-cache add openjdk8-jre-base -RUN +EOF diff --git a/basic-runtimes/libxml.lxcfile b/basic-runtimes/libxml.lxcfile index e92d22d..7de2b67 100644 --- a/basic-runtimes/libxml.lxcfile +++ b/basic-runtimes/libxml.lxcfile @@ -2,6 +2,6 @@ IMAGE build LAYER shared/alpine LAYER shared/libxml -SCRIPT +RUN EOF apk --no-cache add libxml2 libxslt -RUN +EOF diff --git a/basic-runtimes/nodejs.lxcfile b/basic-runtimes/nodejs.lxcfile index 3f66221..cde2a13 100644 --- a/basic-runtimes/nodejs.lxcfile +++ b/basic-runtimes/nodejs.lxcfile @@ -2,6 +2,6 @@ IMAGE build LAYER shared/alpine LAYER shared/nodejs -SCRIPT +RUN EOF apk --no-cache add nodejs -RUN +EOF diff --git a/basic-runtimes/php.lxcfile b/basic-runtimes/php.lxcfile index 47158e4..912e547 100644 --- a/basic-runtimes/php.lxcfile +++ b/basic-runtimes/php.lxcfile @@ -2,6 +2,6 @@ IMAGE build LAYER shared/alpine LAYER shared/php -SCRIPT +RUN EOF apk --no-cache add nginx php7 php7-ctype php7-fpm php7-gd php7-json php7-mbstring php7-mcrypt php7-opcache php7-session -RUN +EOF diff --git a/basic-runtimes/python2.lxcfile b/basic-runtimes/python2.lxcfile index 443b885..dd3efce 100644 --- a/basic-runtimes/python2.lxcfile +++ b/basic-runtimes/python2.lxcfile @@ -3,6 +3,6 @@ LAYER shared/alpine LAYER shared/libxml LAYER shared/python2 -SCRIPT +RUN EOF apk --no-cache add python2 -RUN +EOF diff --git a/basic-runtimes/python3.lxcfile b/basic-runtimes/python3.lxcfile index 155039e..46b840a 100644 --- a/basic-runtimes/python3.lxcfile +++ b/basic-runtimes/python3.lxcfile @@ -3,7 +3,7 @@ LAYER shared/alpine LAYER shared/libxml LAYER shared/python3 -SCRIPT +RUN EOF apk --no-cache add python3 ln -s /usr/bin/python3 /usr/bin/python -RUN +EOF diff --git a/basic-runtimes/ruby.lxcfile b/basic-runtimes/ruby.lxcfile index 1a2243c..82fe408 100644 --- a/basic-runtimes/ruby.lxcfile +++ b/basic-runtimes/ruby.lxcfile @@ -2,7 +2,7 @@ IMAGE build LAYER shared/alpine LAYER shared/ruby -SCRIPT +RUN EOF # Install Ruby runtime dependencies apk --no-cache add gdbm libressl readline zlib @@ -35,4 +35,4 @@ SCRIPT rm -r /usr/src/ruby apk --no-cache del .deps rm -rf /root/.gem -RUN +EOF diff --git a/basic-runtimes/tomcat.lxcfile b/basic-runtimes/tomcat.lxcfile index c37d981..23bfaa8 100644 --- a/basic-runtimes/tomcat.lxcfile +++ b/basic-runtimes/tomcat.lxcfile @@ -3,7 +3,7 @@ LAYER shared/alpine LAYER shared/java LAYER shared/tomcat -SCRIPT +RUN EOF # Install Tomcat 8 wget http://mirror.hosting90.cz/apache/tomcat/tomcat-8/v8.0.53/bin/apache-tomcat-8.0.53.tar.gz -O - | tar xzf - -C /srv mv /srv/apache-tomcat-8.0.53 /srv/tomcat @@ -13,6 +13,6 @@ SCRIPT # Cleanup rm -rf /srv/tomcat/webapps/ROOT /srv/tomcat/webapps/docs /srv/tomcat/webapps/examples /srv/tomcat/webapps/host-manager /srv/tomcat/webapps/manager -RUN +EOF COPY lxc-tomcat diff --git a/build-all.sh b/build-all.sh index fa1cfa7..839c579 100755 --- a/build-all.sh +++ b/build-all.sh @@ -3,8 +3,9 @@ set -e SOURCE_DIR=$(realpath $(dirname "${0}")) -# Alias lxc-build -alias lxc-build=${SOURCE_DIR}/lxc-build +# Install build scripts +cp ${SOURCE_DIR}/zz-extra/lxc-build /usr/bin/lxc-build +cp ${SOURCE_DIR}/zz-extra/fix-apk /usr/bin/fix-apk # Build basic Alpine LXC image mkdir -p /var/lib/lxc/shared/alpine diff --git a/ckan-datapusher/lxcfile b/ckan-datapusher/lxcfile index e6c6f10..448ed47 100644 --- a/ckan-datapusher/lxcfile +++ b/ckan-datapusher/lxcfile @@ -4,7 +4,7 @@ LAYER shared/libxml LAYER shared/python2 LAYER ckan-datapusher/ckan-datapusher -SCRIPT +RUN EOF # Install runtime dependencies apk --no-cache add libffi libressl uwsgi-python @@ -33,7 +33,7 @@ SCRIPT apk --no-cache del .deps find /srv/ckan-datapusher/src -name '.git*' -exec rm -rf {} + rm -rf /root/.cache -RUN +EOF COPY lxc diff --git a/ckan/lxcfile b/ckan/lxcfile index d9fd162..742cae5 100644 --- a/ckan/lxcfile +++ b/ckan/lxcfile @@ -4,7 +4,7 @@ LAYER shared/libxml LAYER shared/python2 LAYER ckan/ckan -SCRIPT +RUN EOF # Add edge/testing repository echo '@edge http://dl-cdn.alpinelinux.org/alpine/edge/testing' >>/etc/apk/repositories @@ -49,7 +49,7 @@ SCRIPT apk --no-cache del .deps find /srv/ckan/src -name '.git*' -exec rm -rf {} + rm -rf /root/.cache -RUN +EOF MOUNT /srv/ckan/conf etc/ckan MOUNT /srv/ckan/data srv/ckan/storage diff --git a/postgres/lxcfile b/postgres/lxcfile index ee04f87..64dbac1 100644 --- a/postgres/lxcfile +++ b/postgres/lxcfile @@ -2,7 +2,7 @@ IMAGE postgres LAYER shared/alpine LAYER postgres/postgres -SCRIPT +RUN EOF # Modify OS user (which will be picked up later by apk add) sed -i 's/postgres:x:70:70/postgres:x:5432:5432/' /etc/passwd sed -i 's/postgres:x:70/postgres:x:5432/' /etc/group @@ -16,7 +16,7 @@ SCRIPT # Create socket directory mkdir /run/postgresql chown postgres:postgres /run/postgresql -RUN +EOF MOUNT /srv/postgres/data var/lib/postgresql diff --git a/redis/lxcfile b/redis/lxcfile index e9a2ee1..b13e99a 100644 --- a/redis/lxcfile +++ b/redis/lxcfile @@ -2,14 +2,14 @@ IMAGE redis LAYER shared/alpine LAYER redis/redis -SCRIPT +RUN EOF # Create OS user (which will be picked up later by apk add) addgroup -S -g 6379 redis adduser -S -u 6379 -h /var/lib/redis -s /bin/false -g redis -G redis redis # Install Redis apk --no-cache add redis -RUN +EOF MOUNT /srv/redis/conf/redis.conf etc/redis.conf MOUNT /srv/redis/data var/lib/redis diff --git a/solr/lxcfile b/solr/lxcfile index 8e2eab8..328e960 100644 --- a/solr/lxcfile +++ b/solr/lxcfile @@ -3,7 +3,7 @@ LAYER shared/alpine LAYER shared/java LAYER solr/solr -SCRIPT +RUN EOF # Install runtime dependencies apk --no-cache add bash lsof @@ -19,7 +19,7 @@ SCRIPT # Make start/stop script visible globally (also defines directory for solr.in.sh) ln -s /opt/solr/bin/solr /usr/bin/solr -RUN +EOF COPY lxc diff --git a/lxc-build b/zz-extra/lxc-build similarity index 86% rename from lxc-build rename to zz-extra/lxc-build index a6236ad..882b28e 100755 --- a/lxc-build +++ b/zz-extra/lxc-build @@ -27,6 +27,7 @@ lxc.mount.entry = /etc/resolv.conf etc/resolv.conf none bind 0 0 lxc.init.cmd = {cmd} lxc.init.uid = {uid} lxc.init.gid = {gid} +lxc.init.cwd = {cwd} # Environment {env} @@ -55,6 +56,7 @@ class LXCImage: self.uid = 0 self.gid = 0 self.cmd = '/bin/true' + self.cwd = '/' if os.path.isfile(build_path): self.lxcfile = os.path.realpath(build_path) @@ -67,22 +69,24 @@ class LXCImage: with open(self.lxcfile, 'r') as fd: lxcfile = [l.strip() for l in fd.readlines()] - in_script = False script = [] + script_eof = None for line in lxcfile: - if line == 'RUN': - in_script = False + if script_eof and line == script_eof: + script_eof = None self.run_script(script) elif in_script: script.append(line) - elif line == 'SCRIPT': + elif line.startswith('RUN'): script = [] - in_script = True + script_eof = line.split()[1] elif line.startswith('IMAGE'): self.set_name(line.split()[1]) elif line.startswith('LAYER'): self.add_layer(line.split()[1]) + elif line.startswith('LAYERFIX'): + self.fix_layers(line.split()[1]) elif line.startswith('COPY'): srcdst = line.split() self.copy_files(srcdst[1], srcdst[2] if len(srcdst) == 3 else '') @@ -97,7 +101,9 @@ class LXCImage: self.set_user(uidgid[1], uidgid[2]) elif line.startswith('CMD'): self.set_cmd(' '.join(line.split()[1:])) - # Add the final layer which can be treated as nonpersistent + elif line.startswith('WORKDIR'): + self.set_cwd(line.split()[1]) + # Add the final layer which can be treated as ephemeral self.add_layer('{}/delta0'.format(self.name)) def rebuild_config(self): @@ -109,7 +115,7 @@ class LXCImage: mount_entries = '\n'.join(['lxc.mount.entry = {} none bind 0 0'.format(m) for m in self.mounts]) env_entries = '\n'.join(['lxc.environment = {}'.format(e) for e in self.env]) with open(os.path.join(LXC_ROOT, self.name, 'config'), 'w') as fd: - fd.write(CONFIG_TEMPLATE.format(name=self.name, rootfs=rootfs_path, mounts=mount_entries, env=env_entries, uid=self.uid, gid=self.gid, cmd=self.cmd)) + fd.write(CONFIG_TEMPLATE.format(name=self.name, rootfs=rootfs_path, mounts=mount_entries, env=env_entries, uid=self.uid, gid=self.gid, cmd=self.cmd, cwd=self.cwd)) def run_script(self, script): sh = os.path.join(self.layers[-1], 'run.sh') @@ -129,6 +135,9 @@ class LXCImage: os.makedirs(layer, 0o755, True) self.rebuild_config() + def fix_layers(self, cmd): + subprocess.run([cmd]+self.layers, check=True) + def copy_files(self, src, dst): src = os.path.join(self.build_dir, src) dst = os.path.join(self.layers[-1], dst) @@ -151,6 +160,10 @@ class LXCImage: self.cmd = cmd self.rebuild_config() + def set_cwd(self, cwd): + self.cwd = cwd + self.rebuild_config() + def copy_tree(src, dst): if not os.path.isdir(src): shutil.copy2(src, dst) @@ -160,19 +173,6 @@ def copy_tree(src, dst): copy_tree(os.path.join(src, name), os.path.join(dst, name)) shutil.copystat(src, dst) -# def fix_world(): -# world_items = [] -# last_world = [] -# for layer in layers[:-1]: -# with open(os.path.join(layer, 'etc/apk/world'), 'r') as fd: -# last_world = fd.read().splitlines() -# world_items.extend(last_world) -# world_items = sorted(set(world_items)) -# if world_items != sorted(last_world): -# os.makedirs(os.path.join(layers[-1], 'etc/apk', 0o755, True)) -# with open(os.path.join(layers[-1], 'etc/apk/world'), 'w') as fd: -# fd.writelines(world_items) - if __name__ == '__main__': if len(sys.argv) != 2: print('Usage: lxc-build \n where the buildpath can be either specific lxcfile or a directory containing one')