From 9dbd7492a1fd6064408ba0da9e6530d595cb725b Mon Sep 17 00:00:00 2001 From: Disassembler Date: Thu, 18 Jan 2018 01:21:11 +0100 Subject: [PATCH] Add the real deal --- Dockerfile | 126 +++++++++++++++++++ README.md | 84 +++++++++++++ docker/etc/nginx/nginx.conf | 81 ++++++++++++ docker/etc/services.d/.s6-svscan/finish | 3 + docker/etc/services.d/nginx/run | 3 + docker/etc/services.d/pandora-cron/run | 5 + docker/etc/services.d/pandora-encoding/run | 5 + docker/etc/services.d/pandora-tasks/run | 5 + docker/etc/services.d/pandora-websocketd/run | 5 + docker/etc/services.d/pandora/run | 5 + 10 files changed, 322 insertions(+) create mode 100644 README.md create mode 100644 docker/etc/nginx/nginx.conf create mode 100755 docker/etc/services.d/.s6-svscan/finish create mode 100755 docker/etc/services.d/nginx/run create mode 100755 docker/etc/services.d/pandora-cron/run create mode 100755 docker/etc/services.d/pandora-encoding/run create mode 100755 docker/etc/services.d/pandora-tasks/run create mode 100755 docker/etc/services.d/pandora-websocketd/run create mode 100755 docker/etc/services.d/pandora/run diff --git a/Dockerfile b/Dockerfile index e69de29..1ca3938 100644 --- a/Dockerfile +++ b/Dockerfile @@ -0,0 +1,126 @@ +FROM alpine:3.7 +MAINTAINER Disassembler + +RUN \ + # Install runtime dependencies + apk --no-cache add \ + ffmpeg \ + imagemagick \ + imlib2 \ + libogg \ + libtheora \ + libvpx \ + libxml2 \ + libxslt \ + mkvtoolnix \ + nginx \ + poppler-utils \ + py3-psycopg2 \ + # py3-pillow \ + py3-numpy \ + py3-geoip \ + py3-lxml \ + python3 \ + s6 \ + && pip3 install \ + pyinotify \ + youtube-dl \ + && ln -s /usr/bin/python3 /usr/bin/python + +RUN \ + # Install build dependencies + apk --no-cache add --virtual .deps \ + autoconf \ + automake \ + build-base \ + flac-dev \ + git \ + imlib2-dev \ + libogg-dev \ + libtheora-dev \ + libtool \ + libvpx-dev \ + libvorbis-dev \ + # Compile liboggz + && wget https://ftp.osuosl.org/pub/xiph/releases/liboggz/liboggz-1.1.1.tar.gz -O /tmp/liboggz.tgz \ + && tar xf /tmp/liboggz.tgz -C /tmp \ + && cd /tmp/liboggz-1.1.1 \ + && ./configure \ + && make -j $(nproc) \ + && make install \ + # Compile libfishsound + && wget https://ftp.osuosl.org/pub/xiph/releases/libfishsound/libfishsound-1.0.0.tar.gz -O /tmp/libfishsound.tgz \ + && tar xf /tmp/libfishsound.tgz -C /tmp/ \ + && cd /tmp/libfishsound-1.0.0 \ + && ./configure \ + && make -j $(nproc) \ + && make install \ + # Compile liboggplay + && git clone --depth 1 git://git.xiph.org/liboggplay.git /tmp/liboggplay \ + && cd /tmp/liboggplay \ + && ./autogen.sh \ + && ./configure \ + && make -j $(nproc) \ + && make install \ + # Compile oxframe (without man pages) + && git clone --depth 1 https://code.0x2620.org/0x2620/oxframe /tmp/oxframe \ + && cd /tmp/oxframe \ + && sed -i '/man\/oxframe/d' Makefile \ + && make \ + && make install \ + && cd / \ + # Clone Pandora git repositories + && git clone --depth 1 https://git.0x2620.org/pandora.git /srv/pandora \ + && git clone --depth 1 https://git.0x2620.org/oxjs.git /srv/pandora/static/oxjs \ + && git clone --depth 1 https://git.0x2620.org/python-ox.git /srv/pandora/src/python-ox \ + && git clone --depth 1 https://git.0x2620.org/oxtimelines.git /srv/pandora/src/oxtimelines \ + # Install python dependencies + && pip3 install -e /srv/pandora/src/python-ox \ + && pip3 install -e /srv/pandora/src/oxtimelines \ + && pip3 install -r /srv/pandora/requirements.txt \ + # Clean build dependencies + && apk del .deps \ + && find /srv/pandora -name '.git*' -exec rm -rf {} + \ + && rm -rf /tmp/lib* /tmp/oxframe \ + && rm -rf /root/.cache + +# TODO: Remove whole following block once the item_icon.py gets fixed +# TODO: Otherwise, if pillow version gets listed in requirements.txt, incorporate following block to the blocks above +RUN \ + apk --no-cache add \ + freetype \ + libjpeg-turbo \ + zlib \ + && apk --no-cache add --virtual .deps \ + build-base \ + freetype-dev \ + libjpeg-turbo-dev \ + python3-dev \ + zlib-dev \ + && pip3 install "pillow<4.2.0" \ + && apk del .deps \ + && rm -rf /root/.cache + +# TODO: Remove following block once the relative paths in extract.py get fixed +RUN \ + mkdir /srv/pandora/bin \ + && ln -s /usr/bin/oxtimelines /srv/pandora/bin/oxtimelines + +RUN \ + # Enable default configuration + cd /srv/pandora/pandora \ + && cp config.pandora.jsonc config.jsonc \ + && cp gunicorn_config.py.in gunicorn_config.py \ + # TODO: Remove following line once the get_version() is fixed + && sed -i 's/version = get_version()/version = "unknown"/' /srv/pandora/static/oxjs/tools/build/build.py \ + # Compile pyc and static files + && ./manage.py update_static \ + && ./manage.py compile_pyc -p /srv/pandora/pandora \ + && ./manage.py collectstatic -l --noinput + +COPY docker/ / + +VOLUME ["/srv/pandora/data"] +EXPOSE 80 + +CMD ["s6-svscan", "/etc/services.d"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..a64a892 --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +# Prepare environment +1. Download or build Docker images for: + - PostgreSQL + - RabbitMQ + +2. Setup PostgreSQL +``` +docker exec -it postgres createuser -P pandora +docker exec postgres createdb -O pandora pandora +echo "CREATE EXTENSION pg_trgm;" | docker exec -i postgres psql pandora +``` + +3. Setup RabbitMQ +``` +docker exec rabbitmq rabbitmqctl add_user pandora RABBITMQ_PWD +docker exec rabbitmq rabbitmqctl add_vhost /pandora +docker exec rabbitmq rabbitmqctl set_permissions -p /pandora pandora ".*" ".*" ".*" +``` + +# Configure Pan.do/ra +1. Build Pan.do/ra Docker image +``` +docker build -t pandora https://gitlab.dasm.cz:8443/Disassembler/pandora-alpine.git +``` + +2. Create empty data directory for persistent storage +``` +mkdir -p /srv/pandora/data +``` + +3. Create file `/srv/pandora/local_settings.py` with the following content: +``` +DATABASES = { + 'default': { + 'HOST': 'postgres', + 'NAME': 'pandora', + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'USER': 'pandora', + 'PASSWORD': 'POSTGRES_PWD', + } +} +BROKER_URL = 'amqp://pandora:RABBITMQ_PWD@rabbitmq:5672//pandora' +DB_GIN_TRGM = True +XACCELREDIRECT = True +``` +Supply the `POSTGRES_PWD` and `RABBITMQ_PWD` according to your values. + +4. Populate database +``` +docker run \ + --link postgres \ + -v /srv/pandora/local_settings.py:/srv/pandora/pandora/local_settings.py \ + pandora \ + /srv/pandora/pandora/manage.py init_db +``` + +5. Start Pan.do/ra Docker image +``` +docker run -it --rm \ + --name pandora \ + --link postgres \ + --link rabbitmq \ + -p 2620:80 \ + -v /srv/pandora/data:/srv/pandora/data \ + -v /srv/pandora/local_settings.py:/srv/pandora/pandora/local_settings.py \ + pandora +``` + +6. Open Pan.do/ra on http://your-machine:2620/ + +# Updating +1. Rebuild Pan.do/ra Docker image +``` +docker build -t pandora https://gitlab.dasm.cz:8443/Disassembler/pandora-alpine.git +``` + +2. Update database +``` +docker run \ + --link postgres \ + -v /srv/pandora/local_settings.py:/srv/pandora/pandora/local_settings.py \ + pandora \ + /srv/pandora/update.py db +``` diff --git a/docker/etc/nginx/nginx.conf b/docker/etc/nginx/nginx.conf new file mode 100644 index 0000000..952b0f4 --- /dev/null +++ b/docker/etc/nginx/nginx.conf @@ -0,0 +1,81 @@ +user nginx; +pid /run/nginx.pid; +worker_processes 1; +error_log /dev/stderr warn; + +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type application/octet-stream; + + access_log off; + server_tokens off; + keepalive_timeout 65; + sendfile on; + tcp_nodelay on; + + server { + listen 80; + server_name localhost; + + location /favicon.ico { + root /srv/pandora/static; + } + + location /static/ { + root /srv/pandora; + autoindex off; + } + location /data/ { + internal; + root /srv/pandora; + } + + location /api/ws/ { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Proxy ""; + proxy_redirect off; + proxy_buffering off; + proxy_read_timeout 999999999; + proxy_pass http://127.0.0.1:2622/; + } + + location / { + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header Host $http_host; + proxy_set_header Proxy ""; + proxy_redirect off; + proxy_buffering off; + proxy_read_timeout 90; #should be in sync with gunicorn timeout + proxy_connect_timeout 90; #should be in sync with gunicorn timeout + if (!-f $request_filename) { + proxy_pass http://127.0.0.1:2620; + break; + } + client_max_body_size 32m; + } + + error_page 400 /; + error_page 403 /403.html; + location /403.html { + root /srv/pandora/static/html; + } + error_page 404 /404.html; + location /404.html { + root /srv/pandora/static/html; + } + + # redirect server error pages to the static page /50x.html + error_page 500 502 503 504 /50x.html; + location /50x.html { + root /srv/pandora/static/html; + } + } +} diff --git a/docker/etc/services.d/.s6-svscan/finish b/docker/etc/services.d/.s6-svscan/finish new file mode 100755 index 0000000..78d5fdc --- /dev/null +++ b/docker/etc/services.d/.s6-svscan/finish @@ -0,0 +1,3 @@ +#!/bin/sh + +/bin/true diff --git a/docker/etc/services.d/nginx/run b/docker/etc/services.d/nginx/run new file mode 100755 index 0000000..dff57ac --- /dev/null +++ b/docker/etc/services.d/nginx/run @@ -0,0 +1,3 @@ +#!/bin/execlineb -P + +nginx -g "daemon off;" diff --git a/docker/etc/services.d/pandora-cron/run b/docker/etc/services.d/pandora-cron/run new file mode 100755 index 0000000..a01ebca --- /dev/null +++ b/docker/etc/services.d/pandora-cron/run @@ -0,0 +1,5 @@ +#!/bin/execlineb -P + +cd /srv/pandora/pandora +fdmove -c 2 1 +./manage.py celerybeat -s /srv/pandora/data/celerybeat-schedule --pidfile /run/pandora-cron.pid -l INFO diff --git a/docker/etc/services.d/pandora-encoding/run b/docker/etc/services.d/pandora-encoding/run new file mode 100755 index 0000000..6530f0c --- /dev/null +++ b/docker/etc/services.d/pandora-encoding/run @@ -0,0 +1,5 @@ +#!/bin/execlineb -P + +cd /srv/pandora/pandora +fdmove -c 2 1 +./manage.py celery worker -Q encoding -n pandora-encoding --pidfile /run/pandora-encoding.pid --maxtasksperchild 500 -l INFO diff --git a/docker/etc/services.d/pandora-tasks/run b/docker/etc/services.d/pandora-tasks/run new file mode 100755 index 0000000..077027c --- /dev/null +++ b/docker/etc/services.d/pandora-tasks/run @@ -0,0 +1,5 @@ +#!/bin/execlineb -P + +cd /srv/pandora/pandora +fdmove -c 2 1 +./manage.py celery worker -Q default,celery -n pandora-default --pidfile /run/pandora-tasks.pid --maxtasksperchild 1000 -l INFO diff --git a/docker/etc/services.d/pandora-websocketd/run b/docker/etc/services.d/pandora-websocketd/run new file mode 100755 index 0000000..dde1494 --- /dev/null +++ b/docker/etc/services.d/pandora-websocketd/run @@ -0,0 +1,5 @@ +#!/bin/execlineb -P + +cd /srv/pandora/pandora +fdmove -c 2 1 +./manage.py websocketd --pidfile /run/pandora-websocketd.pid diff --git a/docker/etc/services.d/pandora/run b/docker/etc/services.d/pandora/run new file mode 100755 index 0000000..1b67b83 --- /dev/null +++ b/docker/etc/services.d/pandora/run @@ -0,0 +1,5 @@ +#!/bin/execlineb -P + +cd /srv/pandora/pandora +fdmove -c 2 1 +gunicorn -c gunicorn_config.py wsgi:application