diff --git a/pandora.sh b/pandora.sh index 7866b85..b5a2794 100755 --- a/pandora.sh +++ b/pandora.sh @@ -1,80 +1,53 @@ -#!/bin/bash +#!/bin/sh SOURCE_DIR=$(realpath $(dirname "${0}"))/pandora -# Add Pandora repository -echo "deb http://ppa.launchpad.net/j/pandora/ubuntu zesty main" > /etc/apt/sources.list.d/pandora.list -apt-key add ${SOURCE_DIR}/pandora.gpg -apt-get -y update - -# Install packages -apt-get -y --no-install-recommends install ffmpeg ghostscript gpac imagemagick mkvtoolnix oxframe poppler-utils python-ox python3-html5lib python3-lxml python3-numpy python3-ox python3-pil python3-pip python3-psycopg2 python3-pyinotify python3-setuptools python3-simplejson python3-virtualenv rabbitmq-server virtualenv youtube-dl - -# Clone Pandora git repositories -git clone --depth 1 https://git.0x2620.org/pandora.git /srv/pandora -git clone --depth 1 https://git.0x2620.org/oxjs.git /srv/pandora/static/oxjs -git clone --depth 1 https://git.0x2620.org/oxtimelines.git /srv/pandora/src/oxtimelines - -# Copy Czech translataion -cp ${SOURCE_DIR}/srv/pandora/static/json/locale.0xdb.cs.json /srv/pandora/static/json/locale.0xdb.cs.json -cp ${SOURCE_DIR}/srv/pandora/static/json/locale.pandora.cs.json /srv/pandora/static/json/locale.pandora.cs.json -cp ${SOURCE_DIR}/srv/pandora/static/oxjs/source/Ox/js/Constants.js /srv/pandora/static/oxjs/source/Ox/js/Constants.js -cp ${SOURCE_DIR}/srv/pandora/static/oxjs/source/Ox/json/locale.cs.json /srv/pandora/static/oxjs/source/Ox/json/locale.cs.json -cp ${SOURCE_DIR}/srv/pandora/static/oxjs/source/UI/json/locale.cs.json /srv/pandora/static/oxjs/source/UI/json/locale.cs.json - -# Create Python virtualenv -virtualenv --system-site-packages -p /usr/bin/python3 /srv/pandora -/srv/pandora/bin/pip install -e /srv/pandora/src/oxtimelines -/srv/pandora/bin/pip install -r /srv/pandora/requirements.txt +# Build Docker container +docker build -t pandora ${SOURCE_DIR} # Create PostgreSQL user and database export PANDORA_PWD=$(head -c 18 /dev/urandom | base64) -envsubst <${SOURCE_DIR}/tmp/pandora-createdb.sql >/tmp/pandora-createdb.sql -sudo -u postgres psql -f /tmp/pandora-createdb.sql -rm -f /tmp/pandora-createdb.sql +envsubst <${SOURCE_DIR}/createdb.sql | docker exec -i postgres psql # Configure RabbitMQ export PANDORA_RABBIT_PWD=$(head -c 18 /dev/urandom | base64 | tr -d "/") -export PANDORA_BROKER_URL="amqp://pandora:${PANDORA_RABBIT_PWD}@localhost:5672//pandora" -rabbitmqctl add_user pandora ${PANDORA_RABBIT_PWD} -rabbitmqctl add_vhost /pandora -rabbitmqctl set_permissions -p /pandora pandora ".*" ".*" ".*" +docker exec rabbitmq rabbitmqctl add_user pandora ${PANDORA_RABBIT_PWD} +docker exec rabbitmq rabbitmqctl add_vhost /pandora +docker exec rabbitmq rabbitmqctl set_permissions -p /pandora pandora ".*" ".*" ".*" # Configure Pandora -mkdir /srv/pandora/data -cp ${SOURCE_DIR}/srv/pandora/pandora/config.jsonc /srv/pandora/pandora/config.jsonc -cp /srv/pandora/pandora/gunicorn_config.py.in /srv/pandora/pandora/gunicorn_config.py -envsubst <${SOURCE_DIR}/srv/pandora/pandora/local_settings.py >/srv/pandora/pandora/local_settings.py +mkdir -p /srv/pandora/conf /srv/pandora/data +chown 8011:8011 /srv/pandora/data +cp ${SOURCE_DIR}/srv/pandora/conf/config.jsonc /srv/pandora/conf/config.jsonc +cp ${SOURCE_DIR}/srv/pandora/conf/gunicorn_config.py /srv/pandora/conf/gunicorn_config.py +envsubst <${SOURCE_DIR}/srv/pandora/conf/local_settings.py >/srv/pandora/conf/local_settings.py # Set "production values" (increases performance) only if the DEBUG environment variable is not set if [ ${DEBUG:-0} -eq 0 ]; then - sed -i 's/DEBUG = True/DEBUG = False/' /srv/pandora/pandora/local_settings.py + sed -i 's/DEBUG = True/DEBUG = False/' /srv/pandora/conf/local_settings.py fi -# Create Pandora OS user -adduser --system --group --home /srv/pandora --shell /bin/false pandora -chown -R pandora:pandora /srv/pandora - # Populate database -sudo -u pandora /srv/pandora/pandora/manage.py init_db +docker run --rm -h pandora --link=postgres -v /srv/pandora/conf:/srv/pandora/conf pandora /srv/pandora/pandora/manage.py migrate --noinput +docker run --rm -h pandora --link=postgres -v /srv/pandora/conf:/srv/pandora/conf pandora /srv/pandora/pandora/manage.py sqlfindindex +docker run --rm -h pandora --link=postgres -v /srv/pandora/conf:/srv/pandora/conf pandora /srv/pandora/pandora/manage.py sync_itemsort +docker run --rm -h pandora --link=postgres -v /srv/pandora/conf:/srv/pandora/conf pandora /srv/pandora/pandora/manage.py sync_documentsort # Create admin account export PANDORA_ADMIN_USER=admin export PANDORA_ADMIN_EMAIL=admin@example.com export PANDORA_ADMIN_PWD=$(head -c 12 /dev/urandom | base64) -export PANDORA_ADMIN_HASH=$(cd /srv/pandora && DJANGO_SETTINGS_MODULE=pandora.settings /srv/pandora/bin/python -c "from django.contrib.auth.hashers import make_password; print(make_password('${PANDORA_ADMIN_PWD}'))") -envsubst <${SOURCE_DIR}/tmp/pandora-adminpwd.sql >/tmp/pandora-adminpwd.sql -sudo -u postgres psql -f /tmp/pandora-adminpwd.sql pandora -rm -f /tmp/pandora-adminpwd.sql +export PANDORA_ADMIN_HASH=$(docker run --rm -h pandora -e DJANGO_SETTINGS_MODULE=pandora.settings -v /srv/pandora/conf:/srv/pandora/conf pandora python3 -c "from django.contrib.auth.hashers import make_password; print(make_password('${PANDORA_ADMIN_PWD}'))") +envsubst <${SOURCE_DIR}/adminpwd.sql | docker exec -i postgres psql pandora -# Create nginx site definition -cp ${SOURCE_DIR}/etc/nginx/sites-available/pandora /etc/nginx/sites-available/pandora -ln -s /etc/nginx/sites-available/pandora /etc/nginx/sites-enabled/pandora +# Configure Pandora service +cp ${SOURCE_DIR}/etc/init.d/pandora /etc/init.d/pandora +rc-update add pandora boot +service pandora start -# Install and start services -/srv/pandora/ctl install -/srv/pandora/ctl start -systemctl restart nginx +# Create nginx app definition +cp ${SOURCE_DIR}/etc/nginx/conf.d/pandora.conf /etc/nginx/conf.d/pandora.conf +service nginx reload # Add portal application definition -portal-app-manager pandora "https://{host}:8001/" "${PANDORA_ADMIN_USER}" "${PANDORA_ADMIN_PWD}" +portal-app-manager pandora "https://{host}:8411/" "${PANDORA_ADMIN_USER}" "${PANDORA_ADMIN_PWD}" diff --git a/pandora/Dockerfile b/pandora/Dockerfile new file mode 100644 index 0000000..8013dd2 --- /dev/null +++ b/pandora/Dockerfile @@ -0,0 +1,97 @@ +FROM alpine:3.7 +MAINTAINER Disassembler + +RUN \ + # Install Python3 runtime + apk --no-cache add python3 \ + && ln -s /usr/bin/python3 /usr/bin/python + +RUN \ + # Install runtime XML dependencies + apk --no-cache add libxml2 libxslt + +RUN \ + # Install runtime dependencies + apk --no-cache add ffmpeg freetype imagemagick imlib2 libogg libjpeg-turbo libtheora libvpx mkvtoolnix nginx poppler-utils py3-psycopg2 py3-numpy py3-geoip py3-lxml s6 zlib \ + && pip3 install pyinotify youtube-dl + +RUN \ + # Install build dependencies + apk --no-cache add --virtual .deps autoconf automake build-base flac-dev freetype-dev git imlib2-dev libjpeg-turbo-dev libogg-dev libtheora-dev libtool libvpx-dev libvorbis-dev python3-dev zlib-dev \ + # Compile liboggz + && wget https://ftp.osuosl.org/pub/xiph/releases/liboggz/liboggz-1.1.1.tar.gz -O /tmp/liboggz.tgz \ + && tar xf /tmp/liboggz.tgz -C /tmp \ + && cd /tmp/liboggz-1.1.1 \ + && ./configure \ + && make -j $(nproc) \ + && make install \ + # Compile libfishsound + && wget https://ftp.osuosl.org/pub/xiph/releases/libfishsound/libfishsound-1.0.0.tar.gz -O /tmp/libfishsound.tgz \ + && tar xf /tmp/libfishsound.tgz -C /tmp/ \ + && cd /tmp/libfishsound-1.0.0 \ + && ./configure \ + && make -j $(nproc) \ + && make install \ + # Compile liboggplay + && git clone --depth 1 git://git.xiph.org/liboggplay.git /tmp/liboggplay \ + && cd /tmp/liboggplay \ + && ./autogen.sh \ + && ./configure \ + && make -j $(nproc) \ + && make install \ + # Compile Oxframe + && git clone --depth 1 https://code.0x2620.org/0x2620/oxframe /tmp/oxframe \ + && cd /tmp/oxframe \ + && sed -i '/man\/oxframe/d' Makefile \ + && make \ + && make install \ + # Clone Pandora git repositories + && git clone --depth 1 https://git.0x2620.org/pandora.git /srv/pandora \ + && git clone --depth 1 https://git.0x2620.org/oxjs.git /srv/pandora/static/oxjs \ + && git clone --depth 1 https://git.0x2620.org/python-ox.git /srv/pandora/src/python-ox \ + && git clone --depth 1 https://git.0x2620.org/oxtimelines.git /srv/pandora/src/oxtimelines \ + && pip3 install -e /srv/pandora/src/python-ox \ + && pip3 install -e /srv/pandora/src/oxtimelines \ + && pip3 install -r /srv/pandora/requirements.txt \ + && pip3 install "pillow<4.2.0" \ + # Compile pyc and static files + && cd /srv/pandora/pandora \ + && ln -s config.pandora.jsonc config.jsonc \ + && ./manage.py update_static \ + && ./manage.py compile_pyc -p /srv/pandora/pandora \ + && ./manage.py collectstatic -l --noinput \ + # Create OS user + && addgroup -S -g 8011 pandora \ + && adduser -S -u 8011 -h /srv/pandora -s /bin/false -g pandora -G pandora pandora \ + && chown -R 8011:8011 /srv/pandora \ + # Cleanup + && apk del .deps \ + && find /srv/pandora -name '.git*' -exec rm -rf {} + \ + && rm -rf /tmp/lib* /tmp/oxframe \ + && rm -rf /root \ + && mkdir /root + +COPY --chown=8011:8011 docker/srv/ /srv/ + +RUN \ + # Update static files for Czech translation + cd /srv/pandora/pandora \ + && ./manage.py update_static \ + # Hackfix missing virtualenv + && mkdir /srv/pandora/bin \ + && ln -s /usr/bin/oxtimelines /srv/pandora/bin/oxtimelines \ + # Create mount with dummy conf files + && mkdir /srv/pandora/conf \ + && rm config.jsonc \ + && ln -s /srv/pandora/conf/config.jsonc config.jsonc \ + && ln -s /srv/pandora/conf/gunicorn_config.py gunicorn_config.py \ + && ln -s /srv/pandora/conf/local_settings.py local_settings.py \ + && chown -R 8011:8011 /srv/pandora + +COPY docker/etc/ /etc/ + +VOLUME ["/srv/pandora/conf", "/srv/pandora/data"] +EXPOSE 8011 + +WORKDIR /srv/pandora +CMD ["s6-svscan", "/etc/services.d"] diff --git a/pandora/tmp/pandora-adminpwd.sql b/pandora/adminpwd.sql similarity index 100% rename from pandora/tmp/pandora-adminpwd.sql rename to pandora/adminpwd.sql diff --git a/pandora/tmp/pandora-createdb.sql b/pandora/createdb.sql similarity index 100% rename from pandora/tmp/pandora-createdb.sql rename to pandora/createdb.sql diff --git a/pandora/docker/etc/nginx/nginx.conf b/pandora/docker/etc/nginx/nginx.conf new file mode 100644 index 0000000..ceae07e --- /dev/null +++ b/pandora/docker/etc/nginx/nginx.conf @@ -0,0 +1,64 @@ +user nginx; +pid /run/nginx.pid; +worker_processes 1; +error_log /dev/stderr warn; + +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type application/octet-stream; + + access_log off; + server_tokens off; + client_max_body_size 100m; + keepalive_timeout 65; + sendfile on; + tcp_nodelay on; + + server { + listen 8011; + server_name localhost; + + location /favicon.ico { + root /srv/pandora/static; + } + + location /static/ { + root /srv/pandora; + autoindex off; + } + + location /data/ { + internal; + root /srv/pandora; + } + + location / { + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Host $host:$server_port; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + proxy_buffering off; + proxy_read_timeout 90; + proxy_connect_timeout 90; + if (!-f $request_filename) { + proxy_pass http://127.0.0.1:2620; + break; + } + } + + error_page 400 /; + error_page 404 /404.html; + location /404.html { + root /srv/pandora/static/html; + } + + error_page 500 502 503 504 /50x.html; + location /50x.html { + root /srv/pandora/static/html; + } + } +} diff --git a/pandora/docker/etc/services.d/.s6-svscan/finish b/pandora/docker/etc/services.d/.s6-svscan/finish new file mode 100755 index 0000000..78d5fdc --- /dev/null +++ b/pandora/docker/etc/services.d/.s6-svscan/finish @@ -0,0 +1,3 @@ +#!/bin/sh + +/bin/true diff --git a/pandora/docker/etc/services.d/nginx/run b/pandora/docker/etc/services.d/nginx/run new file mode 100755 index 0000000..d09b7cd --- /dev/null +++ b/pandora/docker/etc/services.d/nginx/run @@ -0,0 +1,3 @@ +#!/bin/execlineb -P + +/usr/sbin/nginx -g "daemon off;" diff --git a/pandora/docker/etc/services.d/pandora-cron/run b/pandora/docker/etc/services.d/pandora-cron/run new file mode 100755 index 0000000..2988f74 --- /dev/null +++ b/pandora/docker/etc/services.d/pandora-cron/run @@ -0,0 +1,7 @@ +#!/bin/execlineb -P + +cd /srv/pandora/pandora +export HOME /srv/pandora +fdmove -c 2 1 +s6-setuidgid 8011:8011 +./manage.py celerybeat -s /srv/pandora/data/celerybeat-schedule --pidfile pandora-cron.pid -l INFO diff --git a/pandora/docker/etc/services.d/pandora-encoding/run b/pandora/docker/etc/services.d/pandora-encoding/run new file mode 100755 index 0000000..59bc2c7 --- /dev/null +++ b/pandora/docker/etc/services.d/pandora-encoding/run @@ -0,0 +1,7 @@ +#!/bin/execlineb -P + +cd /srv/pandora/pandora +export HOME /srv/pandora +fdmove -c 2 1 +s6-setuidgid 8011:8011 +./manage.py celery worker -Q encoding -n pandora-encoding --pidfile pandora-encoding.pid --maxtasksperchild 500 -l INFO diff --git a/pandora/docker/etc/services.d/pandora-tasks/run b/pandora/docker/etc/services.d/pandora-tasks/run new file mode 100755 index 0000000..bbb41af --- /dev/null +++ b/pandora/docker/etc/services.d/pandora-tasks/run @@ -0,0 +1,7 @@ +#!/bin/execlineb -P + +cd /srv/pandora/pandora +export HOME /srv/pandora +fdmove -c 2 1 +s6-setuidgid 8011:8011 +./manage.py celery worker -Q default,celery -n pandora-default --pidfile pandora-tasks.pid --maxtasksperchild 1000 -l INFO diff --git a/pandora/docker/etc/services.d/pandora/run b/pandora/docker/etc/services.d/pandora/run new file mode 100755 index 0000000..e7ce093 --- /dev/null +++ b/pandora/docker/etc/services.d/pandora/run @@ -0,0 +1,7 @@ +#!/bin/execlineb -P + +cd /srv/pandora/pandora +export HOME /srv/pandora +fdmove -c 2 1 +s6-setuidgid 8011:8011 +gunicorn -c gunicorn_config.py wsgi:application diff --git a/pandora/srv/pandora/static/json/locale.0xdb.cs.json b/pandora/docker/srv/pandora/static/json/locale.0xdb.cs.json similarity index 100% rename from pandora/srv/pandora/static/json/locale.0xdb.cs.json rename to pandora/docker/srv/pandora/static/json/locale.0xdb.cs.json diff --git a/pandora/srv/pandora/static/json/locale.pandora.cs.json b/pandora/docker/srv/pandora/static/json/locale.pandora.cs.json similarity index 100% rename from pandora/srv/pandora/static/json/locale.pandora.cs.json rename to pandora/docker/srv/pandora/static/json/locale.pandora.cs.json diff --git a/pandora/srv/pandora/static/oxjs/source/Ox/js/Constants.js b/pandora/docker/srv/pandora/static/oxjs/source/Ox/js/Constants.js similarity index 100% rename from pandora/srv/pandora/static/oxjs/source/Ox/js/Constants.js rename to pandora/docker/srv/pandora/static/oxjs/source/Ox/js/Constants.js diff --git a/pandora/srv/pandora/static/oxjs/source/Ox/json/locale.cs.json b/pandora/docker/srv/pandora/static/oxjs/source/Ox/json/locale.cs.json similarity index 100% rename from pandora/srv/pandora/static/oxjs/source/Ox/json/locale.cs.json rename to pandora/docker/srv/pandora/static/oxjs/source/Ox/json/locale.cs.json diff --git a/pandora/srv/pandora/static/oxjs/source/UI/json/locale.cs.json b/pandora/docker/srv/pandora/static/oxjs/source/UI/json/locale.cs.json similarity index 100% rename from pandora/srv/pandora/static/oxjs/source/UI/json/locale.cs.json rename to pandora/docker/srv/pandora/static/oxjs/source/UI/json/locale.cs.json diff --git a/pandora/docker/srv/pandora/static/oxjs/tools/build/build.py b/pandora/docker/srv/pandora/static/oxjs/tools/build/build.py new file mode 100755 index 0000000..64f547e --- /dev/null +++ b/pandora/docker/srv/pandora/static/oxjs/tools/build/build.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +#vim: et:ts=4:sw=4:sts=4 +from __future__ import print_function + +import json +import os +import ox +import re +import shutil +import subprocess +import sys +import tarfile +import time + +def get_version(): + revision = None + if os.path.exists('../../.git'): + revision = subprocess.Popen( + ['git', 'rev-list', 'HEAD', '--count'], stdout=subprocess.PIPE + ).communicate()[0].strip().decode('utf-8') + revision = int(revision) - 94 + elif os.path.exists('../../.bzr/branch/last-revision'): + revision = subprocess.Popen( + ['bzr', 'revno'], stdout=subprocess.PIPE + ).communicate()[0].strip().decode('utf-8') + return '0.1.%s' % revision if revision else 'unknown' + +def build_oxjs(downloads=False, geo=False): + + base_path = os.path.dirname(__file__) + if base_path: + os.chdir(base_path) + + root_path = '../../' + source_path = root_path + 'source/' + dev_path = root_path + 'dev.tmp/' + min_path = root_path + 'min.tmp/' + + locales = {} + version = get_version() + year = time.strftime('%Y', time.gmtime()) + comment = ' OxJS %s (c) %s 0x2620, dual-licensed GPL/MIT, see https://oxjs.org for details ' % (version, year) + + # Empty dev and min + for path in [dev_path, min_path]: + if os.path.exists(path): + for item in os.listdir(path): + full_path = '%s%s' % (path, item) + if os.path.isdir(full_path): + if not (geo == False and item == 'Geo'): + shutil.rmtree(full_path) + else: + os.remove(full_path) + + # Ox.UI Theme Data + theme_data = {} + themes = [dirname for dirname in os.listdir(source_path + 'UI/themes/') if not dirname[0] in '._'] + for theme in themes: + theme_data[theme] = read_jsonc(source_path + 'UI/themes/%s/json/theme.jsonc' % theme) + theme_data[theme]['themeClass'] = 'OxTheme' + theme[0].upper() + theme[1:] + + # Ox.UI CSS + css = read_text(source_path + 'UI/css/UI.css') + css = css.replace('$import', '\n'.join([ + '@import url("../themes/%s/css/theme.css?%s");' % (theme, version) for theme in themes + ])) + write_file('%sUI/css/UI.css' % dev_path, css) + write_file('%sUI/css/UI.css' % min_path, css) + + # Ox.UI Theme CSS + css = read_text(source_path + 'UI/css/theme.css') + for theme in themes: + theme_css = parse_css(css, theme_data[theme]) + theme_css = theme_css.replace('.png)', '.png?%s)' % version) + write_file('%sUI/themes/%s/css/theme.css' % (dev_path, theme), theme_css) + write_file('%sUI/themes/%s/css/theme.css' % (min_path, theme), theme_css) + + # Ox.UI SVGs + ui_images = {} + path = source_path + 'UI/svg/' + for filename in [filename for filename in os.listdir(path) if not filename[0] in '._']: + svg = read_text(path + filename) + svg = re.sub('\n\s*', '', svg) + svg = re.sub('', '', svg) + # temporary fix for Chrome SVG bug, remove later! + svg = re.sub('width="256" height="256"', 'width="10" height="10" viewBox="0 0 255 255"', svg) + # end fix + ui_images[filename[:-4]] = svg + if filename.startswith('symbolLoading'): + for theme in themes: + theme_svg = re.sub('#808080', format_hex(theme_data[theme]['symbolDefaultColor']), svg) + write_file('%sUI/themes/%s/svg/%s' % (dev_path, theme, filename), theme_svg) + write_file('%sUI/themes/%s/svg/%s' % (min_path, theme, filename), theme_svg) + + # copy & link + ui_files = {'dev': [], 'min': []} + for path, dirnames, filenames in os.walk(source_path): + for filename in filenames: + if not '_' in path and not filename[0] in '._' \ + and not filename.endswith('~') \ + and not filename.endswith('.css') \ + and not '/UI/svg' in path \ + and (geo or not '/Geo/' in path): + # write copies in min path + source = os.path.join(path, filename) + is_jquery = re.search('^jquery-[\d\.]+\.js$', filename) + is_jquery_min = re.search('^jquery-[\d\.]+\.min\.js$', filename) + is_jquery_plugin = re.search('^jquery\..*?\.js$', filename) + is_jsonc = re.search('\.jsonc$', filename) + if is_jquery or is_jquery_min: + target = os.path.join(path.replace(source_path, min_path), 'jquery.js') + else: + target = os.path.join(path.replace(source_path, min_path), filename) + if is_jquery_plugin: + ui_files['dev'].append(target.replace(min_path, '')) + ui_files['min'].append(target.replace(min_path, '')) + if not '/Ox/js/' in source and not '/UI/js/' in source and not is_jquery: + if re.match('^Ox\..+\.js$', filename) or is_jsonc: + js = read_text(source) + print('minifiy and write', filename, target) + write_file(target, ox.js.minify(js, '' if is_jsonc else comment)) + else: + copy_file(source, target) + # write links in dev path + parts = os.path.join(path.replace(source_path, ''), filename).split('/') + for i, part in enumerate(parts): + if i < len(parts) - 1: + parts[i] = '..' + link_source = '/'.join(parts).replace(filename, os.path.join(path, filename))[3:] + link_target = target.replace(min_path, dev_path) + if not is_jquery_min: + write_link(link_source, link_target) + # locales + match = re.search('/(\w+)/json/locale.(\w+).json', source) + if match: + module = match.group(1) + locale = match.group(2) + if not module in locales: + locales[module] = [] + locales[module].append(locale) + # remove dangling links from dev tree that might + # be left over from renamed or removed files + for path, dirnames, filenames in os.walk(dev_path): + for f in filenames: + f = os.path.join(path, f) + if os.path.islink(f) and not os.path.exists(f): + os.unlink(f) + # Ox.js + filenames = [ + [ + 'Core.js' # has to run first so that Ox is defined + ], + [ + 'Function.js', # getSortValue (Array.js) depends on Ox.cache + 'Polyfill.js' # FIXME: not clear if needed here + ], + [ + 'Array.js', # Ox.slice (Collection.js) depends on Ox.toArray, salt (HTML.js) depends on Ox.range + 'String.js', # salt (HTML.js) depends on Ox.char + 'Type.js' # Ox.typeOf needed in Collection.js FF3.6 for Ox.slice fallback + ], + [ + 'Collection.js', # Ox.PATH (Constants.js) depends on Ox.slice + 'Math.js' # Ox.MAX_LATITUDE (Constants.js) depends on Ox.sinh + ] + ] + js = '' + js_dir = 'Ox/js/' + ox_files = [] + for group in filenames: + ox_files.append([]) + for filename in group: + ox_files[-1].append(js_dir + filename) + ox_files.append([]) + filenames = sum(filenames, []) # flatten + for filename in sorted(os.listdir(source_path + js_dir)): + if not filename in filenames \ + and not filename.startswith('.') \ + and not filename.startswith('_') \ + and not filename.endswith('~'): + filenames.append(filename) + for filename in filenames: + js += read_text(source_path + js_dir + filename) + '\n' + if not js_dir + filename in sum(ox_files, []): + ox_files[-1].append(js_dir + filename) + js = re.sub( + 'Ox.LOCALES = \{\}', + 'Ox.LOCALES = ' + json.dumps(locales, indent=4, sort_keys=True), + js + ) + js = re.sub( + "Ox.VERSION = '([\d\.]+)'", + "Ox.VERSION = '%s'" % version, + js + ) + write_file(dev_path + '/Ox/json/' + 'Ox.json', json.dumps({ + 'files': ox_files, + 'locales': locales, + 'version': version + }, indent=4, sort_keys=True)) + write_file(min_path + 'Ox.js', ox.js.minify(js, comment)) + + # Ox.UI + js = '' + root = source_path + 'UI/' + for path, dirnames, filenames in os.walk(root): + for filename in sorted(filenames): + # jquery gets included by Ox.UI loader + # locale json files are loaded lazily + # Ox.UI.css imports all other css files + # svgs are loaded as URLs or dataURLs + # Ox.UI PNGs are loaded on demand + if path != root and not '_' in path and not filename[0] in '._' \ + and not filename.endswith('~') \ + and not 'jquery' in filename \ + and not 'locale' in filename \ + and not filename.endswith('theme.css') \ + and not filename.endswith('.svg') \ + and not 'UI/png' in path: + ui_files['dev'].append(os.path.join(path.replace(source_path, ''), filename)) + if not '/js/' in path: + ui_files['min'].append(os.path.join(path.replace(source_path, ''), filename)) + if filename.endswith('.js'): + js += read_text(os.path.join(path, filename)) + '\n' + filename = min_path + 'UI/js/UI.js' + write_file(filename, ox.js.minify(js, comment)) + + ui_files['min'].append(filename.replace(min_path, '')) + write_file(min_path + 'UI/json/UI.json', json.dumps({ + 'files': sorted(ui_files['min']), + 'images': ui_images + }, sort_keys=True)) + write_file(dev_path + 'UI/json/UI.json', json.dumps({ + 'files': sorted(ui_files['dev']), + 'images': ui_images + }, indent=4, sort_keys=True)) + ui_files['dev'].append('UI/UI.js') + + # index + data = { + # sum(list, []) is flatten + 'documentation': sorted(sum(ox_files, [])) + sorted(list(filter( + lambda x: re.search('\.js$', x), + ui_files['dev'] + )) + ['%s/%s.js' % (x, x) for x in ['Geo', 'Image', 'Unicode']]), + 'examples': sorted(sum(map( + lambda x: list(filter( + lambda x: not re.search('/[._]', x), + map( + lambda y: x + '/' + y, + os.listdir(root_path + 'examples/' + x) + ) + )), + list(filter( + lambda x: not re.search('^[._]', x), + os.listdir(root_path + 'examples/') + )) + ), [])) if os.path.exists(root_path + 'examples/') else (), + 'readme': list(map( + lambda x: { + 'date': time.strftime( + '%Y-%m-%dT%H:%M:%SZ', + time.gmtime(os.path.getmtime(root_path + 'readme/' + x)) + ), + 'id': x.split('.')[0], + 'title': get_title(root_path + 'readme/' + x) + }, + filter( + lambda x: not re.search('^[._]', x) and re.search('\.html$', x), + os.listdir(root_path + 'readme/') + ) + )) + } + write_file(root_path + 'index.json', json.dumps(data, indent=4, sort_keys=True)) + + # downloads + if downloads: + data = { + 'date': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), + 'size': {'oxjs': os.path.getsize(min_path + 'Ox.js')}, + 'version': version + } + download_path = root_path + 'downloads/' + # source + source_file = download_path + 'OxJS.%s.source.tar.gz' % version + data['size']['source'] = write_tarfile(source_file, root_path, 'OxJS', filter_source) + write_link(source_file.replace(download_path, ''), source_file.replace(version, 'latest')) + # min + min_file = download_path + 'OxJS.%s.min.tar.gz' % version + data['size']['min'] = write_tarfile(min_file, root_path, 'OxJS', filter_min) + write_link(min_file.replace(download_path, ''), min_file.replace(version, 'latest')) + # json + write_file(download_path + 'downloads.json', json.dumps(data, indent=4, sort_keys=True)) + + # legacy + build_path = root_path + 'build/' + if os.path.exists(build_path) and not os.path.islink(build_path[:-1]): + shutil.rmtree(build_path) + write_link('min', build_path[:-1]) + real_dev_path = root_path + 'dev/' + real_min_path = root_path + 'min/' + if os.path.exists(real_dev_path): + shutil.rmtree(real_dev_path) + shutil.move(dev_path,real_dev_path) + if os.path.exists(real_min_path): + shutil.rmtree(real_min_path) + shutil.move(min_path,real_min_path) + + +def copy_file(source, target): + print('copying', source, 'to', target) + write_file(target, read_file(source)) + +def filter_min(tarinfo): + name = tarinfo.name + if name == 'OxJS' or re.search('^OxJS/min', name): + return tarinfo + return None + +def filter_source(tarinfo): + name = tarinfo.name + if re.search('^[._]', name) or re.search('/[._]', name) or re.search('~$', name): + return None + if re.search('^OxJS/downloads', name): + return None + if name == 'OxJS/tools/geo/png/icons.png': + return None + if re.search('^OxJS/tools/geo/png/icons/', name) and ( + not re.search('4096', name) or not os.path.exists( + name.replace('OxJS/', '../../').replace('icons/4096', 'flags') + ) + ): + return None + return tarinfo + +def format_hex(rgb): + return '#%s' % ''.join([hex(c)[-2:].replace('x', '0').upper() for c in rgb]) + +def get_title(file): + match = re.search('

(.+)

', read_text(file)) + return match.groups()[0] if match else 'Untitled' + +def parse_css(css, values): + def sub(match): + key = match.group(1) + index = match.group(2) + value = values[key] if index == None else values[key][int(index[1:-1])] + if isinstance(value, str): + string = value + else: + if isinstance(value[0], int): + value = [value] + string = ', '.join( + ['rgb%s(%s)' % ( + 'a' if len(vals) == 4 else '', + ', '.join([str(val) for val in vals]) + ) for vals in value] + ) + return string + return re.sub('\$(\w+)(\[\d+\])?', sub, css) + +def read_file(file): + print('reading', file) + f = open(file, 'rb') + data = f.read() + f.close() + return data + +def read_text(file): + return read_file(file).decode('utf-8') + +def read_jsonc(file): + return ox.jsonc.loads(read_text(file)) + +def write_file(file, data): + print('writing', file) + write_path(file) + if not isinstance(data, bytes): + data = data.encode('utf-8') + f = open(file, 'wb') + f.write(data) + f.close() + return len(data) + +def write_link(source, target): + print('linking', source, 'to', target) + write_path(target) + # remove files, symlinks *and broken symlinks* + if os.path.exists(target) or os.path.lexists(target): + if os.path.isdir(target) and not os.path.islink(target): + os.rmdir(target) + else: + os.unlink(target) + os.symlink(source, target) + +def write_path(file): + path = os.path.split(file)[0] + if path and not os.path.exists(path): + os.makedirs(path) + +def write_tarfile(file, path, arcname, filter): + print('writing', file) + f = tarfile.open(file, 'w:gz') + f.add(path, arcname=arcname, filter=filter) + f.close() + return os.path.getsize(file) + +if __name__ == '__main__': + build_oxjs(downloads='-downloads' in sys.argv, geo=not '-nogeo' in sys.argv) diff --git a/pandora/etc/init.d/pandora b/pandora/etc/init.d/pandora new file mode 100755 index 0000000..7885f46 --- /dev/null +++ b/pandora/etc/init.d/pandora @@ -0,0 +1,17 @@ +#!/sbin/openrc-run + +description="Pan.do/ra docker container" + +depend() { + need docker net + use dns logger netmount + after postgres rabbitmq +} + +start() { + /usr/bin/docker run -d --rm --name pandora -h pandora --link=postgres --link=rabbitmq -p 127.0.0.1:9011:8011 -v /srv/pandora/conf:/srv/pandora/conf -v /srv/pandora/data:/srv/pandora/data pandora +} + +stop() { + /usr/bin/docker stop pandora +} diff --git a/pandora/etc/nginx/conf.d/pandora.conf b/pandora/etc/nginx/conf.d/pandora.conf new file mode 100644 index 0000000..a01a4e8 --- /dev/null +++ b/pandora/etc/nginx/conf.d/pandora.conf @@ -0,0 +1,14 @@ +server { + listen [::]:8011 ipv6only=off; + listen [::]:8411 ssl http2 ipv6only=off; + + access_log /var/log/nginx/pandora.access.log; + error_log /var/log/nginx/pandora.error.log; + + location / { + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Host $host:$server_port; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://127.0.0.1:9011; + } +} diff --git a/pandora/etc/nginx/sites-available/pandora b/pandora/etc/nginx/sites-available/pandora deleted file mode 100644 index 56ef3fc..0000000 --- a/pandora/etc/nginx/sites-available/pandora +++ /dev/null @@ -1,60 +0,0 @@ -server { - listen 8001 ssl http2; - listen [::]:8001 ssl http2; - - access_log /var/log/nginx/pandora.access.log; - error_log /var/log/nginx/pandora.error.log; - - location /favicon.ico { - root /srv/pandora/static; - } - - location /static/ { - root /srv/pandora; - autoindex off; - } - - location /data/ { - internal; - root /srv/pandora; - } - - location /api/ws/ { - proxy_http_version 1.1; - proxy_set_header Host $http_host; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Proxy ""; - proxy_redirect off; - proxy_buffering off; - proxy_read_timeout 999999999; - proxy_pass http://127.0.0.1:2622/; - } - - location / { - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header Host $http_host; - proxy_set_header Proxy ""; - proxy_redirect off; - proxy_buffering off; - proxy_read_timeout 90; - proxy_connect_timeout 90; - if (!-f $request_filename) { - proxy_pass http://127.0.0.1:2620; - break; - } - client_max_body_size 32m; - } - - error_page 400 /; - error_page 404 /404.html; - location /404.html { - root /srv/pandora/static/html; - } - - error_page 500 502 503 504 /50x.html; - location /50x.html { - root /srv/pandora/static/html; - } -} diff --git a/pandora/pandora.gpg b/pandora/pandora.gpg deleted file mode 100644 index 1325978..0000000 --- a/pandora/pandora.gpg +++ /dev/null @@ -1,13 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1 - -mI0ESXYhEgEEALl9jDTdmgpApPbjN+7b85dC92HisPUp56ifEkKJOBj0X5HhRqxs -Wjx/zlP4/XJGrHnxJyrdPxjSwAXz7bNdeggkN4JWdusTkr5GOXvggQnng0X7f/rX -oJwoEGtYOCODLPs6PC0qjh5yPzJVeiRsKUOZ7YVNnwNwdfS4D8RZvtCrABEBAAG0 -FExhdW5jaHBhZCBQUEEgZm9yIGpeiLYEEwECACAFAkl2IRICGwMGCwkIBwMCBBUC -CAMEFgIDAQIeAQIXgAAKCRAohRM8AZde82FfA/9OB/64/YLaCpizHZ8f6DK3rGgF -e6mX3rFK8yOKGGL06316VhDzfzMiZSauUZ0t+lKHR/KZYeSaFwEoUoblTG/s4IIo -9aBMHWhVXJW6eifKUmTGqEn2/0UxoWQq2C3F6njMkCaP+ALOD5uzaSYGdjqAUAwS -pAAGSEQ4uz6bYSeM4Q== -=SM2a ------END PGP PUBLIC KEY BLOCK----- diff --git a/pandora/srv/pandora/pandora/config.jsonc b/pandora/srv/pandora/conf/config.jsonc similarity index 100% rename from pandora/srv/pandora/pandora/config.jsonc rename to pandora/srv/pandora/conf/config.jsonc diff --git a/pandora/srv/pandora/conf/gunicorn_config.py b/pandora/srv/pandora/conf/gunicorn_config.py new file mode 100644 index 0000000..4e8137f --- /dev/null +++ b/pandora/srv/pandora/conf/gunicorn_config.py @@ -0,0 +1,5 @@ +bind="127.0.0.1:2620" +log_level="info" +max_requests=1000 +timeout=90 +workers=5 diff --git a/pandora/srv/pandora/pandora/local_settings.py b/pandora/srv/pandora/conf/local_settings.py similarity index 75% rename from pandora/srv/pandora/pandora/local_settings.py rename to pandora/srv/pandora/conf/local_settings.py index fb4f92e..3c8b877 100644 --- a/pandora/srv/pandora/pandora/local_settings.py +++ b/pandora/srv/pandora/conf/local_settings.py @@ -1,12 +1,13 @@ DATABASES = { 'default': { + 'HOST': 'postgres', 'NAME': 'pandora', 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'USER': 'pandora', 'PASSWORD': '${PANDORA_PWD}', } } -BROKER_URL = '${PANDORA_BROKER_URL}' +BROKER_URL = 'amqp://pandora:${PANDORA_RABBIT_PWD}@rabbitmq:5672//pandora' DB_GIN_TRGM = True XACCELREDIRECT = True