Disassembler/Spotter-VM
1
0
Fork 0
Code Issues 78 Pull Requests Actions Packages Projects Releases Wiki Activity

Alpinize + Dockerize Pandora

Browse Source
This commit is contained in:
Disassembler 2018-01-18 13:39:39 +01:00
parent 8058938473
commit 642a190514
Signed by: Disassembler
GPG Key ID: 524BD33A0EE29499
24 changed files with 670 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
pandora.sh
View File

@ -1,80 +1,53 @@
#!/bin/bash
#!/bin/sh
SOURCE_DIR=$(realpath $(dirname "${0}"))/pandora
# Add Pandora repository
echo "deb http://ppa.launchpad.net/j/pandora/ubuntu zesty main" > /etc/apt/sources.list.d/pandora.list
apt-key add ${SOURCE_DIR}/pandora.gpg
apt-get -y update
# Install packages
apt-get -y --no-install-recommends install ffmpeg ghostscript gpac imagemagick mkvtoolnix oxframe poppler-utils python-ox python3-html5lib python3-lxml python3-numpy python3-ox python3-pil python3-pip python3-psycopg2 python3-pyinotify python3-setuptools python3-simplejson python3-virtualenv rabbitmq-server virtualenv youtube-dl
# Clone Pandora git repositories
git clone --depth 1 https://git.0x2620.org/pandora.git /srv/pandora
git clone --depth 1 https://git.0x2620.org/oxjs.git /srv/pandora/static/oxjs
git clone --depth 1 https://git.0x2620.org/oxtimelines.git /srv/pandora/src/oxtimelines
# Copy Czech translataion
cp ${SOURCE_DIR}/srv/pandora/static/json/locale.0xdb.cs.json /srv/pandora/static/json/locale.0xdb.cs.json
cp ${SOURCE_DIR}/srv/pandora/static/json/locale.pandora.cs.json /srv/pandora/static/json/locale.pandora.cs.json
cp ${SOURCE_DIR}/srv/pandora/static/oxjs/source/Ox/js/Constants.js /srv/pandora/static/oxjs/source/Ox/js/Constants.js
cp ${SOURCE_DIR}/srv/pandora/static/oxjs/source/Ox/json/locale.cs.json /srv/pandora/static/oxjs/source/Ox/json/locale.cs.json
cp ${SOURCE_DIR}/srv/pandora/static/oxjs/source/UI/json/locale.cs.json /srv/pandora/static/oxjs/source/UI/json/locale.cs.json
# Create Python virtualenv
virtualenv --system-site-packages -p /usr/bin/python3 /srv/pandora
/srv/pandora/bin/pip install -e /srv/pandora/src/oxtimelines
/srv/pandora/bin/pip install -r /srv/pandora/requirements.txt
# Build Docker container
docker build -t pandora ${SOURCE_DIR}
# Create PostgreSQL user and database
export PANDORA_PWD=$(head -c 18 /dev/urandom | base64)
envsubst <${SOURCE_DIR}/tmp/pandora-createdb.sql >/tmp/pandora-createdb.sql
sudo -u postgres psql -f /tmp/pandora-createdb.sql
rm -f /tmp/pandora-createdb.sql
envsubst <${SOURCE_DIR}/createdb.sql | docker exec -i postgres psql
# Configure RabbitMQ
export PANDORA_RABBIT_PWD=$(head -c 18 /dev/urandom | base64 | tr -d "/")
export PANDORA_BROKER_URL="amqp://pandora:${PANDORA_RABBIT_PWD}@localhost:5672//pandora"
rabbitmqctl add_user pandora ${PANDORA_RABBIT_PWD}
rabbitmqctl add_vhost /pandora
rabbitmqctl set_permissions -p /pandora pandora ".*" ".*" ".*"
docker exec rabbitmq rabbitmqctl add_user pandora ${PANDORA_RABBIT_PWD}
docker exec rabbitmq rabbitmqctl add_vhost /pandora
docker exec rabbitmq rabbitmqctl set_permissions -p /pandora pandora ".*" ".*" ".*"
# Configure Pandora
mkdir /srv/pandora/data
cp ${SOURCE_DIR}/srv/pandora/pandora/config.jsonc /srv/pandora/pandora/config.jsonc
cp /srv/pandora/pandora/gunicorn_config.py.in /srv/pandora/pandora/gunicorn_config.py
envsubst <${SOURCE_DIR}/srv/pandora/pandora/local_settings.py >/srv/pandora/pandora/local_settings.py
mkdir -p /srv/pandora/conf /srv/pandora/data
chown 8011:8011 /srv/pandora/data
cp ${SOURCE_DIR}/srv/pandora/conf/config.jsonc /srv/pandora/conf/config.jsonc
cp ${SOURCE_DIR}/srv/pandora/conf/gunicorn_config.py /srv/pandora/conf/gunicorn_config.py
envsubst <${SOURCE_DIR}/srv/pandora/conf/local_settings.py >/srv/pandora/conf/local_settings.py
# Set "production values" (increases performance) only if the DEBUG environment variable is not set
if [ ${DEBUG:-0} -eq 0 ]; then
sed -i 's/DEBUG = True/DEBUG = False/' /srv/pandora/pandora/local_settings.py
sed -i 's/DEBUG = True/DEBUG = False/' /srv/pandora/conf/local_settings.py
fi
# Create Pandora OS user
adduser --system --group --home /srv/pandora --shell /bin/false pandora
chown -R pandora:pandora /srv/pandora
# Populate database
sudo -u pandora /srv/pandora/pandora/manage.py init_db
docker run --rm -h pandora --link=postgres -v /srv/pandora/conf:/srv/pandora/conf pandora /srv/pandora/pandora/manage.py migrate --noinput
docker run --rm -h pandora --link=postgres -v /srv/pandora/conf:/srv/pandora/conf pandora /srv/pandora/pandora/manage.py sqlfindindex
docker run --rm -h pandora --link=postgres -v /srv/pandora/conf:/srv/pandora/conf pandora /srv/pandora/pandora/manage.py sync_itemsort
docker run --rm -h pandora --link=postgres -v /srv/pandora/conf:/srv/pandora/conf pandora /srv/pandora/pandora/manage.py sync_documentsort
# Create admin account
export PANDORA_ADMIN_USER=admin
export PANDORA_ADMIN_EMAIL=admin@example.com
export PANDORA_ADMIN_PWD=$(head -c 12 /dev/urandom | base64)
export PANDORA_ADMIN_HASH=$(cd /srv/pandora && DJANGO_SETTINGS_MODULE=pandora.settings /srv/pandora/bin/python -c "from django.contrib.auth.hashers import make_password; print(make_password('${PANDORA_ADMIN_PWD}'))")
envsubst <${SOURCE_DIR}/tmp/pandora-adminpwd.sql >/tmp/pandora-adminpwd.sql
sudo -u postgres psql -f /tmp/pandora-adminpwd.sql pandora
rm -f /tmp/pandora-adminpwd.sql
export PANDORA_ADMIN_HASH=$(docker run --rm -h pandora -e DJANGO_SETTINGS_MODULE=pandora.settings -v /srv/pandora/conf:/srv/pandora/conf pandora python3 -c "from django.contrib.auth.hashers import make_password; print(make_password('${PANDORA_ADMIN_PWD}'))")
envsubst <${SOURCE_DIR}/adminpwd.sql | docker exec -i postgres psql pandora
# Create nginx site definition
cp ${SOURCE_DIR}/etc/nginx/sites-available/pandora /etc/nginx/sites-available/pandora
ln -s /etc/nginx/sites-available/pandora /etc/nginx/sites-enabled/pandora
# Configure Pandora service
cp ${SOURCE_DIR}/etc/init.d/pandora /etc/init.d/pandora
rc-update add pandora boot
service pandora start
# Install and start services
/srv/pandora/ctl install
/srv/pandora/ctl start
systemctl restart nginx
# Create nginx app definition
cp ${SOURCE_DIR}/etc/nginx/conf.d/pandora.conf /etc/nginx/conf.d/pandora.conf
service nginx reload
# Add portal application definition
portal-app-manager pandora "https://{host}:8001/" "${PANDORA_ADMIN_USER}" "${PANDORA_ADMIN_PWD}"
portal-app-manager pandora "https://{host}:8411/" "${PANDORA_ADMIN_USER}" "${PANDORA_ADMIN_PWD}"

97
pandora/Dockerfile Normal file
View File

@ -0,0 +1,97 @@
FROM alpine:3.7
MAINTAINER Disassembler <disassembler@dasm.cz>
RUN \
# Install Python3 runtime
apk --no-cache add python3 \
&& ln -s /usr/bin/python3 /usr/bin/python
RUN \
# Install runtime XML dependencies
apk --no-cache add libxml2 libxslt
RUN \
# Install runtime dependencies
apk --no-cache add ffmpeg freetype imagemagick imlib2 libogg libjpeg-turbo libtheora libvpx mkvtoolnix nginx poppler-utils py3-psycopg2 py3-numpy py3-geoip py3-lxml s6 zlib \
&& pip3 install pyinotify youtube-dl
RUN \
# Install build dependencies
apk --no-cache add --virtual .deps autoconf automake build-base flac-dev freetype-dev git imlib2-dev libjpeg-turbo-dev libogg-dev libtheora-dev libtool libvpx-dev libvorbis-dev python3-dev zlib-dev \
# Compile liboggz
&& wget https://ftp.osuosl.org/pub/xiph/releases/liboggz/liboggz-1.1.1.tar.gz -O /tmp/liboggz.tgz \
&& tar xf /tmp/liboggz.tgz -C /tmp \
&& cd /tmp/liboggz-1.1.1 \
&& ./configure \
&& make -j $(nproc) \
&& make install \
# Compile libfishsound
&& wget https://ftp.osuosl.org/pub/xiph/releases/libfishsound/libfishsound-1.0.0.tar.gz -O /tmp/libfishsound.tgz \
&& tar xf /tmp/libfishsound.tgz -C /tmp/ \
&& cd /tmp/libfishsound-1.0.0 \
&& ./configure \
&& make -j $(nproc) \
&& make install \
# Compile liboggplay
&& git clone --depth 1 git://git.xiph.org/liboggplay.git /tmp/liboggplay \
&& cd /tmp/liboggplay \
&& ./autogen.sh \
&& ./configure \
&& make -j $(nproc) \
&& make install \
# Compile Oxframe
&& git clone --depth 1 https://code.0x2620.org/0x2620/oxframe /tmp/oxframe \
&& cd /tmp/oxframe \
&& sed -i '/man\/oxframe/d' Makefile \
&& make \
&& make install \
# Clone Pandora git repositories
&& git clone --depth 1 https://git.0x2620.org/pandora.git /srv/pandora \
&& git clone --depth 1 https://git.0x2620.org/oxjs.git /srv/pandora/static/oxjs \
&& git clone --depth 1 https://git.0x2620.org/python-ox.git /srv/pandora/src/python-ox \
&& git clone --depth 1 https://git.0x2620.org/oxtimelines.git /srv/pandora/src/oxtimelines \
&& pip3 install -e /srv/pandora/src/python-ox \
&& pip3 install -e /srv/pandora/src/oxtimelines \
&& pip3 install -r /srv/pandora/requirements.txt \
&& pip3 install "pillow<4.2.0" \
# Compile pyc and static files
&& cd /srv/pandora/pandora \
&& ln -s config.pandora.jsonc config.jsonc \
&& ./manage.py update_static \
&& ./manage.py compile_pyc -p /srv/pandora/pandora \
&& ./manage.py collectstatic -l --noinput \
# Create OS user
&& addgroup -S -g 8011 pandora \
&& adduser -S -u 8011 -h /srv/pandora -s /bin/false -g pandora -G pandora pandora \
&& chown -R 8011:8011 /srv/pandora \
# Cleanup
&& apk del .deps \
&& find /srv/pandora -name '.git*' -exec rm -rf {} + \
&& rm -rf /tmp/lib* /tmp/oxframe \
&& rm -rf /root \
&& mkdir /root
COPY --chown=8011:8011 docker/srv/ /srv/
RUN \
# Update static files for Czech translation
cd /srv/pandora/pandora \
&& ./manage.py update_static \
# Hackfix missing virtualenv
&& mkdir /srv/pandora/bin \
&& ln -s /usr/bin/oxtimelines /srv/pandora/bin/oxtimelines \
# Create mount with dummy conf files
&& mkdir /srv/pandora/conf \
&& rm config.jsonc \
&& ln -s /srv/pandora/conf/config.jsonc config.jsonc \
&& ln -s /srv/pandora/conf/gunicorn_config.py gunicorn_config.py \
&& ln -s /srv/pandora/conf/local_settings.py local_settings.py \
&& chown -R 8011:8011 /srv/pandora
COPY docker/etc/ /etc/
VOLUME ["/srv/pandora/conf", "/srv/pandora/data"]
EXPOSE 8011
WORKDIR /srv/pandora
CMD ["s6-svscan", "/etc/services.d"]

0
pandora/tmp/pandora-adminpwd.sql → pandora/adminpwd.sql
View File

0
pandora/tmp/pandora-createdb.sql → pandora/createdb.sql
View File

64
pandora/docker/etc/nginx/nginx.conf Normal file
View File

@ -0,0 +1,64 @@
user nginx;
pid /run/nginx.pid;
worker_processes 1;
error_log /dev/stderr warn;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
access_log off;
server_tokens off;
client_max_body_size 100m;
keepalive_timeout 65;
sendfile on;
tcp_nodelay on;
server {
listen 8011;
server_name localhost;
location /favicon.ico {
root /srv/pandora/static;
}
location /static/ {
root /srv/pandora;
autoindex off;
}
location /data/ {
internal;
root /srv/pandora;
}
location / {
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Host $host:$server_port;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_redirect off;
proxy_buffering off;
proxy_read_timeout 90;
proxy_connect_timeout 90;
if (!-f $request_filename) {
proxy_pass http://127.0.0.1:2620;
break;
}
}
error_page 400 /;
error_page 404 /404.html;
location /404.html {
root /srv/pandora/static/html;
}
error_page 500 502 503 504 /50x.html;
location /50x.html {
root /srv/pandora/static/html;
}
}
}

3
pandora/docker/etc/services.d/.s6-svscan/finish Executable file
View File

@ -0,0 +1,3 @@
#!/bin/sh
/bin/true

3
pandora/docker/etc/services.d/nginx/run Executable file
View File

@ -0,0 +1,3 @@
#!/bin/execlineb -P
/usr/sbin/nginx -g "daemon off;"

7
pandora/docker/etc/services.d/pandora-cron/run Executable file
View File

@ -0,0 +1,7 @@
#!/bin/execlineb -P
cd /srv/pandora/pandora
export HOME /srv/pandora
fdmove -c 2 1
s6-setuidgid 8011:8011
./manage.py celerybeat -s /srv/pandora/data/celerybeat-schedule --pidfile pandora-cron.pid -l INFO

7
pandora/docker/etc/services.d/pandora-encoding/run Executable file
View File

@ -0,0 +1,7 @@
#!/bin/execlineb -P
cd /srv/pandora/pandora
export HOME /srv/pandora
fdmove -c 2 1
s6-setuidgid 8011:8011
./manage.py celery worker -Q encoding -n pandora-encoding --pidfile pandora-encoding.pid --maxtasksperchild 500 -l INFO

7
pandora/docker/etc/services.d/pandora-tasks/run Executable file
View File

@ -0,0 +1,7 @@
#!/bin/execlineb -P
cd /srv/pandora/pandora
export HOME /srv/pandora
fdmove -c 2 1
s6-setuidgid 8011:8011
./manage.py celery worker -Q default,celery -n pandora-default --pidfile pandora-tasks.pid --maxtasksperchild 1000 -l INFO

7
pandora/docker/etc/services.d/pandora/run Executable file
View File

@ -0,0 +1,7 @@
#!/bin/execlineb -P
cd /srv/pandora/pandora
export HOME /srv/pandora
fdmove -c 2 1
s6-setuidgid 8011:8011
gunicorn -c gunicorn_config.py wsgi:application

0
pandora/srv/pandora/static/json/locale.0xdb.cs.json → pandora/docker/srv/pandora/static/json/locale.0xdb.cs.json
View File

0
pandora/srv/pandora/static/json/locale.pandora.cs.json → pandora/docker/srv/pandora/static/json/locale.pandora.cs.json
View File

0
pandora/srv/pandora/static/oxjs/source/Ox/js/Constants.js → pandora/docker/srv/pandora/static/oxjs/source/Ox/js/Constants.js
View File

0
pandora/srv/pandora/static/oxjs/source/Ox/json/locale.cs.json → pandora/docker/srv/pandora/static/oxjs/source/Ox/json/locale.cs.json
View File

0
pandora/srv/pandora/static/oxjs/source/UI/json/locale.cs.json → pandora/docker/srv/pandora/static/oxjs/source/UI/json/locale.cs.json
View File

410
pandora/docker/srv/pandora/static/oxjs/tools/build/build.py Executable file
View File

@ -0,0 +1,410 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#vim: et:ts=4:sw=4:sts=4
from __future__ import print_function
import json
import os
import ox
import re
import shutil
import subprocess
import sys
import tarfile
import time
def get_version():
revision = None
if os.path.exists('../../.git'):
revision = subprocess.Popen(
['git', 'rev-list', 'HEAD', '--count'], stdout=subprocess.PIPE
).communicate()[0].strip().decode('utf-8')
revision = int(revision) - 94
elif os.path.exists('../../.bzr/branch/last-revision'):
revision = subprocess.Popen(
['bzr', 'revno'], stdout=subprocess.PIPE
).communicate()[0].strip().decode('utf-8')
return '0.1.%s' % revision if revision else 'unknown'
def build_oxjs(downloads=False, geo=False):
base_path = os.path.dirname(__file__)
if base_path:
os.chdir(base_path)
root_path = '../../'
source_path = root_path + 'source/'
dev_path = root_path + 'dev.tmp/'
min_path = root_path + 'min.tmp/'
locales = {}
version = get_version()
year = time.strftime('%Y', time.gmtime())
comment = ' OxJS %s (c) %s 0x2620, dual-licensed GPL/MIT, see https://oxjs.org for details ' % (version, year)
# Empty dev and min
for path in [dev_path, min_path]:
if os.path.exists(path):
for item in os.listdir(path):
full_path = '%s%s' % (path, item)
if os.path.isdir(full_path):
if not (geo == False and item == 'Geo'):
shutil.rmtree(full_path)
else:
os.remove(full_path)
# Ox.UI Theme Data
theme_data = {}
themes = [dirname for dirname in os.listdir(source_path + 'UI/themes/') if not dirname[0] in '._']
for theme in themes:
theme_data[theme] = read_jsonc(source_path + 'UI/themes/%s/json/theme.jsonc' % theme)
theme_data[theme]['themeClass'] = 'OxTheme' + theme[0].upper() + theme[1:]
# Ox.UI CSS
css = read_text(source_path + 'UI/css/UI.css')
css = css.replace('$import', '\n'.join([
'@import url("../themes/%s/css/theme.css?%s");' % (theme, version) for theme in themes
]))
write_file('%sUI/css/UI.css' % dev_path, css)
write_file('%sUI/css/UI.css' % min_path, css)
# Ox.UI Theme CSS
css = read_text(source_path + 'UI/css/theme.css')
for theme in themes:
theme_css = parse_css(css, theme_data[theme])
theme_css = theme_css.replace('.png)', '.png?%s)' % version)
write_file('%sUI/themes/%s/css/theme.css' % (dev_path, theme), theme_css)
write_file('%sUI/themes/%s/css/theme.css' % (min_path, theme), theme_css)
# Ox.UI SVGs
ui_images = {}
path = source_path + 'UI/svg/'
for filename in [filename for filename in os.listdir(path) if not filename[0] in '._']:
svg = read_text(path + filename)
svg = re.sub('\n\s*', '', svg)
svg = re.sub('<!--.+?-->', '', svg)
# temporary fix for Chrome SVG bug, remove later!
svg = re.sub('width="256" height="256"', 'width="10" height="10" viewBox="0 0 255 255"', svg)
# end fix
ui_images[filename[:-4]] = svg
if filename.startswith('symbolLoading'):
for theme in themes:
theme_svg = re.sub('#808080', format_hex(theme_data[theme]['symbolDefaultColor']), svg)
write_file('%sUI/themes/%s/svg/%s' % (dev_path, theme, filename), theme_svg)
write_file('%sUI/themes/%s/svg/%s' % (min_path, theme, filename), theme_svg)
# copy & link
ui_files = {'dev': [], 'min': []}
for path, dirnames, filenames in os.walk(source_path):
for filename in filenames:
if not '_' in path and not filename[0] in '._' \
and not filename.endswith('~') \
and not filename.endswith('.css') \
and not '/UI/svg' in path \
and (geo or not '/Geo/' in path):
# write copies in min path
source = os.path.join(path, filename)
is_jquery = re.search('^jquery-[\d\.]+\.js$', filename)
is_jquery_min = re.search('^jquery-[\d\.]+\.min\.js$', filename)
is_jquery_plugin = re.search('^jquery\..*?\.js$', filename)
is_jsonc = re.search('\.jsonc$', filename)
if is_jquery or is_jquery_min:
target = os.path.join(path.replace(source_path, min_path), 'jquery.js')
else:
target = os.path.join(path.replace(source_path, min_path), filename)
if is_jquery_plugin:
ui_files['dev'].append(target.replace(min_path, ''))
ui_files['min'].append(target.replace(min_path, ''))
if not '/Ox/js/' in source and not '/UI/js/' in source and not is_jquery:
if re.match('^Ox\..+\.js$', filename) or is_jsonc:
js = read_text(source)
print('minifiy and write', filename, target)
write_file(target, ox.js.minify(js, '' if is_jsonc else comment))
else:
copy_file(source, target)
# write links in dev path
parts = os.path.join(path.replace(source_path, ''), filename).split('/')
for i, part in enumerate(parts):
if i < len(parts) - 1:
parts[i] = '..'
link_source = '/'.join(parts).replace(filename, os.path.join(path, filename))[3:]
link_target = target.replace(min_path, dev_path)
if not is_jquery_min:
write_link(link_source, link_target)
# locales
match = re.search('/(\w+)/json/locale.(\w+).json', source)
if match:
module = match.group(1)
locale = match.group(2)
if not module in locales:
locales[module] = []
locales[module].append(locale)
# remove dangling links from dev tree that might
# be left over from renamed or removed files
for path, dirnames, filenames in os.walk(dev_path):
for f in filenames:
f = os.path.join(path, f)
if os.path.islink(f) and not os.path.exists(f):
os.unlink(f)
# Ox.js
filenames = [
[
'Core.js' # has to run first so that Ox is defined
],
[
'Function.js', # getSortValue (Array.js) depends on Ox.cache
'Polyfill.js' # FIXME: not clear if needed here
],
[
'Array.js', # Ox.slice (Collection.js) depends on Ox.toArray, salt (HTML.js) depends on Ox.range
'String.js', # salt (HTML.js) depends on Ox.char
'Type.js' # Ox.typeOf needed in Collection.js FF3.6 for Ox.slice fallback
],
[
'Collection.js', # Ox.PATH (Constants.js) depends on Ox.slice
'Math.js' # Ox.MAX_LATITUDE (Constants.js) depends on Ox.sinh
]
]
js = ''
js_dir = 'Ox/js/'
ox_files = []
for group in filenames:
ox_files.append([])
for filename in group:
ox_files[-1].append(js_dir + filename)
ox_files.append([])
filenames = sum(filenames, []) # flatten
for filename in sorted(os.listdir(source_path + js_dir)):
if not filename in filenames \
and not filename.startswith('.') \
and not filename.startswith('_') \
and not filename.endswith('~'):
filenames.append(filename)
for filename in filenames:
js += read_text(source_path + js_dir + filename) + '\n'
if not js_dir + filename in sum(ox_files, []):
ox_files[-1].append(js_dir + filename)
js = re.sub(
'Ox.LOCALES = \{\}',
'Ox.LOCALES = ' + json.dumps(locales, indent=4, sort_keys=True),
js
)
js = re.sub(
"Ox.VERSION = '([\d\.]+)'",
"Ox.VERSION = '%s'" % version,
js
)
write_file(dev_path + '/Ox/json/' + 'Ox.json', json.dumps({
'files': ox_files,
'locales': locales,
'version': version
}, indent=4, sort_keys=True))
write_file(min_path + 'Ox.js', ox.js.minify(js, comment))
# Ox.UI
js = ''
root = source_path + 'UI/'
for path, dirnames, filenames in os.walk(root):
for filename in sorted(filenames):
# jquery gets included by Ox.UI loader
# locale json files are loaded lazily
# Ox.UI.css imports all other css files
# svgs are loaded as URLs or dataURLs
# Ox.UI PNGs are loaded on demand
if path != root and not '_' in path and not filename[0] in '._' \
and not filename.endswith('~') \
and not 'jquery' in filename \
and not 'locale' in filename \
and not filename.endswith('theme.css') \
and not filename.endswith('.svg') \
and not 'UI/png' in path:
ui_files['dev'].append(os.path.join(path.replace(source_path, ''), filename))
if not '/js/' in path:
ui_files['min'].append(os.path.join(path.replace(source_path, ''), filename))
if filename.endswith('.js'):
js += read_text(os.path.join(path, filename)) + '\n'
filename = min_path + 'UI/js/UI.js'
write_file(filename, ox.js.minify(js, comment))
ui_files['min'].append(filename.replace(min_path, ''))
write_file(min_path + 'UI/json/UI.json', json.dumps({
'files': sorted(ui_files['min']),
'images': ui_images
}, sort_keys=True))
write_file(dev_path + 'UI/json/UI.json', json.dumps({
'files': sorted(ui_files['dev']),
'images': ui_images
}, indent=4, sort_keys=True))
ui_files['dev'].append('UI/UI.js')
# index
data = {
# sum(list, []) is flatten
'documentation': sorted(sum(ox_files, [])) + sorted(list(filter(
lambda x: re.search('\.js$', x),
ui_files['dev']
)) + ['%s/%s.js' % (x, x) for x in ['Geo', 'Image', 'Unicode']]),
'examples': sorted(sum(map(
lambda x: list(filter(
lambda x: not re.search('/[._]', x),
map(
lambda y: x + '/' + y,
os.listdir(root_path + 'examples/' + x)
)
)),
list(filter(
lambda x: not re.search('^[._]', x),
os.listdir(root_path + 'examples/')
))
), [])) if os.path.exists(root_path + 'examples/') else (),
'readme': list(map(
lambda x: {
'date': time.strftime(
'%Y-%m-%dT%H:%M:%SZ',
time.gmtime(os.path.getmtime(root_path + 'readme/' + x))
),
'id': x.split('.')[0],
'title': get_title(root_path + 'readme/' + x)
},
filter(
lambda x: not re.search('^[._]', x) and re.search('\.html$', x),
os.listdir(root_path + 'readme/')
)
))
}
write_file(root_path + 'index.json', json.dumps(data, indent=4, sort_keys=True))
# downloads
if downloads:
data = {
'date': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
'size': {'oxjs': os.path.getsize(min_path + 'Ox.js')},
'version': version
}
download_path = root_path + 'downloads/'
# source
source_file = download_path + 'OxJS.%s.source.tar.gz' % version
data['size']['source'] = write_tarfile(source_file, root_path, 'OxJS', filter_source)
write_link(source_file.replace(download_path, ''), source_file.replace(version, 'latest'))
# min
min_file = download_path + 'OxJS.%s.min.tar.gz' % version
data['size']['min'] = write_tarfile(min_file, root_path, 'OxJS', filter_min)
write_link(min_file.replace(download_path, ''), min_file.replace(version, 'latest'))
# json
write_file(download_path + 'downloads.json', json.dumps(data, indent=4, sort_keys=True))
# legacy
build_path = root_path + 'build/'
if os.path.exists(build_path) and not os.path.islink(build_path[:-1]):
shutil.rmtree(build_path)
write_link('min', build_path[:-1])
real_dev_path = root_path + 'dev/'
real_min_path = root_path + 'min/'
if os.path.exists(real_dev_path):
shutil.rmtree(real_dev_path)
shutil.move(dev_path,real_dev_path)
if os.path.exists(real_min_path):
shutil.rmtree(real_min_path)
shutil.move(min_path,real_min_path)
def copy_file(source, target):
print('copying', source, 'to', target)
write_file(target, read_file(source))
def filter_min(tarinfo):
name = tarinfo.name
if name == 'OxJS' or re.search('^OxJS/min', name):
return tarinfo
return None
def filter_source(tarinfo):
name = tarinfo.name
if re.search('^[._]', name) or re.search('/[._]', name) or re.search('~$', name):
return None
if re.search('^OxJS/downloads', name):
return None
if name == 'OxJS/tools/geo/png/icons.png':
return None
if re.search('^OxJS/tools/geo/png/icons/', name) and (
not re.search('4096', name) or not os.path.exists(
name.replace('OxJS/', '../../').replace('icons/4096', 'flags')
)
):
return None
return tarinfo
def format_hex(rgb):
return '#%s' % ''.join([hex(c)[-2:].replace('x', '0').upper() for c in rgb])
def get_title(file):
match = re.search('<h1>(.+)</h1>', read_text(file))
return match.groups()[0] if match else 'Untitled'
def parse_css(css, values):
def sub(match):
key = match.group(1)
index = match.group(2)
value = values[key] if index == None else values[key][int(index[1:-1])]
if isinstance(value, str):
string = value
else:
if isinstance(value[0], int):
value = [value]
string = ', '.join(
['rgb%s(%s)' % (
'a' if len(vals) == 4 else '',
', '.join([str(val) for val in vals])
) for vals in value]
)
return string
return re.sub('\$(\w+)(\[\d+\])?', sub, css)
def read_file(file):
print('reading', file)
f = open(file, 'rb')
data = f.read()
f.close()
return data
def read_text(file):
return read_file(file).decode('utf-8')
def read_jsonc(file):
return ox.jsonc.loads(read_text(file))
def write_file(file, data):
print('writing', file)
write_path(file)
if not isinstance(data, bytes):
data = data.encode('utf-8')
f = open(file, 'wb')
f.write(data)
f.close()
return len(data)
def write_link(source, target):
print('linking', source, 'to', target)
write_path(target)
# remove files, symlinks *and broken symlinks*
if os.path.exists(target) or os.path.lexists(target):
if os.path.isdir(target) and not os.path.islink(target):
os.rmdir(target)
else:
os.unlink(target)
os.symlink(source, target)
def write_path(file):
path = os.path.split(file)[0]
if path and not os.path.exists(path):
os.makedirs(path)
def write_tarfile(file, path, arcname, filter):
print('writing', file)
f = tarfile.open(file, 'w:gz')
f.add(path, arcname=arcname, filter=filter)
f.close()
return os.path.getsize(file)
if __name__ == '__main__':
build_oxjs(downloads='-downloads' in sys.argv, geo=not '-nogeo' in sys.argv)

17
pandora/etc/init.d/pandora Executable file
View File

@ -0,0 +1,17 @@
#!/sbin/openrc-run
description="Pan.do/ra docker container"
depend() {
need docker net
use dns logger netmount
after postgres rabbitmq
}
start() {
/usr/bin/docker run -d --rm --name pandora -h pandora --link=postgres --link=rabbitmq -p 127.0.0.1:9011:8011 -v /srv/pandora/conf:/srv/pandora/conf -v /srv/pandora/data:/srv/pandora/data pandora
}
stop() {
/usr/bin/docker stop pandora
}

14
pandora/etc/nginx/conf.d/pandora.conf Normal file
View File

@ -0,0 +1,14 @@
server {
listen [::]:8011 ipv6only=off;
listen [::]:8411 ssl http2 ipv6only=off;
access_log /var/log/nginx/pandora.access.log;
error_log /var/log/nginx/pandora.error.log;
location / {
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Host $host:$server_port;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://127.0.0.1:9011;
}
}

60
pandora/etc/nginx/sites-available/pandora
View File

@ -1,60 +0,0 @@
server {
listen 8001 ssl http2;
listen [::]:8001 ssl http2;
access_log /var/log/nginx/pandora.access.log;
error_log /var/log/nginx/pandora.error.log;
location /favicon.ico {
root /srv/pandora/static;
}
location /static/ {
root /srv/pandora;
autoindex off;
}
location /data/ {
internal;
root /srv/pandora;
}
location /api/ws/ {
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Proxy "";
proxy_redirect off;
proxy_buffering off;
proxy_read_timeout 999999999;
proxy_pass http://127.0.0.1:2622/;
}
location / {
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header Host $http_host;
proxy_set_header Proxy "";
proxy_redirect off;
proxy_buffering off;
proxy_read_timeout 90;
proxy_connect_timeout 90;
if (!-f $request_filename) {
proxy_pass http://127.0.0.1:2620;
break;
}
client_max_body_size 32m;
}
error_page 400 /;
error_page 404 /404.html;
location /404.html {
root /srv/pandora/static/html;
}
error_page 500 502 503 504 /50x.html;
location /50x.html {
root /srv/pandora/static/html;
}
}

13
pandora/pandora.gpg
View File

@ -1,13 +0,0 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1
mI0ESXYhEgEEALl9jDTdmgpApPbjN+7b85dC92HisPUp56ifEkKJOBj0X5HhRqxs
Wjx/zlP4/XJGrHnxJyrdPxjSwAXz7bNdeggkN4JWdusTkr5GOXvggQnng0X7f/rX
oJwoEGtYOCODLPs6PC0qjh5yPzJVeiRsKUOZ7YVNnwNwdfS4D8RZvtCrABEBAAG0
FExhdW5jaHBhZCBQUEEgZm9yIGpeiLYEEwECACAFAkl2IRICGwMGCwkIBwMCBBUC
CAMEFgIDAQIeAQIXgAAKCRAohRM8AZde82FfA/9OB/64/YLaCpizHZ8f6DK3rGgF
e6mX3rFK8yOKGGL06316VhDzfzMiZSauUZ0t+lKHR/KZYeSaFwEoUoblTG/s4IIo
9aBMHWhVXJW6eifKUmTGqEn2/0UxoWQq2C3F6njMkCaP+ALOD5uzaSYGdjqAUAwS
pAAGSEQ4uz6bYSeM4Q==
=SM2a
-----END PGP PUBLIC KEY BLOCK-----

0
pandora/srv/pandora/pandora/config.jsonc → pandora/srv/pandora/conf/config.jsonc
View File

5
pandora/srv/pandora/conf/gunicorn_config.py Normal file
View File

@ -0,0 +1,5 @@
bind="127.0.0.1:2620"
log_level="info"
max_requests=1000
timeout=90
workers=5

3
pandora/srv/pandora/pandora/local_settings.py → pandora/srv/pandora/conf/local_settings.py
View File

@ -1,12 +1,13 @@
DATABASES = {
'default': {
'HOST': 'postgres',
'NAME': 'pandora',
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'USER': 'pandora',
'PASSWORD': '${PANDORA_PWD}',
}
}
BROKER_URL = '${PANDORA_BROKER_URL}'
BROKER_URL = 'amqp://pandora:${PANDORA_RABBIT_PWD}@rabbitmq:5672//pandora'
DB_GIN_TRGM = True
XACCELREDIRECT = True