diff --git a/ckan.sh b/ckan.sh index e6733ce..93152a6 100755 --- a/ckan.sh +++ b/ckan.sh @@ -1,104 +1,54 @@ #!/bin/bash SOURCE_DIR=$(realpath $(dirname "${0}"))/ckan +# Requires: Postgres, Redis, Solr, CKAN-Datapusher -# Install dependencies for CKAN -apt-get -y --no-install-recommends install gcc libpq-dev python-dev python-virtualenv redis-server virtualenv - -# Install dependencies for Solr -apt-get -y --no-install-recommends install openjdk-8-jre-headless lsof - -# Install CKAN python virtual environment -mkdir -p /srv/ckan -virtualenv --no-site-packages --python=/usr/bin/python2.7 /srv/ckan -/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckan.git#egg=ckan' -/srv/ckan/bin/pip install -r /srv/ckan/src/ckan/requirements.txt - -# Install Datapusher python virtual environment -mkdir -p /srv/ckan-datapusher -virtualenv --no-site-packages --python=/usr/bin/python2.7 /srv/ckan-datapusher -/srv/ckan-datapusher/bin/pip install -e 'git+https://github.com/ckan/datapusher.git#egg=datapusher' -/srv/ckan-datapusher/bin/pip install -r /srv/ckan-datapusher/src/datapusher/requirements.txt - -# Patch service provider TLS verification for Datapusher -# https://github.com/ckan/ckan-service-provider/issues/36 -patch -d /srv/ckan-datapusher -p0 <${SOURCE_DIR}/ckan-serviceprovider-sslverify.patch - -# Install CKAN extensions -/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-basiccharts#egg=ckanext_basiccharts' -/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-spatial#egg=ckanext_spatial' -/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-geoview#egg=ckanext_geoview' -/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-mapviews#egg=ckanext_mapviews' -/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-pages#egg=ckanext_pages' -/srv/ckan/bin/pip install -e 'git+https://github.com/XVTSolutions/ckanext-spatialUI#egg=ckanext_spatialui' -/srv/ckan/bin/pip install -e 'git+https://github.com/aptivate/ckanext-datasetthumbnail#egg=ckanext_datasetthumbnail' -/srv/ckan/bin/pip install -e 'git+https://github.com/datagvat/ckanext-dgvat_xls#egg=ckanext_dgvat_xls' -/srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-spatial/pip-requirements.txt -/srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-datasetthumbnail/requirements.txt -/srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-dgvat-xls/requirements.txt - -# Download Solr -wget http://archive.apache.org/dist/lucene/solr/6.5.1/solr-6.5.1.tgz -O /tmp/solr-6.5.1.tgz -tar xzf /tmp/solr-6.5.1.tgz -C /opt/ -mv /opt/solr-6.5.1 /opt/solr -rm -f /tmp/solr-6.5.1.tgz - -# Create Solr OS user -adduser --system --group --home /var/lib/solr --shell /bin/false solr -chown -R solr:solr /opt/solr/ - -# Configure Solr -cp ${SOURCE_DIR}/lib/systemd/system/solr.service /lib/systemd/system/solr.service -cp -p /opt/solr/server/solr/solr.xml /var/lib/solr/ -systemctl daemon-reload -systemctl enable solr -systemctl start solr -sleep 5 - -# Configure CKAN Solr core -sudo -u solr /opt/solr/bin/solr create -c ckan -cp ${SOURCE_DIR}/var/lib/solr/ckan/conf/solrconfig.xml /var/lib/solr/ckan/conf/solrconfig.xml -ln -s /srv/ckan/src/ckan/ckan/config/solr/schema.xml /var/lib/solr/ckan/conf/schema.xml -systemctl restart solr +# Build Docker container +docker build -t ckan ${SOURCE_DIR} # Create database export CKAN_PWD=$(head -c 18 /dev/urandom | base64) export CKAN_DS_PWD=$(head -c 18 /dev/urandom | base64) -envsubst <${SOURCE_DIR}/tmp/ckan-createdb.sql >/tmp/ckan-createdb.sql -sudo -u postgres psql -f /tmp/ckan-createdb.sql -rm -f /tmp/ckan-createdb.sql +envsubst <${SOURCE_DIR}/createdb.sql | docker exec -i postgres psql + +# Configure CKAN Solr core +docker exec solr solr create -c ckan +cp ${SOURCE_DIR}/srv/solr/data/ckan/conf/schema.xml /srv/solr/data/ckan/conf/schema.xml +cp ${SOURCE_DIR}/srv/solr/data/ckan/conf/solrconfig.xml /srv/solr/data/ckan/conf/solrconfig.xml +chown 8983:8983 /srv/solr/data/ckan/conf/schema.xml +systemctl restart solr # Configure CKAN export CKAN_SECRET=$(head -c 18 /dev/urandom | base64) -export CKAN_UUID=$(python -c "import uuid; print uuid.uuid4()") -mkdir /etc/ckan -envsubst <${SOURCE_DIR}/etc/ckan/ckan.ini >/etc/ckan/ckan.ini -cp ${SOURCE_DIR}/etc/ckan/datapusher.wsgi /etc/ckan/datapusher.wsgi -cp ${SOURCE_DIR}/etc/ckan/datapusher_settings.py /etc/ckan/datapusher_settings.py -ln -s /srv/ckan/src/ckan/ckan/config/who.ini /etc/ckan/who.ini +export CKAN_UUID=$(cat /proc/sys/kernel/random/uuid) +mkdir -p /srv/ckan/{conf,data} +envsubst <${SOURCE_DIR}/srv/ckan/conf/ckan.ini >/srv/ckan/conf/ckan.ini +cp ${SOURCE_DIR}/srv/ckan/conf/who.ini /srv/ckan/conf/who.ini +chown -R 8003:8003 /srv/ckan/data cp ${SOURCE_DIR}/srv/ckan/update-ip.sh /srv/ckan/update-ip.sh /srv/ckan/update-ip.sh # Populate database -/srv/ckan/bin/paster --plugin=ckan db init -c /etc/ckan/ckan.ini -/srv/ckan/bin/paster --plugin=ckanext-spatial spatial initdb -c /etc/ckan/ckan.ini -/srv/ckan/bin/paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | sudo -u postgres psql - -# Create CKAN OS user -adduser --system --group --home /srv/ckan --shell /bin/false ckan -chown -R ckan:ckan /srv/ckan/ -chown -R ckan:ckan /srv/ckan-datapusher/ +docker run --rm --link=postgres --link=redis --link=solr -v /srv/ckan/conf:/etc/ckan -v /srv/ckan/data:/srv/ckan/storage ckan paster --plugin=ckan db init -c /etc/ckan/ckan.ini +docker run --rm --link=postgres --link=redis --link=solr -v /srv/ckan/conf:/etc/ckan -v /srv/ckan/data:/srv/ckan/storage ckan paster --plugin=ckanext-spatial spatial initdb -c /etc/ckan/ckan.ini +docker run --rm --link=postgres --link=redis --link=solr -v /srv/ckan/conf:/etc/ckan -v /srv/ckan/data:/srv/ckan/storage ckan paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | docker exec -i postgres psql # Create admin account export CKAN_ADMIN_USER="admin" -export CKAN_ADMIN_UUID=$(python -c "import uuid; print uuid.uuid4()") -export CKAN_ADMIN_APIKEY=$(python -c "import uuid; print uuid.uuid4()") +export CKAN_ADMIN_UUID=$(cat /proc/sys/kernel/random/uuid) +export CKAN_ADMIN_APIKEY=$(cat /proc/sys/kernel/random/uuid) export CKAN_ADMIN_PWD=$(head -c 12 /dev/urandom | base64) -export CKAN_ADMIN_HASH=$(/srv/ckan/bin/python ${SOURCE_DIR}/ckan-adminpwd.py ${CKAN_ADMIN_PWD}) +export CKAN_ADMIN_HASH=$(docker run --rm ckan python -c "from passlib.hash import pbkdf2_sha512;print pbkdf2_sha512.encrypt('${CKAN_ADMIN_PWD}')") export CKAN_ADMIN_EMAIL="admin@example.com" -envsubst <${SOURCE_DIR}/tmp/ckan-adminpwd.sql >/tmp/ckan-adminpwd.sql -sudo -u postgres psql -f /tmp/ckan-adminpwd.sql ckan -rm /tmp/ckan-adminpwd.sql +envsubst <${SOURCE_DIR}/adminpwd.sql | docker exec -i postgres psql ckan + +# Configure CKAN service +cp ${SOURCE_DIR}/lib/systemd/system/ckan.service /lib/systemd/system/ckan.service +systemctl daemon-reload +systemctl enable ckan +systemctl start ckan + + # Create uwsgi and nginx app definition cp ${SOURCE_DIR}/etc/uwsgi/apps-available/ckan.ini /etc/uwsgi/apps-available/ckan.ini diff --git a/ckan/Dockerfile b/ckan/Dockerfile new file mode 100644 index 0000000..ea2f49d --- /dev/null +++ b/ckan/Dockerfile @@ -0,0 +1,38 @@ +FROM alpine:3.7 +MAINTAINER Disassembler + +RUN apk --no-cache add python2 + +RUN apk --no-cache add libjpeg-turbo libmagic libpq libxml2 libxslt py2-pip zlib \ + && apk --no-cache add --virtual .deps git build-base libjpeg-turbo-dev libxml2-dev libxslt-dev postgresql-dev python2-dev zlib-dev \ + && echo 'http://repository.fit.cvut.cz/mirrors/alpine/edge/testing' >>/etc/apk/repositories \ + && apk --no-cache add geos \ + && ln -s /lib/ld-musl-x86_64.so.1 /lib/libc.so.1 \ + && mkdir -p /srv/ckan \ + && cd /srv/ckan \ + && pip install setuptools==36.1 \ + && pip install -e 'git+https://github.com/ckan/ckan.git#egg=ckan' \ + && sed -i 's/psycopg2==2.4.5/psycopg2==2.7.1/' /srv/ckan/src/ckan/requirements.txt \ + && pip install -r /srv/ckan/src/ckan/requirements.txt \ + && pip install -e 'git+https://github.com/ckan/ckanext-basiccharts#egg=ckanext_basiccharts' \ + && pip install -e 'git+https://github.com/ckan/ckanext-spatial#egg=ckanext_spatial' \ + && pip install -e 'git+https://github.com/ckan/ckanext-geoview#egg=ckanext_geoview' \ + && pip install -e 'git+https://github.com/ckan/ckanext-mapviews#egg=ckanext_mapviews' \ + && pip install -e 'git+https://github.com/ckan/ckanext-pages#egg=ckanext_pages' \ + && pip install -e 'git+https://github.com/XVTSolutions/ckanext-spatialUI#egg=ckanext_spatialui' \ + && pip install -e 'git+https://github.com/aptivate/ckanext-datasetthumbnail#egg=ckanext_datasetthumbnail' \ + && pip install -e 'git+https://github.com/datagvat/ckanext-dgvat_xls#egg=ckanext_dgvat_xls' \ + && pip install -r /srv/ckan/src/ckanext-spatial/pip-requirements.txt \ + && pip install -r /srv/ckan/src/ckanext-dgvat-xls/requirements.txt \ + && addgroup -S -g 8003 ckan \ + && adduser -S -u 8003 -h /srv/ckan -s /bin/false -g ckan -G ckan ckan \ + && chown -R ckan:ckan /srv/ckan \ + && apk del .deps \ + && find /srv/ckan/src -name '.git*' -exec rm -rf {} + \ + && rm -rf /root/* || true + +VOLUME ["/etc/ckan", "/srv/ckan/storage"] +EXPOSE 8003 + +USER ckan +CMD ["paster", "serve", "/etc/ckan/ckan.ini"] diff --git a/ckan/tmp/ckan-adminpwd.sql b/ckan/adminpwd.sql similarity index 100% rename from ckan/tmp/ckan-adminpwd.sql rename to ckan/adminpwd.sql diff --git a/ckan/ckan-adminpwd.py b/ckan/ckan-adminpwd.py deleted file mode 100644 index 9ea023a..0000000 --- a/ckan/ckan-adminpwd.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/python - -import sys -from passlib.hash import pbkdf2_sha512 - -print pbkdf2_sha512.encrypt(sys.argv[1]) diff --git a/ckan/tmp/ckan-createdb.sql b/ckan/createdb.sql similarity index 54% rename from ckan/tmp/ckan-createdb.sql rename to ckan/createdb.sql index b089479..e5a166c 100644 --- a/ckan/tmp/ckan-createdb.sql +++ b/ckan/createdb.sql @@ -11,14 +11,6 @@ GRANT CONNECT, CREATE, TEMPORARY ON DATABASE ckan_datastore TO ckan; ALTER DATABASE ckan_datastore OWNER TO ckan_datastore; \c ckan -CREATE TEXT SEARCH DICTIONARY cspell (template=ispell, dictfile = czech, afffile=czech, stopwords=czech); -CREATE TEXT SEARCH CONFIGURATION czech (copy=english); -ALTER TEXT SEARCH CONFIGURATION czech ALTER MAPPING FOR word, asciiword WITH cspell, simple; CREATE EXTENSION postgis; GRANT ALL ON geometry_columns TO ckan; GRANT ALL ON spatial_ref_sys TO ckan; - -\c ckan_datastore -CREATE TEXT SEARCH DICTIONARY cspell (template=ispell, dictfile = czech, afffile=czech, stopwords=czech); -CREATE TEXT SEARCH CONFIGURATION czech (copy=english); -ALTER TEXT SEARCH CONFIGURATION czech ALTER MAPPING FOR word, asciiword WITH cspell, simple; diff --git a/ckan/etc/uwsgi/apps-available/ckan.ini b/ckan/etc/uwsgi/apps-available/ckan.ini deleted file mode 100644 index 404d7d4..0000000 --- a/ckan/etc/uwsgi/apps-available/ckan.ini +++ /dev/null @@ -1,12 +0,0 @@ -[uwsgi] -uid = ckan -gid = ckan -chown-socket = www-data:www-data -chdir = /srv/ckan -home = /srv/ckan -master = false -workers = 1 -exec-asap = /srv/ckan/update-ip.sh -disable-logging = true -ini-paste = /etc/ckan/ckan.ini -enable-threads = true diff --git a/ckan/lib/systemd/system/ckan.service b/ckan/lib/systemd/system/ckan.service new file mode 100644 index 0000000..093b1e4 --- /dev/null +++ b/ckan/lib/systemd/system/ckan.service @@ -0,0 +1,11 @@ +[Unit] +Description=CKAN Docker container +After=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/docker run --rm --name ckan --link=postgres --link=redis --link=solr -v /srv/ckan/conf:/etc/ckan -v /srv/ckan/data:/srv/ckan/storage ckan +ExecStop=/usr/bin/docker stop ckan + +[Install] +WantedBy=multi-user.target diff --git a/ckan/lib/systemd/system/solr.service b/ckan/lib/systemd/system/solr.service deleted file mode 100644 index 6fe5404..0000000 --- a/ckan/lib/systemd/system/solr.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Apache Solr -After=network.target - -[Service] -ExecStart=/opt/solr/bin/solr start -f -p 8983 -s /var/lib/solr -ExecStop=/opt/solr/bin/solr stop -p 8983 -User=solr - -[Install] -WantedBy=multi-user.target diff --git a/ckan/ckan-serviceprovider-sslverify.patch b/ckan/serviceprovider-sslverify.patch similarity index 100% rename from ckan/ckan-serviceprovider-sslverify.patch rename to ckan/serviceprovider-sslverify.patch diff --git a/ckan/etc/ckan/ckan.ini b/ckan/srv/ckan/conf/ckan.ini similarity index 95% rename from ckan/etc/ckan/ckan.ini rename to ckan/srv/ckan/conf/ckan.ini index 9abb78f..361a6d3 100644 --- a/ckan/etc/ckan/ckan.ini +++ b/ckan/srv/ckan/conf/ckan.ini @@ -45,13 +45,13 @@ who.log_file = %(cache_dir)s/who_log.ini # who.timeout = 86400 ## Database Settings -sqlalchemy.url = postgresql://ckan:${CKAN_PWD}@localhost/ckan +sqlalchemy.url = postgresql://ckan:${CKAN_PWD}@postgres/ckan -ckan.datastore.write_url = postgresql://ckan:${CKAN_PWD}@localhost/ckan_datastore -ckan.datastore.read_url = postgresql://ckan_datastore:${CKAN_DS_PWD}@localhost/ckan_datastore +ckan.datastore.write_url = postgresql://ckan:${CKAN_PWD}@postgres/ckan_datastore +ckan.datastore.read_url = postgresql://ckan_datastore:${CKAN_DS_PWD}@postgres/ckan_datastore # PostgreSQL' full-text search parameters -ckan.datastore.default_fts_lang = czech +ckan.datastore.default_fts_lang = english ckan.datastore.default_fts_index_method = gist ## Site Settings @@ -76,13 +76,13 @@ ckan.auth.roles_that_cascade_to_sub_groups = admin ## Search Settings ckan.site_id = default -solr_url = http://127.0.0.1:8983/solr/ckan +solr_url = http://solr:8983/solr/ckan ## Redis Settings # URL to your Redis instance, including the database to be used. -ckan.redis.url = redis://127.0.0.1:6379/0 +ckan.redis.url = redis://redis:6379/0 ## CORS Settings @@ -177,7 +177,7 @@ ckan.max_image_size = 10 # Make sure you have set up the DataStore ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet -ckan.datapusher.url = http://127.0.0.1:8098/ +ckan.datapusher.url = http://ckan-datapusher:8004/ #ckan.datapusher.assume_task_stale_after = 3600 # Resource Proxy settings diff --git a/ckan/etc/ckan/datapusher.wsgi b/ckan/srv/ckan/conf/datapusher.wsgi similarity index 100% rename from ckan/etc/ckan/datapusher.wsgi rename to ckan/srv/ckan/conf/datapusher.wsgi diff --git a/ckan/etc/ckan/datapusher_settings.py b/ckan/srv/ckan/conf/datapusher_settings.py similarity index 100% rename from ckan/etc/ckan/datapusher_settings.py rename to ckan/srv/ckan/conf/datapusher_settings.py diff --git a/ckan/srv/ckan/conf/who.ini b/ckan/srv/ckan/conf/who.ini new file mode 100644 index 0000000..885a497 --- /dev/null +++ b/ckan/srv/ckan/conf/who.ini @@ -0,0 +1,37 @@ +[plugin:auth_tkt] +use = ckan.lib.auth_tkt:make_plugin +# If no secret key is defined here, beaker.session.secret will be used +#secret = somesecret + +[plugin:friendlyform] +use = repoze.who.plugins.friendlyform:FriendlyFormPlugin +login_form_url= /user/login +login_handler_path = /login_generic +logout_handler_path = /user/logout +rememberer_name = auth_tkt +post_login_url = /user/logged_in +post_logout_url = /user/logged_out +charset = utf-8 + +#[plugin:basicauth] +#use = repoze.who.plugins.basicauth:make_plugin +#realm = 'CKAN' + +[general] +request_classifier = repoze.who.classifiers:default_request_classifier +challenge_decider = repoze.who.classifiers:default_challenge_decider + +[identifiers] +plugins = + friendlyform;browser + auth_tkt + +[authenticators] +plugins = + auth_tkt + ckan.lib.authenticator:UsernamePasswordAuthenticator + +[challengers] +plugins = + friendlyform;browser +# basicauth diff --git a/ckan/srv/ckan/update-ip.sh b/ckan/srv/ckan/update-ip.sh index 15216bd..5962742 100755 --- a/ckan/srv/ckan/update-ip.sh +++ b/ckan/srv/ckan/update-ip.sh @@ -1,4 +1,4 @@ #!/bin/sh URL=$(ip route get 1 | awk '{print $NF;exit}') -sed -i "s|^ckan\.site_url.*|ckan.site_url = https://${URL}:8003|" /etc/ckan/ckan.ini +sed -i "s|^ckan\.site_url.*|ckan.site_url = http://${URL}:8003|" /srv/ckan/conf/ckan.ini