Add CKAN install script (squashed commit)

01-basic.sh

@@ -127,6 +127,12 @@ systemctl restart nginx
 # Install packages
 apt-get -y --no-install-recommends install postgresql-9.6 postgresql-9.6-postgis postgresql-contrib-9.6
 
+# Install Czech search data
+wget http://postgres.cz/data/czech.tar.gz -O /tmp/czech.tar.gz
+tar xzf /tmp/czech.tar.gz -C /usr/share/postgresql/9.6/tsearch_data --strip-components 1
+chown root:root /usr/share/postgresql/9.6/tsearch_data/czech.*
+rm -f /tmp/czech.tar.gz
+
 # Configure
 cp ${SOURCE_DIR}/basic/etc/postgresql/9.6/main/postgresql.conf /etc/postgresql/9.6/main/postgresql.conf
 cp ${SOURCE_DIR}/basic/etc/postgresql/9.6/main/pg_hba.conf /etc/postgresql/9.6/main/pg_hba.conf

30-ckan.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+SOURCE_DIR=$(realpath $(dirname "${0}"))
+
+# Install dependencies for CKAN
+apt-get -y --no-install-recommends install gcc libpq-dev python-dev python-virtualenv redis-server uuid-runtime virtualenv
+
+# Install dependencies for Solr
+apt-get -y --no-install-recommends install openjdk-8-jre-headless lsof
+
+# Install CKAN python virtual environment
+mkdir -p /srv/ckan
+virtualenv --no-site-packages --python=/usr/bin/python2.7 /srv/ckan
+. /srv/ckan/bin/activate
+pip install -e 'git+https://github.com/ckan/ckan.git#egg=ckan'
+pip install -e 'git+https://github.com/ckan/datapusher.git#egg=datapusher'
+pip install -r /srv/ckan/src/ckan/requirements.txt
+pip install -r /srv/ckan/src/datapusher/requirements.txt
+deactivate
+
+# Donwload Solr
+wget http://archive.apache.org/dist/lucene/solr/6.5.1/solr-6.5.1.tgz -O /tmp/solr-6.5.1.tgz
+tar xzf /tmp/solr-6.5.1.tgz -C /opt/
+mv /opt/solr-6.5.1 /opt/solr
+rm -f /tmp/solr-6.5.1.tgz
+
+# Create database
+export CKAN_PWD=$(head -c 18 /dev/urandom | base64)
+export CKAN_DS_PWD=$(head -c 18 /dev/urandom | base64)
+envsubst <${SOURCE_DIR}/ckan/tmp/ckan-createdb.sql >/tmp/ckan-createdb.sql
+sudo -u postgres psql -f /tmp/ckan-createdb.sql
+rm -f /tmp/ckan-createdb.sql
+
+# Create CKAN OS user
+adduser --system --group --home /srv/ckan --shell /bin/false ckan
+chown -R ckan:ckan /srv/ckan/
+mkdir /var/lib/ckan
+chown ckan:ckan /var/lib/ckan
+
+# Create Solr OS user
+adduser --system --group --home /var/lib/solr --shell /bin/false solr
+chown -R solr:solr /opt/solr/
+
+# Configure Solr
+cp ${SOURCE_DIR}/ckan/lib/systemd/system/solr.service /lib/systemd/system/solr.service
+cp -p /opt/solr/server/solr/solr.xml /var/lib/solr/
+systemctl daemon-reload
+systemctl enable solr
+systemctl start solr
+sleep 5
+
+# Configure CKAN Solr core
+sudo -u solr /opt/solr/bin/solr create -c ckan
+cp ${SOURCE_DIR}/ckan/var/lib/solr/ckan/conf/solrconfig.xml /var/lib/solr/ckan/conf/solrconfig.xml
+ln -s /srv/ckan/src/ckan/ckan/config/solr/schema.xml /var/lib/solr/ckan/conf/schema.xml
+systemctl restart solr
+
+# Configure CKAN
+export CKAN_SECRET=$(head -c 18 /dev/urandom | base64)
+export CKAN_UUID=$(uuidgen -r)
+export CKAN_DSJOB_SECRET=$(uuidgen -r)
+export CKAN_DSJOB_USER=$(uuidgen -r)
+export CKAN_DSJOB_PWD=$(uuidgen -r)
+mkdir /etc/ckan
+envsubst <${SOURCE_DIR}/ckan/etc/ckan/ckan.ini >/etc/ckan/ckan.ini
+envsubst <${SOURCE_DIR}/ckan/etc/ckan/datapusher_settings.py >/etc/ckan/datapusher_settings.py
+cp ${SOURCE_DIR}/ckan/etc/ckan/datapusher.wsgi /etc/ckan/datapusher.wsgi
+ln -s /srv/ckan/src/ckan/ckan/config/who.ini /etc/ckan/who.ini
+cp ${SOURCE_DIR}/ckan/srv/ckan/update-ip.sh /srv/ckan/update-ip.sh
+/srv/ckan/update-ip.sh
+
+# Populate database
+. /srv/ckan/bin/activate
+paster --plugin=ckan db init -c /etc/ckan/ckan.ini
+paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | sudo -u postgres psql
+
+# Create admin account
+export CKAN_ADMIN_USER="admin"
+export CKAN_ADMIN_UUID=$(uuidgen -r)
+export CKAN_ADMIN_APIKEY=$(uuidgen -r)
+export CKAN_ADMIN_PWD=$(head -c 12 /dev/urandom | base64)
+export CKAN_ADMIN_HASH=$(python ${SOURCE_DIR}/ckan/ckan-adminpwd.py ${CKAN_ADMIN_PWD})
+export CKAN_ADMIN_EMAIL="admin@example.com"
+envsubst <${SOURCE_DIR}/ckan/tmp/ckan-adminpwd.sql >/tmp/ckan-adminpwd.sql
+sudo -u postgres psql -f /tmp/ckan-adminpwd.sql ckan
+rm /tmp/ckan-adminpwd.sql
+deactivate
+
+# Create uwsgi and nginx app definition
+cp ${SOURCE_DIR}/ckan/etc/uwsgi/apps-available/ckan.ini /etc/uwsgi/apps-available/ckan.ini
+cp ${SOURCE_DIR}/ckan/etc/uwsgi/apps-available/ckan-datapusher.ini /etc/uwsgi/apps-available/ckan-datapusher.ini
+ln -s /etc/uwsgi/apps-available/ckan.ini /etc/uwsgi/apps-enabled/ckan.ini
+ln -s /etc/uwsgi/apps-available/ckan-datapusher.ini /etc/uwsgi/apps-enabled/ckan-datapusher.ini
+cp ${SOURCE_DIR}/ckan/etc/nginx/apps-available/ckan /etc/nginx/apps-available/ckan
+ln -s /etc/nginx/apps-available/ckan /etc/nginx/apps-enabled/ckan
+cp ${SOURCE_DIR}/ckan/etc/nginx/sites-available/ckan-datapusher /etc/nginx/sites-available/ckan-datapusher
+ln -s /etc/nginx/sites-available/ckan-datapusher /etc/nginx/sites-enabled/ckan-datapusher
+
+# Restart services
+systemctl restart uwsgi
+systemctl restart nginx
+
+# Install cron job
+cp ${SOURCE_DIR}/ckan/srv/ckan/cron-pageviews.sh /srv/ckan/cron-pageviews.sh
+cp ${SOURCE_DIR}/ckan/etc/cron.d/ckan /etc/cron.d/ckan
+
+# Add portal application definition
+${SOURCE_DIR}/basic/portal-app-manager.py ckan "/ckan/" "${CKAN_ADMIN_USER}" "${CKAN_ADMIN_PWD}"
+
+# Remove compilation dependencies
+apt-get -y purge gcc libpq-dev python-dev
+apt-get -y --purge autoremove

ckan/ckan-adminpwd.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python
+
+import sys
+from passlib.hash import pbkdf2_sha512
+
+print pbkdf2_sha512.encrypt(sys.argv[1])

ckan/etc/ckan/ckan.ini

@@ -0,0 +1,224 @@
+#
+# CKAN - Pylons configuration
+#
+# These are some of the configuration options available for your CKAN
+# instance. Check the documentation in 'doc/configuration.rst' or at the
+# following URL for a description of what they do and the full list of
+# available options:
+#
+# http://docs.ckan.org/en/latest/maintaining/configuration.html
+#
+# The %(here)s variable will be replaced with the parent directory of this file
+#
+
+[DEFAULT]
+
+# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PRODUCTION ENVIRONMENT*
+debug = false
+
+[server:main]
+use = egg:Paste#http
+host = 0.0.0.0
+port = 5000
+
+[app:main]
+use = egg:ckan
+full_stack = true
+cache_dir = /tmp/%(ckan.site_id)s/
+beaker.session.key = ckan
+
+# This is the secret token that the beaker library uses to hash the cookie sent
+# to the client. `paster make-config` generates a unique value for this each
+# time it generates a config file.
+beaker.session.secret = ${CKAN_SECRET}
+
+# `paster make-config` generates a unique value for this each time it generates
+# a config file.
+app_instance_uuid = ${CKAN_UUID}
+
+# repoze.who config
+who.config_file = %(here)s/who.ini
+who.log_level = warning
+who.log_file = %(cache_dir)s/who_log.ini
+# Session timeout (user logged out after period of inactivity, in seconds).
+# Inactive by default, so the session doesn't expire.
+# who.timeout = 86400
+
+## Database Settings
+sqlalchemy.url = postgresql://ckan:${CKAN_PWD}@localhost/ckan
+
+ckan.datastore.write_url = postgresql://ckan:${CKAN_PWD}@localhost/ckan_datastore
+ckan.datastore.read_url = postgresql://ckan_datastore:${CKAN_DS_PWD}@localhost/ckan_datastore
+
+# PostgreSQL' full-text search parameters
+ckan.datastore.default_fts_lang = czech
+ckan.datastore.default_fts_index_method = gist
+
+## Site Settings
+
+ckan.site_url = http://127.0.0.1
+ckan.root_path = /ckan/{{LANG}}
+#ckan.use_pylons_response_cleanup_middleware = true
+
+## Authorization Settings
+
+ckan.auth.anon_create_dataset = false
+ckan.auth.create_unowned_dataset = false
+ckan.auth.create_dataset_if_not_in_organization = false
+ckan.auth.user_create_groups = false
+ckan.auth.user_create_organizations = false
+ckan.auth.user_delete_groups = false
+ckan.auth.user_delete_organizations = false
+ckan.auth.create_user_via_api = false
+ckan.auth.create_user_via_web = false
+ckan.auth.roles_that_cascade_to_sub_groups = admin
+
+
+## Search Settings
+
+ckan.site_id = default
+solr_url = http://127.0.0.1:8983/solr/ckan
+
+
+## Redis Settings
+
+# URL to your Redis instance, including the database to be used.
+ckan.redis.url = redis://localhost:6379/0
+
+
+## CORS Settings
+
+# If cors.origin_allow_all is true, all origins are allowed.
+# If false, the cors.origin_whitelist is used.
+# ckan.cors.origin_allow_all = true
+# cors.origin_whitelist is a space separated list of allowed domains.
+# ckan.cors.origin_whitelist = http://example1.com http://example2.com
+
+
+## Plugins Settings
+
+# Note: Add ``datastore`` to enable the CKAN DataStore
+#       Add ``datapusher`` to enable DataPusher
+#		Add ``resource_proxy`` to enable resorce proxying and get around the
+#		same origin policy
+ckan.plugins = stats text_view image_view recline_view datastore datapusher
+
+# Define which views should be created by default
+# (plugins must be loaded in ckan.plugins)
+ckan.views.default_views = image_view text_view recline_view
+
+# Customize which text formats the text_view plugin will show
+#ckan.preview.json_formats = json
+#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss
+#ckan.preview.text_formats = text plain text/plain
+
+# Customize which image formats the image_view plugin will show
+#ckan.preview.image_formats = png jpeg jpg gif
+
+## Front-End Settings
+ckan.site_title = CKAN
+ckan.site_logo = /base/images/ckan-logo.png
+ckan.site_description =
+ckan.favicon = /base/images/ckan.ico
+ckan.gravatar_default = identicon
+ckan.preview.direct = png jpg gif
+ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json
+ckan.display_timezone = server
+
+# package_hide_extras = for_search_index_only
+#package_edit_return_url = http://another.frontend/dataset/<NAME>
+#package_new_return_url = http://another.frontend/dataset/<NAME>
+#ckan.recaptcha.version = 1
+#ckan.recaptcha.publickey =
+#ckan.recaptcha.privatekey =
+#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json
+# ckan.template_footer_end =
+
+
+## Internationalisation Settings
+ckan.locale_default = cs_CZ
+ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv
+ckan.locales_offered = cs_CZ en
+ckan.locales_filtered_out = en_GB
+
+## Feeds Settings
+
+ckan.feeds.authority_name =
+ckan.feeds.date =
+ckan.feeds.author_name =
+ckan.feeds.author_link =
+
+## Storage Settings
+
+ckan.storage_path = /var/lib/ckan
+ckan.max_resource_size = 10
+ckan.max_image_size = 2
+
+## Datapusher settings
+
+# Make sure you have set up the DataStore
+
+ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+ckan.datapusher.url = http://127.0.0.1
+#ckan.datapusher.assume_task_stale_after = 3600
+
+# Resource Proxy settings
+# Preview size limit, default: 1MB
+ckan.resource_proxy.max_file_size = 1048576
+# Size of chunks to read/write.
+ckan.resource_proxy.chunk_size = 4096
+
+## Activity Streams Settings
+
+#ckan.activity_streams_enabled = true
+#ckan.activity_list_limit = 31
+#ckan.activity_streams_email_notifications = true
+#ckan.email_notifications_since = 2 days
+ckan.hide_activity_from_users = %(ckan.site_id)s
+
+
+## Email settings
+
+#email_to = errors@example.com
+#error_email_from = ckan-errors@example.com
+smtp.server = localhost
+smtp.starttls = False
+#smtp.user = username@example.com
+#smtp.password = your_password
+smtp.mail_from = ckan@spotter.ngo
+
+
+## Logging configuration
+[loggers]
+keys = root, ckan, ckanext
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+
+[logger_ckan]
+level = INFO
+handlers = console
+qualname = ckan
+propagate = 0
+
+[logger_ckanext]
+level = DEBUG
+handlers = console
+qualname = ckanext
+propagate = 0
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s

ckan/etc/ckan/datapusher.wsgi

@@ -0,0 +1,13 @@
+#!/usr/bin/python
+
+import os
+
+activate_this = os.path.join('/srv/ckan/bin/activate_this.py')
+execfile(activate_this, dict(__file__=activate_this))
+
+import ckanserviceprovider.web as web
+import datapusher.jobs as jobs
+
+os.environ['JOB_CONFIG'] = '/etc/ckan/datapusher_settings.py'
+web.init()
+application = web.app

ckan/etc/ckan/datapusher_settings.py

@@ -0,0 +1,20 @@
+#!/usr/bin/python
+
+DEBUG = False
+TESTING = False
+SSL_VERIFY = False
+SECRET_KEY = '${CKAN_DSJOB_SECRET}'
+USERNAME = '${CKAN_DSJOB_USER}'
+PASSWORD = '${CKAN_DSJOB_PWD}'
+NAME = 'datapusher'
+
+SQLALCHEMY_DATABASE_URI = 'sqlite:////var/lib/ckan/datapusher-jobs.db'
+
+HOST = '0.0.0.0'
+PORT = 8003
+
+FROM_EMAIL = 'ckan@spotter.ngo'
+#ADMINS = ['yourname@example.com']  # where to send emails
+
+#LOG_FILE = '/tmp/ckan_service.log'
+STDERR = True

ckan/etc/cron.d/ckan

@@ -0,0 +1 @@
+10 * * * * ckan /srv/ckan/cron-pageviews.sh >/dev/null

ckan/etc/nginx/apps-available/ckan

@@ -0,0 +1,8 @@
+location /ckan {
+    uwsgi_pass unix:///run/uwsgi/app/ckan/socket;
+    include uwsgi_params;
+    uwsgi_param SCRIPT_NAME /ckan;
+
+    access_log /var/log/nginx/ckan.access.log;
+    error_log /var/log/nginx/ckan.error.log;
+}

ckan/etc/nginx/sites-available/ckan-datapusher

@@ -0,0 +1,14 @@
+server {
+    listen 8003 ssl http2;
+    listen [::]:8003 ssl http2;
+
+    access_log /var/log/nginx/ckan.access.log;
+    error_log /var/log/nginx/ckan.error.log;
+
+    location / {
+        uwsgi_pass unix:///run/uwsgi/app/ckan-datapusher/socket;
+        include uwsgi_params;
+        uwsgi_param SCRIPT_NAME '';
+        uwsgi_param UWSGI_SCHEME $scheme;
+    }
+}

ckan/etc/uwsgi/apps-available/ckan-datapusher.ini

@@ -0,0 +1,8 @@
+[uwsgi]
+uid = ckan
+gid = ckan
+chown-socket = www-data:www-data
+chdir = /srv/ckan
+exec-asap = /srv/ckan/update-ip.sh
+disable-logging = true
+file = /etc/ckan/datapusher.wsgi

ckan/etc/uwsgi/apps-available/ckan.ini

@@ -0,0 +1,10 @@
+[uwsgi]
+uid = ckan
+gid = ckan
+chown-socket = www-data:www-data
+chdir = /srv/ckan
+home = /srv/ckan
+exec-asap = /srv/ckan/update-ip.sh
+route-run = fixpathinfo:
+disable-logging = true
+ini-paste = /etc/ckan/ckan.ini

ckan/lib/systemd/system/solr.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Apache Solr
+After=network.target
+
+[Service]
+ExecStart=/opt/solr/bin/solr start -f -p 8983 -s /var/lib/solr
+ExecStop=/opt/solr/bin/solr stop -p 8983
+User=solr
+
+[Install]
+WantedBy=multi-user.target

ckan/srv/ckan/cron-pageviews.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+. /srv/ckan/bin/activate
+paster --plugin=ckan tracking update -c /etc/ckan/ckan.ini && \
+paster --plugin=ckan search-index rebuild -r -c /etc/ckan/ckan.ini
+deactivate

ckan/srv/ckan/update-ip.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+IP=$(ip route get 1 | awk '{print $NF;exit}')
+sed -i "s|^ckan\.site_url.*|ckan.site_url = https://${IP}|" /etc/ckan/ckan.ini
+sed -i "s|^ckan\.datapusher\.url.*|ckan.datapusher.url = https://${IP}:8003|" /etc/ckan/ckan.ini

ckan/tmp/ckan-adminpwd.sql

@@ -0,0 +1 @@
+INSERT INTO public.user VALUES ('${CKAN_ADMIN_UUID}', '${CKAN_ADMIN_USER}', '${CKAN_ADMIN_APIKEY}', NOW(), NULL, '${CKAN_ADMIN_HASH}', NULL, '${CKAN_ADMIN_EMAIL}', NULL, TRUE, FALSE, 'active');

ckan/tmp/ckan-createdb.sql

@@ -0,0 +1,21 @@
+CREATE ROLE ckan NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT LOGIN ENCRYPTED PASSWORD '${CKAN_PWD}';
+CREATE ROLE ckan_datastore NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT LOGIN ENCRYPTED PASSWORD '${CKAN_DS_PWD}';
+
+CREATE DATABASE ckan;
+REVOKE ALL ON DATABASE ckan FROM public;
+ALTER DATABASE ckan OWNER TO ckan;
+
+CREATE DATABASE ckan_datastore;
+REVOKE ALL ON DATABASE ckan_datastore FROM public;
+GRANT CONNECT, CREATE, TEMPORARY ON DATABASE ckan_datastore TO ckan;
+ALTER DATABASE ckan_datastore OWNER TO ckan_datastore;
+
+\c ckan
+CREATE TEXT SEARCH DICTIONARY cspell (template=ispell, dictfile = czech, afffile=czech, stopwords=czech);
+CREATE TEXT SEARCH CONFIGURATION czech (copy=english);
+ALTER TEXT SEARCH CONFIGURATION czech ALTER MAPPING FOR word, asciiword WITH cspell, simple;
+
+\c ckan_datastore
+CREATE TEXT SEARCH DICTIONARY cspell (template=ispell, dictfile = czech, afffile=czech, stopwords=czech);
+CREATE TEXT SEARCH CONFIGURATION czech (copy=english);
+ALTER TEXT SEARCH CONFIGURATION czech ALTER MAPPING FOR word, asciiword WITH cspell, simple;