Redo CKAN installation, fixes #148 including related issues

ckan.sh

@@ -20,18 +20,19 @@ virtualenv --no-site-packages --python=/usr/bin/python2.7 /srv/ckan-datapusher
 /srv/ckan-datapusher/bin/pip install -e 'git+https://github.com/ckan/datapusher.git#egg=datapusher'
 /srv/ckan-datapusher/bin/pip install -r /srv/ckan-datapusher/src/datapusher/requirements.txt
 
+# Patch service provider TLS verification for Datapusher
+# https://github.com/ckan/ckan-service-provider/issues/36
+patch -d /srv/ckan-datapusher -p0 <${SOURCE_DIR}/ckan/ckan-serviceprovider-sslverify.patch
+
 # Install CKAN extensions
 /srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-basiccharts#egg=ckanext_basiccharts'
 /srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-spatial#egg=ckanext_spatial'
 /srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-geoview#egg=ckanext_geoview'
 /srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-mapviews#egg=ckanext_mapviews'
-/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-scheming#egg=ckanext_scheming'
 /srv/ckan/bin/pip install -e 'git+https://github.com/XVTSolutions/ckanext-spatialUI#egg=ckanext_spatialui'
 /srv/ckan/bin/pip install -e 'git+https://github.com/aptivate/ckanext-datasetthumbnail#egg=ckanext_datasetthumbnail'
 /srv/ckan/bin/pip install -e 'git+https://github.com/datagvat/ckanext-dgvat_xls#egg=ckanext_dgvat_xls'
 /srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-spatial/pip-requirements.txt
-/srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-geoview/pip-requirements.txt
-/srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-scheming/requirements.txt
 /srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-datasetthumbnail/requirements.txt
 /srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-dgvat-xls/requirements.txt
 
@@ -41,20 +42,6 @@ tar xzf /tmp/solr-6.5.1.tgz -C /opt/
 mv /opt/solr-6.5.1 /opt/solr
 rm -f /tmp/solr-6.5.1.tgz
 
-# Create database
-export CKAN_PWD=$(head -c 18 /dev/urandom | base64)
-export CKAN_DS_PWD=$(head -c 18 /dev/urandom | base64)
-envsubst <${SOURCE_DIR}/ckan/tmp/ckan-createdb.sql >/tmp/ckan-createdb.sql
-sudo -u postgres psql -f /tmp/ckan-createdb.sql
-rm -f /tmp/ckan-createdb.sql
-
-# Create CKAN OS user
-adduser --system --group --home /srv/ckan --shell /bin/false ckan
-chown -R ckan:ckan /srv/ckan/
-mkdir /var/lib/ckan
-chown ckan:ckan /var/lib/ckan
-chown ckan:ckan /var/lib/ckan-datapusher
-
 # Create Solr OS user
 adduser --system --group --home /var/lib/solr --shell /bin/false solr
 chown -R solr:solr /opt/solr/
@@ -73,6 +60,13 @@ cp ${SOURCE_DIR}/ckan/var/lib/solr/ckan/conf/solrconfig.xml /var/lib/solr/ckan/c
 ln -s /srv/ckan/src/ckan/ckan/config/solr/schema.xml /var/lib/solr/ckan/conf/schema.xml
 systemctl restart solr
 
+# Create database
+export CKAN_PWD=$(head -c 18 /dev/urandom | base64)
+export CKAN_DS_PWD=$(head -c 18 /dev/urandom | base64)
+envsubst <${SOURCE_DIR}/ckan/tmp/ckan-createdb.sql >/tmp/ckan-createdb.sql
+sudo -u postgres psql -f /tmp/ckan-createdb.sql
+rm -f /tmp/ckan-createdb.sql
+
 # Configure CKAN
 export CKAN_SECRET=$(head -c 18 /dev/urandom | base64)
 export CKAN_UUID=$(python -c "import uuid; print uuid.uuid4()")
@@ -85,9 +79,14 @@ cp ${SOURCE_DIR}/ckan/srv/ckan/update-ip.sh /srv/ckan/update-ip.sh
 /srv/ckan/update-ip.sh
 
 # Populate database
-sudo -u ckan /srv/ckan/bin/paster --plugin=ckan db init -c /etc/ckan/ckan.ini
-sudo -u ckan /srv/ckan/bin/paster --plugin=ckanext-spatial spatial initdb -c /etc/ckan/ckan.ini
-sudo -u ckan /srv/ckan/bin/paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | sudo -u postgres psql
+/srv/ckan/bin/paster --plugin=ckan db init -c /etc/ckan/ckan.ini
+/srv/ckan/bin/paster --plugin=ckanext-spatial spatial initdb -c /etc/ckan/ckan.ini
+/srv/ckan/bin/paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | sudo -u postgres psql
+
+# Create CKAN OS user
+adduser --system --group --home /srv/ckan --shell /bin/false ckan
+chown -R ckan:ckan /srv/ckan/
+chown -R ckan:ckan /srv/ckan-datapusher/
 
 # Create admin account
 export CKAN_ADMIN_USER="admin"

ckan/ckan-serviceprovider-sslverify.patch

@@ -0,0 +1,11 @@
+diff -ruN orig/python2.7/site-packages/ckanserviceprovider/web.py lib/python2.7/site-packages/ckanserviceprovider/web.py
+--- orig/python2.7/site-packages/ckanserviceprovider/web.py	2017-12-01 08:55:57.595353323 +0100
++++ lib/python2.7/site-packages/ckanserviceprovider/web.py	2017-12-01 08:56:37.819592364 +0100
+@@ -733,6 +733,7 @@
+     try:
+         result = requests.post(
+             result_url,
++            verify=False,
+             data=json.dumps(job_dict, cls=DatetimeJsonEncoder),
+             headers=headers)
+ 

ckan/etc/ckan/ckan.ini

@@ -19,7 +19,7 @@ debug = false
 [server:main]
 use = egg:Paste#http
 host = 0.0.0.0
-port = 5000
+port = 8003
 
 [app:main]
 use = egg:ckan
@@ -100,7 +100,7 @@ ckan.redis.url = redis://localhost:6379/0
 #       Add ``datapusher`` to enable DataPusher
 #		Add ``resource_proxy`` to enable resorce proxying and get around the
 #		same origin policy
-ckan.plugins = stats text_view image_view recline_view resource_proxy datastore datapusher spatial_metadata spatial_query geo_view geojson_view wmts_view navigablemap choroplethmap scheming_datasets spatialUI linechart barchart piechart basicgrid datasetthumbnail dgvat_xls
+ckan.plugins = stats text_view image_view recline_view datastore datapusher resource_proxy linechart barchart piechart basicgrid spatial_metadata spatial_query geo_view geojson_view wmts_view navigablemap choroplethmap spatialUI datasetthumbnail dgvat_xls
 
 # Define which views should be created by default
 # (plugins must be loaded in ckan.plugins)
@@ -114,18 +114,20 @@ ckan.views.default_views = image_view text_view recline_view geo_view geojson_vi
 # Customize which image formats the image_view plugin will show
 #ckan.preview.image_formats = png jpeg jpg gif
 
-# Dataset thumbnail plugin settings
-ckan.datasetthumbnail.show_thumbnail = True
-ckan.datasetthumbnail.auto_generate = True
-
 # GeoView plugin settings
 ckanext.geoview.ol_viewer.formats = wms wfs geojson gml kml arcgis_rest gft
 ckanext.geoview.gapi_key = AIzaSyBvIF3D550tlpL6o1xRrDurGo-81VhHlOw
 
-# Sheming plugin settings
-scheming.dataset_schemas = ckanext.scheming:ckan_dataset.json ckanext.scheming:camel_photos.json
+# Dataset thumbnail plugin settings
+ckan.datasetthumbnail.show_thumbnail = True
+ckan.datasetthumbnail.auto_generate = True
 
 ## Front-End Settings
+
+# Uncomment following configuration to enable using of Bootstrap 2
+#ckan.base_public_folder = public-bs2
+#ckan.base_templates_folder = templates-bs2
+
 ckan.site_title = CKAN
 ckan.site_logo = /base/images/ckan-logo.png
 ckan.site_description =
@@ -160,23 +162,23 @@ ckan.feeds.author_link =
 
 ## Storage Settings
 
-ckan.storage_path = /var/lib/ckan
+ckan.storage_path = /srv/ckan/storage
 ckan.max_resource_size = 100
-ckan.max_image_size = 2
+ckan.max_image_size = 10
 
 ## Datapusher settings
 
 # Make sure you have set up the DataStore
 
 ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
-ckan.datapusher.url = http://127.0.0.1:8098
+ckan.datapusher.url = http://127.0.0.1:8098/
 #ckan.datapusher.assume_task_stale_after = 3600
 
 # Resource Proxy settings
 # Preview size limit, default: 1MB
-ckan.resource_proxy.max_file_size = 1048576
+#ckan.resource_proxy.max_file_size = 1048576
 # Size of chunks to read/write.
-ckan.resource_proxy.chunk_size = 4096
+#ckan.resource_proxy.chunk_size = 4096
 
 ## Activity Streams Settings
 

ckan/etc/ckan/datapusher.wsgi

@@ -1,11 +1,8 @@
 #!/usr/bin/python
 
 import os
-
-activate_this = '/srv/ckan-datapusher/bin/activate_this.py'
-execfile(activate_this, dict(__file__=activate_this))
-
 import ckanserviceprovider.web as web
+
 os.environ['JOB_CONFIG'] = '/etc/ckan/datapusher_settings.py'
 web.init()
 

ckan/etc/ckan/datapusher_settings.py

@@ -10,14 +10,12 @@ PASSWORD = str(uuid.uuid4())
 
 NAME = 'datapusher'
 
-SQLALCHEMY_DATABASE_URI = 'sqlite:////var/lib/ckan/datapusher-jobs.db'
+SQLALCHEMY_DATABASE_URI = 'sqlite:////srv/ckan-datapusher/jobs.db'
 
 HOST = '0.0.0.0'
 PORT = 8098
 SSL_VERIFY = False
 
 FROM_EMAIL = 'ckan@spotter.ngo'
-#ADMINS = ['yourname@example.com']  # where to send emails
 
-#LOG_FILE = '/tmp/ckan_service.log'
 STDERR = True

ckan/etc/uwsgi/apps-available/ckan-datapusher.ini

@@ -3,5 +3,9 @@ uid = ckan
 gid = ckan
 chown-socket = www-data:www-data
 chdir = /srv/ckan-datapusher
+home = /srv/ckan-datapusher
+master = false
+workers = 1
 disable-logging = true
 file = /etc/ckan/datapusher.wsgi
+enable-threads = true

ckan/etc/uwsgi/apps-available/ckan.ini

@@ -4,6 +4,9 @@ gid = ckan
 chown-socket = www-data:www-data
 chdir = /srv/ckan
 home = /srv/ckan
+master = false
+workers = 1
 exec-asap = /srv/ckan/update-ip.sh
 disable-logging = true
 ini-paste = /etc/ckan/ckan.ini
+enable-threads = true

ckan/tmp/ckan-adminpwd.sql

@@ -1 +1 @@
-INSERT INTO public.user VALUES ('${CKAN_ADMIN_UUID}', '${CKAN_ADMIN_USER}', '${CKAN_ADMIN_APIKEY}', NOW(), NULL, '${CKAN_ADMIN_HASH}', NULL, '${CKAN_ADMIN_EMAIL}', NULL, TRUE, FALSE, 'active');
+INSERT INTO public.user (id, name, apikey, created, about, password, fullname, email, reset_key, sysadmin, activity_streams_email_notifications, state) VALUES ('${CKAN_ADMIN_UUID}', '${CKAN_ADMIN_USER}', '${CKAN_ADMIN_APIKEY}', NOW(), NULL, '${CKAN_ADMIN_HASH}', NULL, '${CKAN_ADMIN_EMAIL}', NULL, TRUE, FALSE, 'active');