From ef89d2a562bb0817a52672ade1751411f63bda30 Mon Sep 17 00:00:00 2001 From: Disassembler Date: Fri, 1 Dec 2017 10:17:50 +0100 Subject: [PATCH] Redo CKAN installation, fixes #148 including related issues --- ckan.sh | 39 +++++++++---------- ckan/ckan-serviceprovider-sslverify.patch | 11 ++++++ ckan/etc/ckan/ckan.ini | 28 ++++++------- ckan/etc/ckan/datapusher.wsgi | 5 +-- ckan/etc/ckan/datapusher_settings.py | 4 +- .../uwsgi/apps-available/ckan-datapusher.ini | 4 ++ ckan/etc/uwsgi/apps-available/ckan.ini | 3 ++ ckan/tmp/ckan-adminpwd.sql | 2 +- 8 files changed, 55 insertions(+), 41 deletions(-) create mode 100644 ckan/ckan-serviceprovider-sslverify.patch diff --git a/ckan.sh b/ckan.sh index 7b157d4..d84edd9 100755 --- a/ckan.sh +++ b/ckan.sh @@ -20,18 +20,19 @@ virtualenv --no-site-packages --python=/usr/bin/python2.7 /srv/ckan-datapusher /srv/ckan-datapusher/bin/pip install -e 'git+https://github.com/ckan/datapusher.git#egg=datapusher' /srv/ckan-datapusher/bin/pip install -r /srv/ckan-datapusher/src/datapusher/requirements.txt +# Patch service provider TLS verification for Datapusher +# https://github.com/ckan/ckan-service-provider/issues/36 +patch -d /srv/ckan-datapusher -p0 <${SOURCE_DIR}/ckan/ckan-serviceprovider-sslverify.patch + # Install CKAN extensions /srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-basiccharts#egg=ckanext_basiccharts' /srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-spatial#egg=ckanext_spatial' /srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-geoview#egg=ckanext_geoview' /srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-mapviews#egg=ckanext_mapviews' -/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-scheming#egg=ckanext_scheming' /srv/ckan/bin/pip install -e 'git+https://github.com/XVTSolutions/ckanext-spatialUI#egg=ckanext_spatialui' /srv/ckan/bin/pip install -e 'git+https://github.com/aptivate/ckanext-datasetthumbnail#egg=ckanext_datasetthumbnail' /srv/ckan/bin/pip install -e 'git+https://github.com/datagvat/ckanext-dgvat_xls#egg=ckanext_dgvat_xls' /srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-spatial/pip-requirements.txt -/srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-geoview/pip-requirements.txt -/srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-scheming/requirements.txt /srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-datasetthumbnail/requirements.txt /srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-dgvat-xls/requirements.txt @@ -41,20 +42,6 @@ tar xzf /tmp/solr-6.5.1.tgz -C /opt/ mv /opt/solr-6.5.1 /opt/solr rm -f /tmp/solr-6.5.1.tgz -# Create database -export CKAN_PWD=$(head -c 18 /dev/urandom | base64) -export CKAN_DS_PWD=$(head -c 18 /dev/urandom | base64) -envsubst <${SOURCE_DIR}/ckan/tmp/ckan-createdb.sql >/tmp/ckan-createdb.sql -sudo -u postgres psql -f /tmp/ckan-createdb.sql -rm -f /tmp/ckan-createdb.sql - -# Create CKAN OS user -adduser --system --group --home /srv/ckan --shell /bin/false ckan -chown -R ckan:ckan /srv/ckan/ -mkdir /var/lib/ckan -chown ckan:ckan /var/lib/ckan -chown ckan:ckan /var/lib/ckan-datapusher - # Create Solr OS user adduser --system --group --home /var/lib/solr --shell /bin/false solr chown -R solr:solr /opt/solr/ @@ -73,6 +60,13 @@ cp ${SOURCE_DIR}/ckan/var/lib/solr/ckan/conf/solrconfig.xml /var/lib/solr/ckan/c ln -s /srv/ckan/src/ckan/ckan/config/solr/schema.xml /var/lib/solr/ckan/conf/schema.xml systemctl restart solr +# Create database +export CKAN_PWD=$(head -c 18 /dev/urandom | base64) +export CKAN_DS_PWD=$(head -c 18 /dev/urandom | base64) +envsubst <${SOURCE_DIR}/ckan/tmp/ckan-createdb.sql >/tmp/ckan-createdb.sql +sudo -u postgres psql -f /tmp/ckan-createdb.sql +rm -f /tmp/ckan-createdb.sql + # Configure CKAN export CKAN_SECRET=$(head -c 18 /dev/urandom | base64) export CKAN_UUID=$(python -c "import uuid; print uuid.uuid4()") @@ -85,9 +79,14 @@ cp ${SOURCE_DIR}/ckan/srv/ckan/update-ip.sh /srv/ckan/update-ip.sh /srv/ckan/update-ip.sh # Populate database -sudo -u ckan /srv/ckan/bin/paster --plugin=ckan db init -c /etc/ckan/ckan.ini -sudo -u ckan /srv/ckan/bin/paster --plugin=ckanext-spatial spatial initdb -c /etc/ckan/ckan.ini -sudo -u ckan /srv/ckan/bin/paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | sudo -u postgres psql +/srv/ckan/bin/paster --plugin=ckan db init -c /etc/ckan/ckan.ini +/srv/ckan/bin/paster --plugin=ckanext-spatial spatial initdb -c /etc/ckan/ckan.ini +/srv/ckan/bin/paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | sudo -u postgres psql + +# Create CKAN OS user +adduser --system --group --home /srv/ckan --shell /bin/false ckan +chown -R ckan:ckan /srv/ckan/ +chown -R ckan:ckan /srv/ckan-datapusher/ # Create admin account export CKAN_ADMIN_USER="admin" diff --git a/ckan/ckan-serviceprovider-sslverify.patch b/ckan/ckan-serviceprovider-sslverify.patch new file mode 100644 index 0000000..4ddf13f --- /dev/null +++ b/ckan/ckan-serviceprovider-sslverify.patch @@ -0,0 +1,11 @@ +diff -ruN orig/python2.7/site-packages/ckanserviceprovider/web.py lib/python2.7/site-packages/ckanserviceprovider/web.py +--- orig/python2.7/site-packages/ckanserviceprovider/web.py 2017-12-01 08:55:57.595353323 +0100 ++++ lib/python2.7/site-packages/ckanserviceprovider/web.py 2017-12-01 08:56:37.819592364 +0100 +@@ -733,6 +733,7 @@ + try: + result = requests.post( + result_url, ++ verify=False, + data=json.dumps(job_dict, cls=DatetimeJsonEncoder), + headers=headers) + diff --git a/ckan/etc/ckan/ckan.ini b/ckan/etc/ckan/ckan.ini index 07c92c9..2769d41 100644 --- a/ckan/etc/ckan/ckan.ini +++ b/ckan/etc/ckan/ckan.ini @@ -19,7 +19,7 @@ debug = false [server:main] use = egg:Paste#http host = 0.0.0.0 -port = 5000 +port = 8003 [app:main] use = egg:ckan @@ -100,7 +100,7 @@ ckan.redis.url = redis://localhost:6379/0 # Add ``datapusher`` to enable DataPusher # Add ``resource_proxy`` to enable resorce proxying and get around the # same origin policy -ckan.plugins = stats text_view image_view recline_view resource_proxy datastore datapusher spatial_metadata spatial_query geo_view geojson_view wmts_view navigablemap choroplethmap scheming_datasets spatialUI linechart barchart piechart basicgrid datasetthumbnail dgvat_xls +ckan.plugins = stats text_view image_view recline_view datastore datapusher resource_proxy linechart barchart piechart basicgrid spatial_metadata spatial_query geo_view geojson_view wmts_view navigablemap choroplethmap spatialUI datasetthumbnail dgvat_xls # Define which views should be created by default # (plugins must be loaded in ckan.plugins) @@ -114,18 +114,20 @@ ckan.views.default_views = image_view text_view recline_view geo_view geojson_vi # Customize which image formats the image_view plugin will show #ckan.preview.image_formats = png jpeg jpg gif -# Dataset thumbnail plugin settings -ckan.datasetthumbnail.show_thumbnail = True -ckan.datasetthumbnail.auto_generate = True - # GeoView plugin settings ckanext.geoview.ol_viewer.formats = wms wfs geojson gml kml arcgis_rest gft ckanext.geoview.gapi_key = AIzaSyBvIF3D550tlpL6o1xRrDurGo-81VhHlOw -# Sheming plugin settings -scheming.dataset_schemas = ckanext.scheming:ckan_dataset.json ckanext.scheming:camel_photos.json +# Dataset thumbnail plugin settings +ckan.datasetthumbnail.show_thumbnail = True +ckan.datasetthumbnail.auto_generate = True ## Front-End Settings + +# Uncomment following configuration to enable using of Bootstrap 2 +#ckan.base_public_folder = public-bs2 +#ckan.base_templates_folder = templates-bs2 + ckan.site_title = CKAN ckan.site_logo = /base/images/ckan-logo.png ckan.site_description = @@ -160,23 +162,23 @@ ckan.feeds.author_link = ## Storage Settings -ckan.storage_path = /var/lib/ckan +ckan.storage_path = /srv/ckan/storage ckan.max_resource_size = 100 -ckan.max_image_size = 2 +ckan.max_image_size = 10 ## Datapusher settings # Make sure you have set up the DataStore ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet -ckan.datapusher.url = http://127.0.0.1:8098 +ckan.datapusher.url = http://127.0.0.1:8098/ #ckan.datapusher.assume_task_stale_after = 3600 # Resource Proxy settings # Preview size limit, default: 1MB -ckan.resource_proxy.max_file_size = 1048576 +#ckan.resource_proxy.max_file_size = 1048576 # Size of chunks to read/write. -ckan.resource_proxy.chunk_size = 4096 +#ckan.resource_proxy.chunk_size = 4096 ## Activity Streams Settings diff --git a/ckan/etc/ckan/datapusher.wsgi b/ckan/etc/ckan/datapusher.wsgi index 7728628..f8350ec 100644 --- a/ckan/etc/ckan/datapusher.wsgi +++ b/ckan/etc/ckan/datapusher.wsgi @@ -1,11 +1,8 @@ #!/usr/bin/python import os - -activate_this = '/srv/ckan-datapusher/bin/activate_this.py' -execfile(activate_this, dict(__file__=activate_this)) - import ckanserviceprovider.web as web + os.environ['JOB_CONFIG'] = '/etc/ckan/datapusher_settings.py' web.init() diff --git a/ckan/etc/ckan/datapusher_settings.py b/ckan/etc/ckan/datapusher_settings.py index 2f17757..dc7d53a 100644 --- a/ckan/etc/ckan/datapusher_settings.py +++ b/ckan/etc/ckan/datapusher_settings.py @@ -10,14 +10,12 @@ PASSWORD = str(uuid.uuid4()) NAME = 'datapusher' -SQLALCHEMY_DATABASE_URI = 'sqlite:////var/lib/ckan/datapusher-jobs.db' +SQLALCHEMY_DATABASE_URI = 'sqlite:////srv/ckan-datapusher/jobs.db' HOST = '0.0.0.0' PORT = 8098 SSL_VERIFY = False FROM_EMAIL = 'ckan@spotter.ngo' -#ADMINS = ['yourname@example.com'] # where to send emails -#LOG_FILE = '/tmp/ckan_service.log' STDERR = True diff --git a/ckan/etc/uwsgi/apps-available/ckan-datapusher.ini b/ckan/etc/uwsgi/apps-available/ckan-datapusher.ini index 5aa52f7..47ec146 100644 --- a/ckan/etc/uwsgi/apps-available/ckan-datapusher.ini +++ b/ckan/etc/uwsgi/apps-available/ckan-datapusher.ini @@ -3,5 +3,9 @@ uid = ckan gid = ckan chown-socket = www-data:www-data chdir = /srv/ckan-datapusher +home = /srv/ckan-datapusher +master = false +workers = 1 disable-logging = true file = /etc/ckan/datapusher.wsgi +enable-threads = true diff --git a/ckan/etc/uwsgi/apps-available/ckan.ini b/ckan/etc/uwsgi/apps-available/ckan.ini index 0de8bbf..404d7d4 100644 --- a/ckan/etc/uwsgi/apps-available/ckan.ini +++ b/ckan/etc/uwsgi/apps-available/ckan.ini @@ -4,6 +4,9 @@ gid = ckan chown-socket = www-data:www-data chdir = /srv/ckan home = /srv/ckan +master = false +workers = 1 exec-asap = /srv/ckan/update-ip.sh disable-logging = true ini-paste = /etc/ckan/ckan.ini +enable-threads = true diff --git a/ckan/tmp/ckan-adminpwd.sql b/ckan/tmp/ckan-adminpwd.sql index 3ac23a3..03ebd42 100644 --- a/ckan/tmp/ckan-adminpwd.sql +++ b/ckan/tmp/ckan-adminpwd.sql @@ -1 +1 @@ -INSERT INTO public.user VALUES ('${CKAN_ADMIN_UUID}', '${CKAN_ADMIN_USER}', '${CKAN_ADMIN_APIKEY}', NOW(), NULL, '${CKAN_ADMIN_HASH}', NULL, '${CKAN_ADMIN_EMAIL}', NULL, TRUE, FALSE, 'active'); +INSERT INTO public.user (id, name, apikey, created, about, password, fullname, email, reset_key, sysadmin, activity_streams_email_notifications, state) VALUES ('${CKAN_ADMIN_UUID}', '${CKAN_ADMIN_USER}', '${CKAN_ADMIN_APIKEY}', NOW(), NULL, '${CKAN_ADMIN_HASH}', NULL, '${CKAN_ADMIN_EMAIL}', NULL, TRUE, FALSE, 'active');