Bump CKAN to 2.9.0, use Python 3 runtime

lxc-apps/ckan/app

@@ -1,5 +1,5 @@
 {
-    "version": "2.8.3-200621",
+    "version": "2.9.0-201021",
     "meta": {
         "title": "CKAN",
         "desc-cs": "Datový sklad",
@@ -8,7 +8,7 @@
     },
     "containers": {
         "ckan": {
-            "image": "ckan_2.8.3-200621",
+            "image": "ckan_2.9.0-201021",
             "depends": [
                 "ckan-datapusher",
                 "ckan-redis",
@@ -21,27 +21,27 @@
             }
         },
         "ckan-datapusher": {
-            "image": "ckan-datapusher_0.0.16-200621",
+            "image": "ckan-datapusher_0.0.16-201021",
             "mounts": {
                 "ckan/datapusher_conf": "etc/ckan-datapusher",
                 "ckan/datapusher_data": "srv/ckan-datapusher/data"
             }
         },
         "ckan-redis": {
-            "image": "redis_5.0.7-200621",
+            "image": "redis_5.0.9-201021",
             "mounts": {
                 "ckan/redis_conf/redis.conf": "etc/redis.conf:file",
                 "ckan/redis_data": "var/lib/redis"
             }
         },
         "ckan-solr": {
-            "image": "solr6_6.5.1-200621",
+            "image": "solr6_6.5.1-201021",
             "mounts": {
                 "ckan/solr_data": "var/lib/solr"
             }
         },
         "ckan-postgres": {
-            "image": "postgis_3.0.0-200621",
+            "image": "postgis_3.0.1-201021",
             "mounts": {
                 "ckan/postgres_data": "var/lib/postgresql"
             }

lxc-apps/ckan/ckan-datapusher.image

@@ -1,28 +1,19 @@
-IMAGE ckan-datapusher_0.0.16-200621
+IMAGE ckan-datapusher_0.0.16-201021
-FROM alpine3.10-python2.7_2.7.18-200621
+FROM alpine3.12-python3.8_3.8.5-201021
-# Alpine 3.11 discontinued uwsgi-python2 module
 
 RUN EOF
     # Install runtime dependencies
-    apk --no-cache add libffi uwsgi-python
+    apk --no-cache add uwsgi-python3
 
     # Install build dependencies
-    apk --no-cache add --virtual .deps build-base git libffi-dev libressl-dev libxml2-dev libxslt-dev py2-pip python2-dev
+    apk --no-cache add --virtual .deps build-base git libffi-dev libressl-dev libxml2-dev libxslt-dev python3-dev
 
     # Install CKAN DataPusher
     mkdir -p /srv/ckan-datapusher
     cd /srv/ckan-datapusher
-    pip install -U setuptools
+    pip install -e 'git+https://github.com/ckan/datapusher.git@eb4686030ed6911a5294c62bf275ddbbc4b26114#egg=datapusher'
-    pip install -e 'git+https://github.com/ckan/datapusher.git@e662e3c33e069ac174cdb4fb1d61121f0ba4bb3a#egg=datapusher'
-
-    # Hackfix the X509_STORE_CTX wrapper
-    sed -i 's/\[security\]//' /srv/ckan-datapusher/src/datapusher/requirements.txt
     pip install -r /srv/ckan-datapusher/src/datapusher/requirements.txt
 
-    # Hackfix werkzeug==1.0.0 proxy_fix import
-    # https://github.com/ckan/ckan-service-provider/pull/49
-    sed -i 's/werkzeug\.contrib\.fixers/werkzeug.middleware.proxy_fix/' /usr/lib/python2.7/site-packages/ckanserviceprovider/web.py
-
     # Create OS user
     addgroup -S -g 8080 ckandp
     adduser -S -u 8080 -h /srv/ckan-datapusher -s /bin/false -g ckandp -G ckandp ckandp

lxc-apps/ckan/ckan-datapusher.image.d/bin/add-ca-cert

@@ -6,8 +6,5 @@ with open('/etc/ckan-datapusher/add-ca-cert.env') as f:
     env = dict(tuple(line.split('=')) for line in f.read().splitlines())
 
 cert = ssl.get_server_certificate((env['HOST'], env['PORT']))
-with open('/usr/lib/python2.7/site-packages/requests/cacert.pem', 'a') as f:
+with open('/usr/lib/python3.8/site-packages/certifi/cacert.pem', 'a') as f:
-    f.write(cert)
-
-with open('/usr/lib/python2.7/site-packages/certifi/cacert.pem', 'a') as f:
     f.write(cert)

lxc-apps/ckan/ckan.image

@@ -1,36 +1,22 @@
-IMAGE ckan_2.8.3-200621
+IMAGE ckan_2.9.0-201021
-FROM alpine3.11-python2.7_2.7.18-200621
+FROM alpine3.12-python3.8_3.8.5-201021
 
 RUN EOF
     # Install runtime dependencies
-    apk --no-cache add geos libjpeg-turbo libmagic libpq mailcap py2-pip zlib
+    apk --no-cache add geos libjpeg-turbo libmagic libpq mailcap zlib
 
     # Install build dependencies
-    apk --no-cache add --virtual .deps build-base git libjpeg-turbo-dev libxml2-dev libxslt-dev postgresql-dev python2-dev zlib-dev
+    apk --no-cache add --virtual .deps build-base git libjpeg-turbo-dev libxml2-dev libxslt-dev postgresql-dev python3-dev zlib-dev
 
     # Install CKAN
     mkdir -p /srv/ckan
     cd /srv/ckan
-    pip install -U setuptools
+    pip install -e 'git+https://github.com/ckan/ckan.git@ba0120dc0c798bbc9b6d8e9ad83db01a197ea179#egg=ckan'
+    # Hackfix support for python3.7+ and setuptools 46+
+    sed -i 's/zope\.interface==4\.3\.2/zope.interface==4.7.2/' /srv/ckan/src/ckan/requirements.txt
+    # -I allows dependencies to downgrade six
+    pip install -Ir /srv/ckan/src/ckan/requirements.txt
     pip install flask-debugtoolbar
-    pip install -e 'git+https://github.com/ckan/ckan.git@8e1cc60b2fa11da6843051678b7ee2cc08c2a7a9#egg=ckan'
-    pip install -r /srv/ckan/src/ckan/requirements.txt
-
-    # Install CKAN extensions
-    pip install -e 'git+https://github.com/ckan/ckanext-basiccharts#egg=ckanext_basiccharts'
-    pip install -e 'git+https://github.com/ckan/ckanext-spatial#egg=ckanext_spatial'
-    pip install -e 'git+https://github.com/ckan/ckanext-geoview#egg=ckanext_geoview'
-    pip install -e 'git+https://github.com/ckan/ckanext-mapviews#egg=ckanext_mapviews'
-    pip install -e 'git+https://github.com/XVTSolutions/ckanext-spatialUI#egg=ckanext_spatialui'
-    pip install -e 'git+https://github.com/aptivate/ckanext-datasetthumbnail#egg=ckanext_datasetthumbnail'
-    pip install -e 'git+https://github.com/datagvat/ckanext-dgvat_xls#egg=ckanext_dgvat_xls'
-    pip install -r /srv/ckan/src/ckanext-spatial/pip-requirements.txt
-    pip install -r /srv/ckan/src/ckanext-geoview/pip-requirements.txt
-    pip install -r /srv/ckan/src/ckanext-dgvat-xls/requirements.txt
-
-    # Hackfix support for PostgreSQL 12
-    # https://github.com/sqlalchemy/sqlalchemy/issues/4463
-    sed -i 's/cons\.consrc/pg_get_constraintdef(cons.oid)/' /usr/lib/python2.7/site-packages/sqlalchemy/dialects/postgresql/base.py
 
     # Create OS user
     addgroup -S -g 8080 ckan

lxc-apps/ckan/ckan.image.d/etc/crontabs/ckan

@@ -1,2 +1,2 @@
-0 * * * * paster --plugin=ckan tracking update -c /etc/ckan/ckan.ini >/dev/null
+0 * * * * ckan -c /etc/ckan/ckan.ini tracking update >/dev/null
-0 * * * * paster --plugin=ckan search-index rebuild -r -c /etc/ckan/ckan.ini >/dev/null
+0 * * * * ckan -c /etc/ckan/ckan.ini search-index rebuild -r >/dev/null

lxc-apps/ckan/ckan.image.d/etc/services.d/ckan/run

@@ -2,4 +2,4 @@
 
 fdmove -c 2 1
 s6-setuidgid ckan
-paster serve /etc/ckan/ckan.ini
+ckan -c /etc/ckan/ckan.ini run -H 0.0.0.0 -p 8080

lxc-apps/ckan/install.sh

@@ -6,7 +6,7 @@ POSTGRES_DATA="${VOLUMES_DIR}/ckan/postgres_data"
 REDIS_CONF="${VOLUMES_DIR}/ckan/redis_conf"
 REDIS_DATA="${VOLUMES_DIR}/ckan/redis_data"
 SOLR_DATA="${VOLUMES_DIR}/ckan/solr_data"
-SOLR_LAYER="${LAYERS_DIR}/solr6_6.5.1-200621"
+SOLR_LAYER="${LAYERS_DIR}/solr6_6.5.1-201021"
 DATAPUSHER_CONF="${VOLUMES_DIR}/ckan/datapusher_conf"
 DATAPUSHER_DATA="${VOLUMES_DIR}/ckan/datapusher_data"
 CKAN_CONF="${VOLUMES_DIR}/ckan/ckan_conf"
@@ -60,16 +60,15 @@ envsubst <ckan_conf/ckan.ini | install -o 100000 -g 108080 -m 640 /dev/stdin ${C
 install -o 100000 -g 108080 -m 640 ckan_conf/who.ini ${CKAN_CONF}/who.ini
 
 # Populate database
-spoc-container exec ckan -- paster --plugin=ckan db init -c /etc/ckan/ckan.ini
+spoc-container exec ckan -- ckan -c /etc/ckan/ckan.ini db init
-spoc-container exec ckan -- paster --plugin=ckanext-spatial spatial initdb -c /etc/ckan/ckan.ini
+spoc-container exec ckan -- ckan -c /etc/ckan/ckan.ini datastore set-permissions | spoc-container exec ckan-postgres -- psql
-spoc-container exec ckan -- paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | spoc-container exec ckan-postgres -- psql
 
 # Create admin account
 export CKAN_ADMIN_USER="admin"
 export CKAN_ADMIN_UUID=$(cat /proc/sys/kernel/random/uuid)
 export CKAN_ADMIN_APIKEY=$(cat /proc/sys/kernel/random/uuid)
 export CKAN_ADMIN_PWD=$(head -c 12 /dev/urandom | base64 | tr -d '+/=')
-export CKAN_ADMIN_HASH=$(spoc-container exec ckan -- python -c "from passlib.hash import pbkdf2_sha512;print pbkdf2_sha512.encrypt('${CKAN_ADMIN_PWD}')")
+export CKAN_ADMIN_HASH=$(spoc-container exec ckan -- python -c "from passlib.hash import pbkdf2_sha512;print(pbkdf2_sha512.encrypt('${CKAN_ADMIN_PWD}'))")
 export CKAN_ADMIN_EMAIL="admin@example.com"
 envsubst <adminpwd.sql | spoc-container exec ckan-postgres -- psql ckan
 

lxc-apps/ckan/install/ckan_conf/ckan.ini

@@ -29,11 +29,11 @@ cache_dir = /tmp/%(ckan.site_id)s/
 beaker.session.key = ckan
 
 # This is the secret token that the beaker library uses to hash the cookie sent
-# to the client. `paster make-config` generates a unique value for this each
+# to the client. `ckan generate config` generates a unique value for this each
 # time it generates a config file.
 beaker.session.secret = ${CKAN_SECRET}
 
-# `paster make-config` generates a unique value for this each time it generates
+# `ckan generate config` generates a unique value for this each time it generates
 # a config file.
 app_instance_uuid = ${CKAN_UUID}
 
@@ -75,7 +75,17 @@ ckan.auth.create_user_via_web = true
 ckan.auth.roles_that_cascade_to_sub_groups = admin
 ckan.auth.public_user_details = true
 ckan.auth.public_activity_stream_detail = true
+ckan.auth.allow_dataset_collaborators = true
+ckan.auth.create_default_api_keys = false
 
+## API Token Settings
+api_token.nbytes = 60
+api_token.jwt.encode.secret = string:$app_instance_secret
+api_token.jwt.decode.secret = string:$app_instance_secret
+api_token.jwt.algorithm = HS256
+
+## API Token: expire_api_token plugin
+expire_api_token.default_lifetime = 3600
 
 ## Search Settings
 
@@ -104,34 +114,20 @@ ckan.redis.url = redis://ckan-redis:6379/0
 #       Add ``datapusher`` to enable DataPusher
 #		Add ``resource_proxy`` to enable resorce proxying and get around the
 #		same origin policy
-ckan.plugins = stats text_view image_view recline_view datastore datapusher resource_proxy linechart barchart piechart basicgrid spatial_metadata spatial_query geo_view geojson_view wmts_view navigablemap choroplethmap spatialUI datasetthumbnail dgvat_xls
+ckan.plugins = stats text_view image_view recline_view datastore datapusher resource_proxy
 
 # Define which views should be created by default
 # (plugins must be loaded in ckan.plugins)
-ckan.views.default_views = image_view text_view recline_view geo_view geojson_view wmts_view
+ckan.views.default_views = image_view text_view recline_view
 
 # Customize which text formats the text_view plugin will show
 #ckan.preview.json_formats = json
 #ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss
-#ckan.preview.text_formats = text plain text/plain
+#ckan.preview.text_formats = txt plain text/plain
 
 # Customize which image formats the image_view plugin will show
 #ckan.preview.image_formats = png jpeg jpg gif
 
-# GeoView plugin settings
-ckanext.geoview.ol_viewer.formats = wms wfs geojson gml kml arcgis_rest gft
-ckanext.geoview.gapi_key = 
-
-# Pages plugin settings
-ckanext.pages.organization = true
-ckanext.pages.group = true
-ckanext.pages.allow_html = true
-ckanext.pages.editor = ckeditor
-
-# Dataset thumbnail plugin settings
-ckan.datasetthumbnail.show_thumbnail = true
-ckan.datasetthumbnail.auto_generate = true
-
 ## Front-End Settings
 
 ckan.site_title = CKAN
@@ -215,7 +211,7 @@ ckan.jobs.timeout = 180
 
 ## Logging configuration
 [loggers]
-keys = root, ckan, ckanext
+keys = root, ckan, ckanext, werkzeug
 
 [handlers]
 keys = console
@@ -227,6 +223,12 @@ keys = generic
 level = WARNING
 handlers = console
 
+[logger_werkzeug]
+level = WARNING
+handlers = console
+qualname = werkzeug
+propagate = 0
+
 [logger_ckan]
 level = INFO
 handlers = console

lxc-apps/ckan/install/ckan_conf/who.ini

@@ -1,10 +1,10 @@
 [plugin:auth_tkt]
-use = ckan.lib.auth_tkt:make_plugin
+use = ckan.lib.repoze_plugins.auth_tkt:make_plugin
 # If no secret key is defined here, beaker.session.secret will be used
 #secret = somesecret
 
 [plugin:friendlyform]
-use = repoze.who.plugins.friendlyform:FriendlyFormPlugin
+use = ckan.lib.repoze_plugins.friendly_form:FriendlyFormPlugin
 login_form_url= /user/login
 login_handler_path = /login_generic
 logout_handler_path = /user/logout
@@ -13,10 +13,6 @@ post_login_url = /user/logged_in
 post_logout_url = /user/logged_out
 charset = utf-8
 
-#[plugin:basicauth]
-#use = repoze.who.plugins.basicauth:make_plugin
-#realm = 'CKAN'
-
 [general]
 request_classifier = repoze.who.classifiers:default_request_classifier
 challenge_decider = repoze.who.classifiers:default_challenge_decider
@@ -34,4 +30,3 @@ plugins =
 [challengers]
 plugins =
     friendlyform;browser
-#   basicauth

lxc-apps/ckan/update.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -ev
+
+# Start service containers
+spoc-container start ckan-postgres
+spoc-container start ckan-redis
+spoc-container start ckan-solr
+
+# Run database upgrade
+spoc-container exec ckan -- ckan db upgrade -c /etc/ckan/ckan.ini
+
+# Stop service containers
+spoc-container start ckan-solr
+spoc-container start ckan-redis
+spoc-container start ckan-postgres