Spotter-Cluster/Spotter-VM
2
0
Fork 0
Code Issues 78 Pull Requests Actions Packages Projects Releases Wiki Activity

Bump CKAN to 2.9.0, use Python 3 runtime

Browse Source
This commit is contained in:
Disassembler 2020-10-22 20:56:10 +02:00
parent e39aa41c7c
commit 6ec890aa30
No known key found for this signature in database
GPG Key ID: 524BD33A0EE29499
10 changed files with 67 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lxc-apps/ckan/app
View File

@ -1,5 +1,5 @@
{
"version": "2.8.3-200621",
"version": "2.9.0-201021",
"meta": {
"title": "CKAN",
"desc-cs": "Datový sklad",
@ -8,7 +8,7 @@
},
"containers": {
"ckan": {
"image": "ckan_2.8.3-200621",
"image": "ckan_2.9.0-201021",
"depends": [
"ckan-datapusher",
"ckan-redis",
@ -21,27 +21,27 @@
}
},
"ckan-datapusher": {
"image": "ckan-datapusher_0.0.16-200621",
"image": "ckan-datapusher_0.0.16-201021",
"mounts": {
"ckan/datapusher_conf": "etc/ckan-datapusher",
"ckan/datapusher_data": "srv/ckan-datapusher/data"
}
},
"ckan-redis": {
"image": "redis_5.0.7-200621",
"image": "redis_5.0.9-201021",
"mounts": {
"ckan/redis_conf/redis.conf": "etc/redis.conf:file",
"ckan/redis_data": "var/lib/redis"
}
},
"ckan-solr": {
"image": "solr6_6.5.1-200621",
"image": "solr6_6.5.1-201021",
"mounts": {
"ckan/solr_data": "var/lib/solr"
}
},
"ckan-postgres": {
"image": "postgis_3.0.0-200621",
"image": "postgis_3.0.1-201021",
"mounts": {
"ckan/postgres_data": "var/lib/postgresql"
}

19
lxc-apps/ckan/ckan-datapusher.image
View File

@ -1,28 +1,19 @@
IMAGE ckan-datapusher_0.0.16-200621
FROM alpine3.10-python2.7_2.7.18-200621
# Alpine 3.11 discontinued uwsgi-python2 module
IMAGE ckan-datapusher_0.0.16-201021
FROM alpine3.12-python3.8_3.8.5-201021
RUN EOF
# Install runtime dependencies
apk --no-cache add libffi uwsgi-python
apk --no-cache add uwsgi-python3
# Install build dependencies
apk --no-cache add --virtual .deps build-base git libffi-dev libressl-dev libxml2-dev libxslt-dev py2-pip python2-dev
apk --no-cache add --virtual .deps build-base git libffi-dev libressl-dev libxml2-dev libxslt-dev python3-dev
# Install CKAN DataPusher
mkdir -p /srv/ckan-datapusher
cd /srv/ckan-datapusher
pip install -U setuptools
pip install -e 'git+https://github.com/ckan/datapusher.git@e662e3c33e069ac174cdb4fb1d61121f0ba4bb3a#egg=datapusher'
# Hackfix the X509_STORE_CTX wrapper
sed -i 's/\[security\]//' /srv/ckan-datapusher/src/datapusher/requirements.txt
pip install -e 'git+https://github.com/ckan/datapusher.git@eb4686030ed6911a5294c62bf275ddbbc4b26114#egg=datapusher'
pip install -r /srv/ckan-datapusher/src/datapusher/requirements.txt
# Hackfix werkzeug==1.0.0 proxy_fix import
# https://github.com/ckan/ckan-service-provider/pull/49
sed -i 's/werkzeug\.contrib\.fixers/werkzeug.middleware.proxy_fix/' /usr/lib/python2.7/site-packages/ckanserviceprovider/web.py
# Create OS user
addgroup -S -g 8080 ckandp
adduser -S -u 8080 -h /srv/ckan-datapusher -s /bin/false -g ckandp -G ckandp ckandp

5
lxc-apps/ckan/ckan-datapusher.image.d/bin/add-ca-cert
View File

@ -6,8 +6,5 @@ with open('/etc/ckan-datapusher/add-ca-cert.env') as f:
env = dict(tuple(line.split('=')) for line in f.read().splitlines())
cert = ssl.get_server_certificate((env['HOST'], env['PORT']))
with open('/usr/lib/python2.7/site-packages/requests/cacert.pem', 'a') as f:
f.write(cert)
with open('/usr/lib/python2.7/site-packages/certifi/cacert.pem', 'a') as f:
with open('/usr/lib/python3.8/site-packages/certifi/cacert.pem', 'a') as f:
f.write(cert)

32
lxc-apps/ckan/ckan.image
View File

@ -1,36 +1,22 @@
IMAGE ckan_2.8.3-200621
FROM alpine3.11-python2.7_2.7.18-200621
IMAGE ckan_2.9.0-201021
FROM alpine3.12-python3.8_3.8.5-201021
RUN EOF
# Install runtime dependencies
apk --no-cache add geos libjpeg-turbo libmagic libpq mailcap py2-pip zlib
apk --no-cache add geos libjpeg-turbo libmagic libpq mailcap zlib
# Install build dependencies
apk --no-cache add --virtual .deps build-base git libjpeg-turbo-dev libxml2-dev libxslt-dev postgresql-dev python2-dev zlib-dev
apk --no-cache add --virtual .deps build-base git libjpeg-turbo-dev libxml2-dev libxslt-dev postgresql-dev python3-dev zlib-dev
# Install CKAN
mkdir -p /srv/ckan
cd /srv/ckan
pip install -U setuptools
pip install -e 'git+https://github.com/ckan/ckan.git@ba0120dc0c798bbc9b6d8e9ad83db01a197ea179#egg=ckan'
# Hackfix support for python3.7+ and setuptools 46+
sed -i 's/zope\.interface==4\.3\.2/zope.interface==4.7.2/' /srv/ckan/src/ckan/requirements.txt
# -I allows dependencies to downgrade six
pip install -Ir /srv/ckan/src/ckan/requirements.txt
pip install flask-debugtoolbar
pip install -e 'git+https://github.com/ckan/ckan.git@8e1cc60b2fa11da6843051678b7ee2cc08c2a7a9#egg=ckan'
pip install -r /srv/ckan/src/ckan/requirements.txt
# Install CKAN extensions
pip install -e 'git+https://github.com/ckan/ckanext-basiccharts#egg=ckanext_basiccharts'
pip install -e 'git+https://github.com/ckan/ckanext-spatial#egg=ckanext_spatial'
pip install -e 'git+https://github.com/ckan/ckanext-geoview#egg=ckanext_geoview'
pip install -e 'git+https://github.com/ckan/ckanext-mapviews#egg=ckanext_mapviews'
pip install -e 'git+https://github.com/XVTSolutions/ckanext-spatialUI#egg=ckanext_spatialui'
pip install -e 'git+https://github.com/aptivate/ckanext-datasetthumbnail#egg=ckanext_datasetthumbnail'
pip install -e 'git+https://github.com/datagvat/ckanext-dgvat_xls#egg=ckanext_dgvat_xls'
pip install -r /srv/ckan/src/ckanext-spatial/pip-requirements.txt
pip install -r /srv/ckan/src/ckanext-geoview/pip-requirements.txt
pip install -r /srv/ckan/src/ckanext-dgvat-xls/requirements.txt
# Hackfix support for PostgreSQL 12
# https://github.com/sqlalchemy/sqlalchemy/issues/4463
sed -i 's/cons\.consrc/pg_get_constraintdef(cons.oid)/' /usr/lib/python2.7/site-packages/sqlalchemy/dialects/postgresql/base.py
# Create OS user
addgroup -S -g 8080 ckan

4
lxc-apps/ckan/ckan.image.d/etc/crontabs/ckan
View File

@ -1,2 +1,2 @@
0 * * * * paster --plugin=ckan tracking update -c /etc/ckan/ckan.ini >/dev/null
0 * * * * paster --plugin=ckan search-index rebuild -r -c /etc/ckan/ckan.ini >/dev/null
0 * * * * ckan -c /etc/ckan/ckan.ini tracking update >/dev/null
0 * * * * ckan -c /etc/ckan/ckan.ini search-index rebuild -r >/dev/null

2
lxc-apps/ckan/ckan.image.d/etc/services.d/ckan/run
View File

@ -2,4 +2,4 @@
fdmove -c 2 1
s6-setuidgid ckan
paster serve /etc/ckan/ckan.ini
ckan -c /etc/ckan/ckan.ini run -H 0.0.0.0 -p 8080

9
lxc-apps/ckan/install.sh
View File

@ -6,7 +6,7 @@ POSTGRES_DATA="${VOLUMES_DIR}/ckan/postgres_data"
REDIS_CONF="${VOLUMES_DIR}/ckan/redis_conf"
REDIS_DATA="${VOLUMES_DIR}/ckan/redis_data"
SOLR_DATA="${VOLUMES_DIR}/ckan/solr_data"
SOLR_LAYER="${LAYERS_DIR}/solr6_6.5.1-200621"
SOLR_LAYER="${LAYERS_DIR}/solr6_6.5.1-201021"
DATAPUSHER_CONF="${VOLUMES_DIR}/ckan/datapusher_conf"
DATAPUSHER_DATA="${VOLUMES_DIR}/ckan/datapusher_data"
CKAN_CONF="${VOLUMES_DIR}/ckan/ckan_conf"
@ -60,16 +60,15 @@ envsubst <ckan_conf/ckan.ini | install -o 100000 -g 108080 -m 640 /dev/stdin ${C
install -o 100000 -g 108080 -m 640 ckan_conf/who.ini ${CKAN_CONF}/who.ini
# Populate database
spoc-container exec ckan -- paster --plugin=ckan db init -c /etc/ckan/ckan.ini
spoc-container exec ckan -- paster --plugin=ckanext-spatial spatial initdb -c /etc/ckan/ckan.ini
spoc-container exec ckan -- paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | spoc-container exec ckan-postgres -- psql
spoc-container exec ckan -- ckan -c /etc/ckan/ckan.ini db init
spoc-container exec ckan -- ckan -c /etc/ckan/ckan.ini datastore set-permissions | spoc-container exec ckan-postgres -- psql
# Create admin account
export CKAN_ADMIN_USER="admin"
export CKAN_ADMIN_UUID=$(cat /proc/sys/kernel/random/uuid)
export CKAN_ADMIN_APIKEY=$(cat /proc/sys/kernel/random/uuid)
export CKAN_ADMIN_PWD=$(head -c 12 /dev/urandom | base64 | tr -d '+/=')
export CKAN_ADMIN_HASH=$(spoc-container exec ckan -- python -c "from passlib.hash import pbkdf2_sha512;print pbkdf2_sha512.encrypt('${CKAN_ADMIN_PWD}')")
export CKAN_ADMIN_HASH=$(spoc-container exec ckan -- python -c "from passlib.hash import pbkdf2_sha512;print(pbkdf2_sha512.encrypt('${CKAN_ADMIN_PWD}'))")
export CKAN_ADMIN_EMAIL="admin@example.com"
envsubst <adminpwd.sql | spoc-container exec ckan-postgres -- psql ckan

42
lxc-apps/ckan/install/ckan_conf/ckan.ini
View File

@ -29,11 +29,11 @@ cache_dir = /tmp/%(ckan.site_id)s/
beaker.session.key = ckan
# This is the secret token that the beaker library uses to hash the cookie sent
# to the client. `paster make-config` generates a unique value for this each
# to the client. `ckan generate config` generates a unique value for this each
# time it generates a config file.
beaker.session.secret = ${CKAN_SECRET}
# `paster make-config` generates a unique value for this each time it generates
# `ckan generate config` generates a unique value for this each time it generates
# a config file.
app_instance_uuid = ${CKAN_UUID}
@ -75,7 +75,17 @@ ckan.auth.create_user_via_web = true
ckan.auth.roles_that_cascade_to_sub_groups = admin
ckan.auth.public_user_details = true
ckan.auth.public_activity_stream_detail = true
ckan.auth.allow_dataset_collaborators = true
ckan.auth.create_default_api_keys = false
## API Token Settings
api_token.nbytes = 60
api_token.jwt.encode.secret = string:$app_instance_secret
api_token.jwt.decode.secret = string:$app_instance_secret
api_token.jwt.algorithm = HS256
## API Token: expire_api_token plugin
expire_api_token.default_lifetime = 3600
## Search Settings
@ -104,34 +114,20 @@ ckan.redis.url = redis://ckan-redis:6379/0
# Add ``datapusher`` to enable DataPusher
# Add ``resource_proxy`` to enable resorce proxying and get around the
# same origin policy
ckan.plugins = stats text_view image_view recline_view datastore datapusher resource_proxy linechart barchart piechart basicgrid spatial_metadata spatial_query geo_view geojson_view wmts_view navigablemap choroplethmap spatialUI datasetthumbnail dgvat_xls
ckan.plugins = stats text_view image_view recline_view datastore datapusher resource_proxy
# Define which views should be created by default
# (plugins must be loaded in ckan.plugins)
ckan.views.default_views = image_view text_view recline_view geo_view geojson_view wmts_view
ckan.views.default_views = image_view text_view recline_view
# Customize which text formats the text_view plugin will show
#ckan.preview.json_formats = json
#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss
#ckan.preview.text_formats = text plain text/plain
#ckan.preview.text_formats = txt plain text/plain
# Customize which image formats the image_view plugin will show
#ckan.preview.image_formats = png jpeg jpg gif
# GeoView plugin settings
ckanext.geoview.ol_viewer.formats = wms wfs geojson gml kml arcgis_rest gft
ckanext.geoview.gapi_key =
# Pages plugin settings
ckanext.pages.organization = true
ckanext.pages.group = true
ckanext.pages.allow_html = true
ckanext.pages.editor = ckeditor
# Dataset thumbnail plugin settings
ckan.datasetthumbnail.show_thumbnail = true
ckan.datasetthumbnail.auto_generate = true
## Front-End Settings
ckan.site_title = CKAN
@ -215,7 +211,7 @@ ckan.jobs.timeout = 180
## Logging configuration
[loggers]
keys = root, ckan, ckanext
keys = root, ckan, ckanext, werkzeug
[handlers]
keys = console
@ -227,6 +223,12 @@ keys = generic
level = WARNING
handlers = console
[logger_werkzeug]
level = WARNING
handlers = console
qualname = werkzeug
propagate = 0
[logger_ckan]
level = INFO
handlers = console

9
lxc-apps/ckan/install/ckan_conf/who.ini
View File

@ -1,10 +1,10 @@
[plugin:auth_tkt]
use = ckan.lib.auth_tkt:make_plugin
use = ckan.lib.repoze_plugins.auth_tkt:make_plugin
# If no secret key is defined here, beaker.session.secret will be used
#secret = somesecret
[plugin:friendlyform]
use = repoze.who.plugins.friendlyform:FriendlyFormPlugin
use = ckan.lib.repoze_plugins.friendly_form:FriendlyFormPlugin
login_form_url= /user/login
login_handler_path = /login_generic
logout_handler_path = /user/logout
@ -13,10 +13,6 @@ post_login_url = /user/logged_in
post_logout_url = /user/logged_out
charset = utf-8
#[plugin:basicauth]
#use = repoze.who.plugins.basicauth:make_plugin
#realm = 'CKAN'
[general]
request_classifier = repoze.who.classifiers:default_request_classifier
challenge_decider = repoze.who.classifiers:default_challenge_decider
@ -34,4 +30,3 @@ plugins =
[challengers]
plugins =
friendlyform;browser
# basicauth

15
lxc-apps/ckan/update.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/sh
set -ev
# Start service containers
spoc-container start ckan-postgres
spoc-container start ckan-redis
spoc-container start ckan-solr
# Run database upgrade
spoc-container exec ckan -- ckan db upgrade -c /etc/ckan/ckan.ini
# Stop service containers
spoc-container start ckan-solr
spoc-container start ckan-redis
spoc-container start ckan-postgres