From 6ec890aa3079bec94bd61f705fab9fb6a9b05628 Mon Sep 17 00:00:00 2001 From: Disassembler Date: Thu, 22 Oct 2020 20:56:10 +0200 Subject: [PATCH] Bump CKAN to 2.9.0, use Python 3 runtime --- lxc-apps/ckan/app | 12 +++--- lxc-apps/ckan/ckan-datapusher.image | 19 +++------ .../ckan-datapusher.image.d/bin/add-ca-cert | 5 +-- lxc-apps/ckan/ckan.image | 32 ++++---------- lxc-apps/ckan/ckan.image.d/etc/crontabs/ckan | 4 +- .../ckan/ckan.image.d/etc/services.d/ckan/run | 2 +- lxc-apps/ckan/install.sh | 9 ++-- lxc-apps/ckan/install/ckan_conf/ckan.ini | 42 ++++++++++--------- lxc-apps/ckan/install/ckan_conf/who.ini | 9 +--- lxc-apps/ckan/update.sh | 15 +++++++ 10 files changed, 67 insertions(+), 82 deletions(-) create mode 100755 lxc-apps/ckan/update.sh diff --git a/lxc-apps/ckan/app b/lxc-apps/ckan/app index 1a8ba39..66481cb 100644 --- a/lxc-apps/ckan/app +++ b/lxc-apps/ckan/app @@ -1,5 +1,5 @@ { - "version": "2.8.3-200621", + "version": "2.9.0-201021", "meta": { "title": "CKAN", "desc-cs": "Datový sklad", @@ -8,7 +8,7 @@ }, "containers": { "ckan": { - "image": "ckan_2.8.3-200621", + "image": "ckan_2.9.0-201021", "depends": [ "ckan-datapusher", "ckan-redis", @@ -21,27 +21,27 @@ } }, "ckan-datapusher": { - "image": "ckan-datapusher_0.0.16-200621", + "image": "ckan-datapusher_0.0.16-201021", "mounts": { "ckan/datapusher_conf": "etc/ckan-datapusher", "ckan/datapusher_data": "srv/ckan-datapusher/data" } }, "ckan-redis": { - "image": "redis_5.0.7-200621", + "image": "redis_5.0.9-201021", "mounts": { "ckan/redis_conf/redis.conf": "etc/redis.conf:file", "ckan/redis_data": "var/lib/redis" } }, "ckan-solr": { - "image": "solr6_6.5.1-200621", + "image": "solr6_6.5.1-201021", "mounts": { "ckan/solr_data": "var/lib/solr" } }, "ckan-postgres": { - "image": "postgis_3.0.0-200621", + "image": "postgis_3.0.1-201021", "mounts": { "ckan/postgres_data": "var/lib/postgresql" } diff --git a/lxc-apps/ckan/ckan-datapusher.image b/lxc-apps/ckan/ckan-datapusher.image index 970f024..ce30565 100644 --- a/lxc-apps/ckan/ckan-datapusher.image +++ b/lxc-apps/ckan/ckan-datapusher.image @@ -1,28 +1,19 @@ -IMAGE ckan-datapusher_0.0.16-200621 -FROM alpine3.10-python2.7_2.7.18-200621 -# Alpine 3.11 discontinued uwsgi-python2 module +IMAGE ckan-datapusher_0.0.16-201021 +FROM alpine3.12-python3.8_3.8.5-201021 RUN EOF # Install runtime dependencies - apk --no-cache add libffi uwsgi-python + apk --no-cache add uwsgi-python3 # Install build dependencies - apk --no-cache add --virtual .deps build-base git libffi-dev libressl-dev libxml2-dev libxslt-dev py2-pip python2-dev + apk --no-cache add --virtual .deps build-base git libffi-dev libressl-dev libxml2-dev libxslt-dev python3-dev # Install CKAN DataPusher mkdir -p /srv/ckan-datapusher cd /srv/ckan-datapusher - pip install -U setuptools - pip install -e 'git+https://github.com/ckan/datapusher.git@e662e3c33e069ac174cdb4fb1d61121f0ba4bb3a#egg=datapusher' - - # Hackfix the X509_STORE_CTX wrapper - sed -i 's/\[security\]//' /srv/ckan-datapusher/src/datapusher/requirements.txt + pip install -e 'git+https://github.com/ckan/datapusher.git@eb4686030ed6911a5294c62bf275ddbbc4b26114#egg=datapusher' pip install -r /srv/ckan-datapusher/src/datapusher/requirements.txt - # Hackfix werkzeug==1.0.0 proxy_fix import - # https://github.com/ckan/ckan-service-provider/pull/49 - sed -i 's/werkzeug\.contrib\.fixers/werkzeug.middleware.proxy_fix/' /usr/lib/python2.7/site-packages/ckanserviceprovider/web.py - # Create OS user addgroup -S -g 8080 ckandp adduser -S -u 8080 -h /srv/ckan-datapusher -s /bin/false -g ckandp -G ckandp ckandp diff --git a/lxc-apps/ckan/ckan-datapusher.image.d/bin/add-ca-cert b/lxc-apps/ckan/ckan-datapusher.image.d/bin/add-ca-cert index 49cd105..cee0cfb 100755 --- a/lxc-apps/ckan/ckan-datapusher.image.d/bin/add-ca-cert +++ b/lxc-apps/ckan/ckan-datapusher.image.d/bin/add-ca-cert @@ -6,8 +6,5 @@ with open('/etc/ckan-datapusher/add-ca-cert.env') as f: env = dict(tuple(line.split('=')) for line in f.read().splitlines()) cert = ssl.get_server_certificate((env['HOST'], env['PORT'])) -with open('/usr/lib/python2.7/site-packages/requests/cacert.pem', 'a') as f: - f.write(cert) - -with open('/usr/lib/python2.7/site-packages/certifi/cacert.pem', 'a') as f: +with open('/usr/lib/python3.8/site-packages/certifi/cacert.pem', 'a') as f: f.write(cert) diff --git a/lxc-apps/ckan/ckan.image b/lxc-apps/ckan/ckan.image index 358e919..74e72dd 100644 --- a/lxc-apps/ckan/ckan.image +++ b/lxc-apps/ckan/ckan.image @@ -1,36 +1,22 @@ -IMAGE ckan_2.8.3-200621 -FROM alpine3.11-python2.7_2.7.18-200621 +IMAGE ckan_2.9.0-201021 +FROM alpine3.12-python3.8_3.8.5-201021 RUN EOF # Install runtime dependencies - apk --no-cache add geos libjpeg-turbo libmagic libpq mailcap py2-pip zlib + apk --no-cache add geos libjpeg-turbo libmagic libpq mailcap zlib # Install build dependencies - apk --no-cache add --virtual .deps build-base git libjpeg-turbo-dev libxml2-dev libxslt-dev postgresql-dev python2-dev zlib-dev + apk --no-cache add --virtual .deps build-base git libjpeg-turbo-dev libxml2-dev libxslt-dev postgresql-dev python3-dev zlib-dev # Install CKAN mkdir -p /srv/ckan cd /srv/ckan - pip install -U setuptools + pip install -e 'git+https://github.com/ckan/ckan.git@ba0120dc0c798bbc9b6d8e9ad83db01a197ea179#egg=ckan' + # Hackfix support for python3.7+ and setuptools 46+ + sed -i 's/zope\.interface==4\.3\.2/zope.interface==4.7.2/' /srv/ckan/src/ckan/requirements.txt + # -I allows dependencies to downgrade six + pip install -Ir /srv/ckan/src/ckan/requirements.txt pip install flask-debugtoolbar - pip install -e 'git+https://github.com/ckan/ckan.git@8e1cc60b2fa11da6843051678b7ee2cc08c2a7a9#egg=ckan' - pip install -r /srv/ckan/src/ckan/requirements.txt - - # Install CKAN extensions - pip install -e 'git+https://github.com/ckan/ckanext-basiccharts#egg=ckanext_basiccharts' - pip install -e 'git+https://github.com/ckan/ckanext-spatial#egg=ckanext_spatial' - pip install -e 'git+https://github.com/ckan/ckanext-geoview#egg=ckanext_geoview' - pip install -e 'git+https://github.com/ckan/ckanext-mapviews#egg=ckanext_mapviews' - pip install -e 'git+https://github.com/XVTSolutions/ckanext-spatialUI#egg=ckanext_spatialui' - pip install -e 'git+https://github.com/aptivate/ckanext-datasetthumbnail#egg=ckanext_datasetthumbnail' - pip install -e 'git+https://github.com/datagvat/ckanext-dgvat_xls#egg=ckanext_dgvat_xls' - pip install -r /srv/ckan/src/ckanext-spatial/pip-requirements.txt - pip install -r /srv/ckan/src/ckanext-geoview/pip-requirements.txt - pip install -r /srv/ckan/src/ckanext-dgvat-xls/requirements.txt - - # Hackfix support for PostgreSQL 12 - # https://github.com/sqlalchemy/sqlalchemy/issues/4463 - sed -i 's/cons\.consrc/pg_get_constraintdef(cons.oid)/' /usr/lib/python2.7/site-packages/sqlalchemy/dialects/postgresql/base.py # Create OS user addgroup -S -g 8080 ckan diff --git a/lxc-apps/ckan/ckan.image.d/etc/crontabs/ckan b/lxc-apps/ckan/ckan.image.d/etc/crontabs/ckan index 92f3f37..d2148ab 100644 --- a/lxc-apps/ckan/ckan.image.d/etc/crontabs/ckan +++ b/lxc-apps/ckan/ckan.image.d/etc/crontabs/ckan @@ -1,2 +1,2 @@ -0 * * * * paster --plugin=ckan tracking update -c /etc/ckan/ckan.ini >/dev/null -0 * * * * paster --plugin=ckan search-index rebuild -r -c /etc/ckan/ckan.ini >/dev/null +0 * * * * ckan -c /etc/ckan/ckan.ini tracking update >/dev/null +0 * * * * ckan -c /etc/ckan/ckan.ini search-index rebuild -r >/dev/null diff --git a/lxc-apps/ckan/ckan.image.d/etc/services.d/ckan/run b/lxc-apps/ckan/ckan.image.d/etc/services.d/ckan/run index 1b955e4..469d4e7 100755 --- a/lxc-apps/ckan/ckan.image.d/etc/services.d/ckan/run +++ b/lxc-apps/ckan/ckan.image.d/etc/services.d/ckan/run @@ -2,4 +2,4 @@ fdmove -c 2 1 s6-setuidgid ckan -paster serve /etc/ckan/ckan.ini +ckan -c /etc/ckan/ckan.ini run -H 0.0.0.0 -p 8080 diff --git a/lxc-apps/ckan/install.sh b/lxc-apps/ckan/install.sh index e3b76c3..9e68459 100755 --- a/lxc-apps/ckan/install.sh +++ b/lxc-apps/ckan/install.sh @@ -6,7 +6,7 @@ POSTGRES_DATA="${VOLUMES_DIR}/ckan/postgres_data" REDIS_CONF="${VOLUMES_DIR}/ckan/redis_conf" REDIS_DATA="${VOLUMES_DIR}/ckan/redis_data" SOLR_DATA="${VOLUMES_DIR}/ckan/solr_data" -SOLR_LAYER="${LAYERS_DIR}/solr6_6.5.1-200621" +SOLR_LAYER="${LAYERS_DIR}/solr6_6.5.1-201021" DATAPUSHER_CONF="${VOLUMES_DIR}/ckan/datapusher_conf" DATAPUSHER_DATA="${VOLUMES_DIR}/ckan/datapusher_data" CKAN_CONF="${VOLUMES_DIR}/ckan/ckan_conf" @@ -60,16 +60,15 @@ envsubst