Dockerize CKAN (without DataPusher so far)

ckan.sh

@@ -1,104 +1,54 @@
 #!/bin/bash
 
 SOURCE_DIR=$(realpath $(dirname "${0}"))/ckan
+# Requires: Postgres, Redis, Solr, CKAN-Datapusher
 
-# Install dependencies for CKAN
+# Build Docker container
-apt-get -y --no-install-recommends install gcc libpq-dev python-dev python-virtualenv redis-server virtualenv
+docker build -t ckan ${SOURCE_DIR}
-
-# Install dependencies for Solr
-apt-get -y --no-install-recommends install openjdk-8-jre-headless lsof
-
-# Install CKAN python virtual environment
-mkdir -p /srv/ckan
-virtualenv --no-site-packages --python=/usr/bin/python2.7 /srv/ckan
-/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckan.git#egg=ckan'
-/srv/ckan/bin/pip install -r /srv/ckan/src/ckan/requirements.txt
-
-# Install Datapusher python virtual environment
-mkdir -p /srv/ckan-datapusher
-virtualenv --no-site-packages --python=/usr/bin/python2.7 /srv/ckan-datapusher
-/srv/ckan-datapusher/bin/pip install -e 'git+https://github.com/ckan/datapusher.git#egg=datapusher'
-/srv/ckan-datapusher/bin/pip install -r /srv/ckan-datapusher/src/datapusher/requirements.txt
-
-# Patch service provider TLS verification for Datapusher
-# https://github.com/ckan/ckan-service-provider/issues/36
-patch -d /srv/ckan-datapusher -p0 <${SOURCE_DIR}/ckan-serviceprovider-sslverify.patch
-
-# Install CKAN extensions
-/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-basiccharts#egg=ckanext_basiccharts'
-/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-spatial#egg=ckanext_spatial'
-/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-geoview#egg=ckanext_geoview'
-/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-mapviews#egg=ckanext_mapviews'
-/srv/ckan/bin/pip install -e 'git+https://github.com/ckan/ckanext-pages#egg=ckanext_pages'
-/srv/ckan/bin/pip install -e 'git+https://github.com/XVTSolutions/ckanext-spatialUI#egg=ckanext_spatialui'
-/srv/ckan/bin/pip install -e 'git+https://github.com/aptivate/ckanext-datasetthumbnail#egg=ckanext_datasetthumbnail'
-/srv/ckan/bin/pip install -e 'git+https://github.com/datagvat/ckanext-dgvat_xls#egg=ckanext_dgvat_xls'
-/srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-spatial/pip-requirements.txt
-/srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-datasetthumbnail/requirements.txt
-/srv/ckan/bin/pip install -r /srv/ckan/src/ckanext-dgvat-xls/requirements.txt
-
-# Download Solr
-wget http://archive.apache.org/dist/lucene/solr/6.5.1/solr-6.5.1.tgz -O /tmp/solr-6.5.1.tgz
-tar xzf /tmp/solr-6.5.1.tgz -C /opt/
-mv /opt/solr-6.5.1 /opt/solr
-rm -f /tmp/solr-6.5.1.tgz
-
-# Create Solr OS user
-adduser --system --group --home /var/lib/solr --shell /bin/false solr
-chown -R solr:solr /opt/solr/
-
-# Configure Solr
-cp ${SOURCE_DIR}/lib/systemd/system/solr.service /lib/systemd/system/solr.service
-cp -p /opt/solr/server/solr/solr.xml /var/lib/solr/
-systemctl daemon-reload
-systemctl enable solr
-systemctl start solr
-sleep 5
-
-# Configure CKAN Solr core
-sudo -u solr /opt/solr/bin/solr create -c ckan
-cp ${SOURCE_DIR}/var/lib/solr/ckan/conf/solrconfig.xml /var/lib/solr/ckan/conf/solrconfig.xml
-ln -s /srv/ckan/src/ckan/ckan/config/solr/schema.xml /var/lib/solr/ckan/conf/schema.xml
-systemctl restart solr
 
 # Create database
 export CKAN_PWD=$(head -c 18 /dev/urandom | base64)
 export CKAN_DS_PWD=$(head -c 18 /dev/urandom | base64)
-envsubst <${SOURCE_DIR}/tmp/ckan-createdb.sql >/tmp/ckan-createdb.sql
+envsubst <${SOURCE_DIR}/createdb.sql | docker exec -i postgres psql
-sudo -u postgres psql -f /tmp/ckan-createdb.sql
+
-rm -f /tmp/ckan-createdb.sql
+# Configure CKAN Solr core
+docker exec solr solr create -c ckan
+cp ${SOURCE_DIR}/srv/solr/data/ckan/conf/schema.xml /srv/solr/data/ckan/conf/schema.xml
+cp ${SOURCE_DIR}/srv/solr/data/ckan/conf/solrconfig.xml /srv/solr/data/ckan/conf/solrconfig.xml
+chown 8983:8983 /srv/solr/data/ckan/conf/schema.xml
+systemctl restart solr
 
 # Configure CKAN
 export CKAN_SECRET=$(head -c 18 /dev/urandom | base64)
-export CKAN_UUID=$(python -c "import uuid; print uuid.uuid4()")
+export CKAN_UUID=$(cat /proc/sys/kernel/random/uuid)
-mkdir /etc/ckan
+mkdir -p /srv/ckan/{conf,data}
-envsubst <${SOURCE_DIR}/etc/ckan/ckan.ini >/etc/ckan/ckan.ini
+envsubst <${SOURCE_DIR}/srv/ckan/conf/ckan.ini >/srv/ckan/conf/ckan.ini
-cp ${SOURCE_DIR}/etc/ckan/datapusher.wsgi /etc/ckan/datapusher.wsgi
+cp ${SOURCE_DIR}/srv/ckan/conf/who.ini /srv/ckan/conf/who.ini
-cp ${SOURCE_DIR}/etc/ckan/datapusher_settings.py /etc/ckan/datapusher_settings.py
+chown -R 8003:8003 /srv/ckan/data
-ln -s /srv/ckan/src/ckan/ckan/config/who.ini /etc/ckan/who.ini
 cp ${SOURCE_DIR}/srv/ckan/update-ip.sh /srv/ckan/update-ip.sh
 /srv/ckan/update-ip.sh
 
 # Populate database
-/srv/ckan/bin/paster --plugin=ckan db init -c /etc/ckan/ckan.ini
+docker run --rm --link=postgres --link=redis --link=solr -v /srv/ckan/conf:/etc/ckan -v /srv/ckan/data:/srv/ckan/storage ckan paster --plugin=ckan db init -c /etc/ckan/ckan.ini
-/srv/ckan/bin/paster --plugin=ckanext-spatial spatial initdb -c /etc/ckan/ckan.ini
+docker run --rm --link=postgres --link=redis --link=solr -v /srv/ckan/conf:/etc/ckan -v /srv/ckan/data:/srv/ckan/storage ckan paster --plugin=ckanext-spatial spatial initdb -c /etc/ckan/ckan.ini
-/srv/ckan/bin/paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | sudo -u postgres psql
+docker run --rm --link=postgres --link=redis --link=solr -v /srv/ckan/conf:/etc/ckan -v /srv/ckan/data:/srv/ckan/storage ckan paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | docker exec -i postgres psql
-
-# Create CKAN OS user
-adduser --system --group --home /srv/ckan --shell /bin/false ckan
-chown -R ckan:ckan /srv/ckan/
-chown -R ckan:ckan /srv/ckan-datapusher/
 
 # Create admin account
 export CKAN_ADMIN_USER="admin"
-export CKAN_ADMIN_UUID=$(python -c "import uuid; print uuid.uuid4()")
+export CKAN_ADMIN_UUID=$(cat /proc/sys/kernel/random/uuid)
-export CKAN_ADMIN_APIKEY=$(python -c "import uuid; print uuid.uuid4()")
+export CKAN_ADMIN_APIKEY=$(cat /proc/sys/kernel/random/uuid)
 export CKAN_ADMIN_PWD=$(head -c 12 /dev/urandom | base64)
-export CKAN_ADMIN_HASH=$(/srv/ckan/bin/python ${SOURCE_DIR}/ckan-adminpwd.py ${CKAN_ADMIN_PWD})
+export CKAN_ADMIN_HASH=$(docker run --rm ckan python -c "from passlib.hash import pbkdf2_sha512;print pbkdf2_sha512.encrypt('${CKAN_ADMIN_PWD}')")
 export CKAN_ADMIN_EMAIL="admin@example.com"
-envsubst <${SOURCE_DIR}/tmp/ckan-adminpwd.sql >/tmp/ckan-adminpwd.sql
+envsubst <${SOURCE_DIR}/adminpwd.sql | docker exec -i postgres psql ckan
-sudo -u postgres psql -f /tmp/ckan-adminpwd.sql ckan
+
-rm /tmp/ckan-adminpwd.sql
+# Configure CKAN service
+cp ${SOURCE_DIR}/lib/systemd/system/ckan.service /lib/systemd/system/ckan.service
+systemctl daemon-reload
+systemctl enable ckan
+systemctl start ckan
+
+
 
 # Create uwsgi and nginx app definition
 cp ${SOURCE_DIR}/etc/uwsgi/apps-available/ckan.ini /etc/uwsgi/apps-available/ckan.ini

ckan/Dockerfile

@@ -0,0 +1,38 @@
+FROM alpine:3.7
+MAINTAINER Disassembler <disassembler@dasm.cz>
+
+RUN apk --no-cache add python2
+
+RUN apk --no-cache add libjpeg-turbo libmagic libpq libxml2 libxslt py2-pip zlib \
+ && apk --no-cache add --virtual .deps git build-base libjpeg-turbo-dev libxml2-dev libxslt-dev postgresql-dev python2-dev zlib-dev \
+ && echo 'http://repository.fit.cvut.cz/mirrors/alpine/edge/testing' >>/etc/apk/repositories \
+ && apk --no-cache add geos \
+ && ln -s /lib/ld-musl-x86_64.so.1 /lib/libc.so.1 \
+ && mkdir -p /srv/ckan \
+ && cd /srv/ckan \
+ && pip install setuptools==36.1 \
+ && pip install -e 'git+https://github.com/ckan/ckan.git#egg=ckan' \
+ && sed -i 's/psycopg2==2.4.5/psycopg2==2.7.1/' /srv/ckan/src/ckan/requirements.txt \
+ && pip install -r /srv/ckan/src/ckan/requirements.txt \
+ && pip install -e 'git+https://github.com/ckan/ckanext-basiccharts#egg=ckanext_basiccharts' \
+ && pip install -e 'git+https://github.com/ckan/ckanext-spatial#egg=ckanext_spatial' \
+ && pip install -e 'git+https://github.com/ckan/ckanext-geoview#egg=ckanext_geoview' \
+ && pip install -e 'git+https://github.com/ckan/ckanext-mapviews#egg=ckanext_mapviews' \
+ && pip install -e 'git+https://github.com/ckan/ckanext-pages#egg=ckanext_pages' \
+ && pip install -e 'git+https://github.com/XVTSolutions/ckanext-spatialUI#egg=ckanext_spatialui' \
+ && pip install -e 'git+https://github.com/aptivate/ckanext-datasetthumbnail#egg=ckanext_datasetthumbnail' \
+ && pip install -e 'git+https://github.com/datagvat/ckanext-dgvat_xls#egg=ckanext_dgvat_xls' \
+ && pip install -r /srv/ckan/src/ckanext-spatial/pip-requirements.txt \
+ && pip install -r /srv/ckan/src/ckanext-dgvat-xls/requirements.txt \
+ && addgroup -S -g 8003 ckan \
+ && adduser -S -u 8003 -h /srv/ckan -s /bin/false -g ckan -G ckan ckan \
+ && chown -R ckan:ckan /srv/ckan \
+ && apk del .deps \
+ && find /srv/ckan/src -name '.git*' -exec rm -rf {} + \
+ && rm -rf /root/* || true
+
+VOLUME ["/etc/ckan", "/srv/ckan/storage"]
+EXPOSE 8003
+
+USER ckan
+CMD ["paster", "serve", "/etc/ckan/ckan.ini"]

ckan/ckan-adminpwd.py

@@ -1,6 +0,0 @@
-#!/usr/bin/python
-
-import sys
-from passlib.hash import pbkdf2_sha512
-
-print pbkdf2_sha512.encrypt(sys.argv[1])

ckan/createdb.sql

@@ -11,14 +11,6 @@ GRANT CONNECT, CREATE, TEMPORARY ON DATABASE ckan_datastore TO ckan;
 ALTER DATABASE ckan_datastore OWNER TO ckan_datastore;
 
 \c ckan
-CREATE TEXT SEARCH DICTIONARY cspell (template=ispell, dictfile = czech, afffile=czech, stopwords=czech);
-CREATE TEXT SEARCH CONFIGURATION czech (copy=english);
-ALTER TEXT SEARCH CONFIGURATION czech ALTER MAPPING FOR word, asciiword WITH cspell, simple;
 CREATE EXTENSION postgis;
 GRANT ALL ON geometry_columns TO ckan;
 GRANT ALL ON spatial_ref_sys TO ckan;
-
-\c ckan_datastore
-CREATE TEXT SEARCH DICTIONARY cspell (template=ispell, dictfile = czech, afffile=czech, stopwords=czech);
-CREATE TEXT SEARCH CONFIGURATION czech (copy=english);
-ALTER TEXT SEARCH CONFIGURATION czech ALTER MAPPING FOR word, asciiword WITH cspell, simple;

ckan/etc/uwsgi/apps-available/ckan.ini

@@ -1,12 +0,0 @@
-[uwsgi]
-uid = ckan
-gid = ckan
-chown-socket = www-data:www-data
-chdir = /srv/ckan
-home = /srv/ckan
-master = false
-workers = 1
-exec-asap = /srv/ckan/update-ip.sh
-disable-logging = true
-ini-paste = /etc/ckan/ckan.ini
-enable-threads = true

ckan/lib/systemd/system/ckan.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=CKAN Docker container
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/docker run --rm --name ckan --link=postgres --link=redis --link=solr -v /srv/ckan/conf:/etc/ckan -v /srv/ckan/data:/srv/ckan/storage ckan
+ExecStop=/usr/bin/docker stop ckan
+
+[Install]
+WantedBy=multi-user.target

ckan/lib/systemd/system/solr.service

@@ -1,11 +0,0 @@
-[Unit]
-Description=Apache Solr
-After=network.target
-
-[Service]
-ExecStart=/opt/solr/bin/solr start -f -p 8983 -s /var/lib/solr
-ExecStop=/opt/solr/bin/solr stop -p 8983
-User=solr
-
-[Install]
-WantedBy=multi-user.target

ckan/srv/ckan/conf/ckan.ini

@@ -45,13 +45,13 @@ who.log_file = %(cache_dir)s/who_log.ini
 # who.timeout = 86400
 
 ## Database Settings
-sqlalchemy.url = postgresql://ckan:${CKAN_PWD}@localhost/ckan
+sqlalchemy.url = postgresql://ckan:${CKAN_PWD}@postgres/ckan
 
-ckan.datastore.write_url = postgresql://ckan:${CKAN_PWD}@localhost/ckan_datastore
+ckan.datastore.write_url = postgresql://ckan:${CKAN_PWD}@postgres/ckan_datastore
-ckan.datastore.read_url = postgresql://ckan_datastore:${CKAN_DS_PWD}@localhost/ckan_datastore
+ckan.datastore.read_url = postgresql://ckan_datastore:${CKAN_DS_PWD}@postgres/ckan_datastore
 
 # PostgreSQL' full-text search parameters
-ckan.datastore.default_fts_lang = czech
+ckan.datastore.default_fts_lang = english
 ckan.datastore.default_fts_index_method = gist
 
 ## Site Settings
@@ -76,13 +76,13 @@ ckan.auth.roles_that_cascade_to_sub_groups = admin
 ## Search Settings
 
 ckan.site_id = default
-solr_url = http://127.0.0.1:8983/solr/ckan
+solr_url = http://solr:8983/solr/ckan
 
 
 ## Redis Settings
 
 # URL to your Redis instance, including the database to be used.
-ckan.redis.url = redis://127.0.0.1:6379/0
+ckan.redis.url = redis://redis:6379/0
 
 
 ## CORS Settings
@@ -177,7 +177,7 @@ ckan.max_image_size = 10
 # Make sure you have set up the DataStore
 
 ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
-ckan.datapusher.url = http://127.0.0.1:8098/
+ckan.datapusher.url = http://ckan-datapusher:8004/
 #ckan.datapusher.assume_task_stale_after = 3600
 
 # Resource Proxy settings

ckan/srv/ckan/conf/who.ini

@@ -0,0 +1,37 @@
+[plugin:auth_tkt]
+use = ckan.lib.auth_tkt:make_plugin
+# If no secret key is defined here, beaker.session.secret will be used
+#secret = somesecret
+
+[plugin:friendlyform]
+use = repoze.who.plugins.friendlyform:FriendlyFormPlugin
+login_form_url= /user/login
+login_handler_path = /login_generic
+logout_handler_path = /user/logout
+rememberer_name = auth_tkt
+post_login_url = /user/logged_in
+post_logout_url = /user/logged_out
+charset = utf-8
+
+#[plugin:basicauth]
+#use = repoze.who.plugins.basicauth:make_plugin
+#realm = 'CKAN'
+
+[general]
+request_classifier = repoze.who.classifiers:default_request_classifier
+challenge_decider = repoze.who.classifiers:default_challenge_decider
+
+[identifiers]
+plugins =
+    friendlyform;browser
+    auth_tkt
+
+[authenticators]
+plugins =
+    auth_tkt
+    ckan.lib.authenticator:UsernamePasswordAuthenticator
+
+[challengers]
+plugins =
+    friendlyform;browser
+#   basicauth

ckan/srv/ckan/update-ip.sh

@@ -1,4 +1,4 @@
 #!/bin/sh
 
 URL=$(ip route get 1 | awk '{print $NF;exit}')
-sed -i "s|^ckan\.site_url.*|ckan.site_url = https://${URL}:8003|" /etc/ckan/ckan.ini
+sed -i "s|^ckan\.site_url.*|ckan.site_url = http://${URL}:8003|" /srv/ckan/conf/ckan.ini