diff --git a/README.md b/README.md index a0e632ced..c31ca1fe3 100644 --- a/README.md +++ b/README.md @@ -369,7 +369,28 @@ The base image used in the CKAN Dockerfile and Dockerfile.dev can be changed so ## 13. Replacing DataPusher with XLoader -Check out the wiki page for this: https://github.com/ckan/ckan-docker/wiki/Replacing-DataPusher-with-XLoader +Basically it's just replacing files copied from the xloader directory. Depending on if you are running in Production mode or Development mode, here are the instructions you should use: + +##### For Production (base) + +From the top (root) directory of your docker installation: + + 1. `cd xloader/` + 2. `cp .env ..` + 3. `cp docker-compose.yml ..` + 4. `cp Dockerfile ../ckan` + 5. `cp -pr docker-entrypoint.d/ ../ckan` + +##### For Development + +From the top (root) directory of your docker installation: + + 1. `cd xloader/` + 2. `cp .env ..` + 3. `cp docker-compose-dev.yml ..` + 4. `cp Dockerfile.dev ../ckan` + 5. `cp -pr docker-entrypoint.d/ ../ckan` + Copying and License ------------------- diff --git a/xloader/Dockerfile b/xloader/Dockerfile new file mode 100644 index 000000000..bb8bade07 --- /dev/null +++ b/xloader/Dockerfile @@ -0,0 +1,32 @@ +FROM ckan/ckan-base:2.11 + +ARG XLOADER_VERSION=${XLOADER_VERSION} + +# Install any extensions needed by your CKAN instance +# See Dockerfile.dev for more details and examples + +USER root + +# Install the XLoader extension +RUN pip3 install -U pip && \ + cd ${SRC_DIR} && \ + pip3 install -e git+https://github.com/ckan/ckanext-xloader.git@${XLOADER_VERSION}#egg=ckanext-xloader && \ + pip3 install -r ckanext-xloader/requirements.txt && \ + pip3 install -U requests[security] + +# Copy custom initialization scripts +COPY --chown=ckan-sys:ckan-sys docker-entrypoint.d/* /docker-entrypoint.d/ + +# Apply any patches needed to CKAN core or any of the built extensions (not the +# runtime mounted ones) +COPY --chown=ckan-sys:ckan-sys patches ${APP_DIR}/patches + +USER ckan + +RUN for d in $APP_DIR/patches/*; do \ + if [ -d $d ]; then \ + for f in `ls $d/*.patch | sort -g`; do \ + cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \ + done ; \ + fi ; \ + done diff --git a/xloader/Dockerfile.dev b/xloader/Dockerfile.dev new file mode 100644 index 000000000..bb9efba97 --- /dev/null +++ b/xloader/Dockerfile.dev @@ -0,0 +1,66 @@ +FROM ckan/ckan-dev:2.11 + +ARG XLOADER_VERSION=${XLOADER_VERSION} + +# Install any extensions needed by your CKAN instance +# - Make sure to add the plugins to CKAN__PLUGINS in the .env file +# - Also make sure all provide all extra configuration options, either by: +# * Adding them to the .env file (check the ckanext-envvars syntax for env vars), or +# * Adding extra configuration scripts to /docker-entrypoint.d folder) to update +# the CKAN config file (ckan.ini) with the `ckan config-tool` command +# +# See README > Extending the base images for more details +# +# For instance: +# +### XLoader ### +#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@master#egg=ckanext-xloader' && \ +# pip3 install -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \ +# pip3 install -U requests[security] + +### Harvester ### +#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest' && \ +# pip3 install -r ${APP_DIR}/src/ckanext-harvest/pip-requirements.txt +# will also require gather_consumer and fetch_consumer processes running (please see https://github.com/ckan/ckanext-harvest) + +### Scheming ### +#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@master#egg=ckanext-scheming' + +### Pages ### +#RUN pip3 install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages + +### DCAT ### +#RUN pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \ +# pip3 install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt + +# Clone the extension(s) your are writing for your own project in the `src` folder +# to get them mounted in this image at runtime + +# Install any extensions needed by your CKAN instance +# See Dockerfile.dev for more details and examples + +USER root + +# Install the XLoader extension +RUN pip3 install -U pip && \ + cd ${SRC_DIR} && \ + pip3 install -e git+https://github.com/ckan/ckanext-xloader.git@${XLOADER_VERSION}#egg=ckanext-xloader && \ + pip3 install -r ckanext-xloader/requirements.txt && \ + pip3 install -U requests[security] + +# Copy custom initialization scripts +COPY --chown=ckan-sys:ckan-sys docker-entrypoint.d/* /docker-entrypoint.d/ + +# Apply any patches needed to CKAN core or any of the built extensions (not the +# runtime mounted ones) +COPY --chown=ckan-sys:ckan-sys patches ${APP_DIR}/patches + +USER ckan + +RUN for d in $APP_DIR/patches/*; do \ + if [ -d $d ]; then \ + for f in `ls $d/*.patch | sort -g`; do \ + cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \ + done ; \ + fi ; \ + done diff --git a/xloader/docker-compose.dev.yml b/xloader/docker-compose.dev.yml new file mode 100755 index 000000000..a3a710a16 --- /dev/null +++ b/xloader/docker-compose.dev.yml @@ -0,0 +1,81 @@ +volumes: + ckan_storage: + pg_data: + solr_data: + pip_cache: + site_packages: + local_bin: + home_dir: + +services: + + ckan-dev: + build: + context: ckan/ + dockerfile: Dockerfile.dev + args: + - TZ=${TZ} + - XLOADER_VERSION=${XLOADER_VERSION} + env_file: + - .env + links: + - db + - solr + - redis + ports: + - "0.0.0.0:${CKAN_PORT_HOST}:5000" + volumes: + - ckan_storage:/var/lib/ckan + - ./src:/srv/app/src_extensions + - pip_cache:/root/.cache/pip + - site_packages:/usr/local/lib/python3.10/site-packages + - local_bin:/usr/local/bin + - home_dir:/srv/app/ + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] + interval: 60s + timeout: 10s + retries: 3 + + xloader: + image: ckan/ckan-base-xloader:${XLOADER_VERSION} + env_file: + - .env + restart: unless-stopped + depends_on: + ckan-dev: + condition: service_healthy + + db: + build: + context: postgresql/ + environment: + - POSTGRES_USER + - POSTGRES_PASSWORD + - POSTGRES_DB + - CKAN_DB_USER + - CKAN_DB_PASSWORD + - CKAN_DB + - DATASTORE_READONLY_USER + - DATASTORE_READONLY_PASSWORD + - DATASTORE_DB + volumes: + - pg_data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] + + solr: + image: ckan/ckan-solr:${SOLR_IMAGE_VERSION} + volumes: + - solr_data:/var/solr + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"] + + redis: + image: redis:${REDIS_VERSION} + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "-e", "QUIT"] diff --git a/xloader/docker-compose.yml b/xloader/docker-compose.yml new file mode 100755 index 000000000..672ce06c6 --- /dev/null +++ b/xloader/docker-compose.yml @@ -0,0 +1,116 @@ +volumes: + ckan_storage: + pg_data: + solr_data: + pip_cache: + site_packages: + +services: + + nginx: + build: + context: nginx/ + dockerfile: Dockerfile + networks: + - webnet + - ckannet + depends_on: + ckan: + condition: service_healthy + ports: + - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}" + + ckan: + build: + context: ckan/ + dockerfile: Dockerfile + args: + - TZ=${TZ} + - XLOADER_VERSION=${XLOADER_VERSION} + networks: + - ckannet + - dbnet + - solrnet + - redisnet + env_file: + - .env + depends_on: + db: + condition: service_healthy + solr: + condition: service_healthy + redis: + condition: service_healthy + volumes: + - ckan_storage:/var/lib/ckan + - pip_cache:/root/.cache/pip + - site_packages:/usr/lib/python3.10/site-packages + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000/api/action/status_show"] + interval: 60s + timeout: 10s + retries: 3 + + xloader: + image: ckan/ckan-base-xloader:${XLOADER_VERSION} + env_file: + - .env + networks: + - ckannet + - dbnet + - solrnet + - redisnet + restart: unless-stopped + depends_on: + ckan: + condition: service_healthy + + db: + build: + context: postgresql/ + networks: + - dbnet + environment: + - POSTGRES_USER + - POSTGRES_PASSWORD + - POSTGRES_DB + - CKAN_DB_USER + - CKAN_DB_PASSWORD + - CKAN_DB + - DATASTORE_READONLY_USER + - DATASTORE_READONLY_PASSWORD + - DATASTORE_DB + volumes: + - pg_data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] + + solr: + networks: + - solrnet + image: ckan/ckan-solr:${SOLR_IMAGE_VERSION} + volumes: + - solr_data:/var/solr + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"] + + redis: + image: redis:${REDIS_VERSION} + networks: + - redisnet + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "-e", "QUIT"] + +networks: + webnet: + ckannet: + solrnet: + internal: true + dbnet: + internal: true + redisnet: + internal: true diff --git a/xloader/docker-entrypoint.d/02_setup_xloader.sh b/xloader/docker-entrypoint.d/02_setup_xloader.sh new file mode 100644 index 000000000..0cb5701a4 --- /dev/null +++ b/xloader/docker-entrypoint.d/02_setup_xloader.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [[ $CKAN__PLUGINS == *"xloader"* ]]; then + # Datapusher settings have been configured in the .env file + # Set API token if necessary + if [ -z "$CKAN__XLOADER__API_TOKEN" ] ; then + echo "Set up ckanext.xloader.api_token in the CKAN config file" + ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add ckan_admin xloader | tail -n 1 | tr -d '\t')" + fi +else + echo "Not configuring xloader" +fi diff --git a/xloader/docker-entrypoint.d/README.md b/xloader/docker-entrypoint.d/README.md new file mode 100644 index 000000000..27fa2922d --- /dev/null +++ b/xloader/docker-entrypoint.d/README.md @@ -0,0 +1,4 @@ +Use scripts in this folder to run extra initialization steps in your custom CKAN images. +Any file with `.sh` or `.py` extension will be executed just after the main initialization +script (`prerun.py`) is executed and just before the web server and supervisor processes are +started.