Merge pull request ckan#25 from qld-gov-au/QOL-9055-py3-amazon-linux-2

QOL-9055 Add Python 3 compatibility and continuous integration testing

Author: ThrawnCA

.ahoy.yml

@@ -80,7 +80,7 @@ commands:
     usage: Install a site.
     cmd: |
       ahoy title "Installing a fresh site"
-      ahoy cli "./scripts/init.sh"
+      ahoy cli '$APP_DIR/scripts/init.sh'
 
   clean:
     usage: Remove containers and all build files.
@@ -109,6 +109,13 @@ commands:
       ahoy cli "flake8 ${@:-ckanext}" || \
       [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
 
+  copy-local-files:
+    usage: Update files from local repo.
+    cmd: |
+      docker cp . $(docker-compose ps -q ckan):/app/
+      docker cp .docker/scripts/ckan_cli $(docker-compose ps -q ckan):/app/ckan/default/bin/
+      ahoy cli 'chmod u+x $VENV_DIR/bin/ckan_cli; cp .docker/test.ini $CKAN_INI'
+
   test-unit:
     usage: Run unit tests.
     cmd: |
@@ -120,7 +127,7 @@ commands:
     cmd: |
       ahoy start-ckan-job-worker &
       sleep 5 &&
-      ahoy cli "behave ${*:-test/features} --tags=-format_autocomplete" || \
+      ahoy cli "behave -k ${*:-test/features} --tags=-format_autocomplete" || \
       [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       ahoy stop-ckan-job-worker
 

.circleci/config.yml

@@ -23,7 +23,10 @@ jobs:
           at: /workspace
       - checkout
       - *step_setup_remote_docker
-      - run: .circleci/build.sh
+      - run:
+          command: .circleci/build.sh
+          environment:
+            CKAN_VERSION: ckan-2.9.5
       - run: .circleci/test.sh
       - run:
           name: Process artifacts

.docker/Dockerfile.ckan .docker/Dockerfile-py2.ckan.docker/Dockerfile.ckan renamed to .docker/Dockerfile-py2.ckan

@@ -2,7 +2,8 @@ FROM amazeeio/python:2.7-ckan-21.8.0
 
 ARG SITE_URL=http://ckan:3000/
 ARG CKAN_REPO=ckan/ckan
-ARG CKAN_VERSION=2.8.8
+ARG CKAN_VERSION=ckan-2.9.5
+ENV PYTHON_VERSION=py2
 ENV SITE_URL="${SITE_URL}"
 ENV VENV_DIR=/app/ckan/default
 ENV APP_DIR=/app
@@ -11,16 +12,16 @@ ENV CKAN_INI=/app/ckan/default/production.ini
 WORKDIR "${APP_DIR}"
 
 ENV DOCKERIZE_VERSION v0.6.1
-RUN echo 'http://dl-4.alpinelinux.org/alpine/v3.11/main' >> /etc/apk/repositories && apk add --no-cache curl build-base \
-    && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+RUN apk add --no-cache curl build-base postgresql-client \
+    && curl -sLO https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
     && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
     && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
 
 # Install CKAN.
 
 RUN . ${VENV_DIR}/bin/activate \
     && pip install setuptools==36.1 \
-    && pip install -e "git+https://github.com/${CKAN_REPO}.git@ckan-${CKAN_VERSION}#egg=ckan" \
+    && pip install -e "git+https://github.com/${CKAN_REPO}.git@${CKAN_VERSION}#egg=ckan" \
     && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \
     && ((test -f "${VENV_DIR}/src/ckan/requirements-py2.txt" && \
         pip install -r "${VENV_DIR}/src/ckan/requirements-py2.txt") || \
@@ -32,15 +33,13 @@ RUN . ${VENV_DIR}/bin/activate \
 
 COPY .docker/test.ini $CKAN_INI
 
-# Add current extension and files.
 COPY . ${APP_DIR}/
 
 COPY .docker/scripts ${APP_DIR}/scripts
 
 COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/
 
-RUN chmod +x ${APP_DIR}/scripts/*.sh\
-    && chmod +x ${VENV_DIR}/bin/ckan_cli
+RUN chmod +x ${APP_DIR}/scripts/*.sh ${VENV_DIR}/bin/ckan_cli
 
 # Init current extension.
 RUN ${APP_DIR}/scripts/init-ext.sh

.docker/Dockerfile-py3.ckan

@@ -0,0 +1,47 @@
+FROM amazeeio/python:3.8-22.7.0
+
+ARG SITE_URL=http://ckan:3000/
+ARG CKAN_REPO=ckan/ckan
+ARG CKAN_VERSION=ckan-2.9.5
+ENV PYTHON_VERSION=py3
+ENV SITE_URL="${SITE_URL}"
+ENV VENV_DIR=/app/ckan/default
+ENV APP_DIR=/app
+ENV CKAN_INI=/app/ckan/default/production.ini
+
+WORKDIR "${APP_DIR}"
+
+ENV DOCKERIZE_VERSION v0.6.1
+RUN apk add --no-cache curl build-base git libxml2-dev libxslt-dev postgresql postgresql-client postgresql-dev \
+    && curl -sLO https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+    && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+    && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
+
+# Install CKAN.
+
+RUN mkdir -p ${VENV_DIR} \
+    && virtualenv ${VENV_DIR} \
+    && . ${VENV_DIR}/bin/activate \
+    && pip install setuptools==44.1.0 \
+    && pip install -e "git+https://github.com/${CKAN_REPO}.git@${CKAN_VERSION}#egg=ckan" \
+    && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \
+    && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \
+    && deactivate \
+    && ln -s ${APP_DIR}/ckan /usr/lib/ckan \
+    && fix-permissions ${APP_DIR}/ckan
+
+COPY .docker/test.ini $CKAN_INI
+
+COPY . ${APP_DIR}/
+
+COPY .docker/scripts ${APP_DIR}/scripts
+
+COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/
+
+RUN chmod +x ${APP_DIR}/scripts/*.sh ${VENV_DIR}/bin/ckan_cli
+
+# Init current extension.
+RUN ${APP_DIR}/scripts/init-ext.sh
+
+ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"]
+CMD ["/app/scripts/serve.sh"]

.docker/scripts/ckan_cli

@@ -57,15 +57,15 @@ else
 fi
 
 if [ "$COMMAND" = "ckan" ]; then
-    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2
     # adjust args to match ckan expectations
     COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/')
+    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
     exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS
 elif [ "$COMMAND" = "paster" ]; then
-    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2
     # adjust args to match paster expectations
     COMMAND=$1
+    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
     if [ "$1" = "show" ]; then shift; fi
     exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI}

.docker/scripts/create-test-data.sh

@@ -4,16 +4,15 @@
 #
 set -e
 
+CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
+CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
+CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"
 CKAN_ACTION_URL=http://ckan:3000/api/action
 
 if [ "$VENV_DIR" != "" ]; then
   . ${VENV_DIR}/bin/activate
 fi
 
-CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
-CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
-CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"
-
 add_user_if_needed () {
     echo "Adding user '$2' ($1) with email address [$3]"
     ckan_cli user show "$1" | grep "$1" || ckan_cli user add "$1"\
@@ -25,7 +24,6 @@ add_user_if_needed () {
 add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL"
 ckan_cli sysadmin add "${CKAN_USER_NAME}"
 
-# We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data
 API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/')
 if [ "$API_KEY" = "None" ]; then
     echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..."
@@ -45,17 +43,18 @@ add_user_if_needed test_org_admin "Test Admin" test_org_admin@localhost
 add_user_if_needed test_org_editor "Test Editor" test_org_editor@localhost
 add_user_if_needed test_org_member "Test Member" test_org_member@localhost
 
-echo "Creating ${TEST_ORG_TITLE} Organisation:"
+echo "Creating ${TEST_ORG_TITLE} organisation:"
 
 TEST_ORG=$( \
     curl -LsH "Authorization: ${API_KEY}" \
     --data "name=${TEST_ORG_NAME}&title=${TEST_ORG_TITLE}" \
     ${CKAN_ACTION_URL}/organization_create
 )
+echo "${TEST_ORG_TITLE} organisation is: ${TEST_ORG}"
 
-TEST_ORG_ID=$(echo $TEST_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/')
+TEST_ORG_ID=$(echo $TEST_ORG | python $APP_DIR/scripts/extract-id.py)
 
-echo "Assigning test users to ${TEST_ORG_TITLE} Organisation:"
+echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):"
 
 curl -LsH "Authorization: ${API_KEY}" \
     --data "id=${TEST_ORG_ID}&object=test_org_admin&object_type=user&capacity=admin" \

.docker/scripts/doctor.sh

@@ -75,7 +75,7 @@ main() {
     #      - container:amazeeio-ssh-agent
     #    ```
     # 4. When CLI container starts, the volume is mounted and an entrypoint script
-    #    adds SHH key into agent.
+    #    adds SSH key into agent.
     #    @see https://github.com/amazeeio/lagoon/blob/master/images/php/cli/10-ssh-agent.sh
     #
     #  Running `ssh-add -L` within CLI container should show that the SSH key

.docker/scripts/extract-id.py

@@ -0,0 +1,5 @@
+# encoding: utf-8
+import json
+import sys
+
+print(json.loads(sys.stdin.read())['result']['id'])

.docker/scripts/init-ext.sh

@@ -7,9 +7,8 @@ set -e
 if [ "$VENV_DIR" != "" ]; then
   . ${VENV_DIR}/bin/activate
 fi
-pip install -r "requirements.txt"
-pip install -r "dev-requirements.txt"
-if [ "$CKAN_VERSION" = "2.8.8" ]; then
+pip install -r "requirements.txt" -r "dev-requirements.txt"
+if [ "$CKAN_VERSION" = "ckan-2.8.8" ]; then
     pip install -r "dev-requirements-2.8.txt"
 fi
 python setup.py develop

.docker/test.ini

@@ -55,21 +55,6 @@ ckan.datastore.default_fts_index_method = gist
 ## Site Settings.
 ckan.site_url = http://ckan:3000/
 
-## Authorization Settings
-
-ckan.auth.anon_create_dataset = false
-ckan.auth.create_unowned_dataset = false
-ckan.auth.create_dataset_if_not_in_organization = false
-ckan.auth.user_create_groups = false
-ckan.auth.user_create_organizations = false
-ckan.auth.user_delete_groups = true
-ckan.auth.user_delete_organizations = true
-ckan.auth.create_user_via_api = false
-ckan.auth.create_user_via_web = true
-ckan.auth.roles_that_cascade_to_sub_groups = admin
-ckan.auth.public_user_details = False
-
-
 ## Search Settings
 
 ckan.site_id = default
@@ -97,7 +82,6 @@ ckan.redis.url = redis://redis:6379
 #       Add ``datapusher`` to enable DataPusher
 #       Add ``resource_proxy`` to enable resource proxying and get around the
 #       same origin policy
-# @todo:setup Cleanup the list to use only required plugins.
 ckan.plugins = validation scheming_datasets
 
 scheming.dataset_schemas = ckanext.validation.examples:ckan_default_schema.json