Update renderers and fix warning about AcceptValidHeader.best_match deprecation (C4-627)

Makefile

@@ -95,11 +95,17 @@ deploy2:  # spins up waittress to serve the application
 	pserve development.ini
 
 psql-dev:  # starts psql with the url after 'sqlalchemy.url =' in development.ini
-	@psql `grep 'sqlalchemy[.]url =' development.ini | sed -E 's/^.* = (.*)/\1/'`
+	@scripts/psql-start dev
 
-kibana-start:
+psql-test:  # starts psql with a url constructed from data in 'ps aux'.
+	@scripts/psql-start test
+
+kibana-start:  # starts a dev version of kibana (default port)
 	scripts/kibana-start
 
+kibana-start-test:  # starts a test version of kibana (port chosen for active tests)
+	scripts/kibana-start test
+
 kibana-stop:
 	scripts/kibana-stop
 
@@ -114,19 +120,27 @@ clean-python:
 	pip freeze | xargs pip uninstall -y
 
 test:
-	make test-unit
 	make test-npm
+	make test-unit
+
+retest:
+	bin/test  -vv --last-failed
 
 test-any:
 	bin/test -vv --timeout=200
 
-
 test-npm:
 	bin/test -vv --timeout=200 -m "working and not manual and not integratedx and not performance and not broken and not sloppy and workbook"
 
 test-unit:
 	bin/test -vv --timeout=200 -m "working and not manual and not integratedx and not performance and not broken and not sloppy and not workbook"
 
+test-performance:
+	bin/test -vv --timeout=200 -m "working and not manual and not integratedx and performance and not broken and not sloppy"
+
+test-integrated:
+	bin/test -vv --timeout=200 -m "working and not manual and (integrated or integratedx) and not performance and not broken and not sloppy"
+
 travis-test:  # Actually, we don't normally use this. Instead the GA workflow sets up two parallel tests.
 	make travis-test-npm
 	make travis-test-unit
@@ -154,11 +168,14 @@ info:
 	   $(info - Use 'make configure' to install poetry. You should not have to do this directly.)
 	   $(info - Use 'make deploy1' to spin up postgres/elasticsearch and load inserts.)
 	   $(info - Use 'make deploy2' to spin up the application server.)
-	   $(info - Use 'make kibana-start' to start kibana, and 'make kibana-stop' to stop it.)
+	   $(info - Use 'make kibana-start' to start kibana on the default local ES port, and 'make kibana-stop' to stop it.)
+	   $(info - Use 'make kibana-start-test' to start kibana on the port being used for active testing, and 'make kibana-stop' to stop it.)
 	   $(info - Use 'make kill' to kill postgres and elasticsearch proccesses. Please use with care.)
 	   $(info - Use 'make moto-setup' to install moto, for less flaky tests. Implied by 'make build'.)
 	   $(info - Use 'make npm-setup' to build the front-end. Implied by 'make build'.)
 	   $(info - Use 'make psql-dev' to start psql on data associated with an active 'make deploy1'.)
+	   $(info - Use 'make psql-test' to start psql on data associated with an active test.)
+	   $(info - Use 'make retest' to run failing tests from the previous test run.)
 	   $(info - Use 'make test' to run tests with normal options similar to what we use on GitHub Actions.)
 	   $(info - Use 'make test-any' to run tests without marker constraints (i.e., with no '-m' option).)
 	   $(info - Use 'make update' to update dependencies (and the lock file).)

scripts/kibana-start

@@ -1,5 +1,60 @@
 #!/bin/bash
 
+docker_kibana_image=docker.elastic.co/kibana/kibana-oss:6.8.9
+docker_kibana_port=5601
+local_kibana_port=${docker_kibana_port}
+default_es_port=9200
+local_es_port=${default_es_port}
+
+if [ "$1" = "test" ]; then
+
+    # This deletes all the output that doesn't match our pattern and includes only what does.
+    # Ref: https://stackoverflow.com/questions/6011661/regexp-sed-suppress-no-match-output
+    port=`ps aux | sed -E '/.*elasticsearch.*-Ehttp[.]port=([0-9]+)[^0-9].*/!d;s//\1/'`
+
+    if [ -z "${port}" ]; then
+        echo "Cannot find test port."
+        exit 1
+    elif [ `echo "${port}" | wc -l` -gt 1 ]; then
+        echo "Found multiple test ports:"
+        echo "${port}"
+        exit 1
+    fi
+
+    if [[ "${port}" =~ ^[0-9]+$ ]]; then
+        local_es_port="${port}"
+        echo "Using local_es_port=${local_es_port}."
+    else
+        echo "Port format is wrong: ${port}"
+        exit 1
+    fi
+
+elif [[ "$1" =~ ^[0-9]+$ ]]; then
+
+    local_es_port=$1
+    echo "Using local_es_port=${local_es_port}."
+
+elif [ $# -gt 0 -a "$1" != "dev" ]; then
+
+    echo "Syntax: $0 [ <es-port> | test | dev ]"
+    echo " Starts kibana. By default, or if 'dev' is given, uses the standard port ${default_es_port}."
+    echo " If es-port is an integer, that port is used."
+    echo " If es-port is the word 'test', a test port is found using 'ps aux'."
+    echo " Note that this will choose its own http port, which will be 5601 for es-port=9200,"
+    echo " or else a generated port number 10000+(<es-port> % 55536) otherwise."
+    exit 1
+
+fi
+
+if [ "${local_es_port}" != "${default_es_port}" ]; then
+    # 0-1023 are reserved ports
+    # 1024-65535 are available for custom use, but usualy 1024-9999 are assigned manually
+    # we'll generate a port in the range 10000-65535. that might sometimes collide, but probably VERY rarely.
+    local_kibana_port=$(( $local_es_port % 55536 + 10000 ))
+fi
+
+echo "Using local_kibana_port=${local_kibana_port}"
+
 docker --version
 
 if [ $? -ne 0 ]; then
@@ -11,25 +66,38 @@ fi
 existing_network=`docker network ls | grep localnet`
 
 if [ -z "${existing_network}" ]; then
+    echo "creating localnet --driver=bridge"
     docker network create localnet --driver=bridge
 else
     echo "docker localnet is already set up. From 'docker network':"
-    echo " ${existing_network}"
+    # echo " ${existing_network}"
 fi
 
-existing_kibana=`docker ps | egrep 'kibana:[0-9]*.[0-9]+.*[ ].*'`
+# This pattern is structured so that if we need to extract the container id (e.g., to kill it), it will be in \1.
+# But I figured out a way not to have to kill the old one, so that part of the pattern isn't used any more.
+# I retained the first group just for possible future use. At this point, this just detects whether we have a
+# handler for $local_kibana_port at all. This pattern also anticipates kibana images named 'kibana' or 'kibana-oss'.
+# I other image names come up, for example if aws creates its own naming convention, it will need adjusting.
+# -kmp 31-Jan-2021
+kibana_docker_pattern="([0-9a-f]+) .*kibana(-oss)?:[0-9]+[.][0-9]+[.].*0[.]0[.]0[.]0[:]${local_kibana_port}-[>]"
+
+existing_kibana=`docker ps | egrep "${kibana_docker_pattern}"`
 
 if [ -z "${existing_kibana}" ]; then
 
-    docker run -d --network localnet -p 5601:5601 -e ELASTICSEARCH_URL=http://host.docker.internal:9200 kibana:5.6.16
+    echo "Kibana is not already running for ES port ${local_es_port}"
+    docker_es_url="http://host.docker.internal:${local_es_port}"
+    docker run -d --network localnet -p ${local_kibana_port}:${docker_kibana_port} -e ELASTICSEARCH_URL=${docker_es_url} ${docker_kibana_image}
+
+    echo "Waiting for kibana to start..."
+    sleep 5
 
 else
-    echo "Kibana is already running. From 'docker ps':"
+    echo "Kibana is already listening on port ${local_kibana_port} for elasticsearch on port ${local_es_port}:"
     echo " ${existing_kibana}"
 fi
 
-local_kibana_url="http://localhost:5601/app/kibana#/dev_tools/console?_g=()"
+local_kibana_url="http://localhost:${local_kibana_port}/app/kibana#/dev_tools/console?_g=()"
 
 echo "Opening kibana in browser at '${local_kibana_url}'..."
 open "${local_kibana_url}" &
-

scripts/psql-start

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+port=$1
+
+dev_url_line=`grep 'sqlalchemy[.]url =' development.ini`
+
+dev_url=`echo "${dev_url_line}" | sed -E 's/^.* = (.*)$/\1/'`
+dev_port=`echo "${dev_url_line}" | sed -E 's|^.* = .*:([0-9]+)/postgres[?].*$|\1|'`
+
+
+# echo "dev_url=${dev_url}"
+# echo "dev_port=${dev_port}"
+
+
+# There seem be two processes, one for postgres and one for postgres-engine.
+# The relevant data can be obtained from either, but matching both 
+# the match for postgres[^-] excludes the matches on postgres-engine so we
+# can assume the match is unique.
+
+if [ "$port" = 'test' ]; then
+
+    test_process=`ps aux | grep '.*[p]ostgres -D.*/private[a-zA-Z0-9_/-]*/postgresql[^-]'`
+
+    if [ -z "${test_process}" ]; then
+
+        echo "No test process found."
+	exit 1
+
+    else
+
+        test_url=`echo "$test_process" | sed -E 's|^.*postgres[ ]+-D[ ]+([/a-zA-Z0-9_-]+)[ ]+.*-p[ ]+([0-9]+)([^0-9].*)?$|postgresql://postgres@localhost:\2/postgres?host=\1|'`
+        psql "${test_url}"
+
+        # psql `ps aux | grep '.*[p]ostgres -D.*/private[a-zA-Z0-9_/-]*/postgresql[^-]' | sed -E 's|^.*postgres[ ]+-D[ ]+([/a-zA-Z0-9_-]+)[ ]+.*-p[ ]+([0-9]+)([^0-9].*)?$|postgresql://postgres@localhost:\2/postgres?host=\1|'`
+
+    fi
+
+elif [ "$port" = 'dev' -o "$port" = "$dev_port" ]; then
+
+    dev_url=`grep 'sqlalchemy[.]url =' development.ini | sed -E 's/^.* = (.*)/\1/'`
+    psql "${dev_url}"
+
+elif [[ "${port}" =~ ^[0-9]+$ ]]; then
+
+    port_process=`ps aux | grep ".*[p]ostgres -D.*/private[a-zA-Z0-9_/-]*/postgresql[^-].*-p[ ]+${port}.*"`
+
+    if [ -z "${port_process}" ]; then
+
+        echo "No postgres process found on port ${port}."
+	exit 1
+
+    else
+
+        port_url=`echo "$test_process" | sed -E 's|^.*postgres[ ]+-D[ ]+([/a-zA-Z0-9_-]+)[ ]+.*-p[ ]+([0-9]+)([^0-9].*)?$|postgresql://postgres@localhost:\2/postgres?host=\1|'`
+        psql "${port_url}"
+
+        # psql `ps aux | grep '.*[p]ostgres -D.*/private[a-zA-Z0-9_/-]*/postgresql[^-]' | sed -E 's|^.*postgres[ ]+-D[ ]+([/a-zA-Z0-9_-]+)[ ]+.*-p[ ]+([0-9]+)([^0-9].*)?$|postgresql://postgres@localhost:\2/postgres?host=\1|'`
+
+    fi
+
+else
+
+    echo "Syntax: $0 [ <port> | test | dev ]"
+    echo ""
+    echo "Starts psql for debugging in a way that corresponds to the given port."
+    echo "The port can be an integer or one of the special tokens 'dev' or 'test'."
+    echo "If 'dev' is given, the port from development.ini (currently '${dev_port}') is used."
+    echo "If 'test' is given, the port will be found from data in 'ps aux'."
+
+fi