Move fourfront from pytest 2.9 to 3.10

.ebextensions/13_run_npm.config

@@ -31,6 +31,7 @@ files:
 
             echo "Finished NPM build in 99_run_npm." >> /var/log/deploy.log
             echo "Finished NPM build in 99_run_npm."
+
             echo "Restarting Apache"
 
             sudo service httpd restart

.gitignore

@@ -1,4 +1,5 @@
 .coverage
+*.srctr*
 coverage.xml
 beanstalk.cfg
 /coverage
@@ -16,6 +17,7 @@ beanstalk.cfg
 /develop-eggs/
 /downloads/
 /eggs/
+/ped_files_data/
 /.eggs/
 /extends/
 /jsconfig.json
@@ -66,8 +68,17 @@ elasticsearch-*.deb
 # Emacs backup files
 *~
 
+# PyCharm metadata
+.idea/
+
 # Virtual environments
+# Conventions vary, including env, venv, or longer names like cgap_env,
+# but typically they end in "env".
 *env/
 
-# PyCharm
-.idea/
+# Saved configurations from Elastic beanstalk that have been downloaded.
+.elasticbeanstalk/saved_configs/
+.elasticbeanstalk/app_versions/
+
+# Used for some kinds of debugging in dcicutils, snovault, cgap & ff.
+DEBUGLOG-*

Makefile

@@ -1,3 +1,5 @@
+SHELL=/bin/bash
+
 clean:  # clear node modules, eggs, npm build stuff
 	make clean-python-caches
 	make clean-npm-caches
@@ -38,7 +40,7 @@ macpoetry-install:  # Same as 'poetry install' except that on OSX Catalina, an e
 
 configure:  # does any pre-requisite installs
 	pip install --upgrade pip
-	pip install poetry==1.0.10  # pinned to avoid build problems we cannot fix in pyproject.toml
+	pip install poetry==1.1.4  # poetry latest as of 1/25/2021 seemed to work but apparantly does not
 
 build:  # builds
 	make configure
@@ -93,11 +95,17 @@ deploy2:  # spins up waittress to serve the application
 	pserve development.ini
 
 psql-dev:  # starts psql with the url after 'sqlalchemy.url =' in development.ini
-	@psql `grep 'sqlalchemy[.]url =' development.ini | sed -E 's/^.* = (.*)/\1/'`
+	@scripts/psql-start dev
+
+psql-test:  # starts psql with a url constructed from data in 'ps aux'.
+	@scripts/psql-start test
 
-kibana-start:
+kibana-start:  # starts a dev version of kibana (default port)
 	scripts/kibana-start
 
+kibana-start-test:  # starts a test version of kibana (port chosen for active tests)
+	scripts/kibana-start test
+
 kibana-stop:
 	scripts/kibana-stop
 
@@ -140,11 +148,14 @@ info:
 	   $(info - Use 'make configure' to install poetry. You should not have to do this directly.)
 	   $(info - Use 'make deploy1' to spin up postgres/elasticsearch and load inserts.)
 	   $(info - Use 'make deploy2' to spin up the application server.)
-	   $(info - Use 'make psql-dev' to start psql on data associated with an active 'make deploy1'.)
-	   $(info - Use 'make kibana-start' to start kibana, and 'make kibana-stop' to stop it.)
+	   $(info - Use 'make kibana-start' to start kibana on the default local ES port, and 'make kibana-stop' to stop it.)
+	   $(info - Use 'make kibana-start-test' to start kibana on the port being used for active testing, and 'make kibana-stop' to stop it.)
 	   $(info - Use 'make kill' to kill postgres and elasticsearch proccesses. Please use with care.)
 	   $(info - Use 'make moto-setup' to install moto, for less flaky tests. Implied by 'make build'.)
 	   $(info - Use 'make npm-setup' to build the front-end. Implied by 'make build'.)
+	   $(info - Use 'make psql-dev' to start psql on data associated with an active 'make deploy1'.)
+	   $(info - Use 'make psql-test' to start psql on data associated with an active test.)
+	   $(info - Use 'make retest' to run failing tests from the previous test run.)
 	   $(info - Use 'make test' to run tests with normal options we use on travis ('-m "working and not performance"').)
 	   $(info - Use 'make test-any' to run tests without marker constraints (i.e., with no '-m' option).)
 	   $(info - Use 'make update' to update dependencies (and the lock file).)

conftest.py

@@ -1,4 +1,6 @@
 import pytest
+import tempfile
+
 
 def pytest_addoption(parser):
     parser.addoption("--es", action="store", default="", dest='es',
@@ -15,3 +17,10 @@ def remote_es(request):
 @pytest.fixture(scope='session')
 def aws_auth(request):
     return request.config.getoption("--aws-auth")
+
+
+def pytest_configure():
+    # This adjustment is important to set the default choice of temporary filenames to a nice short name
+    # because without it some of the filenames we generate end up being too long, and critical functionality
+    # ends up failing. Some socket-related filenames, for example, seem to have length limits. -kmp 5-Jun-2020
+    tempfile.tempdir = '/tmp'

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 # Note: Various modules refer to this system as "encoded", not "fourfront".
 name = "encoded"
-version = "2.3.21"
+version = "2.3.22"
 description = "4DN-DCIC Fourfront"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"
@@ -36,96 +36,92 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = ">=3.6.1,<3.7"
-boto3 = "^1.10.50"
-botocore = "^1.13.50"
+awscli = ">=1.19.9"
+boto3 = "^1.17.9"
+botocore = "^1.20.9"
 certifi = ">=2020.11.8"
 chardet = "3.0.4"
-# TODO: These next three are not used in cgap. Are they really needed? -kmp 29-Apr-2020
-# I think .recipe is a buildout thing so I'm going to try commenting this out as a start. Delete later. -kmp 15-Dec-2020
-# "collective.recipe.cmd" = "0.11"
-# "collective.recipe.modwsgi" = "2.1"
-# "collective.recipe.template" = "1.13"
 colorama = "0.3.3"
-dcicsnovault = "4.1.0"
-dcicutils = "^1.4.0"
+dcicsnovault = "4.3.1"
+dcicutils = "^1.10.0"
 docutils = "0.12"
-elasticsearch = "6.4.0"
+elasticsearch = "6.8.1"
 elasticsearch-dsl = "^6.4.0"
 execnet = "1.4.1"
 future = "^0.15.2"
-humanfriendly = "1.44.5"
+humanfriendly = "^1.44.9"
 hupper = "1.5"
-idna = "2.7"
+idna = ">=2.7,<3"
 isodate = "0.5.4"
 jmespath = "0.9.0"
-jsonschema-serialize-fork = "2.1.1"
+jsonschema_serialize_fork = "^2.1.1"
 keepalive = "0.5"
 loremipsum = "1.0.5"
-netaddr = ">=0.7.18,<1"
-passlib = "1.6.5"
+netaddr = ">=0.8.0,<1"
+passlib = "^1.7.4"
 PasteDeploy = "1.5.2"
 pbkdf2 = "1.3"
 Pillow = "^6.2.2"  # 6.x.x should work for CGAP -Will 11/18/2020
 plaster = "1.0"
 plaster-pastedeploy = "0.6"
-psutil = "^5.6.6"
-psycopg2-binary = "2.7.7"
-py = "1.4.31"
+psutil = "^5.8.0"
+psycopg2-binary = "^2.8.6"
+py = ">=1.8.1"  # was "1.4.31" (used by pytest, not sure if elsewhere)
 pyasn1 = "0.1.9"
 PyBrowserID = "^0.10.0"
 pycparser = "2.14"
 PyJWT = "1.5.3"
-pyparsing = "2.1.1"
+pyparsing = "^2.4.7"
 pyramid = "1.10.4"
-pyramid-localroles = ">=0.1,<1"
-pyramid-multiauth = "0.8.0"
-pyramid-retry = "1.0"
-pyramid-tm = "2.2.1"
-pyramid-translogger = "^0.1"
-python-dateutil = "2.7.3"
+pyramid_localroles = ">=0.1,<1"
+pyramid_multiauth = ">=0.9.0,<1"
+pyramid-retry = "^1.0"
+pyramid-tm = "^2.4"
+pyramid_translogger = "^0.1"
+python-dateutil = "^2.8.1"
 # python-magic is presently pinned to 0.4.15 in lockstep with dcicsnovault's requirements. See explanation there.
 python-magic = "0.4.15"
 pytz = ">=2020.4"
 rdflib = "^4.2.2"
-rdflib-jsonld = "0.3"
+rdflib-jsonld = ">=0.5.0,<1.0.0"
 requests = "^2.23.0"
-rfc3987 = "^1.3.6"
+rfc3987 = "^1.3.8"
 rsa = "3.3"
-rutter = ">=0.2,<1"
-s3transfer = "^0.2.0"
+rutter = ">=0.3,<1"
+s3transfer = ">=0.3.0,<0.4.0"
 simplejson = "^3.17.0"
-SPARQLWrapper = "1.7.6"
+SPARQLWrapper = "^1.8.5"
 SQLAlchemy = "1.3.16"  # Pinned because >=1.3.17 is broken for us (circular constraints prevent deletes)
-structlog = ">=18.1.0,<20"
+structlog = ">=19.2.0,<20"
 submit4dn = "0.9.7"
-subprocess-middleware = ">=0.3,<1"
+subprocess_middleware = ">=0.3,<1"
 # Useful for picking apart pyproject.toml
 toml = ">=0.10.1,<1"
-transaction = "2.4.0"
+transaction = "^2.4.0"
 translationstring = "1.3"
 uptime = ">=3.0.1,<4"
 urllib3 = "^1.24.3"
 venusian = "^1.2.0"
 waitress = "1.2.0"  # 1.4.3 had lots of problems, so pin this -kmp 18-May-2020
-WebOb = "1.8.5"
-WebTest = "^2.0.21"
+WebOb = "^1.8.6"
+WebTest = "^2.0.35"
 wheel = "0.29.0"
 WSGIProxy2 = "0.4.2"
-xlrd = "^1.0.0"
+xlrd = "^1.2.0"
 xlwt = "1.2.0"
-"zope.deprecation" = "4.4.0"
-"zope.interface" = "4.6.0"
+"zope.deprecation" = "^4.4.0"
+"zope.interface" = "^4.7.2"
 "zope.sqlalchemy" = "1.3"
 sentry-sdk = "^0.16.5"
 
 [tool.poetry.dev-dependencies]
 # PyCharm says boto3-stubs contains useful type hints
-boto3-stubs = ">=1.16.15.0"
-coverage = ">=5.2"
+boto3-stubs = ">=1.17.8"
+coverage = ">=5.3.1"
 codacy-coverage = ">=1.3.11"
-coveralls = ">=2.1.1"
-flake8 = "^3.7.8"
-flaky = "3.6.1"
+coveralls = ">=3.0.0"
+flake8 = ">=3.8.4"
+flaky = ">=3.7.0"
 # flask only for moto[server]
 flask = ">=1.1.1"
 # Here AND ELSEWHERE (related scripts), we pin use of moto 1.3.7.
@@ -140,15 +136,22 @@ flask = ">=1.1.1"
 #       If not, get a reference to an open bug report here so we can check back periodically.
 moto = "1.3.7"
 # TODO: Investigate whether a major version upgrade is allowable for 'pytest', which is several versions behind.
-pytest = "2.9.2"
-pytest-cov = "2.2.1"
-pytest-exact-fixtures = "0.1"
-pytest-instafail = "0.3.0"
-pytest-mock = "0.11.0"
-pytest-timeout = "1.0.0"
-pytest-xdist = "1.14"
-# responses = "^0"
-"repoze.debug" = "1.0.2"
+pytest = ">=3.10,<4"
+pytest-cov = ">=2.2.1"
+pytest-exact-fixtures = ">=0.1"
+pytest-instafail = ">=0.3.0"
+pytest-mock = ">=0.11.0"
+pytest-timeout = ">=1.0.0"
+pytest-xdist = ">=1.14"
+"repoze.debug" = ">=1.0.2"
+
+# In pytest 6.0, we'll be able to use this instead of a separate pytest.ini configuration.
+# -kmp 18-Jan-2021
+#
+# [tool.pytest.ini_options]
+# minversion = "6.0"
+# ...etc.
+# See details at https://pytest.org/en/stable/customize.html
 
 [tool.poetry.scripts]
 # snovault commands

pytest.ini

@@ -1,18 +1,27 @@
 [pytest]
 addopts =
-    --pyargs encoded.tests deploy.tests
-    -p encoded.tests deploy.tests
+   -p encoded.tests.datafixtures
+   -p snovault.tests.serverfixtures
     --instafail
 markers =
+    action_fail: mark a test that fails consistently on GA (and thus skipped)
     broken: mark as a 'broken' test that does not run on Travis
     es: mark a test as an elastic search test (deselect with '-m "not es"')
-    indexing: mark a test as an indexing test (deselect with '-m "not indexing"')
+    file_operation: a test that utilizes files
+    indexing: mark a test as an indexing test, or really just not-workbook (deselect with '-m "not indexing"')
     ingestion: mark a test as an ingestion test (deselect with '-m "not ingestion"')
+    integrated: an integration test
+    integratedx: an excludable integration test, redundantly testing functionality also covered by a unit test
+    manual: a test that is only ever intended to be run manually
     performance: mark a test as a performance test (deselect with '-m "not performance"')
     schema: mark a test as a schema-related test (deselect with '-m "not schema"')
+    search: mark a test a search module test (deselect with '-m "not search"')
     setone: (deprecated) instead of '-m "setone"', please use '-m "not indexing"', TODO: refs and scripts to be rewritten
     slow: mark a test as slow (deselect with '-m "not slow"')
     storage: mark a test as about storage (deselect with '-m "not storage"')
+    unit: a proper unit test
     working: mark a test as working (deselect with '-m "not working"')
-    action_fail: known to fail on Github Action build
 norecursedirs = *env site-packages .cache .git .idea *.egg-info
+testpaths =
+    src/encoded
+    deploy