ADX-949 ckanext-validation upgrade

.gitignore

@@ -84,6 +84,7 @@ celerybeat-schedule
 # Environments
 .env
 .venv
+venv
 env/
 venv/
 ENV/

.gitmodules

@@ -25,9 +25,6 @@
 [submodule "ckan-extensions/ckanext-unaids"]
 	path = submodules/ckanext-unaids
 	url = git@github.com:/fjelltopp/ckanext-unaids.git
-[submodule "ckan-extensions/ckanext-validation"]
-	path = submodules/ckanext-validation
-	url = git@github.com:/fjelltopp/ckanext-validation.git
 [submodule "ckan-extensions/ckanext-blob-storage"]
 	path = submodules/ckanext-blob-storage
 	url = git@github.com:/fjelltopp/ckanext-blob-storage.git
@@ -46,3 +43,6 @@
 [submodule "submodules/giftless"]
 	path = submodules/giftless
 	url = git@github.com:/fjelltopp/giftless.git
+[submodule "ckan-extensions/ckanext-validation"]
+    path = submodules/ckanext-validation
+	url = git@github.com:fjelltopp/ckanext-validation.git

Pipfile

@@ -17,15 +17,14 @@ fanstatic = "==1.1"
 feedgen = "==0.9.0"
 flask-babel = "==1.0.0"
 flask-multistatic = "==1.0"
-flask = "==1.1.1"
+flask = ">=1.1.1"
 funcsigs = "==1.0.2"
 idna = "==3.3"
 itsdangerous = "==1.1.0"
-jinja2 = "==2.11"
 lxml = ">=2.3"
 mako = "==1.1.5"
 markdown = "==2.6.7"
-markupsafe = "==1.1.1"
+markupsafe = "==2.0.1"
 nose = "==1.3.7"
 passlib = "==1.6.5"
 polib = "==1.0.7"
@@ -86,7 +85,7 @@ importlib-resources = "*"
 cffi = "==1.14.5"
 cryptography = "~=3.4.7"
 pycparser = "==2.20"
-typing-extensions = "==4.2.0"
+typing-extensions = ">=4.3.0"
 typing = "==3.7.4.3"
 ckantoolkit = ">=0.0.3"
 python-slugify = "==4.0.0"
@@ -105,6 +104,10 @@ python-dotenv = "==1.0.0"
 python-jose = "==3.3.0"
 setuptools = "==67.2.0"
 raven = "==6.10.0"
+frictionless = {editable = true, ref = "jinja2_2_10_1", git = "https://github.com/fjelltopp/frictionless-py", extras=["ckan"]}
+frictionless-geojson = {editable = true, ref = "v1.1.0rc1", git = "https://github.com/fjelltopp/frictionless-geojson"}
+jinja2 = "==2.10.1"
+tableschema = "*"
 
 [dev-packages]
 beautifulsoup4 = "==4.9.1"

ckan/adx_config.ini

@@ -165,12 +165,14 @@ ckanext.authz_service.jwt_private_key = -----BEGIN RSA PRIVATE KEY-----
 ckanext.authz_service.jwt_public_key_file = /etc/ckan/jwt_public_key.pem
 ckanext.bulk_file_uploader_default_fields = {"restricted_allowed_orgs": "unaids"}
 
-ckanext.validation.schema_directory = /usr/lib/adx/submodules/unaids_data_specifications/table_schemas
+ckanext.unaids.schema_directory = /usr/lib/adx/submodules/unaids_data_specifications/table_schemas
 ckanext.validation.run_on_create_async = True
 ckanext.validation.run_on_update_async = True
+ckanext.validation.run_on_create_sync = False
+ckanext.validation.run_on_update_sync = False
 ckanext.validation.allow_invalid_data = True
-ckanext.validation.default_validation_options = {"row_limit": 50000, "error_limit": 100, "skip_checks": ["extra-header"] }
-ckanext.validation.formats = csv xlsx xls shp geojson
+ckanext.validation.formats = csv xlsx xls geojson
+ckanext.validation.default_validation_options = {"limit_rows": 50000, "limit_errors": 100, "skip_errors": ["extra-header"], "schema_sync": true}
 
 ckanext.unaids.auth0_domain =
 ckanext.unaids.oauth2_api_audience =

ckan/bootstrap.sh

@@ -19,6 +19,8 @@ fi
 rm -rf /usr/lib/adx/submodules/ckan/ckan/pastertemplates/template/ckanext_+project_shortname+.egg-info
 cd /usr/lib/adx/ || exit 1
 pipenv sync --dev
+echo "installed python packages"
+pipenv run python -m pip freeze
 echo "show current dir"
 ls -la
 echo "Show local/bin"