diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c1b0bf37..71593d04 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,9 +58,10 @@ jobs: run: | ckan -c test.ini db init - name: Run tests - run: pytest --ckan-ini=test.ini --cov=ckanext.validation --cov-report=xml --cov-append --disable-warnings ckanext/validation/tests -vv + # run: pytest --ckan-ini=test.ini --cov=ckanext.validation --cov-report=xml --cov-append --disable-warnings ckanext/validation/tests -vv + run: pytest --ckan-ini=test.ini --disable-warnings ckanext/validation/tests -vv - - name: Upload coverage report to codecov - uses: codecov/codecov-action@v1 - with: - file: ./coverage.xml + #- name: Upload coverage report to codecov + # uses: codecov/codecov-action@v1 + # with: + # file: ./coverage.xml diff --git a/ckanext/validation/logic.py b/ckanext/validation/logic.py index 213e4648..50a82762 100644 --- a/ckanext/validation/logic.py +++ b/ckanext/validation/logic.py @@ -5,6 +5,7 @@ import json from sqlalchemy.orm.exc import NoResultFound +from frictionless import system, Resource import ckan.plugins as plugins import ckan.lib.uploader as uploader @@ -174,6 +175,42 @@ def resource_validation_show(context, data_dict): return _validation_dictize(validation) +def resource_table_schema_infer(context, data_dict): + ''' + Use frictionless framework to infer a resource schema + ''' + + t.check_access('resource_create', context, data_dict) + + t.get_or_bust(data_dict, 'resource_id') + + store_schema = data_dict.get('store_schema', True) + + resource = t.get_action('resource_show')( + {}, {u'id': data_dict['resource_id']}) + + source = None + if resource.get('url_type') == 'upload': + upload = uploader.get_resource_uploader(resource) + if isinstance(upload, uploader.ResourceUpload): + source = upload.get_path(resource['id']) + + if not source: + source = resource['url'] + + with system.use_context(trusted=True): + # TODO: check for valid formats + fric_resource = Resource({'path': source, 'format': resource.get('format', 'csv').lower()}) + fric_resource.infer() + resource['schema'] = fric_resource.schema.to_json() + + # TODO: check for exception + if store_schema: + t.get_action('resource_update')( + context, resource) + + return {u'schema': fric_resource.schema.to_dict()} + def resource_validation_delete(context, data_dict): u''' diff --git a/ckanext/validation/plugin/__init__.py b/ckanext/validation/plugin/__init__.py index d648709e..b9153ff5 100644 --- a/ckanext/validation/plugin/__init__.py +++ b/ckanext/validation/plugin/__init__.py @@ -6,6 +6,7 @@ from werkzeug.datastructures import FileStorage as FlaskFileStorage import ckan.plugins as p +import ckan.lib.uploader as uploader import ckantoolkit as t from ckanext.validation import settings @@ -17,6 +18,7 @@ auth_resource_validation_delete, auth_resource_validation_run_batch, resource_create as custom_resource_create, resource_update as custom_resource_update, + resource_table_schema_infer ) from ckanext.validation.helpers import ( get_validation_badge, @@ -34,6 +36,7 @@ get_create_mode_from_config, get_update_mode_from_config, ) + from ckanext.validation.interfaces import IDataValidation from ckanext.validation import blueprints, cli @@ -89,6 +92,7 @@ def get_actions(self): u'resource_validation_run_batch': resource_validation_run_batch, u'resource_create': custom_resource_create, u'resource_update': custom_resource_update, + u'resource_table_schema_infer': resource_table_schema_infer } return new_actions