diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index c7eb91a..4f61636 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -212,6 +212,7 @@ def reply(action, error, response): # do not send resource_views & datastore_fields to package actions resource_views = {} datastore_fields = {} + group_users = [] if thing == 'datasets' and obj.get('resources'): for r in obj['resources']: # NOTE: will only work with existing Resource IDs in the input, @@ -220,10 +221,15 @@ def reply(action, error, response): resource_views[r['id']] = r.pop('resource_views', []) if arguments['--datastore-fields']: datastore_fields[r['id']] = r.pop('datastore_fields', []) + if thing in ('groups', 'organizations') and obj.get('users'): + group_users = obj.pop('users', []) + if not arguments['--append-users']: + obj['users'] = group_users if existing: r = ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) else: + # FIXME: add ignore_not_sysadmin to creator_user_id to ckan core.... r = ckan.call_action(thing_create, obj, requests_kwargs=requests_kwargs) if thing == 'datasets' and 'resources' in obj: @@ -235,14 +241,16 @@ def reply(action, error, response): _upload_resources(ckan, obj, arguments) if arguments['--resource-views'] and resource_views: # check if it is needed to create resource views when creating/updating packages created_views, updated_views, skipped_views = _load_resource_views(ckan, resource_views, arguments) - if thing in ['groups','organizations'] and 'image_display_url' in obj: # load images for groups and organizations - if arguments['--upload-logo']: + if thing in ('groups', 'organizations'): + if 'image_display_url' in obj and arguments['--upload-logo']: # load images for groups and organizations users = obj['users'] obj = _upload_logo(ckan,obj) obj.pop('image_upload') obj['users'] = users ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) + if arguments['--append-users'] and group_users: # check if it is needed to append group/org users + set_members = _load_group_members(ckan, group_users, arguments, thing, r['id']) if thing == 'users' and arguments['--api-tokens'] and api_token_list: # check if it is needed to create user api tokens when creating/updating users created_tokens = _load_user_api_tokens(ckan, api_token_list, arguments) except ValidationError as e: @@ -269,6 +277,11 @@ def reply(action, error, response): log_obj['created_datastore_tables'] = created_tables if skipped_tables: log_obj['skipped_datastore_tables'] = skipped_tables + if thing in ('groups', 'organizations'): + if arguments['--append-users'] and group_users and set_members: + log_obj['set_members'] = set_members + elif group_users: + log_obj['set_members'] = ['%s[%s]' % (m['name'], m['capacity']) for m in group_users] reply(act, None, log_obj) def _worker_command_line(thing, arguments): @@ -295,6 +308,7 @@ def b(name): + b('--api-tokens') + b('--datastore-fields') + b('--resource-views') + + b('--append-users') ) @@ -456,3 +470,27 @@ def _load_user_api_tokens(ckan, api_token_list, arguments): requests_kwargs=requests_kwargs) created_tokens.append(token['name']) return created_tokens + + +def _load_group_members(ckan, user_list, arguments, thing, obj_id): + """ + Appends users as members for Groups/Organizations + """ + requests_kwargs = None + if arguments['--insecure']: + requests_kwargs = {'verify': False} + set_members = [] + action = 'group_member_create' if thing == 'group' \ + else 'organization_member_create' + for user in user_list: + # exceptions handled in load_things_worker + ckan.call_action( + action, + { + 'id': obj_id, + 'username': user['name'], + 'role': user['capacity'], + }, + requests_kwargs=requests_kwargs) + set_members.append('%s[%s]' % (user['name'], user['capacity'])) + return set_members diff --git a/ckanapi/cli/main.py b/ckanapi/cli/main.py index 4fef28f..6d10061 100644 --- a/ckanapi/cli/main.py +++ b/ckanapi/cli/main.py @@ -30,7 +30,7 @@ [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] ckanapi load (groups | organizations) [--upload-logo] [-I JSONL_INPUT] [-s START] [-m MAX] - [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwzU] + [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwzUA] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] ckanapi load users [-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE] @@ -92,6 +92,8 @@ -u --ckan-user=USER perform actions as user with this name, uses the site sysadmin user when not specified -U --include-users include users of a group/organization + -A --append-users adds users to a group/organization instead of truncating them. + Mutually exclusive with -U --api-tokens export API Token information along with user metadata as api_token_list lists (dump). create API Tokens for users (load). diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index ec534fd..2e98786 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -30,23 +30,26 @@ def call_action(self, name, data_dict, requests_kwargs=None): '30ish': {'id': '34', 'title': "Thirty-four"}, '34': {'id': '34', 'title': "Thirty-four"}, '46': {'id': '46', 'name': '46', 'title': 'Forty-six'}, - }, + }, 'group_show': { 'ab': {'title': "ABBA"}, - }, + }, 'organization_show': { 'cd': {'id': 'cd', 'title': "Super Trouper"}, 'used': {'users': ['people']}, 'unused': {'users': ['people']}, - }, + 'mems': {'id': 'mems', 'name': 'mems', + "users": [{"capacity": "admin", "name": "test-user-admin"}, + {"capacity": "editor", "name": "test-user"}]}, + }, 'package_create': { None: {'id': 'some-generated-uuid', 'name': 'something-new'}, '46': {'id': '46', 'name': '46'}, - }, + }, 'package_update': { '34': {'id': '34', 'name': 'something-updated'}, '46': {'id': '46', 'name': '46'}, - }, + }, 'resource_view_show': { '456': {'description': 'Test view', 'package_id': '46', 'resource_id': '456'} }, @@ -66,15 +69,20 @@ def call_action(self, name, data_dict, requests_kwargs=None): }, 'group_update': { 'ab': {'id': 'ab', 'name': 'group-updated'}, - }, + }, 'organization_update': { 'cd': {'id': 'cd', 'name': 'org-updated'}, 'used': {'id': 'used', 'name': 'users-unchanged'}, 'unused': {'id': 'unused', 'name': 'users-cleared'}, - }, + 'mems': {'id': 'mems', 'name': 'mems', "users": [{"capacity": "admin", "name": "test-user-admin"}]}, + }, 'organization_create': { None: {'id': 'some-generated-uuid', 'name': 'org-created'}, - }, + 'mems': {'id': 'mems', 'name': 'mems', "users": [{"capacity": "editor", "name": "test-user"}]}, + }, + 'organization_member_create': { + 'mems': {'id': 'mems', 'role': 'admin', 'username': 'test-user-admin'} + }, 'user_show': { 'test_user': {'id': 'test_user', 'name': 'test_user'}, }, @@ -105,6 +113,7 @@ def test_create_with_no_resources(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'), stdout=self.stdout) @@ -123,6 +132,7 @@ def test_create_with_corrupted_resources(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five","resources":[{"id":"123"}]}\n'), stdout=self.stdout) @@ -141,6 +151,7 @@ def test_create_with_complete_resources(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO( b'{"name": "45","title":"Forty-five",' @@ -184,6 +195,7 @@ def test_create_with_resource_views(self): '--insecure': False, '--resource-views': True, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(json.dumps(payload).encode()), stdout=self.stdout) @@ -221,6 +233,7 @@ def test_create_with_resource_datastore_fields(self): '--insecure': False, '--resource-views': False, '--datastore-fields': True, + '--append-users': False, }, stdin=BytesIO(json.dumps(payload).encode()), stdout=self.stdout) @@ -258,6 +271,7 @@ def test_create_with_bad_resource_datastore_fields(self): '--insecure': False, '--resource-views': False, '--datastore-fields': True, + '--append-users': False, }, stdin=BytesIO(json.dumps(payload).encode()), stdout=self.stdout) @@ -276,6 +290,7 @@ def test_create_only(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'), stdout=self.stdout) @@ -294,6 +309,7 @@ def test_create_empty_dict(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{}\n'), stdout=self.stdout) @@ -311,6 +327,7 @@ def test_create_bad_option(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'), stdout=self.stdout) @@ -328,6 +345,7 @@ def test_update_with_no_resources(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{"name": "30ish","title":"3.4 times ten"}\n'), stdout=self.stdout) @@ -346,6 +364,7 @@ def test_update_with_corrupted_resources(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{"name": "30ish","title":"3.4 times ten","resources":[{"id":"123"}]}\n'), stdout=self.stdout) @@ -364,6 +383,7 @@ def test_update_with_complete_resources(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO( b'{"name": "30ish","title":"3.4 times ten",' @@ -407,6 +427,7 @@ def test_update_with_resource_views(self): '--insecure': False, '--resource-views': True, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(json.dumps(payload).encode()), stdout=self.stdout) @@ -448,6 +469,7 @@ def test_update_with_new_resource_views(self): '--insecure': False, '--resource-views': True, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(json.dumps(payload).encode()), stdout=self.stdout) @@ -485,6 +507,7 @@ def test_update_with_resource_datastore_fields(self): '--insecure': False, '--resource-views': False, '--datastore-fields': True, + '--append-users': False, }, stdin=BytesIO(json.dumps(payload).encode()), stdout=self.stdout) @@ -522,6 +545,7 @@ def test_update_with_bad_resource_datastore_fields(self): '--insecure': False, '--resource-views': False, '--datastore-fields': True, + '--append-users': False, }, stdin=BytesIO(json.dumps(payload).encode()), stdout=self.stdout) @@ -559,6 +583,7 @@ def test_update_with_resource_datastore_fields_no_resource_id(self): '--insecure': False, '--resource-views': False, '--datastore-fields': True, + '--append-users': False, }, stdin=BytesIO(json.dumps(payload).encode()), stdout=self.stdout) @@ -595,6 +620,7 @@ def test_update_with_resource_views_no_resource_id(self): '--insecure': False, '--resource-views': True, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(json.dumps(payload).encode()), stdout=self.stdout) @@ -608,6 +634,7 @@ def test_update_only(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{"name": "34","title":"3.4 times ten"}\n'), stdout=self.stdout) @@ -626,6 +653,7 @@ def test_update_bad_option(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{"name": "34","title":"3.4 times ten"}\n'), stdout=self.stdout) @@ -644,6 +672,7 @@ def test_update_unauthorized(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{"name": "seekrit", "title": "Things"}\n'), stdout=self.stdout) @@ -662,6 +691,7 @@ def test_update_group(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{"id": "ab","title":"a balloon"}\n'), stdout=self.stdout) @@ -680,6 +710,7 @@ def test_update_organization_two(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO( b'{"name": "cd", "title": "Go"}\n' @@ -706,6 +737,7 @@ def test_update_organization_with_users_unchanged(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{"id": "used", "title": "here"}\n'), stdout=self.stdout) @@ -724,6 +756,7 @@ def test_update_organization_with_users_cleared(self): '--insecure': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, stdin=BytesIO(b'{"id": "unused", "users": []}\n'), stdout=self.stdout) @@ -734,6 +767,47 @@ def test_update_organization_with_users_cleared(self): self.assertEqual(error, None) self.assertEqual(data, {'id': 'unused', 'name': 'users-cleared'}) + def test_update_organization_with_users_appended(self): + load_things_worker(self.ckan, 'organizations', { + '--create-only': True, + '--update-only': False, + '--upload-resources': False, + '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, + '--append-users': False, + }, + stdin=BytesIO(b'{"id": "mems", "name": "mems", "users": [{"capacity": "editor", "name": "test-user"}]}\n'), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'create') + self.assertEqual(error, None) + self.assertEqual(data, {'id': 'mems', 'name': 'mems', "set_members": ["test-user[editor]"]}) + + self.stdout.seek(0) + self.stdout.truncate(0) + + load_things_worker(self.ckan, 'organizations', { + '--create-only': False, + '--update-only': True, + '--upload-resources': False, + '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, + '--append-users': True, + }, + stdin=BytesIO(b'{"id": "mems", "name": "mems", "users": [{"capacity": "admin", "name": "test-user-admin"}]}\n'), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'update') + self.assertEqual(error, None) + self.assertEqual(data, {'id': 'mems', 'name': 'mems', "set_members": ["test-user-admin[admin]"]}) + def test_parent_load_two(self): load_things(self.ckan, 'datasets', { '--quiet': False, @@ -756,6 +830,7 @@ def test_parent_load_two(self): '--api-tokens': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -794,6 +869,7 @@ def test_parent_load_start_max(self): '--api-tokens': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -835,6 +911,7 @@ def test_parent_parallel_limit(self): '--api-tokens': False, '--resource-views': False, '--datastore-fields': False, + '--append-users': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO(