Skip to content

Dump & Load User API Tokens #230

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wardi merged 10 commits into from
May 23, 2026
Merged

Dump & Load User API Tokens #230

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5ed3718
feat(dev): api tokens;
JVickery-TBS May 7, 2026
2ce3ba6
feat(dev): api tokens;
JVickery-TBS May 11, 2026
baa0c7c
feat(tests): coverage;
JVickery-TBS May 11, 2026
c0f4cb6
feat(tests): coverage;
JVickery-TBS May 11, 2026
a3e80b2
feat(tests): coverage;
JVickery-TBS May 22, 2026
d5d102a
feat(tests): coverage;
JVickery-TBS May 22, 2026
dc06c41
feat(tests): coverage;
JVickery-TBS May 22, 2026
2da12ec
feat(tests): coverage;
JVickery-TBS May 22, 2026
61b3821
feat(tests): coverage;
JVickery-TBS May 22, 2026
a15b0fc
feat(tests): coverage;
JVickery-TBS May 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions ckanapi/cli/dump.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,8 +204,11 @@ def reply(error, record=None):
if thing == 'datasets' and arguments['--resource-views']:
for res in obj.get('resources', []):
populate_res_views(ckan, res)
if thing == 'users' and arguments['--api-tokens']:
populate_api_tokens(ckan, obj)
reply(None, obj)


def _worker_command_line(thing, arguments):
"""
Create a worker command line suitable for Popen with only the
Expand All @@ -227,6 +230,7 @@ def b(name):
+ b('--datastore-fields')
+ b('--resource-views')
+ b('--include-users')
+ b('--api-tokens')
+ ['value-here-to-make-docopt-happy']
)

Expand All @@ -248,3 +252,19 @@ def populate_res_views(ckan, res):
return # return if the resource views list is empty
res['resource_views'] = views


def populate_api_tokens(ckan, user):
"""
Update user dict in-place with api_token_list
"""
try:
tokens = ckan.call_action('api_token_list', {
'user_id': user['name'],
'limit': 0})
Comment thread
JVickery-TBS marked this conversation as resolved.
Outdated
except CKANAPIError:
return
except NotFound:
return # with localckan we'll get the real CKAN exception not a CKANAPIError subclass
Comment on lines +261 to +264

@wardi wardi May 23, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The way this is written multiple exceptions would be caught in the RemoteCKAN case but only NotFound will be caught in the LocalCKAN case, that's strange.

Suggested change
except CKANAPIError:
return
except NotFound:
return # with localckan we'll get the real CKAN exception not a CKANAPIError subclass
except NotFound:
return # with localckan we'll get the real CKAN exception not a CKANAPIError subclass

But, would we ever expect a NotFound? Won't that only happen if the user doesn't exist? If so it's better to not catch and ignore any exceptions.

@wardi wardi May 23, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But, I'll merge as-is and we can clean it up later.

if not tokens:
return # return if the user api token list is empty
user['api_token_list'] = tokens
28 changes: 27 additions & 1 deletion ckanapi/cli/load.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,21 +208,24 @@ def reply(action, error, response):

act = 'update' if existing else 'create'
try:
api_token_list = obj.pop('api_token_list', None) # do not send api_token_list to user actions
if existing:
r = ckan.call_action(thing_update, obj,
requests_kwargs=requests_kwargs)
else:
r = ckan.call_action(thing_create, obj)
if thing == 'datasets' and 'resources' in obj:# check if it is needed to upload resources when creating/updating packages
_upload_resources(ckan,obj,arguments)
elif thing in ['groups','organizations'] and 'image_display_url' in obj: #load images for groups and organizations
if thing in ['groups','organizations'] and 'image_display_url' in obj: #load images for groups and organizations
if arguments['--upload-logo']:
users = obj['users']
obj = _upload_logo(ckan,obj)
obj.pop('image_upload')
obj['users'] = users
ckan.call_action(thing_update, obj,
requests_kwargs=requests_kwargs)
if thing == 'users' and arguments['--api-tokens'] and api_token_list: # check if it is needed to create user api tokens when creating/updating users
_load_user_api_tokens(ckan, api_token_list, arguments)
except ValidationError as e:
reply(act, 'ValidationError', e.error_dict)
except SearchIndexError as e:
Expand All @@ -234,6 +237,7 @@ def reply(action, error, response):
else:
reply(act, None, r.get('name',r.get('id')))


def _worker_command_line(thing, arguments):
"""
Create a worker command line suitable for Popen with only the
Expand All @@ -255,6 +259,7 @@ def b(name):
+ b('--update-only')
+ b('--upload-resources')
+ b('--upload-logo')
+ b('--api-tokens')
)


Expand Down Expand Up @@ -309,3 +314,24 @@ def _upload_logo(ckan,obj_orig):
obj['image_upload'] = (new_url, f.raw)
ckan.action.group_update(**obj)
return obj


def _load_user_api_tokens(ckan, api_token_list, arguments):
"""
Loads user API Tokens from api_token_list
"""
requests_kwargs = None
if arguments['--insecure']:
requests_kwargs = {'verify': False}
for token in api_token_list:
# exceptions handled in load_things_worker
ckan.call_action(
'api_token_create',
{
'id': token['id'],
'created_at': token['created_at'],
'last_access': token['last_access'],
'name': token['name'],
'user': token['user_id']
},
Comment thread
wardi marked this conversation as resolved.
requests_kwargs=requests_kwargs)
15 changes: 13 additions & 2 deletions ckanapi/cli/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,14 @@
(ID_OR_NAME ... | [-I JSONL_INPUT] [-s START] [-m MAX])
[-p PROCESSES] [-l LOG_FILE] [-qwz]
[[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
ckanapi dump (datasets | groups | organizations | users | related)
ckanapi dump (datasets | groups | organizations | related)
(ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY])
[-p PROCESSES] [-dqwzRU --include-private --include-drafts --include-deleted]
[[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]]
ckanapi dump users
(ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY])
[-p PROCESSES] [-qwz --api-tokens]
[[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]]
ckanapi load datasets
[--upload-resources] [-I JSONL_INPUT] [-s START] [-m MAX]
[-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz]
Expand All @@ -24,7 +28,11 @@
[--upload-logo] [-I JSONL_INPUT] [-s START] [-m MAX]
[-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwzU]
[[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
ckanapi load (users | related)
ckanapi load users
[-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE]
[-n | -o] [-qwz --api-tokens]
Comment thread
JVickery-TBS marked this conversation as resolved.
[[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
ckanapi load related
[-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE]
[-n | -o] [-qwz]
[[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
Expand Down Expand Up @@ -76,6 +84,9 @@
-u --ckan-user=USER perform actions as user with this name, uses the
site sysadmin user when not specified
-U --include-users include users of a group/organization
--api-tokens export API Token information along with
user metadata as api_token_list lists (dump).
create API Tokens for users (load).
--upload-logo upload logo image of a group/organization if the
image is stored in the original server, otherwise
its image url will be used
Expand Down
9 changes: 8 additions & 1 deletion ckanapi/tests/test_cli_dump.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ def test_parent_dump_all(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
},
worker_pool=self._mock_worker_pool,
stdout=self.stdout,
Expand Down Expand Up @@ -177,6 +178,7 @@ def test_parent_parallel_limit(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
},
worker_pool=self._mock_worker_pool,
stdout=self.stdout,
Expand Down Expand Up @@ -206,6 +208,7 @@ def test_parent_id_argument(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
},

worker_pool=self._mock_worker_pool,
Expand Down Expand Up @@ -237,6 +240,7 @@ def test_parent_maintain_order(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
},
worker_pool=self._mock_worker_pool_reversed,
stdout=self.stdout,
Expand Down Expand Up @@ -272,6 +276,7 @@ def test_parent_datapackages(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
},
worker_pool=self._worker_pool_with_data,
stdout=self.stdout,
Expand Down Expand Up @@ -326,6 +331,7 @@ def test_resource_views(self):
'--resource-views': True,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
},
worker_pool=self._worker_pool_with_resource_views,
stdout=self.stdout,
Expand Down Expand Up @@ -379,6 +385,7 @@ def test_include_params_default(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
})

action = ckan.method_calls[0].args[0]
Expand Down Expand Up @@ -412,7 +419,7 @@ def test_include_params_true(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,

'--api-tokens': False,
'--include-private': True,
'--include-drafts': True,
'--include-deleted': True,
Expand Down
3 changes: 3 additions & 0 deletions ckanapi/tests/test_cli_load.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,7 @@ def test_parent_load_two(self):
'--upload-resources': False,
'--upload-logo': False,
'--insecure': False,
'--api-tokens': False,
},
worker_pool=self._mock_worker_pool,
stdin=BytesIO(
Expand Down Expand Up @@ -380,6 +381,7 @@ def test_parent_load_start_max(self):
'--upload-resources': False,
'--upload-logo': False,
'--insecure': False,
'--api-tokens': False,
},
worker_pool=self._mock_worker_pool,
stdin=BytesIO(
Expand Down Expand Up @@ -418,6 +420,7 @@ def test_parent_parallel_limit(self):
'--upload-resources': False,
'--upload-logo': False,
'--insecure': False,
'--api-tokens': False,
},
worker_pool=self._mock_worker_pool,
stdin=BytesIO(
Expand Down
Loading