Resource View & DataStore Fields for Loading Objects

ckanapi/cli/load.py

@@ -208,14 +208,30 @@ def reply(action, error, response):
 
             act = 'update' if existing else 'create'
             try:
+                # do not send resource_views & datastore_fields to package actions
+                resource_views = []
+                datastore_fields = {}
+                if thing == 'datasets' and obj.get('resources'):
+                    for r in obj['resources']:
+                        resource_views += r.pop('resource_views', [])
+                        # FIXME: assuming passed resources have an id
+                        datastore_fields[r['id']] = r.pop('datastore_fields', [])
                 if existing:
                     r = ckan.call_action(thing_update, obj,
                                          requests_kwargs=requests_kwargs)
                 else:
-                    r = ckan.call_action(thing_create, obj)
-                if thing == 'datasets' and 'resources' in obj:# check if it is needed to upload resources when creating/updating packages
-                    _upload_resources(ckan,obj,arguments)
-                elif thing in ['groups','organizations'] and 'image_display_url' in obj:   #load images for groups and organizations
+                    r = ckan.call_action(thing_create, obj,
+                                         requests_kwargs=requests_kwargs)
+                if thing == 'datasets' and 'resources' in obj:
+                    # NOTE: order is important as Resource uploads may be dependant on DS Fields (XLoader/DataPusher),
+                    #       and Resource Views may be dependant on DS and Upload.
+                    if arguments['--datastore-fields'] and datastore_fields:  # check if it is needed to update datastore resource fields when creating/updating packages
+                        created_tables, skipped_tables = _load_datastore_resource_fields(ckan, datastore_fields, arguments)
+                    if arguments['--upload-resources']:  # check if it is needed to upload resources when creating/updating packages
+                        _upload_resources(ckan, obj, arguments)
+                    if arguments['--resource-views'] and resource_views:  # check if it is needed to create resource views when creating/updating packages
+                        created_views, updated_views, skipped_views = _load_resource_views(ckan, resource_views, arguments)
+                if thing in ['groups','organizations'] and 'image_display_url' in obj:  # load images for groups and organizations
                     if arguments['--upload-logo']:
                         users = obj['users']
                         obj = _upload_logo(ckan,obj)
@@ -232,7 +248,20 @@ def reply(action, error, response):
             except NotFound:
                 reply(act, 'NotFound', obj)
             else:
-                reply(act, None, r.get('name',r.get('id')))
+                log_obj = {}
+                if arguments['--resource-views'] and resource_views:
+                    if created_views:
+                        log_obj['created_resource_views'] = created_views
+                    if updated_views:
+                        log_obj['updated_resource_views'] = updated_views
+                    if skipped_views:
+                        log_obj['skipped_resource_views'] = skipped_views
+                if arguments['--datastore-fields'] and datastore_fields:
+                    if created_tables:
+                        log_obj['created_datastore_tables'] = created_tables
+                    if skipped_tables:
+                        log_obj['skipped_datastore_tables'] = skipped_tables
+                reply(act, None, log_obj if log_obj else r.get('name', r.get('id')))
 
 def _worker_command_line(thing, arguments):
     """
@@ -255,6 +284,8 @@ def b(name):
         + b('--update-only')
         + b('--upload-resources')
         + b('--upload-logo')
+        + b('--datastore-fields')
+        + b('--resource-views')
         )
 
 
@@ -274,10 +305,9 @@ def _copy_from_existing_for_update(obj, existing, thing):
         if 'users' not in obj and 'users' in existing:
             obj['users'] = existing['users']
 
-def _upload_resources(ckan,obj,arguments):
+
+def _upload_resources(ckan, obj, arguments):
     resources = obj['resources']
-    if not arguments['--upload-resources']:
-        return
     requests_kwargs = None
     if arguments['--insecure']:
         requests_kwargs = {'verify': False}
@@ -293,6 +323,88 @@ def _upload_resources(ckan,obj,arguments):
             requests_kwargs=requests_kwargs)
 
 
+def _load_resource_views(ckan, resource_views, arguments):
+    """
+    Loads resource views
+    """
+    created = []
+    updated = []
+    skipped = []
+    requests_kwargs = None
+    if arguments['--insecure']:
+        requests_kwargs = {'verify': False}
+    for view in resource_views:
+        existing = None
+        if not arguments['--create-only']:
+            if view.get('id'):
+                try:
+                    existing = ckan.call_action('resource_view_show',
+                        {'id': view['id']},
+                        requests_kwargs=requests_kwargs)
+                except NotFound:
+                    pass
+
+            if existing:
+                _copy_from_existing_for_update(view, existing, 'resource_view')
+
+        if not existing and arguments['--update-only']:
+            skipped.append(view.get('id', view.get('view_type')))
+            continue
+
+        if existing:
+            # exceptions handled in load_things_worker
+            ckan.call_action('resource_view_update', view,
+                             requests_kwargs=requests_kwargs)
+            updated.append(view.get('id', view.get('view_type')))
+        else:
+            # exceptions handled in load_things_worker
+            ckan.call_action('resource_view_create', view,
+                             requests_kwargs=requests_kwargs)
+            created.append(view.get('id', view.get('view_type')))
+
+    return created, updated, skipped
+
+
+def _load_datastore_resource_fields(ckan, datastore_fields, arguments):
+    """
+    Load datastore tables for Resources
+    """
+    created = []
+    skipped = []
+    requests_kwargs = None
+    if arguments['--insecure']:
+        requests_kwargs = {'verify': False}
+    for rid, ds_fields in datastore_fields.items():
+        if not ds_fields:
+            continue
+        existing = None
+        try:
+            existing = ckan.call_action('datastore_search',
+                {'resource_id': rid, 'limit': 0},
+                requests_kwargs=requests_kwargs)
+        except NotFound:
+            pass
+
+        try:
+            ckan.call_action(
+                'datastore_create',
+                {
+                    'resource_id': rid,
+                    'fields': ds_fields,
+                    'force': True
+                },
+                requests_kwargs=requests_kwargs)
+            created.append(rid)
+        except ValidationError as e:
+            if not existing:
+                # exceptions handled in load_things_worker
+                # raise normal exception for non-existing tables
+                raise e
+            skipped.append('%s: %s' % (rid, str(e)))
+
+    return created, skipped
+
+
 def _upload_logo(ckan,obj_orig):
     obj = obj_orig.copy()
     for key in obj_orig.keys():

ckanapi/cli/main.py

@@ -12,13 +12,17 @@
           (ID_OR_NAME ... | [-I JSONL_INPUT] [-s START] [-m MAX])
           [-p PROCESSES] [-l LOG_FILE] [-qwz]
           [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
-  ckanapi dump (datasets | groups | organizations | users | related)
+  ckanapi dump datasets
           (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY])
-          [-p PROCESSES] [-dqwzRU --include-private --include-drafts --include-deleted]
+          [-p PROCESSES] [-qwz --include-private --include-drafts --include-deleted --datastore-fields --resource-views]
+          [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]]
+  ckanapi dump (groups | organizations | users | related)
+          (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY])
+          [-p PROCESSES] [-qwzU]
           [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]]
   ckanapi load datasets
           [--upload-resources] [-I JSONL_INPUT] [-s START] [-m MAX]
-          [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz]
+          [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz --datastore-fields --resource-views]
           [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
   ckanapi load (groups | organizations)
           [--upload-logo] [-I JSONL_INPUT] [-s START] [-m MAX]
@@ -43,7 +47,8 @@
   -c --config=CONFIG        CKAN configuration file for local actions,
                             defaults to $CKAN_INI or development.ini
   -d --datastore-fields     export datastore field information along with
-                            resource metadata as datastore_fields lists
+                            resource metadata as datastore_fields lists (dump).
+                            load datastore field information for resources (load).
   --include-private         include private datasets in the dump
   --include-drafts          include draft datasets in the dump
   --include-deleted         include deleted datasets in the dump
@@ -70,7 +75,8 @@
   -q --quiet                don't display progress messages
   -r --remote=URL           URL of CKAN server for remote actions
   -R --resource-views       export resource views information along with
-                            resource metadata as resource_views lists
+                            resource metadata as resource_views lists (dump).
+                            create/update resource views for resources (load).
   -s --start-record=START   start from record number START, where the first
                             record is number 1 [default: 1]
   -u --ckan-user=USER       perform actions as user with this name, uses the

ckanapi/tests/test_cli_load.py

@@ -66,6 +66,8 @@ def test_create_with_no_resources(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'),
             stdout=self.stdout)
@@ -82,6 +84,8 @@ def test_create_with_corrupted_resources(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{"name": "45","title":"Forty-five","resources":[{"id":"123"}]}\n'),
             stdout=self.stdout)
@@ -98,6 +102,8 @@ def test_create_with_complete_resources(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(
                  b'{"name": "45","title":"Forty-five",'
@@ -116,6 +122,8 @@ def test_create_only(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'),
             stdout=self.stdout)
@@ -132,6 +140,8 @@ def test_create_empty_dict(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{}\n'),
             stdout=self.stdout)
@@ -147,6 +157,8 @@ def test_create_bad_option(self):
                 '--create-only': False,
                 '--update-only': True,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'),
             stdout=self.stdout)
@@ -162,6 +174,8 @@ def test_update_with_no_resources(self):
                 '--create-only': False,
                 '--update-only': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{"name": "30ish","title":"3.4 times ten"}\n'),
             stdout=self.stdout)
@@ -178,6 +192,8 @@ def test_update_with_corrupted_resources(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{"name": "30ish","title":"3.4 times ten","resources":[{"id":"123"}]}\n'),
             stdout=self.stdout)
@@ -194,6 +210,8 @@ def test_update_with_complete_resources(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(
                  b'{"name": "30ish","title":"3.4 times ten",'
@@ -212,6 +230,8 @@ def test_update_only(self):
                 '--update-only': True,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{"name": "34","title":"3.4 times ten"}\n'),
             stdout=self.stdout)
@@ -228,6 +248,8 @@ def test_update_bad_option(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{"name": "34","title":"3.4 times ten"}\n'),
             stdout=self.stdout)
@@ -244,6 +266,8 @@ def test_update_unauthorized(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{"name": "seekrit", "title": "Things"}\n'),
             stdout=self.stdout)
@@ -260,6 +284,8 @@ def test_update_group(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{"id": "ab","title":"a balloon"}\n'),
             stdout=self.stdout)
@@ -276,6 +302,8 @@ def test_update_organization_two(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(
                 b'{"name": "cd", "title": "Go"}\n'
@@ -300,6 +328,8 @@ def test_update_organization_with_users_unchanged(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{"id": "used", "title": "here"}\n'),
             stdout=self.stdout)
@@ -316,6 +346,8 @@ def test_update_organization_with_users_cleared(self):
                 '--update-only': False,
                 '--upload-resources': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
                 },
             stdin=BytesIO(b'{"id": "unused", "users": []}\n'),
             stdout=self.stdout)
@@ -345,6 +377,8 @@ def test_parent_load_two(self):
                 '--upload-resources': False,
                 '--upload-logo': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
             },
             worker_pool=self._mock_worker_pool,
             stdin=BytesIO(
@@ -380,6 +414,8 @@ def test_parent_load_start_max(self):
                 '--upload-resources': False,
                 '--upload-logo': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
             },
             worker_pool=self._mock_worker_pool,
             stdin=BytesIO(
@@ -418,6 +454,8 @@ def test_parent_parallel_limit(self):
                 '--upload-resources': False,
                 '--upload-logo': False,
                 '--insecure': False,
+                '--resource-views': False,
+                '--datastore-fields': False,
             },
             worker_pool=self._mock_worker_pool,
             stdin=BytesIO(