diff --git a/plugins/flows/base/create_cachedb_file_plugin/chunk_utils.py b/plugins/flows/base/create_cachedb_file_plugin/chunk_utils.py index cff55c97ec..424479ffce 100644 --- a/plugins/flows/base/create_cachedb_file_plugin/chunk_utils.py +++ b/plugins/flows/base/create_cachedb_file_plugin/chunk_utils.py @@ -81,6 +81,9 @@ def resolve_chunk_count(row_count: int, config: ChunkConfig) -> int: if row_count <= 0: return 1 n = -(-row_count // config.target_chunk_rows) # ceil division + # Large tables require at least two chunks; one is still an unbounded copy. + if row_count >= config.small_table_threshold: + n = max(2, n) n = min(n, config.max_chunks) n = min(n, max(1, row_count // config.min_chunk_rows)) return max(1, n) diff --git a/plugins/flows/base/create_cachedb_file_plugin/copy.py b/plugins/flows/base/create_cachedb_file_plugin/copy.py index 95b868ec5b..bf2f0a6d1d 100644 --- a/plugins/flows/base/create_cachedb_file_plugin/copy.py +++ b/plugins/flows/base/create_cachedb_file_plugin/copy.py @@ -363,6 +363,7 @@ def copy_table_chunk(write_conn: Any, copy_params: CopyParameters, query_columns logger.info( f"Chunk {chunk_index + 1}/{total_chunks} for '{query_columns.table}': {predicate}" ) + insert_sql = None try: # DELETE before INSERT, in that order. DuckDB over pgwire autocommits # every statement, so a crash between the INSERT and the progress @@ -373,8 +374,17 @@ def copy_table_chunk(write_conn: Any, copy_params: CopyParameters, query_columns # never remove another chunk's rows. delete_seconds = execute_statement(write_conn, f"DELETE FROM {target} WHERE {predicate};") select_sql = create_select_query(copy_params, query_columns, source_schema, predicate) - insert_seconds = execute_statement(write_conn, f"INSERT INTO {target} {select_sql};") + target_columns = ", ".join( + f'"{column}"' for column in query_columns.columns_to_copy + ) + insert_sql = f"INSERT INTO {target} ({target_columns}) {select_sql};" + insert_seconds = execute_statement(write_conn, insert_sql) except Exception as exc: + if insert_sql is not None: + logger.error( + f"Chunk {chunk_index + 1}/{total_chunks} INSERT failed for " + f"'{query_columns.table}'. SQL: {insert_sql}" + ) raise ChunkCopyError( f"Chunk {chunk_index + 1}/{total_chunks} of '{query_columns.table}' " f"failed ({predicate}): {exc}" @@ -751,4 +761,3 @@ def copy_indexes(write_conn: Any, read_conn: Any, copy_params: CopyParameters, q logger.info( f"Primary Key Index '{pk_index_name}' copied for table '{table}' in schema '{target_database}'.'{target_schema}'." ) - diff --git a/plugins/flows/base/create_cachedb_file_plugin/filter.py b/plugins/flows/base/create_cachedb_file_plugin/filter.py index 2d066dd12e..e17dfc6ae8 100644 --- a/plugins/flows/base/create_cachedb_file_plugin/filter.py +++ b/plugins/flows/base/create_cachedb_file_plugin/filter.py @@ -70,6 +70,7 @@ "concept": {"column_name": "concept_id"}, "concept_ancestor": {"column_name": "ancestor_concept_id"}, "concept_class": {"column_name": "concept_class_id"}, + "concept_hierarchy": {"column_name": "concept_id"}, "concept_recommended": {"column_name": "concept_id_1"}, "concept_relationship": {"column_name": "concept_id_1"}, "concept_synonym": {"column_name": "concept_id"},