From ed0b533f84efecab3439e50e6fcd5a7634dd5474 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Wed, 13 Aug 2025 12:10:39 -0500 Subject: [PATCH 01/24] documentation for core staging tables --- documentation_blocks/avvan_source_relation.md | 3 + documentation_blocks/bonterra_missing.md | 3 + documentation_blocks/segment_by_key.md | 3 + documentation_blocks/source_schema.md | 3 + documentation_blocks/source_table.md | 3 + documentation_blocks/vendor.md | 3 + documentation_blocks/vendor_unique_id.md | 3 + .../staging/ngpvan/_staging_ngpvan_models.yml | 585 --------------- models/staging/ngpvan/_stg_ngpvan.yaml | 697 ++++++++++++++++++ 9 files changed, 718 insertions(+), 585 deletions(-) create mode 100644 documentation_blocks/avvan_source_relation.md create mode 100644 documentation_blocks/bonterra_missing.md create mode 100644 documentation_blocks/segment_by_key.md create mode 100644 documentation_blocks/source_schema.md create mode 100644 documentation_blocks/source_table.md create mode 100644 documentation_blocks/vendor.md create mode 100644 documentation_blocks/vendor_unique_id.md delete mode 100644 models/staging/ngpvan/_staging_ngpvan_models.yml create mode 100644 models/staging/ngpvan/_stg_ngpvan.yaml diff --git a/documentation_blocks/avvan_source_relation.md b/documentation_blocks/avvan_source_relation.md new file mode 100644 index 0000000..72ffcd9 --- /dev/null +++ b/documentation_blocks/avvan_source_relation.md @@ -0,0 +1,3 @@ +{% docs _avvan_source_relation %} +Concatenated string of America Votes source database, schema and table. Null for values unrelated to America Votes data +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/bonterra_missing.md b/documentation_blocks/bonterra_missing.md new file mode 100644 index 0000000..1a6a484 --- /dev/null +++ b/documentation_blocks/bonterra_missing.md @@ -0,0 +1,3 @@ +{% docs bonterra_missing %} +Documentation for this column is not provided by Bonterra. +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/segment_by_key.md b/documentation_blocks/segment_by_key.md new file mode 100644 index 0000000..ebaa8f8 --- /dev/null +++ b/documentation_blocks/segment_by_key.md @@ -0,0 +1,3 @@ +{% docs segment_by_key %} +Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data ownership and sharing arrangements. +{% enddocs %} diff --git a/documentation_blocks/source_schema.md b/documentation_blocks/source_schema.md new file mode 100644 index 0000000..43d6053 --- /dev/null +++ b/documentation_blocks/source_schema.md @@ -0,0 +1,3 @@ +{% docs source_schema %} +Denotes the schema component of the _dbt_source_relation field +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/source_table.md b/documentation_blocks/source_table.md new file mode 100644 index 0000000..0ee0f6f --- /dev/null +++ b/documentation_blocks/source_table.md @@ -0,0 +1,3 @@ +{% docs source_table %} +Denotes the table component of the _dbt_source_relation field +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/vendor.md b/documentation_blocks/vendor.md new file mode 100644 index 0000000..6006061 --- /dev/null +++ b/documentation_blocks/vendor.md @@ -0,0 +1,3 @@ +{% docs vendor %} +Denotes the name of the vendor as it relates to TMC's segmentation processes +{% enddocs %} diff --git a/documentation_blocks/vendor_unique_id.md b/documentation_blocks/vendor_unique_id.md new file mode 100644 index 0000000..e1e6bb2 --- /dev/null +++ b/documentation_blocks/vendor_unique_id.md @@ -0,0 +1,3 @@ +{% docs vendor_unique_id %} +Surrogate primary key uniquely identifying this record across all members data. +{% enddocs %} \ No newline at end of file diff --git a/models/staging/ngpvan/_staging_ngpvan_models.yml b/models/staging/ngpvan/_staging_ngpvan_models.yml deleted file mode 100644 index b1f411b..0000000 --- a/models/staging/ngpvan/_staging_ngpvan_models.yml +++ /dev/null @@ -1,585 +0,0 @@ -version: 2 - -models: - - name: stg_ngpvan__activist_codes - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__activist_codes' }}" - tags: ['ngpvan_core'] - columns: - - name: activist_code_id - - name: van_state_id - - name: activist_code_type - - name: activist_code_name - - name: activist_code_description - - name: report_question - - name: democrat_points - - name: republican_points - - name: independent_points - - name: committee_id - - name: action_type_id - - name: campaign_id - - name: is_active - - name: is_archived - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: segmented_activist_code_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__activist_code_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__campaigns - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__campaigns' }}" - tags: ['ngpvan_core'] - columns: - - name: campaign_id - - name: campaign_name - - name: campaign_description - - name: committee_id - - name: campaign_type_id - - name: campaign_type - - name: created_at - - name: created_by_user_id - - name: is_active - - name: is_archived - - name: _avvan_source_relation - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: segmented_campaign_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__campaign_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__codes - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__codes' }}" - tags: ['ngpvan_core'] - columns: - - name: van_state_code - - name: code_id - - name: code_name - - name: committee_id - - name: committee_name - - name: is_active - - name: is_archived - - name: parent_code_id - - name: created_by_user_id - - name: utc_created_at - - name: static_full_name - - name: code_type_id - - name: code_type - - name: utc_modified_at - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: database_mode - - name: is_myvoters - - name: segment_by - - name: segmented_code_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__code_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__code_types - config: - enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__code_types' }}" - tags: ['ngpvan_core', 'ngpvan_lookup'] - columns: - - name: code_type_id - - name: code_type - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__code_type_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique - - - name: stg_ngpvan__committees - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__committees' }}" - tags: ['ngpvan_core'] - columns: - - name: committee_id - - name: van_state_id - - name: committee_name - - name: committee_short_name - - name: committee_type - - name: is_active - - name: is_archived - - name: parent_committee_id - - name: master_committee_id - - name: _avvan_source_relation - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__committee_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__contacts_activist_codes - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_activist_codes' }}" - tags: ['ngpvan_core'] - columns: - - name: van_state_code - - name: contacts_activist_code_id - - name: van_id - - name: activist_code_id - - name: committee_id - - name: utc_created_at - - name: input_type_id - - name: input_type - - name: contact_method_type_id - - name: contact_method_type - - name: canvassed_by_username - - name: canvassed_by_user_id - - name: campaign_id - - name: content_id - - name: utc_modified_at - - name: contacts_contact_id - - name: utc_canvassed_at - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: database_mode - - name: is_myvoters - - name: segment_by - - name: unique_contacts_activist_code_id - - name: segmented_contacts_activist_code_id - - name: segmented_activist_code_id - - name: segmented_van_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__contacts_activist_code_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__contacts_codes - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_codes' }}" - tags: ['ngpvan_core'] - columns: - - name: van_state_code - - name: contacts_code_id - - name: van_id - - name: code_id - - name: committee_id - - name: utc_created_at - - name: input_type_id - - name: input_type - - name: utc_modified_at - - name: created_by_user_id - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: database_mode - - name: is_myvoters - - name: segment_by - - name: unique_contacts_code_id - - name: segmented_contacts_code_id - - name: segmented_van_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__contacts_code_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__contacts_contacts - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_contacts' }}" - tags: ['ngpvan_core'] - columns: - - name: van_state_code - - name: contacts_contact_id - - name: van_id - - name: result_id - - name: result_name - - name: committee_id - - name: utc_created_at - - name: utc_canvassed_at - - name: input_type_id - - name: input_type - - name: contact_method_type_id - - name: contact_method_type - - name: canvassed_by_username - - name: canvassed_by_user_id - - name: campaign_id - - name: content_id - - name: created_by_user_id - - name: contacts_phone_id - - name: team_id - - name: division_id - - name: utc_modified_at - - name: _avvan_source_relation - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: database_mode - - name: is_myvoters - - name: segment_by - - name: unique_contacts_contact_id - - name: segmented_contacts_contact_id - - name: segmented_van_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__contacts_contact_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__contacts_notes - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_notes' }}" - tags: ['ngpvan_core'] - columns: - - name: van_state_code - - name: contacts_note_id - - name: van_id - - name: created_by_user_id - - name: utc_created_at - - name: committee_id - - name: suppressed_by_user_id - - name: utc_suppressed_at - - name: note_text - - name: datefollowup - - name: followup_completed_by_user_id - - name: datefollowupcompleted - - name: contacts_contact_id - - name: utc_modified_at - - name: note_category - - name: contacts_online_form_id - - name: contacts_story_id - - name: note_category_id - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: database_mode - - name: is_myvoters - - name: segment_by - - name: unique_contacts_note_id - - name: segmented_contacts_note_id - - name: segmented_van_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__contacts_note_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__contacts_survey_responses - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_survey_responses' }}" - tags: ['ngpvan_core'] - columns: - - name: van_state_code - - name: contacts_survey_response_id - - name: van_id - - name: contacts_contact_id - - name: survey_question_id - - name: survey_response_id - - name: utc_created_at - - name: utc_canvassed_at - - name: input_type_id - - name: input_type - - name: contact_method_type_id - - name: contact_method_type - - name: committee_id - - name: canvassed_by_username - - name: canvassed_by_user_id - - name: campaign_id - - name: content_id - - name: utc_modified_at - - name: team_id - - name: division_id - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: database_mode - - name: is_myvoters - - name: segment_by - - name: unique_contacts_survey_response_id - - name: segmented_contacts_survey_response_id - - name: segmented_van_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__contacts_survey_response_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 and contacts_survey_response_id is not null - - - name: stg_ngpvan__contact_types - config: - enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contact_types' }}" - tags: ['ngpvan_core', 'ngpvan_lookup'] - columns: - - name: contact_method_type_id - - name: contact_method_type - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__contact_type_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique - - - name: stg_ngpvan__input_types - config: - enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__input_types' }}" - tags: ['ngpvan_core', 'ngpvan_lookup'] - columns: - - name: input_type_id - - name: input_type - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__input_type_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique - - - name: stg_ngpvan__public_users - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__public_users' }}" - - - name: stg_ngpvan__results - config: - enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__results' }}" - tags: ['ngpvan_core', 'ngpvan_lookup'] - columns: - - name: result_id - - name: result_name - - name: result_description - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__result_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique - - - name: stg_ngpvan__survey_questions - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__survey_questions' }}" - tags: ['ngpvan_core'] - columns: - - name: survey_question_id - - name: van_state_id - - name: election_cycle - - name: survey_question_type - - name: survey_question_name - - name: survey_question_text - - name: master_survey_question_id - - name: committee_id - - name: is_active - - name: is_archived - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: segmented_survey_question_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__survey_question_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__survey_responses - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__survey_responses' }}" - tags: ['ngpvan_core'] - columns: - - name: survey_question_id - - name: survey_response_id - - name: survey_response - - name: democrat_points - - name: republican_points - - name: independent_points - - name: master_survey_response_id - - name: committee_id - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: segmented_survey_response_id - - name: segmented_survey_question_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__survey_response_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__users - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__users' }}" - columns: - - name: user_id - - name: username - - name: first_name - - name: last_name - - name: public_username - - name: address_line_1 - - name: city - - name: state - - name: zip_code - - name: email_address - - name: home_phone - - name: cell_phone - - name: _avvan_source_relation - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__user_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique - - - name: stg_ngpvan__users_committees - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__users_committees' }}" - columns: - - name: user_id, - - name: committee_id, - - name: state_id, - - name: created_by_user_id, - - name: utc_created_at, - - name: utc_modified_at, - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__users_committee_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__users_user_groups - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__users_user_groups' }}" - columns: - - name: user_id - - name: user_group_id - - name: user_group_name - - name: created_at - - name: created_by_user_id - - name: _avvan_source_relation - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__users_user_group_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique \ No newline at end of file diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml new file mode 100644 index 0000000..0c67959 --- /dev/null +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -0,0 +1,697 @@ +version: 2 + +models: + - name: stg_ngpvan__activist_codes + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____activist_codes' }}" + tags: + - ngpvan_core + columns: + - name: vendor_unique_stg_ngpvan__activist_code_id + description: '{{ doc("bonterra_missing") }}' + - name: activist_code_id + description: '{{ doc("bonterra_missing") }}' + - name: van_state_id + description: '{{ doc("bonterra_missing") }}' + - name: activist_code_type + description: '{{ doc("bonterra_missing") }}' + - name: activist_code_description + description: '{{ doc("bonterra_missing") }}' + - name: report_question + description: '{{ doc("bonterra_missing") }}' + - name: democrat_points + description: '{{ doc("bonterra_missing") }}' + - name: republican_points + description: '{{ doc("bonterra_missing") }}' + - name: independent_points + description: '{{ doc("bonterra_missing") }}' + - name: committee_id + description: '{{ doc("bonterra_missing") }}' + - name: campaign_id + description: '{{ doc("bonterra_missing") }}' + - name: segment_by + description: '{{ doc("bonterra_missing") }}' + - name: segmented_activist_code_id + description: '{{ doc("bonterra_missing") }}' + - name: vendor + description: '{{ doc("bonterra_missing") }}' + - name: segment_by_key + description: '{{ doc("bonterra_missing") }}' + - name: activist_code_name + description: '{{ doc("bonterra_missing") }}' + - name: action_type_id + description: '{{ doc("bonterra_missing") }}' + - name: is_active + description: '{{ doc("bonterra_missing") }}' + - name: is_archived + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: source_schema + description: '{{ doc("bonterra_missing") }}' + - name: source_table + description: '{{ doc("bonterra_missing") }}' + - name: stg_ngpvan__campaigns + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____campaigns' }}" + tags: + - ngpvan_core + columns: + - name: campaign_id + description: '{{ doc("bonterra_missing") }}' + - name: campaign_name + description: '{{ doc("bonterra_missing") }}' + - name: campaign_description + description: '{{ doc("bonterra_missing") }}' + - name: committee_id + description: '{{ doc("bonterra_missing") }}' + - name: campaign_type_id + description: '{{ doc("bonterra_missing") }}' + - name: campaign_type + description: '{{ doc("bonterra_missing") }}' + - name: created_at + description: '{{ doc("bonterra_missing") }}' + - name: created_by_user_id + description: '{{ doc("bonterra_missing") }}' + - name: is_active + description: '{{ doc("bonterra_missing") }}' + - name: is_archived + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: source_schema + description: '{{ doc("bonterra_missing") }}' + - name: source_table + description: '{{ doc("bonterra_missing") }}' + - name: segment_by + description: '{{ doc("bonterra_missing") }}' + - name: segmented_campaign_id + description: '{{ doc("bonterra_missing") }}' + - name: vendor + description: '{{ doc("bonterra_missing") }}' + - name: segment_by_key + description: '{{ doc("bonterra_missing") }}' + - name: vendor_unique_stg_ngpvan__campaign_id + description: '{{ doc("bonterra_missing") }}' + - name: stg_ngpvan__code_types + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '____code_types' }}" + tags: + - ngpvan_core + - ngpvan_lookup + columns: + - name: code_type_id + description: '{{ doc("bonterra_missing") }}' + - name: code_type + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: source_schema + description: '{{ doc("bonterra_missing") }}' + - name: source_table + description: '{{ doc("bonterra_missing") }}' + - name: segment_by + description: '{{ doc("bonterra_missing") }}' + - name: vendor + description: '{{ doc("bonterra_missing") }}' + - name: vendor_unique_stg_ngpvan__code_type_id + description: '{{ doc("bonterra_missing") }}' + - name: stg_ngpvan__codes + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____codes' }}" + tags: + - ngpvan_core + columns: + - name: sourcestateid + description: + SourceDatabase information to identify the source of the bridged + data + - name: sourcedatabasemodeid + description: '{{ doc("bonterra_missing") }}' + - name: codeid + description: Unique ID for Code + - name: codename + description: Code Name in VAN + - name: committeeid + description: Committee in which record lives + - name: isactive + description: A Flag for if the activity is active + - name: parentcodeid + description: + If the Code is in a nested structure, this will be the direct parent + of the Code + - name: createdby + description: User ID of who/what created record (person or automated process) + - name: datecreated + description: Date record in table was created + - name: staticfullname + description: + Name of the code. Codes can apply to many items in Votebuilder, like + Organizations, Events and Locations. Not to be confused with Activist Codes, + which only apply to individuals. They give users a lot more flexibility to create + custom groupings of information, and are similar to hashtags in that respect. + - name: codetypeid + description: Source Code vs Tag + - name: codedescription + description: System stores description of code for reference purposes + - name: contacttypeid + description: Unique ID of Type of Contact + - name: campaignid + description: Unique ID of Campain (if applied) + - name: stg_ngpvan__committees + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____committees' }}" + tags: + - ngpvan_core + columns: + - name: committee_id + description: '{{ doc("bonterra_missing") }}' + - name: van_state_id + description: '{{ doc("bonterra_missing") }}' + - name: committee_name + description: '{{ doc("bonterra_missing") }}' + - name: committee_short_name + description: '{{ doc("bonterra_missing") }}' + - name: committee_type + description: '{{ doc("bonterra_missing") }}' + - name: is_active + description: '{{ doc("bonterra_missing") }}' + - name: is_archived + description: '{{ doc("bonterra_missing") }}' + - name: parent_committee_id + description: '{{ doc("bonterra_missing") }}' + - name: master_committee_id + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: source_schema + description: '{{ doc("bonterra_missing") }}' + - name: source_table + description: '{{ doc("bonterra_missing") }}' + - name: segment_by + description: '{{ doc("bonterra_missing") }}' + - name: vendor + description: '{{ doc("bonterra_missing") }}' + - name: segment_by_key + description: '{{ doc("bonterra_missing") }}' + - name: vendor_unique_stg_ngpvan__committee_id + description: '{{ doc("bonterra_missing") }}' + - name: stg_ngpvan__contact_types + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '____contact_types' }}" + tags: + - ngpvan_core + - ngpvan_lookup + columns: + - name: contact_method_type_id + description: '{{ doc("bonterra_missing") }}' + - name: contact_method_type + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: source_schema + description: '{{ doc("bonterra_missing") }}' + - name: source_table + description: '{{ doc("bonterra_missing") }}' + - name: segment_by + description: '{{ doc("bonterra_missing") }}' + - name: vendor + description: '{{ doc("bonterra_missing") }}' + - name: vendor_unique_stg_ngpvan__contact_type_id + description: '{{ doc("bonterra_missing") }}' + - name: stg_ngpvan__contacts_activist_codes + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____contacts_activist_codes' }}" + tags: + - ngpvan_core + columns: + - name: sourcestateid + description: + SourceDatabase information to identify the source of the bridged + data + - name: sourcedatabasemodeid + description: '{{ doc("bonterra_missing") }}' + - name: contactsactivistcodeid + description: Unique ID for Activist Code applied to Specific Contact + - name: vanid + description: + Distinct VAN ID belonging to contact record. My C IDs begin with + 100 and are Distinct to committee and My V IDs are unique to voter file state. + - name: activistcodeid + description: Distinct ID belonging to Activist Code + - name: committeeid + description: Committee in which record lives + - name: datecreated + description: Date record in table was created + - name: inputtypeid + description: The ID of how the record was entered + - name: contacttypeid + description: Unique ID of Type of Contact + - name: username + description: Name of User who created record + - name: canvassedby + description: User ID of canvasser + - name: campaignid + description: Unique ID of Campain (if applied) + - name: contentid + description: Unique ID of Content (if applied) + - name: personcommitteeid + description: Committee that owns the person record + - name: contactscontactid + description: Unique ID for Contact History + - name: datecanvassed + description: Date contact was solicited or canvassed + - name: stg_ngpvan__contacts_codes + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____contacts_codes' }}" + tags: + - ngpvan_core + columns: + - name: sourcestateid + description: + SourceDatabase information to identify the source of the bridged + data + - name: sourcedatabasemodeid + description: '{{ doc("bonterra_missing") }}' + - name: contactscodeid + description: nan + - name: vanid + description: + Distinct VAN ID belonging to contact record. My C IDs begin with + 100 and are Distinct to committee and My V IDs are unique to voter file state. + - name: codeid + description: Unique ID for Code + - name: committeeid + description: Committee in which record lives + - name: datecreated + description: Date record in table was created + - name: inputtypeid + description: The ID of how the record was entered + - name: contacttypeid + description: Unique ID of Type of Contact + - name: username + description: Name of User who created record + - name: canvassedby + description: User ID of canvasser + - name: campaignid + description: Unique ID of Campain (if applied) + - name: contentid + description: Unique ID of Content (if applied) + - name: personcommitteeid + description: Committee that owns the person record + - name: contactscontactid + description: Unique ID for Contact History + - name: datecanvassed + description: Date contact was solicited or canvassed + - name: stg_ngpvan__contacts_contacts + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____contacts_contacts' }}" + tags: + - ngpvan_core + columns: + - name: sourcestateid + description: + SourceDatabase information to identify the source of the bridged + data + - name: sourcedatabasemodeid + description: '{{ doc("bonterra_missing") }}' + - name: contactscontactid + description: Unique ID for Contact History + - name: vanid + description: + Distinct VAN ID belonging to contact record. My C IDs begin with + 100 and are Distinct to committee and My V IDs are unique to voter file state. + - name: datecanvassed + description: Date contact was solicited or canvassed + - name: contacttypeid + description: Unique ID of Type of Contact + - name: inputtypeid + description: The ID of how the record was entered + - name: resultid + description: Result of contact interaction + - name: committeeid + description: Committee in which record lives + - name: canvassedby + description: User ID of canvasser + - name: username + description: Name of User who created record + - name: createdby + description: User ID of who/what created record (person or automated process) + - name: datecreated + description: Date record in table was created + - name: contactscontactnotestext + description: '{{ doc("bonterra_missing") }}' + - name: callduration + description: '{{ doc("bonterra_missing") }}' + - name: contactsphoneid + description: Unique identifier of Phone Number + - name: contactscontactsfull + description: '{{ doc("bonterra_missing") }}' + - name: liveid + description: '{{ doc("bonterra_missing") }}' + - name: deleted + description: '{{ doc("bonterra_missing") }}' + - name: personcommitteeid + description: Committee that owns the person record + - name: campaignid + description: Unique ID of Campain (if applied) + - name: contentid + description: Unique ID of Content (if applied) + - name: eventid + description: Unique ID of Event + - name: eventshiftid + description: Unique ID of Event Shift assigned to Contact Record in the event + - name: locationid + description: Unique ID of Location of Event + - name: modeofcontactid + description: '{{ doc("bonterra_missing") }}' + - name: taskassignmentid + description: '{{ doc("bonterra_missing") }}' + - name: phoneoptinstatus + description: '{{ doc("bonterra_missing") }}' + - name: phoneoptintypeid + description: '{{ doc("bonterra_missing") }}' + - name: phoneoptinsourceid + description: '{{ doc("bonterra_missing") }}' + - name: stg_ngpvan__contacts_notes + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____contacts_notes' }}" + tags: + - ngpvan_core + columns: + - name: sourcestateid + description: + SourceDatabase information to identify the source of the bridged + data + - name: sourcedatabasemodeid + description: '{{ doc("bonterra_missing") }}' + - name: contactsnoteid + description: Unique id for the note applied to the contact + - name: vanid + description: + Distinct VAN ID belonging to contact record. My C IDs begin with + 100 and are Distinct to committee and My V IDs are unique to voter file state. + - name: notetext + description: Text of Note entered on Organization + - name: notecategoryid + description: '{{ doc("bonterra_missing") }}' + - name: isviewrestricted + description: '{{ doc("bonterra_missing") }}' + - name: contactscontactid + description: Unique ID for Contact History + - name: committeeid + description: Committee in which record lives + - name: createdby + description: User ID of who/what created record (person or automated process) + - name: datecreated + description: Date record in table was created + - name: personcommitteeid + description: Committee that owns the person record + - name: modifiedby + description: The user ID of the persion that modified the record + - name: datemodified + description: + DateModified refers to when the row was last replicated. This doesn't + necessarily mean any value in the row was changed. + - name: stg_ngpvan__contacts_survey_responses + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____contacts_survey_responses' }}" + tags: + - ngpvan_core + columns: + - name: sourcestateid + description: + SourceDatabase information to identify the source of the bridged + data + - name: sourcedatabasemodeid + description: '{{ doc("bonterra_missing") }}' + - name: contactssurveyresponseid + description: Unique ID for Survey Response applied to Contact + - name: vanid + description: + Distinct VAN ID belonging to contact record. My C IDs begin with + 100 and are Distinct to committee and My V IDs are unique to voter file state. + - name: surveyquestionid + description: Unique ID of Survey Question - link to Survey Question + - name: surveyresponseid + description: Unique ID of Survey Response - link to Survey Response + - name: contacttypeid + description: Unique ID of Type of Contact + - name: inputtypeid + description: The ID of how the record was entered + - name: datecanvassed + description: Date contact was solicited or canvassed + - name: committeeid + description: Committee in which record lives + - name: canvassedby + description: User ID of canvasser + - name: username + description: Name of User who created record + - name: createdby + description: User ID of who/what created record (person or automated process) + - name: datecreated + description: Date record in table was created + - name: personcommitteeid + description: Committee that owns the person record + - name: contactscontactid + description: Unique ID for Contact History + - name: campaignid + description: Unique ID of Campain (if applied) + - name: contentid + description: Unique ID of Content (if applied) + - name: stg_ngpvan__input_types + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '____input_types' }}" + tags: + - ngpvan_core + - ngpvan_lookup + columns: + - name: input_type_id + description: '{{ doc("bonterra_missing") }}' + - name: input_type + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: source_schema + description: '{{ doc("bonterra_missing") }}' + - name: source_table + description: '{{ doc("bonterra_missing") }}' + - name: segment_by + description: '{{ doc("bonterra_missing") }}' + - name: vendor + description: '{{ doc("bonterra_missing") }}' + - name: vendor_unique_stg_ngpvan__input_type_id + description: '{{ doc("bonterra_missing") }}' + - name: stg_ngpvan__public_users + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____public_users' }}" + tags: + - ngpvan_core + columns: + - name: public_user_id + description: '{{ doc("bonterra_missing") }}' + - name: public_username + description: '{{ doc("bonterra_missing") }}' + - name: van_id + description: '{{ doc("bonterra_missing") }}' + - name: committee_id + description: '{{ doc("bonterra_missing") }}' + - name: created_at + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: source_schema + description: '{{ doc("bonterra_missing") }}' + - name: source_table + description: '{{ doc("bonterra_missing") }}' + - name: segment_by + description: '{{ doc("bonterra_missing") }}' + - name: vendor + description: '{{ doc("bonterra_missing") }}' + - name: segment_by_key + description: '{{ doc("bonterra_missing") }}' + - name: vendor_unique_stg_ngpvan__public_user_id + description: '{{ doc("bonterra_missing") }}' + - name: stg_ngpvan__results + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '____results' }}" + tags: + - ngpvan_core + - ngpvan_lookup + columns: + - name: result_id + description: '{{ doc("bonterra_missing") }}' + - name: result_name + description: '{{ doc("bonterra_missing") }}' + - name: result_description + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: source_schema + description: '{{ doc("bonterra_missing") }}' + - name: source_table + description: '{{ doc("bonterra_missing") }}' + - name: segment_by + description: '{{ doc("bonterra_missing") }}' + - name: vendor + description: '{{ doc("bonterra_missing") }}' + - name: vendor_unique_stg_ngpvan__result_id + description: '{{ doc("bonterra_missing") }}' + - name: stg_ngpvan__survey_questions + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____survey_questions' }}" + tags: + - ngpvan_core + columns: + - name: survey_question_id + description: '{{ doc("bonterra_missing") }}' + - name: van_state_id + description: '{{ doc("bonterra_missing") }}' + - name: election_cycle + description: '{{ doc("bonterra_missing") }}' + - name: survey_question_type + description: '{{ doc("bonterra_missing") }}' + - name: survey_question_name + description: '{{ doc("bonterra_missing") }}' + - name: survey_question_text + description: '{{ doc("bonterra_missing") }}' + - name: master_survey_question_id + description: '{{ doc("bonterra_missing") }}' + - name: committee_id + description: '{{ doc("bonterra_missing") }}' + - name: is_active + description: '{{ doc("bonterra_missing") }}' + - name: is_archived + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: source_schema + description: '{{ doc("bonterra_missing") }}' + - name: source_table + description: '{{ doc("bonterra_missing") }}' + - name: segment_by + description: '{{ doc("bonterra_missing") }}' + - name: segmented_survey_question_id + description: '{{ doc("bonterra_missing") }}' + - name: vendor + description: '{{ doc("bonterra_missing") }}' + - name: segment_by_key + description: '{{ doc("bonterra_missing") }}' + - name: vendor_unique_stg_ngpvan__survey_question_id + description: '{{ doc("bonterra_missing") }}' + - name: stg_ngpvan__survey_responses + description: '{{ doc("bonterra_missing") }}' + tests: + - test_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____survey_responses' }}" + tags: + - ngpvan_core + columns: + - name: survey_question_id + description: '{{ doc("bonterra_missing") }}' + - name: survey_response_id + description: '{{ doc("bonterra_missing") }}' + - name: survey_response + description: '{{ doc("bonterra_missing") }}' + - name: democrat_points + description: '{{ doc("bonterra_missing") }}' + - name: republican_points + description: '{{ doc("bonterra_missing") }}' + - name: independent_points + description: '{{ doc("bonterra_missing") }}' + - name: master_survey_response_id + description: '{{ doc("bonterra_missing") }}' + - name: committee_id + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("bonterra_missing") }}' + - name: source_schema + description: '{{ doc("bonterra_missing") }}' + - name: source_table + description: '{{ doc("bonterra_missing") }}' + - name: segment_by + description: '{{ doc("bonterra_missing") }}' + - name: segmented_survey_response_id + description: '{{ doc("bonterra_missing") }}' + - name: segmented_survey_question_id + description: '{{ doc("bonterra_missing") }}' + - name: vendor + description: '{{ doc("bonterra_missing") }}' + - name: segment_by_key + description: '{{ doc("bonterra_missing") }}' + - name: vendor_unique_stg_ngpvan__survey_response_id + description: '{{ doc("bonterra_missing") }}' + + - name: stg_ngpvan__users + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__users' }}" + + - name: stg_ngpvan__users_user_groups + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__users_user_groups' }}" From 5484924e57e7778c72ba025ff5a2df5a904ea52a Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Wed, 13 Aug 2025 15:09:59 -0500 Subject: [PATCH 02/24] fix yaml --- models/staging/ngpvan/_stg_ngpvan.yaml | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index 0c67959..9225405 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -6,7 +6,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____activist_codes' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__activist_codes' }}" tags: - ngpvan_core columns: @@ -61,7 +61,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____campaigns' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__campaigns' }}" tags: - ngpvan_core columns: @@ -108,7 +108,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '____code_types' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '__code_types' }}" tags: - ngpvan_core - ngpvan_lookup @@ -136,7 +136,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____codes' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__codes' }}" tags: - ngpvan_core columns: @@ -181,7 +181,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____committees' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__committees' }}" tags: - ngpvan_core columns: @@ -224,7 +224,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '____contact_types' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '__contact_types' }}" tags: - ngpvan_core - ngpvan_lookup @@ -252,7 +252,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____contacts_activist_codes' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_activist_codes' }}" tags: - ngpvan_core columns: @@ -297,7 +297,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____contacts_codes' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_codes' }}" tags: - ngpvan_core columns: @@ -342,7 +342,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____contacts_contacts' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_contacts' }}" tags: - ngpvan_core columns: @@ -415,7 +415,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____contacts_notes' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_notes' }}" tags: - ngpvan_core columns: @@ -458,7 +458,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____contacts_survey_responses' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_survey_responses' }}" tags: - ngpvan_core columns: @@ -507,7 +507,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '____input_types' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '__input_types' }}" tags: - ngpvan_core - ngpvan_lookup @@ -535,7 +535,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____public_users' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__public_users' }}" tags: - ngpvan_core columns: @@ -570,7 +570,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '____results' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '__results' }}" tags: - ngpvan_core - ngpvan_lookup @@ -600,7 +600,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____survey_questions' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__survey_questions' }}" tags: - ngpvan_core columns: @@ -647,7 +647,7 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '____survey_responses' }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__survey_responses' }}" tags: - ngpvan_core columns: From ebc88fbf5fed293fcaf9d825dc771fdcb434fc16 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Wed, 13 Aug 2025 15:11:30 -0500 Subject: [PATCH 03/24] fix references --- models/staging/ngpvan/_stg_ngpvan.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index 9225405..bfdda86 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -108,7 +108,8 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '__code_types' }}" + enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__code_types' }}" tags: - ngpvan_core - ngpvan_lookup @@ -224,7 +225,8 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '__contact_types' }}" + enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contact_types' }}" tags: - ngpvan_core - ngpvan_lookup @@ -507,7 +509,8 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '__input_types' }}" + enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__input_types' }}" tags: - ngpvan_core - ngpvan_lookup @@ -570,7 +573,8 @@ models: tests: - test_fully_documented config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['lookup_tables'] ~ '__results' }}" + enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__results' }}" tags: - ngpvan_core - ngpvan_lookup From 160f09456254803edac0698a2b334575bbfe8d46 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 08:58:13 -0500 Subject: [PATCH 04/24] add test --- .../test_table_fully_documented.sql | 19 +++++++++++ models/staging/ngpvan/_stg_ngpvan.yaml | 32 +++++++++---------- 2 files changed, 35 insertions(+), 16 deletions(-) create mode 100644 macros/custom_tests/test_table_fully_documented.sql diff --git a/macros/custom_tests/test_table_fully_documented.sql b/macros/custom_tests/test_table_fully_documented.sql new file mode 100644 index 0000000..0651bc1 --- /dev/null +++ b/macros/custom_tests/test_table_fully_documented.sql @@ -0,0 +1,19 @@ +{% test table_fully_documented(model) %} + {% set table_name = model.identifier %} + SELECT + column_name + FROM + {{ model.database }}.{{ model.schema }}.INFORMATION_SCHEMA.COLUMN_FIELD_PATHS + WHERE + (description IS NULL OR description = '') + AND table_name = '{{ table_name }}' + UNION ALL + SELECT '{{ table_name }}' + FROM UNNEST([1]) AS dummy + WHERE NOT EXISTS ( + SELECT 1 + FROM {{ model.database }}.{{ model.schema }}.INFORMATION_SCHEMA.TABLE_OPTIONS + WHERE option_name = 'description' + AND table_name = '{{ table_name }}' + ) +{% endtest %} \ No newline at end of file diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index bfdda86..dcf5237 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -4,7 +4,7 @@ models: - name: stg_ngpvan__activist_codes description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__activist_codes' }}" tags: @@ -59,7 +59,7 @@ models: - name: stg_ngpvan__campaigns description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__campaigns' }}" tags: @@ -106,7 +106,7 @@ models: - name: stg_ngpvan__code_types description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__code_types' }}" @@ -135,7 +135,7 @@ models: - name: stg_ngpvan__codes description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__codes' }}" tags: @@ -180,7 +180,7 @@ models: - name: stg_ngpvan__committees description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__committees' }}" tags: @@ -223,7 +223,7 @@ models: - name: stg_ngpvan__contact_types description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contact_types' }}" @@ -252,7 +252,7 @@ models: - name: stg_ngpvan__contacts_activist_codes description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_activist_codes' }}" tags: @@ -297,7 +297,7 @@ models: - name: stg_ngpvan__contacts_codes description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_codes' }}" tags: @@ -342,7 +342,7 @@ models: - name: stg_ngpvan__contacts_contacts description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_contacts' }}" tags: @@ -415,7 +415,7 @@ models: - name: stg_ngpvan__contacts_notes description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_notes' }}" tags: @@ -458,7 +458,7 @@ models: - name: stg_ngpvan__contacts_survey_responses description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_survey_responses' }}" tags: @@ -507,7 +507,7 @@ models: - name: stg_ngpvan__input_types description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__input_types' }}" @@ -536,7 +536,7 @@ models: - name: stg_ngpvan__public_users description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__public_users' }}" tags: @@ -571,7 +571,7 @@ models: - name: stg_ngpvan__results description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__results' }}" @@ -602,7 +602,7 @@ models: - name: stg_ngpvan__survey_questions description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__survey_questions' }}" tags: @@ -649,7 +649,7 @@ models: - name: stg_ngpvan__survey_responses description: '{{ doc("bonterra_missing") }}' tests: - - test_fully_documented + - table_fully_documented config: alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__survey_responses' }}" tags: From ffdd258e8634f5a81e103a2821704f0c244232b1 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 09:56:36 -0500 Subject: [PATCH 05/24] fix variable names --- models/staging/ngpvan/_stg_ngpvan.yaml | 192 +++++++++++-------------- 1 file changed, 84 insertions(+), 108 deletions(-) diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index dcf5237..bf056d9 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -151,7 +151,7 @@ models: description: Unique ID for Code - name: codename description: Code Name in VAN - - name: committeeid + - name: committee_id description: Committee in which record lives - name: isactive description: A Flag for if the activity is active @@ -159,9 +159,9 @@ models: description: If the Code is in a nested structure, this will be the direct parent of the Code - - name: createdby + - name: created_by description: User ID of who/what created record (person or automated process) - - name: datecreated + - name: date_created description: Date record in table was created - name: staticfullname description: @@ -173,7 +173,7 @@ models: description: Source Code vs Tag - name: codedescription description: System stores description of code for reference purposes - - name: contacttypeid + - name: contact_type_id description: Unique ID of Type of Contact - name: campaignid description: Unique ID of Campain (if applied) @@ -266,33 +266,33 @@ models: description: '{{ doc("bonterra_missing") }}' - name: contactsactivistcodeid description: Unique ID for Activist Code applied to Specific Contact - - name: vanid + - name: van_id description: Distinct VAN ID belonging to contact record. My C IDs begin with 100 and are Distinct to committee and My V IDs are unique to voter file state. - name: activistcodeid description: Distinct ID belonging to Activist Code - - name: committeeid + - name: committee_id description: Committee in which record lives - - name: datecreated + - name: date_created description: Date record in table was created - - name: inputtypeid + - name: input_type_id description: The ID of how the record was entered - - name: contacttypeid + - name: contact_type_id description: Unique ID of Type of Contact - - name: username + - name: user_name description: Name of User who created record - - name: canvassedby + - name: canvassed_by description: User ID of canvasser - name: campaignid description: Unique ID of Campain (if applied) - name: contentid description: Unique ID of Content (if applied) - - name: personcommitteeid + - name: personcommittee_id description: Committee that owns the person record - - name: contactscontactid + - name: contacts_contacts_id description: Unique ID for Contact History - - name: datecanvassed + - name: date_canvassed description: Date contact was solicited or canvassed - name: stg_ngpvan__contacts_codes description: '{{ doc("bonterra_missing") }}' @@ -303,41 +303,35 @@ models: tags: - ngpvan_core columns: - - name: sourcestateid - description: - SourceDatabase information to identify the source of the bridged - data - - name: sourcedatabasemodeid - description: '{{ doc("bonterra_missing") }}' - - name: contactscodeid + - name: contacts_code_id description: nan - - name: vanid + - name: van_id description: Distinct VAN ID belonging to contact record. My C IDs begin with 100 and are Distinct to committee and My V IDs are unique to voter file state. - - name: codeid + - name: code_id description: Unique ID for Code - - name: committeeid + - name: committee_id description: Committee in which record lives - - name: datecreated + - name: date_created description: Date record in table was created - - name: inputtypeid + - name: input_type_id description: The ID of how the record was entered - - name: contacttypeid + - name: contact_type_id description: Unique ID of Type of Contact - - name: username + - name: user_name description: Name of User who created record - - name: canvassedby + - name: canvassed_by description: User ID of canvasser - - name: campaignid + - name: campaign_id description: Unique ID of Campain (if applied) - - name: contentid + - name: content_id description: Unique ID of Content (if applied) - - name: personcommitteeid + - name: person_committee_id description: Committee that owns the person record - - name: contactscontactid + - name: contacts_contact_id description: Unique ID for Contact History - - name: datecanvassed + - name: date_canvassed description: Date contact was solicited or canvassed - name: stg_ngpvan__contacts_contacts description: '{{ doc("bonterra_missing") }}' @@ -348,69 +342,63 @@ models: tags: - ngpvan_core columns: - - name: sourcestateid - description: - SourceDatabase information to identify the source of the bridged - data - - name: sourcedatabasemodeid - description: '{{ doc("bonterra_missing") }}' - - name: contactscontactid + - name: contacts_contacts_id description: Unique ID for Contact History - - name: vanid + - name: van_id description: Distinct VAN ID belonging to contact record. My C IDs begin with 100 and are Distinct to committee and My V IDs are unique to voter file state. - - name: datecanvassed + - name: date_canvassed description: Date contact was solicited or canvassed - - name: contacttypeid + - name: contact_type_id description: Unique ID of Type of Contact - - name: inputtypeid + - name: input_type_id description: The ID of how the record was entered - - name: resultid + - name: result_id description: Result of contact interaction - - name: committeeid + - name: committee_id description: Committee in which record lives - - name: canvassedby + - name: canvassed_by description: User ID of canvasser - - name: username + - name: user_name description: Name of User who created record - - name: createdby + - name: created_by description: User ID of who/what created record (person or automated process) - - name: datecreated + - name: date_created description: Date record in table was created - - name: contactscontactnotestext + - name: contacts_contact_notes_text description: '{{ doc("bonterra_missing") }}' - - name: callduration + - name: call_duration description: '{{ doc("bonterra_missing") }}' - - name: contactsphoneid + - name: contacts_phone_id description: Unique identifier of Phone Number - - name: contactscontactsfull + - name: contacts_contacts_full description: '{{ doc("bonterra_missing") }}' - - name: liveid + - name: live_id description: '{{ doc("bonterra_missing") }}' - name: deleted description: '{{ doc("bonterra_missing") }}' - - name: personcommitteeid + - name: person_committee_id description: Committee that owns the person record - - name: campaignid + - name: campaign_id description: Unique ID of Campain (if applied) - - name: contentid + - name: content_id description: Unique ID of Content (if applied) - - name: eventid + - name: event_id description: Unique ID of Event - - name: eventshiftid + - name: event_shift_id description: Unique ID of Event Shift assigned to Contact Record in the event - - name: locationid + - name: location_id description: Unique ID of Location of Event - - name: modeofcontactid + - name: mode_of_contact_id description: '{{ doc("bonterra_missing") }}' - - name: taskassignmentid + - name: task_assignment_id description: '{{ doc("bonterra_missing") }}' - - name: phoneoptinstatus + - name: phone_opt_in_status description: '{{ doc("bonterra_missing") }}' - - name: phoneoptintypeid + - name: phone_opt_in_type_id description: '{{ doc("bonterra_missing") }}' - - name: phoneoptinsourceid + - name: phone_opt_in_source_id description: '{{ doc("bonterra_missing") }}' - name: stg_ngpvan__contacts_notes description: '{{ doc("bonterra_missing") }}' @@ -421,37 +409,31 @@ models: tags: - ngpvan_core columns: - - name: sourcestateid - description: - SourceDatabase information to identify the source of the bridged - data - - name: sourcedatabasemodeid - description: '{{ doc("bonterra_missing") }}' - - name: contactsnoteid + - name: contacts_note_id description: Unique id for the note applied to the contact - - name: vanid + - name: van_id description: Distinct VAN ID belonging to contact record. My C IDs begin with 100 and are Distinct to committee and My V IDs are unique to voter file state. - - name: notetext + - name: note_text description: Text of Note entered on Organization - - name: notecategoryid + - name: note_category_id description: '{{ doc("bonterra_missing") }}' - - name: isviewrestricted + - name: is_view_restricted description: '{{ doc("bonterra_missing") }}' - - name: contactscontactid + - name: contacts_contacts_id description: Unique ID for Contact History - - name: committeeid + - name: committee_id description: Committee in which record lives - - name: createdby + - name: created_by description: User ID of who/what created record (person or automated process) - - name: datecreated + - name: date_created description: Date record in table was created - - name: personcommitteeid + - name: person_committee_id description: Committee that owns the person record - - name: modifiedby + - name: modified_by description: The user ID of the persion that modified the record - - name: datemodified + - name: date_modified description: DateModified refers to when the row was last replicated. This doesn't necessarily mean any value in the row was changed. @@ -464,45 +446,39 @@ models: tags: - ngpvan_core columns: - - name: sourcestateid - description: - SourceDatabase information to identify the source of the bridged - data - - name: sourcedatabasemodeid - description: '{{ doc("bonterra_missing") }}' - - name: contactssurveyresponseid + - name: contacts_survey_response_id description: Unique ID for Survey Response applied to Contact - - name: vanid + - name: van_id description: Distinct VAN ID belonging to contact record. My C IDs begin with 100 and are Distinct to committee and My V IDs are unique to voter file state. - - name: surveyquestionid + - name: survey_question_id description: Unique ID of Survey Question - link to Survey Question - - name: surveyresponseid + - name: survey_response_id description: Unique ID of Survey Response - link to Survey Response - - name: contacttypeid + - name: contact_type_id description: Unique ID of Type of Contact - - name: inputtypeid + - name: input_type_id description: The ID of how the record was entered - - name: datecanvassed + - name: date_canvassed description: Date contact was solicited or canvassed - - name: committeeid + - name: committee_id description: Committee in which record lives - - name: canvassedby + - name: canvassed_by description: User ID of canvasser - - name: username + - name: user_name description: Name of User who created record - - name: createdby + - name: created_by description: User ID of who/what created record (person or automated process) - - name: datecreated + - name: date_created description: Date record in table was created - - name: personcommitteeid + - name: person_committee_id description: Committee that owns the person record - - name: contactscontactid + - name: contacts_contacts_id description: Unique ID for Contact History - - name: campaignid + - name: campaign_id description: Unique ID of Campain (if applied) - - name: contentid + - name: content_id description: Unique ID of Content (if applied) - name: stg_ngpvan__input_types description: '{{ doc("bonterra_missing") }}' @@ -544,7 +520,7 @@ models: columns: - name: public_user_id description: '{{ doc("bonterra_missing") }}' - - name: public_username + - name: public_user_name description: '{{ doc("bonterra_missing") }}' - name: van_id description: '{{ doc("bonterra_missing") }}' From eafb22c4c24c34b0f5bf0c2ffb3d954a5f1b91e5 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:47:49 -0500 Subject: [PATCH 06/24] try to fix codes --- documentation_blocks/database_mode.md | 3 + documentation_blocks/is_myvoters.md | 3 + models/staging/ngpvan/_stg_ngpvan.yaml | 308 ++++++++++-------- .../staging/ngpvan/core/stg_ngpvan__codes.sql | 4 +- 4 files changed, 188 insertions(+), 130 deletions(-) create mode 100644 documentation_blocks/database_mode.md create mode 100644 documentation_blocks/is_myvoters.md diff --git a/documentation_blocks/database_mode.md b/documentation_blocks/database_mode.md new file mode 100644 index 0000000..54a917f --- /dev/null +++ b/documentation_blocks/database_mode.md @@ -0,0 +1,3 @@ +{% docs database_mode %} +Boolean to indicate if this row was synced via database +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/is_myvoters.md b/documentation_blocks/is_myvoters.md new file mode 100644 index 0000000..4ca0d77 --- /dev/null +++ b/documentation_blocks/is_myvoters.md @@ -0,0 +1,3 @@ +{% docs is_myvoters %} +Boolean to indicate if this row was synced via database +{% enddocs %} \ No newline at end of file diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index bf056d9..58882fc 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -11,7 +11,7 @@ models: - ngpvan_core columns: - name: vendor_unique_stg_ngpvan__activist_code_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor_unique_id") }}' - name: activist_code_id description: '{{ doc("bonterra_missing") }}' - name: van_state_id @@ -37,9 +37,9 @@ models: - name: segmented_activist_code_id description: '{{ doc("bonterra_missing") }}' - name: vendor - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor") }}' - name: segment_by_key - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by_key") }}' - name: activist_code_name description: '{{ doc("bonterra_missing") }}' - name: action_type_id @@ -49,13 +49,13 @@ models: - name: is_archived description: '{{ doc("bonterra_missing") }}' - name: _avvan_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("dbt_source_relation") }}' - name: source_schema - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_schema") }}' - name: source_table - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_table") }}' - name: stg_ngpvan__campaigns description: '{{ doc("bonterra_missing") }}' tests: @@ -86,23 +86,23 @@ models: - name: is_archived description: '{{ doc("bonterra_missing") }}' - name: _avvan_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("dbt_source_relation") }}' - name: source_schema - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_schema") }}' - name: source_table - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by") }}' - name: segmented_campaign_id - description: '{{ doc("bonterra_missing") }}' + description: Combination of segment by and campaign id - name: vendor - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor") }}' - name: segment_by_key - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by_key") }}' - name: vendor_unique_stg_ngpvan__campaign_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor_unique_id") }}' - name: stg_ngpvan__code_types description: '{{ doc("bonterra_missing") }}' tests: @@ -119,19 +119,19 @@ models: - name: code_type description: '{{ doc("bonterra_missing") }}' - name: _avvan_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("dbt_source_relation") }}' - name: source_schema - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_schema") }}' - name: source_table - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by") }}' - name: vendor - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__code_type_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor_unique_id") }}' - name: stg_ngpvan__codes description: '{{ doc("bonterra_missing") }}' tests: @@ -205,21 +205,21 @@ models: - name: master_committee_id description: '{{ doc("bonterra_missing") }}' - name: _avvan_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("dbt_source_relation") }}' - name: source_schema - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_schema") }}' - name: source_table - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by") }}' - name: vendor - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor") }}' - name: segment_by_key - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by_key") }}' - name: vendor_unique_stg_ngpvan__committee_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor_unique_id") }}' - name: stg_ngpvan__contact_types description: '{{ doc("bonterra_missing") }}' tests: @@ -236,19 +236,19 @@ models: - name: contact_method_type description: '{{ doc("bonterra_missing") }}' - name: _avvan_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("dbt_source_relation") }}' - name: source_schema - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_schema") }}' - name: source_table - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by") }}' - name: vendor - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__contact_type_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor_unique_id") }}' - name: stg_ngpvan__contacts_activist_codes description: '{{ doc("bonterra_missing") }}' tests: @@ -258,42 +258,56 @@ models: tags: - ngpvan_core columns: - - name: sourcestateid - description: - SourceDatabase information to identify the source of the bridged - data - - name: sourcedatabasemodeid - description: '{{ doc("bonterra_missing") }}' - - name: contactsactivistcodeid + - name: contacts_activist_code_id description: Unique ID for Activist Code applied to Specific Contact - name: van_id description: Distinct VAN ID belonging to contact record. My C IDs begin with 100 and are Distinct to committee and My V IDs are unique to voter file state. - - name: activistcodeid + - name: activist_code_id description: Distinct ID belonging to Activist Code - name: committee_id description: Committee in which record lives - - name: date_created - description: Date record in table was created + - name: utc_created_at + description: Date record in table was created, normalized to UTC - name: input_type_id description: The ID of how the record was entered + - name: input_type + description: '{{ doc("bonterra_missing") }}' + - name: contact_method_type_id + description: '{{ doc("bonterra_missing") }}' + - name: contact_method_type + description: '{{ doc("bonterra_missing") }}' - name: contact_type_id description: Unique ID of Type of Contact - - name: user_name + - name: canvased_by_username description: Name of User who created record - - name: canvassed_by + - name: canvassed_by_user_id description: User ID of canvasser - - name: campaignid + - name: campaign_id description: Unique ID of Campain (if applied) - - name: contentid + - name: content_id description: Unique ID of Content (if applied) - - name: personcommittee_id - description: Committee that owns the person record + - name: utc_modified_at + description: '{{ doc("bonterra_missing") }}' - name: contacts_contacts_id description: Unique ID for Contact History - - name: date_canvassed - description: Date contact was solicited or canvassed + - name: utc_canvassed_at + description: Date contact was solicited or canvassed, normalized to UTC + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: vendor + description: '{{ doc("vendor") }}' + - name: vendor_unique_stg_ngpvan__contacts_activist_code_id + description: '{{ doc("vendor_unique_id") }}' - name: stg_ngpvan__contacts_codes description: '{{ doc("bonterra_missing") }}' tests: @@ -303,8 +317,10 @@ models: tags: - ngpvan_core columns: + - name: van_state_code + description: '{{ doc("bonterra_missing") }}' - name: contacts_code_id - description: nan + description: '{{ doc("bonterra_missing") }}' - name: van_id description: Distinct VAN ID belonging to contact record. My C IDs begin with @@ -313,26 +329,40 @@ models: description: Unique ID for Code - name: committee_id description: Committee in which record lives - - name: date_created - description: Date record in table was created + - name: utc_created_at + description: Date record in table was created, normalized to UTC - name: input_type_id description: The ID of how the record was entered - - name: contact_type_id - description: Unique ID of Type of Contact - - name: user_name - description: Name of User who created record - - name: canvassed_by - description: User ID of canvasser - - name: campaign_id - description: Unique ID of Campain (if applied) - - name: content_id - description: Unique ID of Content (if applied) - - name: person_committee_id - description: Committee that owns the person record - - name: contacts_contact_id - description: Unique ID for Contact History - - name: date_canvassed - description: Date contact was solicited or canvassed + - name: input_type + description: '{{ doc("bonterra_missing") }}' + - name: utc_modified_at + description: '{{ doc("bonterra_missing") }}' + - name: created_by_user_id + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: database_mode + description: '{{ doc("database_mode") }}' + - name: is_myvoters + description: '{{ doc("is_myvoters") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: unique_contacts_code_id + description: Unique key constructed by combining segment by, state code, and contacts code id + - name: segmented_contacts_code_id + description: Combination of segment by and contacts code id + - name: segmented_van_id + description: Combination of segment by and van id + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' - name: stg_ngpvan__contacts_contacts description: '{{ doc("bonterra_missing") }}' tests: @@ -342,64 +372,76 @@ models: tags: - ngpvan_core columns: + - name: van_state_code + description: '{{ doc("bonterra_missing") }}' - name: contacts_contacts_id description: Unique ID for Contact History - name: van_id description: Distinct VAN ID belonging to contact record. My C IDs begin with 100 and are Distinct to committee and My V IDs are unique to voter file state. - - name: date_canvassed - description: Date contact was solicited or canvassed - - name: contact_type_id - description: Unique ID of Type of Contact - - name: input_type_id - description: The ID of how the record was entered - name: result_id description: Result of contact interaction + - name: result_name + description: '{{ doc("bonterra_missing") }}' - name: committee_id description: Committee in which record lives - - name: canvassed_by - description: User ID of canvasser - - name: user_name - description: Name of User who created record - - name: created_by - description: User ID of who/what created record (person or automated process) - - name: date_created - description: Date record in table was created - - name: contacts_contact_notes_text - description: '{{ doc("bonterra_missing") }}' - - name: call_duration - description: '{{ doc("bonterra_missing") }}' - - name: contacts_phone_id - description: Unique identifier of Phone Number - - name: contacts_contacts_full - description: '{{ doc("bonterra_missing") }}' - - name: live_id + - name: utc_created_at + description: Date record in table was created, normalized to UTC + - name: utc_canvassed_at + description: Date contact was solicited or canvassed, normalized to UTC + - name: input_type_id + description: The ID of how the record was entered + - name: input_type description: '{{ doc("bonterra_missing") }}' - - name: deleted + - name: contact_method_type_id + description: Unique ID of Type of Contact + - name: contact_method_type description: '{{ doc("bonterra_missing") }}' - - name: person_committee_id - description: Committee that owns the person record + - name: canvassed_by_user_name + description: Name of User who created record + - name: canvassed_by_user_id + description: User ID of canvasser - name: campaign_id description: Unique ID of Campain (if applied) - name: content_id description: Unique ID of Content (if applied) - - name: event_id - description: Unique ID of Event - - name: event_shift_id - description: Unique ID of Event Shift assigned to Contact Record in the event - - name: location_id - description: Unique ID of Location of Event - - name: mode_of_contact_id - description: '{{ doc("bonterra_missing") }}' - - name: task_assignment_id - description: '{{ doc("bonterra_missing") }}' - - name: phone_opt_in_status + - name: created_by_user_id + description: User ID of who/what created record (person or automated process) + - name: contacts_phone_id + description: Unique identifier of Phone Number + - name: team_id description: '{{ doc("bonterra_missing") }}' - - name: phone_opt_in_type_id + - name: division_id description: '{{ doc("bonterra_missing") }}' - - name: phone_opt_in_source_id + - name: utc_modified_at description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: database_mode + description: '{{ doc("database_mode") }}' + - name: is_myvoters + description: '{{ doc("is_myvoters") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: unique_contacts_contact_id + description: Unique key constructed by combining segment by, state code, and contacts contact id + - name: segmented_contacts_contact_id + description: Combination of segment by and contacts contact id + - name: segmented_van_id + description: Combination of segment by and van id + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: vendor_unique_stg_ngpvan__contacts_contact_id + description: '{{ doc("vendor_unique_id") }}' - name: stg_ngpvan__contacts_notes description: '{{ doc("bonterra_missing") }}' tests: @@ -409,34 +451,44 @@ models: tags: - ngpvan_core columns: + - name: van_state_code + description: '{{ doc("bonterra_missing") }}' - name: contacts_note_id description: Unique id for the note applied to the contact - name: van_id description: Distinct VAN ID belonging to contact record. My C IDs begin with 100 and are Distinct to committee and My V IDs are unique to voter file state. + - name: created_by_user_id + description: User ID of who/what created record (person or automated process) + - name: utc_created_at + description: Date record in table was created, normalized to UTC + - name: committee_id + description: Committee in which record lives + - name: suppressed_by_user_id + description: '{{ doc("bonterra_missing") }}' + - name: utc_suppressed_at + description: '{{ doc("bonterra_missing") }}' - name: note_text description: Text of Note entered on Organization - - name: note_category_id + - name: datefollowup description: '{{ doc("bonterra_missing") }}' - - name: is_view_restricted + - name: followup_completed_by_user_id + description: '{{ doc("bonterra_missing") }}' + - name: datefollowupcompleted description: '{{ doc("bonterra_missing") }}' - name: contacts_contacts_id description: Unique ID for Contact History - - name: committee_id - description: Committee in which record lives - - name: created_by - description: User ID of who/what created record (person or automated process) - - name: date_created - description: Date record in table was created - - name: person_committee_id - description: Committee that owns the person record - - name: modified_by - description: The user ID of the persion that modified the record - - name: date_modified + - name: utc_modified_at description: DateModified refers to when the row was last replicated. This doesn't - necessarily mean any value in the row was changed. + necessarily mean any value in the row was changed. Normalized to UTC + - name: note_category + description: '{{ doc("bonterra_missing") }}' + - name: contacts_online_form_id + description: '{{ doc("bonterra_missing") }}' + - name: contacts_story_id + - name: stg_ngpvan__contacts_survey_responses description: '{{ doc("bonterra_missing") }}' tests: diff --git a/models/staging/ngpvan/core/stg_ngpvan__codes.sql b/models/staging/ngpvan/core/stg_ngpvan__codes.sql index 1925593..bddd550 100644 --- a/models/staging/ngpvan/core/stg_ngpvan__codes.sql +++ b/models/staging/ngpvan/core/stg_ngpvan__codes.sql @@ -5,11 +5,11 @@ WITH ), committees AS ( - SELECT * FROM {{ ref('base_ngpvan__committees') }} + SELECT *(EXCEPT _avvan_source_relation) FROM {{ ref('base_ngpvan__committees') }} ), codetypes AS ( - SELECT * FROM {{ ref('base_ngpvan__codetypes') }} + SELECT *(EXCEPT _avvan_source_relation) FROM {{ ref('base_ngpvan__codetypes') }} ), renamed AS ( From 6935ebd8c9f7bd4beb12647967849a2b3243c6f3 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:49:03 -0500 Subject: [PATCH 07/24] fix typo --- documentation_blocks/avvan_source_relation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation_blocks/avvan_source_relation.md b/documentation_blocks/avvan_source_relation.md index 72ffcd9..1937e8b 100644 --- a/documentation_blocks/avvan_source_relation.md +++ b/documentation_blocks/avvan_source_relation.md @@ -1,3 +1,3 @@ -{% docs _avvan_source_relation %} +{% docs avvan_source_relation %} Concatenated string of America Votes source database, schema and table. Null for values unrelated to America Votes data {% enddocs %} \ No newline at end of file From 1caa9ad1e34b6afb308613a17247814d55bf8513 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:55:48 -0500 Subject: [PATCH 08/24] fine we get rid of it --- models/staging/ngpvan/_stg_ngpvan.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index 58882fc..fe236da 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -85,8 +85,6 @@ models: description: '{{ doc("bonterra_missing") }}' - name: is_archived description: '{{ doc("bonterra_missing") }}' - - name: _avvan_source_relation - description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema From 36b805e3b60a39581aede53136e9f9a18719e809 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:59:05 -0500 Subject: [PATCH 09/24] just get rid of all of them --- models/staging/ngpvan/_stg_ngpvan.yaml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index fe236da..8f207f8 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -48,8 +48,6 @@ models: description: '{{ doc("bonterra_missing") }}' - name: is_archived description: '{{ doc("bonterra_missing") }}' - - name: _avvan_source_relation - description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema @@ -116,8 +114,6 @@ models: description: '{{ doc("bonterra_missing") }}' - name: code_type description: '{{ doc("bonterra_missing") }}' - - name: _avvan_source_relation - description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema @@ -202,8 +198,6 @@ models: description: '{{ doc("bonterra_missing") }}' - name: master_committee_id description: '{{ doc("bonterra_missing") }}' - - name: _avvan_source_relation - description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema @@ -233,8 +227,6 @@ models: description: '{{ doc("bonterra_missing") }}' - name: contact_method_type description: '{{ doc("bonterra_missing") }}' - - name: _avvan_source_relation - description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema @@ -292,8 +284,6 @@ models: description: Unique ID for Contact History - name: utc_canvassed_at description: Date contact was solicited or canvassed, normalized to UTC - - name: _avvan_source_relation - description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema @@ -337,8 +327,6 @@ models: description: '{{ doc("bonterra_missing") }}' - name: created_by_user_id description: '{{ doc("bonterra_missing") }}' - - name: _avvan_source_relation - description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema @@ -414,8 +402,6 @@ models: description: '{{ doc("bonterra_missing") }}' - name: utc_modified_at description: '{{ doc("bonterra_missing") }}' - - name: _avvan_source_relation - description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema From d6243be0820f65057d680a11de592257046fe944 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:59:58 -0500 Subject: [PATCH 10/24] add docs paths --- dbt_project.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index fd6762a..565e50e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,8 +1,8 @@ -name: 'dbt_ngpvan' -version: '1.0.0' +name: "dbt_ngpvan" +version: "1.0.0" config-version: 2 -profile: 'dbt_ngpvan' +profile: "dbt_ngpvan" model-paths: ["models"] analysis-paths: ["analyses"] @@ -10,6 +10,7 @@ test-paths: ["tests"] seed-paths: ["seeds"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] +docs-paths: ["documentation_blocks"] target-path: "target" clean-targets: @@ -29,7 +30,7 @@ models: +materialized: table tests: - +severity: warn # all tests + +severity: warn # all tests vars: dbt_ngpvan_config: @@ -60,4 +61,4 @@ vars: lookup_tables: true additional_fields: [] enable_incremental_models: true - ngpvan_default_incremental_window_days: 3 \ No newline at end of file + ngpvan_default_incremental_window_days: 3 From 9f2df9f49e27f1b88019f75aa3d3aad115f477fc Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:02:52 -0500 Subject: [PATCH 11/24] remove blocks --- models/staging/ngpvan/_stg_ngpvan.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index 8f207f8..1a15558 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -90,7 +90,7 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("segment_by") }}' + description: - name: segmented_campaign_id description: Combination of segment by and campaign id - name: vendor @@ -121,7 +121,7 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("segment_by") }}' + description: - name: vendor description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__code_type_id @@ -205,7 +205,7 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("segment_by") }}' + description: - name: vendor description: '{{ doc("vendor") }}' - name: segment_by_key @@ -234,7 +234,7 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("segment_by") }}' + description: - name: vendor description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__contact_type_id @@ -291,7 +291,7 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("segment_by") }}' + description: - name: vendor description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__contacts_activist_code_id @@ -338,7 +338,7 @@ models: - name: is_myvoters description: '{{ doc("is_myvoters") }}' - name: segment_by - description: '{{ doc("segment_by") }}' + description: - name: unique_contacts_code_id description: Unique key constructed by combining segment by, state code, and contacts code id - name: segmented_contacts_code_id @@ -413,7 +413,7 @@ models: - name: is_myvoters description: '{{ doc("is_myvoters") }}' - name: segment_by - description: '{{ doc("segment_by") }}' + description: - name: unique_contacts_contact_id description: Unique key constructed by combining segment by, state code, and contacts contact id - name: segmented_contacts_contact_id From 3ddfcfcb0f6801363470911016199270a649c3b2 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:14:47 -0500 Subject: [PATCH 12/24] not none --- models/staging/ngpvan/_stg_ngpvan.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index 1a15558..0801198 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -291,7 +291,7 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: + description: "" - name: vendor description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__contacts_activist_code_id @@ -338,7 +338,7 @@ models: - name: is_myvoters description: '{{ doc("is_myvoters") }}' - name: segment_by - description: + description: "" - name: unique_contacts_code_id description: Unique key constructed by combining segment by, state code, and contacts code id - name: segmented_contacts_code_id @@ -413,7 +413,7 @@ models: - name: is_myvoters description: '{{ doc("is_myvoters") }}' - name: segment_by - description: + description: "" - name: unique_contacts_contact_id description: Unique key constructed by combining segment by, state code, and contacts contact id - name: segmented_contacts_contact_id From 92fb25f0704104e98946b844af8d8651f81def8c Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:18:22 -0500 Subject: [PATCH 13/24] more nulls --- models/staging/ngpvan/_stg_ngpvan.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index 0801198..52929ae 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -90,7 +90,7 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: + description: "" - name: segmented_campaign_id description: Combination of segment by and campaign id - name: vendor @@ -121,7 +121,7 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: + description: "" - name: vendor description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__code_type_id @@ -205,7 +205,7 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: + description: "" - name: vendor description: '{{ doc("vendor") }}' - name: segment_by_key @@ -234,7 +234,7 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: + description: "" - name: vendor description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__contact_type_id From 1ab71a6dade2400a483dd122a204c8ca9595a3f9 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 15:00:48 -0500 Subject: [PATCH 14/24] add space --- models/staging/ngpvan/core/stg_ngpvan__codes.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/staging/ngpvan/core/stg_ngpvan__codes.sql b/models/staging/ngpvan/core/stg_ngpvan__codes.sql index bddd550..cf99d7a 100644 --- a/models/staging/ngpvan/core/stg_ngpvan__codes.sql +++ b/models/staging/ngpvan/core/stg_ngpvan__codes.sql @@ -5,11 +5,11 @@ WITH ), committees AS ( - SELECT *(EXCEPT _avvan_source_relation) FROM {{ ref('base_ngpvan__committees') }} + SELECT * (EXCEPT _avvan_source_relation) FROM {{ ref('base_ngpvan__committees') }} ), codetypes AS ( - SELECT *(EXCEPT _avvan_source_relation) FROM {{ ref('base_ngpvan__codetypes') }} + SELECT * (EXCEPT _avvan_source_relation) FROM {{ ref('base_ngpvan__codetypes') }} ), renamed AS ( From 56557e44bf22fbee67374e2f281b155039e27dbc Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 15:05:57 -0500 Subject: [PATCH 15/24] move punctuation --- models/staging/ngpvan/core/stg_ngpvan__codes.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/staging/ngpvan/core/stg_ngpvan__codes.sql b/models/staging/ngpvan/core/stg_ngpvan__codes.sql index cf99d7a..c06cdad 100644 --- a/models/staging/ngpvan/core/stg_ngpvan__codes.sql +++ b/models/staging/ngpvan/core/stg_ngpvan__codes.sql @@ -5,11 +5,11 @@ WITH ), committees AS ( - SELECT * (EXCEPT _avvan_source_relation) FROM {{ ref('base_ngpvan__committees') }} + SELECT * EXCEPT (_avvan_source_relation) FROM {{ ref('base_ngpvan__committees') }} ), codetypes AS ( - SELECT * (EXCEPT _avvan_source_relation) FROM {{ ref('base_ngpvan__codetypes') }} + SELECT * EXCEPT (_avvan_source_relation) FROM {{ ref('base_ngpvan__codetypes') }} ), renamed AS ( From c77c307df3addfcb907113af8903cda7359fdbb3 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 15:10:15 -0500 Subject: [PATCH 16/24] one more fix --- models/staging/ngpvan/core/stg_ngpvan__codes.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/ngpvan/core/stg_ngpvan__codes.sql b/models/staging/ngpvan/core/stg_ngpvan__codes.sql index c06cdad..03ead3e 100644 --- a/models/staging/ngpvan/core/stg_ngpvan__codes.sql +++ b/models/staging/ngpvan/core/stg_ngpvan__codes.sql @@ -9,7 +9,7 @@ WITH ), codetypes AS ( - SELECT * EXCEPT (_avvan_source_relation) FROM {{ ref('base_ngpvan__codetypes') }} + SELECT * FROM {{ ref('base_ngpvan__codetypes') }} ), renamed AS ( From b27767dbc8a309331fb7c13694db68ef3f5be42d Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:32:06 -0500 Subject: [PATCH 17/24] fix missing docs --- documentation_blocks/segment_by.md | 3 + models/staging/ngpvan/_stg_ngpvan.yaml | 280 +++++++++++++++++-------- 2 files changed, 192 insertions(+), 91 deletions(-) create mode 100644 documentation_blocks/segment_by.md diff --git a/documentation_blocks/segment_by.md b/documentation_blocks/segment_by.md new file mode 100644 index 0000000..16d0465 --- /dev/null +++ b/documentation_blocks/segment_by.md @@ -0,0 +1,3 @@ +{% docs source_schema %} +The value used for segmentation +{% enddocs %} \ No newline at end of file diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index 52929ae..e64a7b9 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -54,6 +54,8 @@ models: description: '{{ doc("source_schema") }}' - name: source_table description: '{{ doc("source_table") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' - name: stg_ngpvan__campaigns description: '{{ doc("bonterra_missing") }}' tests: @@ -90,7 +92,9 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: "" + description: '{{ doc("segment_by") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' - name: segmented_campaign_id description: Combination of segment by and campaign id - name: vendor @@ -121,7 +125,9 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: "" + description: '{{ doc("segment_by") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' - name: vendor description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__code_type_id @@ -135,42 +141,58 @@ models: tags: - ngpvan_core columns: - - name: sourcestateid - description: - SourceDatabase information to identify the source of the bridged - data - - name: sourcedatabasemodeid + - name: van_state_code description: '{{ doc("bonterra_missing") }}' - - name: codeid + - name: code_id description: Unique ID for Code - - name: codename + - name: code_name description: Code Name in VAN - name: committee_id description: Committee in which record lives - - name: isactive - description: A Flag for if the activity is active - - name: parentcodeid + - name: is_active + description: A Flag for if the activity is active, converted to boolean + - name: is_archived + description: A Flag for if the activity is archived, derived from the raw isactive value + - name: parent_code_id description: If the Code is in a nested structure, this will be the direct parent of the Code - - name: created_by + - name: created_by_user_id description: User ID of who/what created record (person or automated process) - - name: date_created - description: Date record in table was created - - name: staticfullname + - name: utc_created_at + description: Date record in table was created, normalized to UTC + - name: static_full_name description: Name of the code. Codes can apply to many items in Votebuilder, like Organizations, Events and Locations. Not to be confused with Activist Codes, which only apply to individuals. They give users a lot more flexibility to create custom groupings of information, and are similar to hashtags in that respect. - - name: codetypeid + - name: code_type_id description: Source Code vs Tag - - name: codedescription - description: System stores description of code for reference purposes - - name: contact_type_id - description: Unique ID of Type of Contact - - name: campaignid - description: Unique ID of Campain (if applied) + - name: code_type + description: '{{ doc("bonterra_missing") }}' + - name: utc_modified_at + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: vendor + description: '{{ doc("vendor") }}' + - name: vendor_unique_stg_ngpvan__code_id + description: '{{ doc("vendor_unique_id") }}' + - name: database_mode + description: '{{ doc("database_mode") }}' + - name: is_myvoters + description: '{{ doc("is_myvoters") }}' + - name: segmented_code_id + description: Combination of segment by and code id - name: stg_ngpvan__committees description: '{{ doc("bonterra_missing") }}' tests: @@ -198,6 +220,8 @@ models: description: '{{ doc("bonterra_missing") }}' - name: master_committee_id description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema @@ -205,7 +229,7 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: "" + description: '{{ doc("segment_by") }}' - name: vendor description: '{{ doc("vendor") }}' - name: segment_by_key @@ -233,12 +257,14 @@ models: description: '{{ doc("source_schema") }}' - name: source_table description: '{{ doc("source_table") }}' - - name: segment_by - description: "" - name: vendor description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__contact_type_id description: '{{ doc("vendor_unique_id") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' - name: stg_ngpvan__contacts_activist_codes description: '{{ doc("bonterra_missing") }}' tests: @@ -248,6 +274,8 @@ models: tags: - ngpvan_core columns: + - name: van_state_code + description: '{{ doc("bonterra_missing") }}' - name: contacts_activist_code_id description: Unique ID for Activist Code applied to Specific Contact - name: van_id @@ -264,13 +292,11 @@ models: description: The ID of how the record was entered - name: input_type description: '{{ doc("bonterra_missing") }}' - - name: contact_method_type_id - description: '{{ doc("bonterra_missing") }}' - name: contact_method_type description: '{{ doc("bonterra_missing") }}' - - name: contact_type_id + - name: contact_method_type_id description: Unique ID of Type of Contact - - name: canvased_by_username + - name: canvassed_by_username description: Name of User who created record - name: canvassed_by_user_id description: User ID of canvasser @@ -280,7 +306,7 @@ models: description: Unique ID of Content (if applied) - name: utc_modified_at description: '{{ doc("bonterra_missing") }}' - - name: contacts_contacts_id + - name: contacts_contact_id description: Unique ID for Contact History - name: utc_canvassed_at description: Date contact was solicited or canvassed, normalized to UTC @@ -291,11 +317,15 @@ models: - name: source_table description: '{{ doc("source_table") }}' - name: segment_by - description: "" + description: '{{ doc("segment_by") }}' - name: vendor description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__contacts_activist_code_id description: '{{ doc("vendor_unique_id") }}' + - name: database_mode + description: '{{ doc("database_mode") }}' + - name: is_myvoters + description: '{{ doc("is_myvoters") }}' - name: stg_ngpvan__contacts_codes description: '{{ doc("bonterra_missing") }}' tests: @@ -338,7 +368,7 @@ models: - name: is_myvoters description: '{{ doc("is_myvoters") }}' - name: segment_by - description: "" + description: '{{ doc("segment_by") }}' - name: unique_contacts_code_id description: Unique key constructed by combining segment by, state code, and contacts code id - name: segmented_contacts_code_id @@ -349,6 +379,10 @@ models: description: '{{ doc("vendor") }}' - name: segment_by_key description: '{{ doc("segment_by_key") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: vendor_unique_stg_ngpvan__contacts_code_id + description: '{{ doc("vendor_unique_id") }}' - name: stg_ngpvan__contacts_contacts description: '{{ doc("bonterra_missing") }}' tests: @@ -360,7 +394,7 @@ models: columns: - name: van_state_code description: '{{ doc("bonterra_missing") }}' - - name: contacts_contacts_id + - name: contacts_contact_id description: Unique ID for Contact History - name: van_id description: @@ -384,7 +418,7 @@ models: description: Unique ID of Type of Contact - name: contact_method_type description: '{{ doc("bonterra_missing") }}' - - name: canvassed_by_user_name + - name: canvassed_by_username description: Name of User who created record - name: canvassed_by_user_id description: User ID of canvasser @@ -413,7 +447,7 @@ models: - name: is_myvoters description: '{{ doc("is_myvoters") }}' - name: segment_by - description: "" + description: '{{ doc("segment_by") }}' - name: unique_contacts_contact_id description: Unique key constructed by combining segment by, state code, and contacts contact id - name: segmented_contacts_contact_id @@ -426,6 +460,8 @@ models: description: '{{ doc("segment_by_key") }}' - name: vendor_unique_stg_ngpvan__contacts_contact_id description: '{{ doc("vendor_unique_id") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' - name: stg_ngpvan__contacts_notes description: '{{ doc("bonterra_missing") }}' tests: @@ -461,7 +497,7 @@ models: description: '{{ doc("bonterra_missing") }}' - name: datefollowupcompleted description: '{{ doc("bonterra_missing") }}' - - name: contacts_contacts_id + - name: contacts_contact_id description: Unique ID for Contact History - name: utc_modified_at description: @@ -472,7 +508,35 @@ models: - name: contacts_online_form_id description: '{{ doc("bonterra_missing") }}' - name: contacts_story_id - + description: '{{ doc("bonterra_missing") }}' + - name: note_category_id + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: database_mode + description: '{{ doc("database_mode") }}' + - name: is_myvoters + description: '{{ doc("is_myvoters") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: unique_contacts_contact_id + description: Unique key constructed by combining segment by, state code, and contacts contact id + - name: segmented_contacts_contact_id + description: Combination of segment by and contacts contact id + - name: segmented_van_id + description: Combination of segment by and van id + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: vendor_unique_stg_ngpvan__contacts_note_id + description: '{{ doc("vendor_unique_id") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' - name: stg_ngpvan__contacts_survey_responses description: '{{ doc("bonterra_missing") }}' tests: @@ -482,6 +546,8 @@ models: tags: - ngpvan_core columns: + - name: van_state_code + description: '{{ doc("bonterra_missing") }}' - name: contacts_survey_response_id description: Unique ID for Survey Response applied to Contact - name: van_id @@ -492,30 +558,62 @@ models: description: Unique ID of Survey Question - link to Survey Question - name: survey_response_id description: Unique ID of Survey Response - link to Survey Response - - name: contact_type_id + - name: contact_method_type_id description: Unique ID of Type of Contact + - name: contact_method_type + description: '{{ doc("bonterra_missing") }}' - name: input_type_id description: The ID of how the record was entered - - name: date_canvassed - description: Date contact was solicited or canvassed + - name: input_type + description: '{{ doc("bonterra_missing") }}' + - name: utc_canvassed_at + description: Date contact was solicited or canvassed, normalized to UTC - name: committee_id description: Committee in which record lives - - name: canvassed_by + - name: canvassed_by_user_id description: User ID of canvasser - - name: user_name + - name: canvassed_by_username description: Name of User who created record - - name: created_by - description: User ID of who/what created record (person or automated process) - - name: date_created - description: Date record in table was created - - name: person_committee_id - description: Committee that owns the person record - - name: contacts_contacts_id + - name: utc_created_at + description: Date record in table was created, normalized to UTC + - name: contacts_contact_id description: Unique ID for Contact History - name: campaign_id description: Unique ID of Campain (if applied) - name: content_id description: Unique ID of Content (if applied) + - name: utc_modified_at + description: '{{ doc("bonterra_missing") }}' + - name: team_id + description: '{{ doc("bonterra_missing") }}' + - name: division_id + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: database_mode + description: '{{ doc("database_mode") }}' + - name: is_myvoters + description: '{{ doc("is_myvoters") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: unique_contacts_contact_id + description: Unique key constructed by combining segment by, state code, and contacts contact id + - name: segmented_contacts_contact_id + description: Combination of segment by and contacts contact id + - name: segmented_van_id + description: Combination of segment by and van id + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: vendor_unique_stg_ngpvan__contacts_survey_response_id + description: '{{ doc("vendor_unique_id") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' - name: stg_ngpvan__input_types description: '{{ doc("bonterra_missing") }}' tests: @@ -532,19 +630,19 @@ models: - name: input_type description: '{{ doc("bonterra_missing") }}' - name: _avvan_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("dbt_source_relation") }}' - name: source_schema - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_schema") }}' - name: source_table - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by") }}' - name: vendor - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__input_type_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor_unique_id") }}' - name: stg_ngpvan__public_users description: '{{ doc("bonterra_missing") }}' tests: @@ -565,21 +663,21 @@ models: - name: created_at description: '{{ doc("bonterra_missing") }}' - name: _avvan_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("dbt_source_relation") }}' - name: source_schema - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_schema`") }}' - name: source_table - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by") }}' - name: vendor - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor") }}' - name: segment_by_key - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by_key") }}' - name: vendor_unique_stg_ngpvan__public_user_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor_unique_id") }}' - name: stg_ngpvan__results description: '{{ doc("bonterra_missing") }}' tests: @@ -598,19 +696,19 @@ models: - name: result_description description: '{{ doc("bonterra_missing") }}' - name: _avvan_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("dbt_source_relation") }}' - name: source_schema - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_schema") }}' - name: source_table - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by") }}' - name: vendor - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor") }}' - name: vendor_unique_stg_ngpvan__result_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor_unique_id") }}' - name: stg_ngpvan__survey_questions description: '{{ doc("bonterra_missing") }}' tests: @@ -641,23 +739,23 @@ models: - name: is_archived description: '{{ doc("bonterra_missing") }}' - name: _avvan_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("dbt_source_relation") }}' - name: source_schema - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_schema") }}' - name: source_table - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by") }}' - name: segmented_survey_question_id - description: '{{ doc("bonterra_missing") }}' + description: Combination of segment by and survey question id - name: vendor - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor") }}' - name: segment_by_key - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by_key") }}' - name: vendor_unique_stg_ngpvan__survey_question_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor_unique_id") }}' - name: stg_ngpvan__survey_responses description: '{{ doc("bonterra_missing") }}' tests: @@ -684,25 +782,25 @@ models: - name: committee_id description: '{{ doc("bonterra_missing") }}' - name: _avvan_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("dbt_source_relation") }}' - name: source_schema - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_schema") }}' - name: source_table - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("source_table") }}' - name: segment_by - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by") }}' - name: segmented_survey_response_id - description: '{{ doc("bonterra_missing") }}' + description: Combination of segment by and survey response id - name: segmented_survey_question_id - description: '{{ doc("bonterra_missing") }}' + description: Combination of segment by and survey question id - name: vendor - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor") }}' - name: segment_by_key - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("segment_by_key") }}' - name: vendor_unique_stg_ngpvan__survey_response_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("vendor_unique_id") }}' - name: stg_ngpvan__users config: From db21adff36120028b14ae600dc5b0f791984861d Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:33:01 -0500 Subject: [PATCH 18/24] fix doc name --- documentation_blocks/segment_by.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation_blocks/segment_by.md b/documentation_blocks/segment_by.md index 16d0465..6867395 100644 --- a/documentation_blocks/segment_by.md +++ b/documentation_blocks/segment_by.md @@ -1,3 +1,3 @@ -{% docs source_schema %} +{% docs segment_by %} The value used for segmentation {% enddocs %} \ No newline at end of file From ccdcb0b610cc861599619896ece8b4c46ab4b4f1 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:34:25 -0500 Subject: [PATCH 19/24] for christs sake --- models/staging/ngpvan/_stg_ngpvan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index e64a7b9..705f899 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -667,7 +667,7 @@ models: - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema - description: '{{ doc("source_schema`") }}' + description: '{{ doc("source_schema") }}' - name: source_table description: '{{ doc("source_table") }}' - name: segment_by From dcd6488088632f6ed6584e81bce51d8cc0eae807 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Fri, 15 Aug 2025 10:06:13 -0500 Subject: [PATCH 20/24] add missing descriptions --- models/staging/ngpvan/_stg_ngpvan.yaml | 34 +++++++++++++++++++------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index 705f899..86d4e23 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -149,6 +149,8 @@ models: description: Code Name in VAN - name: committee_id description: Committee in which record lives + - name: committee_name + description: '{{ doc("bonterra_missing") }}' - name: is_active description: A Flag for if the activity is active, converted to boolean - name: is_archived @@ -193,6 +195,8 @@ models: description: '{{ doc("is_myvoters") }}' - name: segmented_code_id description: Combination of segment by and code id + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' - name: stg_ngpvan__committees description: '{{ doc("bonterra_missing") }}' tests: @@ -326,6 +330,18 @@ models: description: '{{ doc("database_mode") }}' - name: is_myvoters description: '{{ doc("is_myvoters") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: unique_contacts_activist_code_id + description: Combination of segment by, state code, and contacts activist code id + - name: segmented_contacts_activist_code_id + description: Combination of segment by and contacts activist code id + - name: segmented_activist_code_id + description: Combination of segment by and activist code id + - name: segmented_van_id + description: Combination of segment by and van id + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' - name: stg_ngpvan__contacts_codes description: '{{ doc("bonterra_missing") }}' tests: @@ -523,10 +539,10 @@ models: description: '{{ doc("is_myvoters") }}' - name: segment_by description: '{{ doc("segment_by") }}' - - name: unique_contacts_contact_id - description: Unique key constructed by combining segment by, state code, and contacts contact id - - name: segmented_contacts_contact_id - description: Combination of segment by and contacts contact id + - name: unique_contacts_note_id + description: Unique key constructed by combining segment by, state code, and contacts note id + - name: segmented_contacts_note_id + description: Combination of segment by and contacts note id - name: segmented_van_id description: Combination of segment by and van id - name: vendor @@ -600,10 +616,10 @@ models: description: '{{ doc("is_myvoters") }}' - name: segment_by description: '{{ doc("segment_by") }}' - - name: unique_contacts_contact_id - description: Unique key constructed by combining segment by, state code, and contacts contact id - - name: segmented_contacts_contact_id - description: Combination of segment by and contacts contact id + - name: unique_contacts_survey_response_id + description: Unique key constructed by combining segment by, state code, and contacts survey response id + - name: segmented_contacts_survey_response_id + description: Combination of segment by and contacts survey response id - name: segmented_van_id description: Combination of segment by and van id - name: vendor @@ -654,7 +670,7 @@ models: columns: - name: public_user_id description: '{{ doc("bonterra_missing") }}' - - name: public_user_name + - name: public_username description: '{{ doc("bonterra_missing") }}' - name: van_id description: '{{ doc("bonterra_missing") }}' From 4eb28a8594272e1824d221f67c9e46cacff856fa Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Tue, 19 Aug 2025 10:31:27 -0500 Subject: [PATCH 21/24] add more defs --- .../bonterra_definitions/code_id.md | 3 + .../bonterra_definitions/committee_id.md | 3 + models/staging/ngpvan/_stg_ngpvan.yaml | 82 +++++++++---------- 3 files changed, 47 insertions(+), 41 deletions(-) create mode 100644 documentation_blocks/bonterra_definitions/code_id.md create mode 100644 documentation_blocks/bonterra_definitions/committee_id.md diff --git a/documentation_blocks/bonterra_definitions/code_id.md b/documentation_blocks/bonterra_definitions/code_id.md new file mode 100644 index 0000000..cb07ffe --- /dev/null +++ b/documentation_blocks/bonterra_definitions/code_id.md @@ -0,0 +1,3 @@ +{% docs code_id %} +Unique ID for Code +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/bonterra_definitions/committee_id.md b/documentation_blocks/bonterra_definitions/committee_id.md new file mode 100644 index 0000000..1bb5ede --- /dev/null +++ b/documentation_blocks/bonterra_definitions/committee_id.md @@ -0,0 +1,3 @@ +{% docs committee_id %} +Committee in which record lives +{% enddocs %} \ No newline at end of file diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index 86d4e23..65df16a 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -13,7 +13,7 @@ models: - name: vendor_unique_stg_ngpvan__activist_code_id description: '{{ doc("vendor_unique_id") }}' - name: activist_code_id - description: '{{ doc("bonterra_missing") }}' + description: Distinct ID belonging to Activist Code - name: van_state_id description: '{{ doc("bonterra_missing") }}' - name: activist_code_type @@ -29,13 +29,13 @@ models: - name: independent_points description: '{{ doc("bonterra_missing") }}' - name: committee_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("committee_id") }}' - name: campaign_id - description: '{{ doc("bonterra_missing") }}' + description: Unique ID of Campaign (if applied) - name: segment_by description: '{{ doc("bonterra_missing") }}' - name: segmented_activist_code_id - description: '{{ doc("bonterra_missing") }}' + description: Combination of segment by and activist code id - name: vendor description: '{{ doc("vendor") }}' - name: segment_by_key @@ -45,9 +45,9 @@ models: - name: action_type_id description: '{{ doc("bonterra_missing") }}' - name: is_active - description: '{{ doc("bonterra_missing") }}' + description: A Flag for if the activity is active, converted to boolean - name: is_archived - description: '{{ doc("bonterra_missing") }}' + description: A Flag for if the activity is archived, derived from the raw isactive value - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema @@ -66,25 +66,25 @@ models: - ngpvan_core columns: - name: campaign_id - description: '{{ doc("bonterra_missing") }}' + description: Unique ID of Campaign (if applied) - name: campaign_name description: '{{ doc("bonterra_missing") }}' - name: campaign_description description: '{{ doc("bonterra_missing") }}' - name: committee_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("committee_id") }}' - name: campaign_type_id description: '{{ doc("bonterra_missing") }}' - name: campaign_type description: '{{ doc("bonterra_missing") }}' - name: created_at - description: '{{ doc("bonterra_missing") }}' + description: Date record in table was created, normalized to UTC - name: created_by_user_id - description: '{{ doc("bonterra_missing") }}' + description: User ID of who/what created record (person or automated process) - name: is_active - description: '{{ doc("bonterra_missing") }}' + description: A Flag for if the activity is active, converted to boolean - name: is_archived - description: '{{ doc("bonterra_missing") }}' + description: A Flag for if the activity is archived, derived from the raw isactive value - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema @@ -115,7 +115,7 @@ models: - ngpvan_lookup columns: - name: code_type_id - description: '{{ doc("bonterra_missing") }}' + description: Source Code vs Tag - name: code_type description: '{{ doc("bonterra_missing") }}' - name: _dbt_source_relation @@ -144,11 +144,11 @@ models: - name: van_state_code description: '{{ doc("bonterra_missing") }}' - name: code_id - description: Unique ID for Code + description: '{{ doc("code_id") }}' - name: code_name description: Code Name in VAN - name: committee_id - description: Committee in which record lives + description: '{{ doc("committee_id") }}' - name: committee_name description: '{{ doc("bonterra_missing") }}' - name: is_active @@ -207,7 +207,7 @@ models: - ngpvan_core columns: - name: committee_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("committee_id") }}' - name: van_state_id description: '{{ doc("bonterra_missing") }}' - name: committee_name @@ -217,9 +217,9 @@ models: - name: committee_type description: '{{ doc("bonterra_missing") }}' - name: is_active - description: '{{ doc("bonterra_missing") }}' + description: A Flag for if the activity is active, converted to boolean - name: is_archived - description: '{{ doc("bonterra_missing") }}' + description: A Flag for if the activity is archived, derived from the raw isactive value - name: parent_committee_id description: '{{ doc("bonterra_missing") }}' - name: master_committee_id @@ -252,7 +252,7 @@ models: - ngpvan_lookup columns: - name: contact_method_type_id - description: '{{ doc("bonterra_missing") }}' + description: Unique ID of Type of Contact - name: contact_method_type description: '{{ doc("bonterra_missing") }}' - name: _dbt_source_relation @@ -289,7 +289,7 @@ models: - name: activist_code_id description: Distinct ID belonging to Activist Code - name: committee_id - description: Committee in which record lives + description: '{{ doc("committee_id") }}' - name: utc_created_at description: Date record in table was created, normalized to UTC - name: input_type_id @@ -305,7 +305,7 @@ models: - name: canvassed_by_user_id description: User ID of canvasser - name: campaign_id - description: Unique ID of Campain (if applied) + description: Unique ID of Campaign (if applied) - name: content_id description: Unique ID of Content (if applied) - name: utc_modified_at @@ -360,9 +360,9 @@ models: Distinct VAN ID belonging to contact record. My C IDs begin with 100 and are Distinct to committee and My V IDs are unique to voter file state. - name: code_id - description: Unique ID for Code + description: '{{ doc("code_id") }}' - name: committee_id - description: Committee in which record lives + description: '{{ doc("committee_id") }}' - name: utc_created_at description: Date record in table was created, normalized to UTC - name: input_type_id @@ -372,7 +372,7 @@ models: - name: utc_modified_at description: '{{ doc("bonterra_missing") }}' - name: created_by_user_id - description: '{{ doc("bonterra_missing") }}' + description: User ID of who/what created record (person or automated process) - name: _dbt_source_relation description: '{{ doc("dbt_source_relation") }}' - name: source_schema @@ -421,7 +421,7 @@ models: - name: result_name description: '{{ doc("bonterra_missing") }}' - name: committee_id - description: Committee in which record lives + description: '{{ doc("committee_id") }}' - name: utc_created_at description: Date record in table was created, normalized to UTC - name: utc_canvassed_at @@ -439,7 +439,7 @@ models: - name: canvassed_by_user_id description: User ID of canvasser - name: campaign_id - description: Unique ID of Campain (if applied) + description: Unique ID of Campaign (if applied) - name: content_id description: Unique ID of Content (if applied) - name: created_by_user_id @@ -500,7 +500,7 @@ models: - name: utc_created_at description: Date record in table was created, normalized to UTC - name: committee_id - description: Committee in which record lives + description: '{{ doc("committee_id") }}' - name: suppressed_by_user_id description: '{{ doc("bonterra_missing") }}' - name: utc_suppressed_at @@ -585,7 +585,7 @@ models: - name: utc_canvassed_at description: Date contact was solicited or canvassed, normalized to UTC - name: committee_id - description: Committee in which record lives + description: '{{ doc("committee_id") }}' - name: canvassed_by_user_id description: User ID of canvasser - name: canvassed_by_username @@ -595,7 +595,7 @@ models: - name: contacts_contact_id description: Unique ID for Contact History - name: campaign_id - description: Unique ID of Campain (if applied) + description: Unique ID of Campaign (if applied) - name: content_id description: Unique ID of Content (if applied) - name: utc_modified_at @@ -642,7 +642,7 @@ models: - ngpvan_lookup columns: - name: input_type_id - description: '{{ doc("bonterra_missing") }}' + description: The ID of how the record was entered - name: input_type description: '{{ doc("bonterra_missing") }}' - name: _avvan_source_relation @@ -673,11 +673,11 @@ models: - name: public_username description: '{{ doc("bonterra_missing") }}' - name: van_id - description: '{{ doc("bonterra_missing") }}' + description: Distinct VAN ID belonging to contact record. My C IDs begin with 100 and are Distinct to committee and My V IDs are unique to voter file state. - name: committee_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("committee_id") }}' - name: created_at - description: '{{ doc("bonterra_missing") }}' + description: Date record in table was created, normalized to UTC - name: _avvan_source_relation description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation @@ -706,7 +706,7 @@ models: - ngpvan_lookup columns: - name: result_id - description: '{{ doc("bonterra_missing") }}' + description: Result of contact interaction - name: result_name description: '{{ doc("bonterra_missing") }}' - name: result_description @@ -735,7 +735,7 @@ models: - ngpvan_core columns: - name: survey_question_id - description: '{{ doc("bonterra_missing") }}' + description: Unique ID of Survey Question - link to Survey Question - name: van_state_id description: '{{ doc("bonterra_missing") }}' - name: election_cycle @@ -749,11 +749,11 @@ models: - name: master_survey_question_id description: '{{ doc("bonterra_missing") }}' - name: committee_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("committee_id") }}' - name: is_active - description: '{{ doc("bonterra_missing") }}' + description: A Flag for if the activity is active, converted to boolean - name: is_archived - description: '{{ doc("bonterra_missing") }}' + description: A Flag for if the activity is archived, derived from the raw isactive value - name: _avvan_source_relation description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation @@ -782,9 +782,9 @@ models: - ngpvan_core columns: - name: survey_question_id - description: '{{ doc("bonterra_missing") }}' + description: Unique ID of Survey Question - link to Survey Question - name: survey_response_id - description: '{{ doc("bonterra_missing") }}' + description: Unique ID of Survey Response - link to Survey Response - name: survey_response description: '{{ doc("bonterra_missing") }}' - name: democrat_points @@ -796,7 +796,7 @@ models: - name: master_survey_response_id description: '{{ doc("bonterra_missing") }}' - name: committee_id - description: '{{ doc("bonterra_missing") }}' + description: '{{ doc("committee_id") }}' - name: _avvan_source_relation description: '{{ doc("avvan_source_relation") }}' - name: _dbt_source_relation From fd3ec421889cdd29a855cd4417106726ca9d6ff4 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Mon, 25 Aug 2025 10:15:37 -0500 Subject: [PATCH 22/24] fix a couple descriptions --- .../bonterra_definitions/code_id.md | 4 +- models/staging/ngpvan/_stg_ngpvan.yaml | 45 +++++++++++++++---- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/documentation_blocks/bonterra_definitions/code_id.md b/documentation_blocks/bonterra_definitions/code_id.md index cb07ffe..5d31ed0 100644 --- a/documentation_blocks/bonterra_definitions/code_id.md +++ b/documentation_blocks/bonterra_definitions/code_id.md @@ -1,3 +1,5 @@ {% docs code_id %} -Unique ID for Code +Unique ID for Code. Note - this ID may not be unique across committees. +If you receive data from multiple committees, it's safer to the use the +"vendor_unique" column in the table. {% enddocs %} \ No newline at end of file diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index 65df16a..493a8a2 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -31,7 +31,10 @@ models: - name: committee_id description: '{{ doc("committee_id") }}' - name: campaign_id - description: Unique ID of Campaign (if applied) + description: | + Unique ID of Campaign (if applied). Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. - name: segment_by description: '{{ doc("bonterra_missing") }}' - name: segmented_activist_code_id @@ -66,7 +69,10 @@ models: - ngpvan_core columns: - name: campaign_id - description: Unique ID of Campaign (if applied) + description: | + Unique ID of Campaign (if applied). Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. - name: campaign_name description: '{{ doc("bonterra_missing") }}' - name: campaign_description @@ -252,7 +258,10 @@ models: - ngpvan_lookup columns: - name: contact_method_type_id - description: Unique ID of Type of Contact + description: | + Unique ID of Type of Contact. Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. - name: contact_method_type description: '{{ doc("bonterra_missing") }}' - name: _dbt_source_relation @@ -299,13 +308,19 @@ models: - name: contact_method_type description: '{{ doc("bonterra_missing") }}' - name: contact_method_type_id - description: Unique ID of Type of Contact + description: | + Unique ID of Type of Contact. Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. - name: canvassed_by_username description: Name of User who created record - name: canvassed_by_user_id description: User ID of canvasser - name: campaign_id - description: Unique ID of Campaign (if applied) + description: | + Unique ID of Campaign (if applied). Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. - name: content_id description: Unique ID of Content (if applied) - name: utc_modified_at @@ -431,7 +446,10 @@ models: - name: input_type description: '{{ doc("bonterra_missing") }}' - name: contact_method_type_id - description: Unique ID of Type of Contact + description: | + Unique ID of Type of Contact. Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. - name: contact_method_type description: '{{ doc("bonterra_missing") }}' - name: canvassed_by_username @@ -439,7 +457,10 @@ models: - name: canvassed_by_user_id description: User ID of canvasser - name: campaign_id - description: Unique ID of Campaign (if applied) + description: | + Unique ID of Campaign (if applied). Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. - name: content_id description: Unique ID of Content (if applied) - name: created_by_user_id @@ -575,7 +596,10 @@ models: - name: survey_response_id description: Unique ID of Survey Response - link to Survey Response - name: contact_method_type_id - description: Unique ID of Type of Contact + description: | + Unique ID of Type of Contact. Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. - name: contact_method_type description: '{{ doc("bonterra_missing") }}' - name: input_type_id @@ -595,7 +619,10 @@ models: - name: contacts_contact_id description: Unique ID for Contact History - name: campaign_id - description: Unique ID of Campaign (if applied) + description: | + Unique ID of Campaign (if applied). Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. - name: content_id description: Unique ID of Content (if applied) - name: utc_modified_at From 6c3d24bbb1eaeae2dcce928ab22a260875154191 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Fri, 26 Sep 2025 12:48:46 -0500 Subject: [PATCH 23/24] remove one bad row (#24) --- models/staging/ngpvan/_stg_ngpvan.yaml | 5 +++++ models/staging/ngpvan/core/stg_ngpvan__committees.sql | 2 ++ 2 files changed, 7 insertions(+) diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml index 493a8a2..6e928b6 100644 --- a/models/staging/ngpvan/_stg_ngpvan.yaml +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -246,6 +246,11 @@ models: description: '{{ doc("segment_by_key") }}' - name: vendor_unique_stg_ngpvan__committee_id description: '{{ doc("vendor_unique_id") }}' + tests: + - not_null + - unique: + config: + severity: error - name: stg_ngpvan__contact_types description: '{{ doc("bonterra_missing") }}' tests: diff --git a/models/staging/ngpvan/core/stg_ngpvan__committees.sql b/models/staging/ngpvan/core/stg_ngpvan__committees.sql index 004e343..7cd407b 100644 --- a/models/staging/ngpvan/core/stg_ngpvan__committees.sql +++ b/models/staging/ngpvan/core/stg_ngpvan__committees.sql @@ -33,4 +33,6 @@ WITH SELECT DISTINCT * FROM renamed +-- hard-coding out one bad committee from AV while we work with them to remove from raw +WHERE NOT (committee_id = 15701 AND committee_short_name IS NULL) From 8cfbc7b8fe3025eafecbe1abee4f1cba51b95fe1 Mon Sep 17 00:00:00 2001 From: hannah-marcus28 <103459472+hannah-marcus28@users.noreply.github.com> Date: Wed, 5 Nov 2025 14:37:32 -0500 Subject: [PATCH 24/24] Adding more join keys to intermediate models (#25) * adding more join keys to intermediate models * adding missing segment_by column --- .gitignore | 1 + .../ngpvan/int_ngpvan__01__activist_codes.sql | 21 +++++++++--- .../ngpvan/int_ngpvan__01__admins.sql | 7 ++-- .../int_ngpvan__01__contact_attempts.sql | 16 +++++++--- .../int_ngpvan__01__survey_responses.sql | 32 +++++++++++++------ 5 files changed, 57 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index f242830..0344fcf 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ development.env *.ipynb profiles.yml dbt_project_tmc_rs.yml +dbt_internal_packages/ diff --git a/models/intermediate/ngpvan/int_ngpvan__01__activist_codes.sql b/models/intermediate/ngpvan/int_ngpvan__01__activist_codes.sql index 4fa4670..55e93da 100644 --- a/models/intermediate/ngpvan/int_ngpvan__01__activist_codes.sql +++ b/models/intermediate/ngpvan/int_ngpvan__01__activist_codes.sql @@ -33,14 +33,17 @@ WITH committee_id, committee_name, committee_short_name, - committee_type + committee_type, + segment_by FROM {{ ref("stg_ngpvan__committees") }} ), campaigns AS ( SELECT campaign_id, - campaign_name + campaign_name, + committee_id, + segment_by FROM {{ ref("stg_ngpvan__campaigns") }} ), @@ -88,9 +91,17 @@ WITH {{ ngpvan__int__additional_fields() }} FROM contacts - LEFT JOIN codes USING (activist_code_id) - LEFT JOIN committees USING (committee_id) - LEFT JOIN campaigns USING (campaign_id) + LEFT JOIN codes + ON contacts.activist_code_id = codes.activist_code_id + AND contacts.committee_id = codes.activist_code_committee_id + AND contacts.segment_by = codes.segment_by + LEFT JOIN committees + ON contacts.committee_id = committees.committee_id + AND contacts.segment_by = committees.segment_by + LEFT JOIN campaigns + ON contacts.campaign_id = campaigns.campaign_id + AND contacts.committee_id = campaigns.committee_id + AND contacts.segment_by = campaigns.segment_by ) SELECT * FROM activist_codes \ No newline at end of file diff --git a/models/intermediate/ngpvan/int_ngpvan__01__admins.sql b/models/intermediate/ngpvan/int_ngpvan__01__admins.sql index cc0d496..b5fa18f 100644 --- a/models/intermediate/ngpvan/int_ngpvan__01__admins.sql +++ b/models/intermediate/ngpvan/int_ngpvan__01__admins.sql @@ -29,7 +29,8 @@ WITH SELECT user_id, user_group_id, - user_group_name + user_group_name, + segment_by FROM {{ ref("stg_ngpvan__users_user_groups") }} ), @@ -95,7 +96,9 @@ WITH {{- ngpvan__int__additional_fields() }} FROM all_users users - LEFT JOIN user_groups USING (user_id) + LEFT JOIN user_groups + ON users.user_id = user_groups.user_id + AND users.segment_by = user_groups.segment_by GROUP BY users.user_id, users.username, diff --git a/models/intermediate/ngpvan/int_ngpvan__01__contact_attempts.sql b/models/intermediate/ngpvan/int_ngpvan__01__contact_attempts.sql index 9737b5b..c33124d 100644 --- a/models/intermediate/ngpvan/int_ngpvan__01__contact_attempts.sql +++ b/models/intermediate/ngpvan/int_ngpvan__01__contact_attempts.sql @@ -9,14 +9,17 @@ WITH committee_id, committee_name, committee_short_name, - committee_type + committee_type, + segment_by FROM {{ ref("stg_ngpvan__committees") }} ), campaigns AS ( SELECT campaign_id, - campaign_name + campaign_name, + committee_id, + segment_by FROM {{ ref("stg_ngpvan__campaigns") }} ), @@ -61,8 +64,13 @@ WITH {{ ngpvan__int__additional_fields() }} FROM contacts - LEFT JOIN committees USING (committee_id) - LEFT JOIN campaigns USING (campaign_id) + LEFT JOIN committees + ON contacts.committee_id = committees.committee_id + AND contacts.segment_by = committees.segment_by + LEFT JOIN campaigns + ON contacts.campaign_id = campaigns.campaign_id + AND contacts.committee_id = campaigns.committee_id + AND contacts.segment_by = campaigns.segment_by ) SELECT * FROM contact_attempts \ No newline at end of file diff --git a/models/intermediate/ngpvan/int_ngpvan__01__survey_responses.sql b/models/intermediate/ngpvan/int_ngpvan__01__survey_responses.sql index 8e9414b..b6d6d43 100644 --- a/models/intermediate/ngpvan/int_ngpvan__01__survey_responses.sql +++ b/models/intermediate/ngpvan/int_ngpvan__01__survey_responses.sql @@ -32,7 +32,8 @@ WITH master_survey_question_id, committee_id AS survey_question_committee_id, is_active, - is_archived + is_archived, + segment_by FROM {{ ref("stg_ngpvan__survey_questions") }} ), @@ -41,14 +42,17 @@ WITH committee_id, committee_name, committee_short_name, - committee_type + committee_type, + segment_by FROM {{ ref("stg_ngpvan__committees") }} ), campaigns AS ( SELECT campaign_id, - campaign_name + campaign_name, + committee_id, + segment_by FROM {{ ref("stg_ngpvan__campaigns") }} ), @@ -100,13 +104,23 @@ WITH contacts.source_table, contacts.segment_by {{ ngpvan__int__additional_fields() }} - - FROM contacts - LEFT JOIN responses USING (survey_question_id, survey_response_id) - LEFT JOIN questions USING (survey_question_id) - LEFT JOIN committees ON (committees.committee_id = COALESCE(contacts.committee_id, questions.survey_question_committee_id, responses.survey_response_committee_id)) - LEFT JOIN campaigns USING (campaign_id) + LEFT JOIN responses + ON contacts.survey_question_id = responses.survey_question_id + AND contacts.survey_response_id = responses.survey_response_id + AND contacts.committee_id = responses.survey_response_committee_id + AND contacts.segment_by = responses.segment_by + LEFT JOIN questions + ON contacts.survey_question_id = questions.survey_question_id + AND contacts.committee_id = questions.survey_question_committee_id + AND contacts.segment_by = questions.segment_by + LEFT JOIN committees + ON (committees.committee_id = COALESCE(contacts.committee_id, questions.survey_question_committee_id, responses.survey_response_committee_id)) + AND committees.segment_by = contacts.segment_by + LEFT JOIN campaigns + ON contacts.campaign_id = campaigns.campaign_id + AND contacts.committee_id = campaigns.committee_id + AND contacts.segment_by = campaigns.segment_by ) SELECT * FROM survey_responses