diff --git a/.gitignore b/.gitignore index f242830..0344fcf 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ development.env *.ipynb profiles.yml dbt_project_tmc_rs.yml +dbt_internal_packages/ diff --git a/dbt_project.yml b/dbt_project.yml index fd6762a..565e50e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,8 +1,8 @@ -name: 'dbt_ngpvan' -version: '1.0.0' +name: "dbt_ngpvan" +version: "1.0.0" config-version: 2 -profile: 'dbt_ngpvan' +profile: "dbt_ngpvan" model-paths: ["models"] analysis-paths: ["analyses"] @@ -10,6 +10,7 @@ test-paths: ["tests"] seed-paths: ["seeds"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] +docs-paths: ["documentation_blocks"] target-path: "target" clean-targets: @@ -29,7 +30,7 @@ models: +materialized: table tests: - +severity: warn # all tests + +severity: warn # all tests vars: dbt_ngpvan_config: @@ -60,4 +61,4 @@ vars: lookup_tables: true additional_fields: [] enable_incremental_models: true - ngpvan_default_incremental_window_days: 3 \ No newline at end of file + ngpvan_default_incremental_window_days: 3 diff --git a/documentation_blocks/avvan_source_relation.md b/documentation_blocks/avvan_source_relation.md new file mode 100644 index 0000000..1937e8b --- /dev/null +++ b/documentation_blocks/avvan_source_relation.md @@ -0,0 +1,3 @@ +{% docs avvan_source_relation %} +Concatenated string of America Votes source database, schema and table. Null for values unrelated to America Votes data +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/bonterra_definitions/code_id.md b/documentation_blocks/bonterra_definitions/code_id.md new file mode 100644 index 0000000..5d31ed0 --- /dev/null +++ b/documentation_blocks/bonterra_definitions/code_id.md @@ -0,0 +1,5 @@ +{% docs code_id %} +Unique ID for Code. Note - this ID may not be unique across committees. +If you receive data from multiple committees, it's safer to the use the +"vendor_unique" column in the table. +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/bonterra_definitions/committee_id.md b/documentation_blocks/bonterra_definitions/committee_id.md new file mode 100644 index 0000000..1bb5ede --- /dev/null +++ b/documentation_blocks/bonterra_definitions/committee_id.md @@ -0,0 +1,3 @@ +{% docs committee_id %} +Committee in which record lives +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/bonterra_missing.md b/documentation_blocks/bonterra_missing.md new file mode 100644 index 0000000..1a6a484 --- /dev/null +++ b/documentation_blocks/bonterra_missing.md @@ -0,0 +1,3 @@ +{% docs bonterra_missing %} +Documentation for this column is not provided by Bonterra. +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/database_mode.md b/documentation_blocks/database_mode.md new file mode 100644 index 0000000..54a917f --- /dev/null +++ b/documentation_blocks/database_mode.md @@ -0,0 +1,3 @@ +{% docs database_mode %} +Boolean to indicate if this row was synced via database +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/is_myvoters.md b/documentation_blocks/is_myvoters.md new file mode 100644 index 0000000..4ca0d77 --- /dev/null +++ b/documentation_blocks/is_myvoters.md @@ -0,0 +1,3 @@ +{% docs is_myvoters %} +Boolean to indicate if this row was synced via database +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/segment_by.md b/documentation_blocks/segment_by.md new file mode 100644 index 0000000..6867395 --- /dev/null +++ b/documentation_blocks/segment_by.md @@ -0,0 +1,3 @@ +{% docs segment_by %} +The value used for segmentation +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/segment_by_key.md b/documentation_blocks/segment_by_key.md new file mode 100644 index 0000000..ebaa8f8 --- /dev/null +++ b/documentation_blocks/segment_by_key.md @@ -0,0 +1,3 @@ +{% docs segment_by_key %} +Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data ownership and sharing arrangements. +{% enddocs %} diff --git a/documentation_blocks/source_schema.md b/documentation_blocks/source_schema.md new file mode 100644 index 0000000..43d6053 --- /dev/null +++ b/documentation_blocks/source_schema.md @@ -0,0 +1,3 @@ +{% docs source_schema %} +Denotes the schema component of the _dbt_source_relation field +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/source_table.md b/documentation_blocks/source_table.md new file mode 100644 index 0000000..0ee0f6f --- /dev/null +++ b/documentation_blocks/source_table.md @@ -0,0 +1,3 @@ +{% docs source_table %} +Denotes the table component of the _dbt_source_relation field +{% enddocs %} \ No newline at end of file diff --git a/documentation_blocks/vendor.md b/documentation_blocks/vendor.md new file mode 100644 index 0000000..6006061 --- /dev/null +++ b/documentation_blocks/vendor.md @@ -0,0 +1,3 @@ +{% docs vendor %} +Denotes the name of the vendor as it relates to TMC's segmentation processes +{% enddocs %} diff --git a/documentation_blocks/vendor_unique_id.md b/documentation_blocks/vendor_unique_id.md new file mode 100644 index 0000000..e1e6bb2 --- /dev/null +++ b/documentation_blocks/vendor_unique_id.md @@ -0,0 +1,3 @@ +{% docs vendor_unique_id %} +Surrogate primary key uniquely identifying this record across all members data. +{% enddocs %} \ No newline at end of file diff --git a/macros/custom_tests/test_table_fully_documented.sql b/macros/custom_tests/test_table_fully_documented.sql new file mode 100644 index 0000000..0651bc1 --- /dev/null +++ b/macros/custom_tests/test_table_fully_documented.sql @@ -0,0 +1,19 @@ +{% test table_fully_documented(model) %} + {% set table_name = model.identifier %} + SELECT + column_name + FROM + {{ model.database }}.{{ model.schema }}.INFORMATION_SCHEMA.COLUMN_FIELD_PATHS + WHERE + (description IS NULL OR description = '') + AND table_name = '{{ table_name }}' + UNION ALL + SELECT '{{ table_name }}' + FROM UNNEST([1]) AS dummy + WHERE NOT EXISTS ( + SELECT 1 + FROM {{ model.database }}.{{ model.schema }}.INFORMATION_SCHEMA.TABLE_OPTIONS + WHERE option_name = 'description' + AND table_name = '{{ table_name }}' + ) +{% endtest %} \ No newline at end of file diff --git a/models/intermediate/ngpvan/int_ngpvan__01__activist_codes.sql b/models/intermediate/ngpvan/int_ngpvan__01__activist_codes.sql index 4fa4670..55e93da 100644 --- a/models/intermediate/ngpvan/int_ngpvan__01__activist_codes.sql +++ b/models/intermediate/ngpvan/int_ngpvan__01__activist_codes.sql @@ -33,14 +33,17 @@ WITH committee_id, committee_name, committee_short_name, - committee_type + committee_type, + segment_by FROM {{ ref("stg_ngpvan__committees") }} ), campaigns AS ( SELECT campaign_id, - campaign_name + campaign_name, + committee_id, + segment_by FROM {{ ref("stg_ngpvan__campaigns") }} ), @@ -88,9 +91,17 @@ WITH {{ ngpvan__int__additional_fields() }} FROM contacts - LEFT JOIN codes USING (activist_code_id) - LEFT JOIN committees USING (committee_id) - LEFT JOIN campaigns USING (campaign_id) + LEFT JOIN codes + ON contacts.activist_code_id = codes.activist_code_id + AND contacts.committee_id = codes.activist_code_committee_id + AND contacts.segment_by = codes.segment_by + LEFT JOIN committees + ON contacts.committee_id = committees.committee_id + AND contacts.segment_by = committees.segment_by + LEFT JOIN campaigns + ON contacts.campaign_id = campaigns.campaign_id + AND contacts.committee_id = campaigns.committee_id + AND contacts.segment_by = campaigns.segment_by ) SELECT * FROM activist_codes \ No newline at end of file diff --git a/models/intermediate/ngpvan/int_ngpvan__01__admins.sql b/models/intermediate/ngpvan/int_ngpvan__01__admins.sql index cc0d496..b5fa18f 100644 --- a/models/intermediate/ngpvan/int_ngpvan__01__admins.sql +++ b/models/intermediate/ngpvan/int_ngpvan__01__admins.sql @@ -29,7 +29,8 @@ WITH SELECT user_id, user_group_id, - user_group_name + user_group_name, + segment_by FROM {{ ref("stg_ngpvan__users_user_groups") }} ), @@ -95,7 +96,9 @@ WITH {{- ngpvan__int__additional_fields() }} FROM all_users users - LEFT JOIN user_groups USING (user_id) + LEFT JOIN user_groups + ON users.user_id = user_groups.user_id + AND users.segment_by = user_groups.segment_by GROUP BY users.user_id, users.username, diff --git a/models/intermediate/ngpvan/int_ngpvan__01__contact_attempts.sql b/models/intermediate/ngpvan/int_ngpvan__01__contact_attempts.sql index 9737b5b..c33124d 100644 --- a/models/intermediate/ngpvan/int_ngpvan__01__contact_attempts.sql +++ b/models/intermediate/ngpvan/int_ngpvan__01__contact_attempts.sql @@ -9,14 +9,17 @@ WITH committee_id, committee_name, committee_short_name, - committee_type + committee_type, + segment_by FROM {{ ref("stg_ngpvan__committees") }} ), campaigns AS ( SELECT campaign_id, - campaign_name + campaign_name, + committee_id, + segment_by FROM {{ ref("stg_ngpvan__campaigns") }} ), @@ -61,8 +64,13 @@ WITH {{ ngpvan__int__additional_fields() }} FROM contacts - LEFT JOIN committees USING (committee_id) - LEFT JOIN campaigns USING (campaign_id) + LEFT JOIN committees + ON contacts.committee_id = committees.committee_id + AND contacts.segment_by = committees.segment_by + LEFT JOIN campaigns + ON contacts.campaign_id = campaigns.campaign_id + AND contacts.committee_id = campaigns.committee_id + AND contacts.segment_by = campaigns.segment_by ) SELECT * FROM contact_attempts \ No newline at end of file diff --git a/models/intermediate/ngpvan/int_ngpvan__01__survey_responses.sql b/models/intermediate/ngpvan/int_ngpvan__01__survey_responses.sql index 8e9414b..b6d6d43 100644 --- a/models/intermediate/ngpvan/int_ngpvan__01__survey_responses.sql +++ b/models/intermediate/ngpvan/int_ngpvan__01__survey_responses.sql @@ -32,7 +32,8 @@ WITH master_survey_question_id, committee_id AS survey_question_committee_id, is_active, - is_archived + is_archived, + segment_by FROM {{ ref("stg_ngpvan__survey_questions") }} ), @@ -41,14 +42,17 @@ WITH committee_id, committee_name, committee_short_name, - committee_type + committee_type, + segment_by FROM {{ ref("stg_ngpvan__committees") }} ), campaigns AS ( SELECT campaign_id, - campaign_name + campaign_name, + committee_id, + segment_by FROM {{ ref("stg_ngpvan__campaigns") }} ), @@ -100,13 +104,23 @@ WITH contacts.source_table, contacts.segment_by {{ ngpvan__int__additional_fields() }} - - FROM contacts - LEFT JOIN responses USING (survey_question_id, survey_response_id) - LEFT JOIN questions USING (survey_question_id) - LEFT JOIN committees ON (committees.committee_id = COALESCE(contacts.committee_id, questions.survey_question_committee_id, responses.survey_response_committee_id)) - LEFT JOIN campaigns USING (campaign_id) + LEFT JOIN responses + ON contacts.survey_question_id = responses.survey_question_id + AND contacts.survey_response_id = responses.survey_response_id + AND contacts.committee_id = responses.survey_response_committee_id + AND contacts.segment_by = responses.segment_by + LEFT JOIN questions + ON contacts.survey_question_id = questions.survey_question_id + AND contacts.committee_id = questions.survey_question_committee_id + AND contacts.segment_by = questions.segment_by + LEFT JOIN committees + ON (committees.committee_id = COALESCE(contacts.committee_id, questions.survey_question_committee_id, responses.survey_response_committee_id)) + AND committees.segment_by = contacts.segment_by + LEFT JOIN campaigns + ON contacts.campaign_id = campaigns.campaign_id + AND contacts.committee_id = campaigns.committee_id + AND contacts.segment_by = campaigns.segment_by ) SELECT * FROM survey_responses diff --git a/models/staging/ngpvan/_staging_ngpvan_models.yml b/models/staging/ngpvan/_staging_ngpvan_models.yml deleted file mode 100644 index b1f411b..0000000 --- a/models/staging/ngpvan/_staging_ngpvan_models.yml +++ /dev/null @@ -1,585 +0,0 @@ -version: 2 - -models: - - name: stg_ngpvan__activist_codes - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__activist_codes' }}" - tags: ['ngpvan_core'] - columns: - - name: activist_code_id - - name: van_state_id - - name: activist_code_type - - name: activist_code_name - - name: activist_code_description - - name: report_question - - name: democrat_points - - name: republican_points - - name: independent_points - - name: committee_id - - name: action_type_id - - name: campaign_id - - name: is_active - - name: is_archived - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: segmented_activist_code_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__activist_code_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__campaigns - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__campaigns' }}" - tags: ['ngpvan_core'] - columns: - - name: campaign_id - - name: campaign_name - - name: campaign_description - - name: committee_id - - name: campaign_type_id - - name: campaign_type - - name: created_at - - name: created_by_user_id - - name: is_active - - name: is_archived - - name: _avvan_source_relation - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: segmented_campaign_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__campaign_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__codes - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__codes' }}" - tags: ['ngpvan_core'] - columns: - - name: van_state_code - - name: code_id - - name: code_name - - name: committee_id - - name: committee_name - - name: is_active - - name: is_archived - - name: parent_code_id - - name: created_by_user_id - - name: utc_created_at - - name: static_full_name - - name: code_type_id - - name: code_type - - name: utc_modified_at - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: database_mode - - name: is_myvoters - - name: segment_by - - name: segmented_code_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__code_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__code_types - config: - enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__code_types' }}" - tags: ['ngpvan_core', 'ngpvan_lookup'] - columns: - - name: code_type_id - - name: code_type - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__code_type_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique - - - name: stg_ngpvan__committees - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__committees' }}" - tags: ['ngpvan_core'] - columns: - - name: committee_id - - name: van_state_id - - name: committee_name - - name: committee_short_name - - name: committee_type - - name: is_active - - name: is_archived - - name: parent_committee_id - - name: master_committee_id - - name: _avvan_source_relation - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__committee_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__contacts_activist_codes - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_activist_codes' }}" - tags: ['ngpvan_core'] - columns: - - name: van_state_code - - name: contacts_activist_code_id - - name: van_id - - name: activist_code_id - - name: committee_id - - name: utc_created_at - - name: input_type_id - - name: input_type - - name: contact_method_type_id - - name: contact_method_type - - name: canvassed_by_username - - name: canvassed_by_user_id - - name: campaign_id - - name: content_id - - name: utc_modified_at - - name: contacts_contact_id - - name: utc_canvassed_at - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: database_mode - - name: is_myvoters - - name: segment_by - - name: unique_contacts_activist_code_id - - name: segmented_contacts_activist_code_id - - name: segmented_activist_code_id - - name: segmented_van_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__contacts_activist_code_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__contacts_codes - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_codes' }}" - tags: ['ngpvan_core'] - columns: - - name: van_state_code - - name: contacts_code_id - - name: van_id - - name: code_id - - name: committee_id - - name: utc_created_at - - name: input_type_id - - name: input_type - - name: utc_modified_at - - name: created_by_user_id - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: database_mode - - name: is_myvoters - - name: segment_by - - name: unique_contacts_code_id - - name: segmented_contacts_code_id - - name: segmented_van_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__contacts_code_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__contacts_contacts - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_contacts' }}" - tags: ['ngpvan_core'] - columns: - - name: van_state_code - - name: contacts_contact_id - - name: van_id - - name: result_id - - name: result_name - - name: committee_id - - name: utc_created_at - - name: utc_canvassed_at - - name: input_type_id - - name: input_type - - name: contact_method_type_id - - name: contact_method_type - - name: canvassed_by_username - - name: canvassed_by_user_id - - name: campaign_id - - name: content_id - - name: created_by_user_id - - name: contacts_phone_id - - name: team_id - - name: division_id - - name: utc_modified_at - - name: _avvan_source_relation - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: database_mode - - name: is_myvoters - - name: segment_by - - name: unique_contacts_contact_id - - name: segmented_contacts_contact_id - - name: segmented_van_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__contacts_contact_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__contacts_notes - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_notes' }}" - tags: ['ngpvan_core'] - columns: - - name: van_state_code - - name: contacts_note_id - - name: van_id - - name: created_by_user_id - - name: utc_created_at - - name: committee_id - - name: suppressed_by_user_id - - name: utc_suppressed_at - - name: note_text - - name: datefollowup - - name: followup_completed_by_user_id - - name: datefollowupcompleted - - name: contacts_contact_id - - name: utc_modified_at - - name: note_category - - name: contacts_online_form_id - - name: contacts_story_id - - name: note_category_id - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: database_mode - - name: is_myvoters - - name: segment_by - - name: unique_contacts_note_id - - name: segmented_contacts_note_id - - name: segmented_van_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__contacts_note_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__contacts_survey_responses - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_survey_responses' }}" - tags: ['ngpvan_core'] - columns: - - name: van_state_code - - name: contacts_survey_response_id - - name: van_id - - name: contacts_contact_id - - name: survey_question_id - - name: survey_response_id - - name: utc_created_at - - name: utc_canvassed_at - - name: input_type_id - - name: input_type - - name: contact_method_type_id - - name: contact_method_type - - name: committee_id - - name: canvassed_by_username - - name: canvassed_by_user_id - - name: campaign_id - - name: content_id - - name: utc_modified_at - - name: team_id - - name: division_id - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: database_mode - - name: is_myvoters - - name: segment_by - - name: unique_contacts_survey_response_id - - name: segmented_contacts_survey_response_id - - name: segmented_van_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__contacts_survey_response_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 and contacts_survey_response_id is not null - - - name: stg_ngpvan__contact_types - config: - enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contact_types' }}" - tags: ['ngpvan_core', 'ngpvan_lookup'] - columns: - - name: contact_method_type_id - - name: contact_method_type - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__contact_type_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique - - - name: stg_ngpvan__input_types - config: - enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__input_types' }}" - tags: ['ngpvan_core', 'ngpvan_lookup'] - columns: - - name: input_type_id - - name: input_type - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__input_type_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique - - - name: stg_ngpvan__public_users - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__public_users' }}" - - - name: stg_ngpvan__results - config: - enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__results' }}" - tags: ['ngpvan_core', 'ngpvan_lookup'] - columns: - - name: result_id - - name: result_name - - name: result_description - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__result_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique - - - name: stg_ngpvan__survey_questions - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__survey_questions' }}" - tags: ['ngpvan_core'] - columns: - - name: survey_question_id - - name: van_state_id - - name: election_cycle - - name: survey_question_type - - name: survey_question_name - - name: survey_question_text - - name: master_survey_question_id - - name: committee_id - - name: is_active - - name: is_archived - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: segmented_survey_question_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__survey_question_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__survey_responses - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__survey_responses' }}" - tags: ['ngpvan_core'] - columns: - - name: survey_question_id - - name: survey_response_id - - name: survey_response - - name: democrat_points - - name: republican_points - - name: independent_points - - name: master_survey_response_id - - name: committee_id - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: segmented_survey_response_id - - name: segmented_survey_question_id - - name: vendor - - name: segment_by_key - description: "Surrogate key identifying the segment of which this record is a part. Segments are then exposed to members according to data owernship and sharing arrangements." - tests: - - not_null - - name: vendor_unique_stg_ngpvan__survey_response_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__users - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__users' }}" - columns: - - name: user_id - - name: username - - name: first_name - - name: last_name - - name: public_username - - name: address_line_1 - - name: city - - name: state - - name: zip_code - - name: email_address - - name: home_phone - - name: cell_phone - - name: _avvan_source_relation - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__user_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique - - - name: stg_ngpvan__users_committees - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__users_committees' }}" - columns: - - name: user_id, - - name: committee_id, - - name: state_id, - - name: created_by_user_id, - - name: utc_created_at, - - name: utc_modified_at, - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__users_committee_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique: - config: - where: source_schema != 'raw_bonterra__tmc_corrected' and committee_id != 68149 - - - name: stg_ngpvan__users_user_groups - config: - alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__users_user_groups' }}" - columns: - - name: user_id - - name: user_group_id - - name: user_group_name - - name: created_at - - name: created_by_user_id - - name: _avvan_source_relation - - name: _dbt_source_relation - - name: source_schema - - name: source_table - - name: segment_by - - name: vendor - - name: vendor_unique_stg_ngpvan__users_user_group_id - description: "Surrogate primary key uniquely identifying this record across all members data." - tests: - - not_null - - unique \ No newline at end of file diff --git a/models/staging/ngpvan/_stg_ngpvan.yaml b/models/staging/ngpvan/_stg_ngpvan.yaml new file mode 100644 index 0000000..6e928b6 --- /dev/null +++ b/models/staging/ngpvan/_stg_ngpvan.yaml @@ -0,0 +1,859 @@ +version: 2 + +models: + - name: stg_ngpvan__activist_codes + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__activist_codes' }}" + tags: + - ngpvan_core + columns: + - name: vendor_unique_stg_ngpvan__activist_code_id + description: '{{ doc("vendor_unique_id") }}' + - name: activist_code_id + description: Distinct ID belonging to Activist Code + - name: van_state_id + description: '{{ doc("bonterra_missing") }}' + - name: activist_code_type + description: '{{ doc("bonterra_missing") }}' + - name: activist_code_description + description: '{{ doc("bonterra_missing") }}' + - name: report_question + description: '{{ doc("bonterra_missing") }}' + - name: democrat_points + description: '{{ doc("bonterra_missing") }}' + - name: republican_points + description: '{{ doc("bonterra_missing") }}' + - name: independent_points + description: '{{ doc("bonterra_missing") }}' + - name: committee_id + description: '{{ doc("committee_id") }}' + - name: campaign_id + description: | + Unique ID of Campaign (if applied). Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. + - name: segment_by + description: '{{ doc("bonterra_missing") }}' + - name: segmented_activist_code_id + description: Combination of segment by and activist code id + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: activist_code_name + description: '{{ doc("bonterra_missing") }}' + - name: action_type_id + description: '{{ doc("bonterra_missing") }}' + - name: is_active + description: A Flag for if the activity is active, converted to boolean + - name: is_archived + description: A Flag for if the activity is archived, derived from the raw isactive value + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: stg_ngpvan__campaigns + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__campaigns' }}" + tags: + - ngpvan_core + columns: + - name: campaign_id + description: | + Unique ID of Campaign (if applied). Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. + - name: campaign_name + description: '{{ doc("bonterra_missing") }}' + - name: campaign_description + description: '{{ doc("bonterra_missing") }}' + - name: committee_id + description: '{{ doc("committee_id") }}' + - name: campaign_type_id + description: '{{ doc("bonterra_missing") }}' + - name: campaign_type + description: '{{ doc("bonterra_missing") }}' + - name: created_at + description: Date record in table was created, normalized to UTC + - name: created_by_user_id + description: User ID of who/what created record (person or automated process) + - name: is_active + description: A Flag for if the activity is active, converted to boolean + - name: is_archived + description: A Flag for if the activity is archived, derived from the raw isactive value + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: segmented_campaign_id + description: Combination of segment by and campaign id + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: vendor_unique_stg_ngpvan__campaign_id + description: '{{ doc("vendor_unique_id") }}' + - name: stg_ngpvan__code_types + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__code_types' }}" + tags: + - ngpvan_core + - ngpvan_lookup + columns: + - name: code_type_id + description: Source Code vs Tag + - name: code_type + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: vendor + description: '{{ doc("vendor") }}' + - name: vendor_unique_stg_ngpvan__code_type_id + description: '{{ doc("vendor_unique_id") }}' + - name: stg_ngpvan__codes + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__codes' }}" + tags: + - ngpvan_core + columns: + - name: van_state_code + description: '{{ doc("bonterra_missing") }}' + - name: code_id + description: '{{ doc("code_id") }}' + - name: code_name + description: Code Name in VAN + - name: committee_id + description: '{{ doc("committee_id") }}' + - name: committee_name + description: '{{ doc("bonterra_missing") }}' + - name: is_active + description: A Flag for if the activity is active, converted to boolean + - name: is_archived + description: A Flag for if the activity is archived, derived from the raw isactive value + - name: parent_code_id + description: + If the Code is in a nested structure, this will be the direct parent + of the Code + - name: created_by_user_id + description: User ID of who/what created record (person or automated process) + - name: utc_created_at + description: Date record in table was created, normalized to UTC + - name: static_full_name + description: + Name of the code. Codes can apply to many items in Votebuilder, like + Organizations, Events and Locations. Not to be confused with Activist Codes, + which only apply to individuals. They give users a lot more flexibility to create + custom groupings of information, and are similar to hashtags in that respect. + - name: code_type_id + description: Source Code vs Tag + - name: code_type + description: '{{ doc("bonterra_missing") }}' + - name: utc_modified_at + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: vendor + description: '{{ doc("vendor") }}' + - name: vendor_unique_stg_ngpvan__code_id + description: '{{ doc("vendor_unique_id") }}' + - name: database_mode + description: '{{ doc("database_mode") }}' + - name: is_myvoters + description: '{{ doc("is_myvoters") }}' + - name: segmented_code_id + description: Combination of segment by and code id + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: stg_ngpvan__committees + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__committees' }}" + tags: + - ngpvan_core + columns: + - name: committee_id + description: '{{ doc("committee_id") }}' + - name: van_state_id + description: '{{ doc("bonterra_missing") }}' + - name: committee_name + description: '{{ doc("bonterra_missing") }}' + - name: committee_short_name + description: '{{ doc("bonterra_missing") }}' + - name: committee_type + description: '{{ doc("bonterra_missing") }}' + - name: is_active + description: A Flag for if the activity is active, converted to boolean + - name: is_archived + description: A Flag for if the activity is archived, derived from the raw isactive value + - name: parent_committee_id + description: '{{ doc("bonterra_missing") }}' + - name: master_committee_id + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: vendor_unique_stg_ngpvan__committee_id + description: '{{ doc("vendor_unique_id") }}' + tests: + - not_null + - unique: + config: + severity: error + - name: stg_ngpvan__contact_types + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contact_types' }}" + tags: + - ngpvan_core + - ngpvan_lookup + columns: + - name: contact_method_type_id + description: | + Unique ID of Type of Contact. Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. + - name: contact_method_type + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: vendor + description: '{{ doc("vendor") }}' + - name: vendor_unique_stg_ngpvan__contact_type_id + description: '{{ doc("vendor_unique_id") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: stg_ngpvan__contacts_activist_codes + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_activist_codes' }}" + tags: + - ngpvan_core + columns: + - name: van_state_code + description: '{{ doc("bonterra_missing") }}' + - name: contacts_activist_code_id + description: Unique ID for Activist Code applied to Specific Contact + - name: van_id + description: + Distinct VAN ID belonging to contact record. My C IDs begin with + 100 and are Distinct to committee and My V IDs are unique to voter file state. + - name: activist_code_id + description: Distinct ID belonging to Activist Code + - name: committee_id + description: '{{ doc("committee_id") }}' + - name: utc_created_at + description: Date record in table was created, normalized to UTC + - name: input_type_id + description: The ID of how the record was entered + - name: input_type + description: '{{ doc("bonterra_missing") }}' + - name: contact_method_type + description: '{{ doc("bonterra_missing") }}' + - name: contact_method_type_id + description: | + Unique ID of Type of Contact. Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. + - name: canvassed_by_username + description: Name of User who created record + - name: canvassed_by_user_id + description: User ID of canvasser + - name: campaign_id + description: | + Unique ID of Campaign (if applied). Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. + - name: content_id + description: Unique ID of Content (if applied) + - name: utc_modified_at + description: '{{ doc("bonterra_missing") }}' + - name: contacts_contact_id + description: Unique ID for Contact History + - name: utc_canvassed_at + description: Date contact was solicited or canvassed, normalized to UTC + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: vendor + description: '{{ doc("vendor") }}' + - name: vendor_unique_stg_ngpvan__contacts_activist_code_id + description: '{{ doc("vendor_unique_id") }}' + - name: database_mode + description: '{{ doc("database_mode") }}' + - name: is_myvoters + description: '{{ doc("is_myvoters") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: unique_contacts_activist_code_id + description: Combination of segment by, state code, and contacts activist code id + - name: segmented_contacts_activist_code_id + description: Combination of segment by and contacts activist code id + - name: segmented_activist_code_id + description: Combination of segment by and activist code id + - name: segmented_van_id + description: Combination of segment by and van id + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: stg_ngpvan__contacts_codes + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_codes' }}" + tags: + - ngpvan_core + columns: + - name: van_state_code + description: '{{ doc("bonterra_missing") }}' + - name: contacts_code_id + description: '{{ doc("bonterra_missing") }}' + - name: van_id + description: + Distinct VAN ID belonging to contact record. My C IDs begin with + 100 and are Distinct to committee and My V IDs are unique to voter file state. + - name: code_id + description: '{{ doc("code_id") }}' + - name: committee_id + description: '{{ doc("committee_id") }}' + - name: utc_created_at + description: Date record in table was created, normalized to UTC + - name: input_type_id + description: The ID of how the record was entered + - name: input_type + description: '{{ doc("bonterra_missing") }}' + - name: utc_modified_at + description: '{{ doc("bonterra_missing") }}' + - name: created_by_user_id + description: User ID of who/what created record (person or automated process) + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: database_mode + description: '{{ doc("database_mode") }}' + - name: is_myvoters + description: '{{ doc("is_myvoters") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: unique_contacts_code_id + description: Unique key constructed by combining segment by, state code, and contacts code id + - name: segmented_contacts_code_id + description: Combination of segment by and contacts code id + - name: segmented_van_id + description: Combination of segment by and van id + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: vendor_unique_stg_ngpvan__contacts_code_id + description: '{{ doc("vendor_unique_id") }}' + - name: stg_ngpvan__contacts_contacts + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_contacts' }}" + tags: + - ngpvan_core + columns: + - name: van_state_code + description: '{{ doc("bonterra_missing") }}' + - name: contacts_contact_id + description: Unique ID for Contact History + - name: van_id + description: + Distinct VAN ID belonging to contact record. My C IDs begin with + 100 and are Distinct to committee and My V IDs are unique to voter file state. + - name: result_id + description: Result of contact interaction + - name: result_name + description: '{{ doc("bonterra_missing") }}' + - name: committee_id + description: '{{ doc("committee_id") }}' + - name: utc_created_at + description: Date record in table was created, normalized to UTC + - name: utc_canvassed_at + description: Date contact was solicited or canvassed, normalized to UTC + - name: input_type_id + description: The ID of how the record was entered + - name: input_type + description: '{{ doc("bonterra_missing") }}' + - name: contact_method_type_id + description: | + Unique ID of Type of Contact. Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. + - name: contact_method_type + description: '{{ doc("bonterra_missing") }}' + - name: canvassed_by_username + description: Name of User who created record + - name: canvassed_by_user_id + description: User ID of canvasser + - name: campaign_id + description: | + Unique ID of Campaign (if applied). Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. + - name: content_id + description: Unique ID of Content (if applied) + - name: created_by_user_id + description: User ID of who/what created record (person or automated process) + - name: contacts_phone_id + description: Unique identifier of Phone Number + - name: team_id + description: '{{ doc("bonterra_missing") }}' + - name: division_id + description: '{{ doc("bonterra_missing") }}' + - name: utc_modified_at + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: database_mode + description: '{{ doc("database_mode") }}' + - name: is_myvoters + description: '{{ doc("is_myvoters") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: unique_contacts_contact_id + description: Unique key constructed by combining segment by, state code, and contacts contact id + - name: segmented_contacts_contact_id + description: Combination of segment by and contacts contact id + - name: segmented_van_id + description: Combination of segment by and van id + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: vendor_unique_stg_ngpvan__contacts_contact_id + description: '{{ doc("vendor_unique_id") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: stg_ngpvan__contacts_notes + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_notes' }}" + tags: + - ngpvan_core + columns: + - name: van_state_code + description: '{{ doc("bonterra_missing") }}' + - name: contacts_note_id + description: Unique id for the note applied to the contact + - name: van_id + description: + Distinct VAN ID belonging to contact record. My C IDs begin with + 100 and are Distinct to committee and My V IDs are unique to voter file state. + - name: created_by_user_id + description: User ID of who/what created record (person or automated process) + - name: utc_created_at + description: Date record in table was created, normalized to UTC + - name: committee_id + description: '{{ doc("committee_id") }}' + - name: suppressed_by_user_id + description: '{{ doc("bonterra_missing") }}' + - name: utc_suppressed_at + description: '{{ doc("bonterra_missing") }}' + - name: note_text + description: Text of Note entered on Organization + - name: datefollowup + description: '{{ doc("bonterra_missing") }}' + - name: followup_completed_by_user_id + description: '{{ doc("bonterra_missing") }}' + - name: datefollowupcompleted + description: '{{ doc("bonterra_missing") }}' + - name: contacts_contact_id + description: Unique ID for Contact History + - name: utc_modified_at + description: + DateModified refers to when the row was last replicated. This doesn't + necessarily mean any value in the row was changed. Normalized to UTC + - name: note_category + description: '{{ doc("bonterra_missing") }}' + - name: contacts_online_form_id + description: '{{ doc("bonterra_missing") }}' + - name: contacts_story_id + description: '{{ doc("bonterra_missing") }}' + - name: note_category_id + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: database_mode + description: '{{ doc("database_mode") }}' + - name: is_myvoters + description: '{{ doc("is_myvoters") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: unique_contacts_note_id + description: Unique key constructed by combining segment by, state code, and contacts note id + - name: segmented_contacts_note_id + description: Combination of segment by and contacts note id + - name: segmented_van_id + description: Combination of segment by and van id + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: vendor_unique_stg_ngpvan__contacts_note_id + description: '{{ doc("vendor_unique_id") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: stg_ngpvan__contacts_survey_responses + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__contacts_survey_responses' }}" + tags: + - ngpvan_core + columns: + - name: van_state_code + description: '{{ doc("bonterra_missing") }}' + - name: contacts_survey_response_id + description: Unique ID for Survey Response applied to Contact + - name: van_id + description: + Distinct VAN ID belonging to contact record. My C IDs begin with + 100 and are Distinct to committee and My V IDs are unique to voter file state. + - name: survey_question_id + description: Unique ID of Survey Question - link to Survey Question + - name: survey_response_id + description: Unique ID of Survey Response - link to Survey Response + - name: contact_method_type_id + description: | + Unique ID of Type of Contact. Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. + - name: contact_method_type + description: '{{ doc("bonterra_missing") }}' + - name: input_type_id + description: The ID of how the record was entered + - name: input_type + description: '{{ doc("bonterra_missing") }}' + - name: utc_canvassed_at + description: Date contact was solicited or canvassed, normalized to UTC + - name: committee_id + description: '{{ doc("committee_id") }}' + - name: canvassed_by_user_id + description: User ID of canvasser + - name: canvassed_by_username + description: Name of User who created record + - name: utc_created_at + description: Date record in table was created, normalized to UTC + - name: contacts_contact_id + description: Unique ID for Contact History + - name: campaign_id + description: | + Unique ID of Campaign (if applied). Note - this ID may not be unique across committees. + If you receive data from multiple committees, it's safer to the use the + "vendor_unique" column in the table. + - name: content_id + description: Unique ID of Content (if applied) + - name: utc_modified_at + description: '{{ doc("bonterra_missing") }}' + - name: team_id + description: '{{ doc("bonterra_missing") }}' + - name: division_id + description: '{{ doc("bonterra_missing") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: database_mode + description: '{{ doc("database_mode") }}' + - name: is_myvoters + description: '{{ doc("is_myvoters") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: unique_contacts_survey_response_id + description: Unique key constructed by combining segment by, state code, and contacts survey response id + - name: segmented_contacts_survey_response_id + description: Combination of segment by and contacts survey response id + - name: segmented_van_id + description: Combination of segment by and van id + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: vendor_unique_stg_ngpvan__contacts_survey_response_id + description: '{{ doc("vendor_unique_id") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: stg_ngpvan__input_types + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__input_types' }}" + tags: + - ngpvan_core + - ngpvan_lookup + columns: + - name: input_type_id + description: The ID of how the record was entered + - name: input_type + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: vendor + description: '{{ doc("vendor") }}' + - name: vendor_unique_stg_ngpvan__input_type_id + description: '{{ doc("vendor_unique_id") }}' + - name: stg_ngpvan__public_users + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__public_users' }}" + tags: + - ngpvan_core + columns: + - name: public_user_id + description: '{{ doc("bonterra_missing") }}' + - name: public_username + description: '{{ doc("bonterra_missing") }}' + - name: van_id + description: Distinct VAN ID belonging to contact record. My C IDs begin with 100 and are Distinct to committee and My V IDs are unique to voter file state. + - name: committee_id + description: '{{ doc("committee_id") }}' + - name: created_at + description: Date record in table was created, normalized to UTC + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: vendor_unique_stg_ngpvan__public_user_id + description: '{{ doc("vendor_unique_id") }}' + - name: stg_ngpvan__results + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + enabled: "{{ var('dbt_ngpvan_config')['lookup_tables'] }}" + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__results' }}" + tags: + - ngpvan_core + - ngpvan_lookup + columns: + - name: result_id + description: Result of contact interaction + - name: result_name + description: '{{ doc("bonterra_missing") }}' + - name: result_description + description: '{{ doc("bonterra_missing") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: vendor + description: '{{ doc("vendor") }}' + - name: vendor_unique_stg_ngpvan__result_id + description: '{{ doc("vendor_unique_id") }}' + - name: stg_ngpvan__survey_questions + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__survey_questions' }}" + tags: + - ngpvan_core + columns: + - name: survey_question_id + description: Unique ID of Survey Question - link to Survey Question + - name: van_state_id + description: '{{ doc("bonterra_missing") }}' + - name: election_cycle + description: '{{ doc("bonterra_missing") }}' + - name: survey_question_type + description: '{{ doc("bonterra_missing") }}' + - name: survey_question_name + description: '{{ doc("bonterra_missing") }}' + - name: survey_question_text + description: '{{ doc("bonterra_missing") }}' + - name: master_survey_question_id + description: '{{ doc("bonterra_missing") }}' + - name: committee_id + description: '{{ doc("committee_id") }}' + - name: is_active + description: A Flag for if the activity is active, converted to boolean + - name: is_archived + description: A Flag for if the activity is archived, derived from the raw isactive value + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: segmented_survey_question_id + description: Combination of segment by and survey question id + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: vendor_unique_stg_ngpvan__survey_question_id + description: '{{ doc("vendor_unique_id") }}' + - name: stg_ngpvan__survey_responses + description: '{{ doc("bonterra_missing") }}' + tests: + - table_fully_documented + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__survey_responses' }}" + tags: + - ngpvan_core + columns: + - name: survey_question_id + description: Unique ID of Survey Question - link to Survey Question + - name: survey_response_id + description: Unique ID of Survey Response - link to Survey Response + - name: survey_response + description: '{{ doc("bonterra_missing") }}' + - name: democrat_points + description: '{{ doc("bonterra_missing") }}' + - name: republican_points + description: '{{ doc("bonterra_missing") }}' + - name: independent_points + description: '{{ doc("bonterra_missing") }}' + - name: master_survey_response_id + description: '{{ doc("bonterra_missing") }}' + - name: committee_id + description: '{{ doc("committee_id") }}' + - name: _avvan_source_relation + description: '{{ doc("avvan_source_relation") }}' + - name: _dbt_source_relation + description: '{{ doc("dbt_source_relation") }}' + - name: source_schema + description: '{{ doc("source_schema") }}' + - name: source_table + description: '{{ doc("source_table") }}' + - name: segment_by + description: '{{ doc("segment_by") }}' + - name: segmented_survey_response_id + description: Combination of segment by and survey response id + - name: segmented_survey_question_id + description: Combination of segment by and survey question id + - name: vendor + description: '{{ doc("vendor") }}' + - name: segment_by_key + description: '{{ doc("segment_by_key") }}' + - name: vendor_unique_stg_ngpvan__survey_response_id + description: '{{ doc("vendor_unique_id") }}' + + - name: stg_ngpvan__users + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__users' }}" + + - name: stg_ngpvan__users_user_groups + config: + alias: "{{ 'stg_' ~ var('dbt_ngpvan_config')['vendor_name'] ~ '__users_user_groups' }}" diff --git a/models/staging/ngpvan/core/stg_ngpvan__codes.sql b/models/staging/ngpvan/core/stg_ngpvan__codes.sql index 1925593..03ead3e 100644 --- a/models/staging/ngpvan/core/stg_ngpvan__codes.sql +++ b/models/staging/ngpvan/core/stg_ngpvan__codes.sql @@ -5,7 +5,7 @@ WITH ), committees AS ( - SELECT * FROM {{ ref('base_ngpvan__committees') }} + SELECT * EXCEPT (_avvan_source_relation) FROM {{ ref('base_ngpvan__committees') }} ), codetypes AS ( diff --git a/models/staging/ngpvan/core/stg_ngpvan__committees.sql b/models/staging/ngpvan/core/stg_ngpvan__committees.sql index 004e343..7cd407b 100644 --- a/models/staging/ngpvan/core/stg_ngpvan__committees.sql +++ b/models/staging/ngpvan/core/stg_ngpvan__committees.sql @@ -33,4 +33,6 @@ WITH SELECT DISTINCT * FROM renamed +-- hard-coding out one bad committee from AV while we work with them to remove from raw +WHERE NOT (committee_id = 15701 AND committee_short_name IS NULL)