diff --git a/macros/agg_automation_activities.sql b/macros/agg_automation_activities.sql new file mode 100644 index 00000000..f3f76b76 --- /dev/null +++ b/macros/agg_automation_activities.sql @@ -0,0 +1,107 @@ +{% macro agg_automation_activities(by) %} + +{% set id_cols = [] %} +{% for col in by %} + {% do id_cols.append(col ~ "_id" ) %} +{% endfor %} + +with activities as ( + + select * + from {{ ref('mailchimp__automation_activities')}} + +), recipients as ( + + select * + from {{ ref('int_mailchimp__automation_recipients') }} + +), unsubscribes as ( + + select * + from {{ ref('int_mailchimp__automation_unsubscribes') }} + + +-- aggregate automation opens and clicks by {{ by }} + +), pivoted as ( + + select + {% for id_col in id_cols %} + {{ id_col }}, + {% endfor %} + sum(case when action_type = 'open' then 1 end) as opens, + sum(case when action_type = 'click' then 1 end) as clicks, + count(distinct case when action_type = 'open' then member_id end) as unique_opens, + count(distinct case when action_type = 'click' then member_id end) as unique_clicks + from activities + where + {% for id_col in id_cols %} + {% if not loop.first %}and {% endif %}{{ id_col }} is not null + {% endfor %} + group by + {% for id_col in id_cols %} + {% if not loop.first %}, {% endif %}{{ id_col }} + {% endfor %} + +), sends as ( + + select + {% for id_col in id_cols %} + {{ id_col }}, + {% endfor %} + count(*) as sends + from recipients + where + {% for id_col in id_cols %} + {% if not loop.first %}and {% endif %}{{ id_col }} is not null + {% endfor %} + group by + {% for id_col in id_cols %} + {% if not loop.first %}, {% endif %}{{ id_col }} + {% endfor %} + +), unsubscribes_xf as ( + + select + {% for id_col in id_cols %} + {{ id_col }}, + {% endfor %} + count(*) as unsubscribes + from unsubscribes + where + {% for id_col in id_cols %} + {% if not loop.first %}and {% endif %}{{ id_col }} is not null + {% endfor %} + group by + {% for id_col in id_cols %} + {% if not loop.first %}, {% endif %}{{ id_col }} + {% endfor %} + +), joined as ( + + select + {% for id_col in id_cols %} + sends.{{ id_col }}, + {% endfor %} + pivoted.opens, + pivoted.clicks, + pivoted.unique_opens, + pivoted.unique_clicks, + sends.sends, + unsubscribes_xf.unsubscribes + from sends + left join pivoted on + {% for id_col in id_cols %} + {% if not loop.first %}and {% endif %}sends.{{ id_col }} = pivoted.{{ id_col }} + {% endfor %} + full outer join unsubscribes_xf on + {% for id_col in id_cols %} + {% if not loop.first %}and {% endif %}sends.{{ id_col }} = unsubscribes_xf.{{ id_col }} + {% endfor %} + +) + +select * +from joined + +{% endmacro %} diff --git a/models/intermediate/int_mailchimp__campaign_activities_by_segment.sql b/macros/agg_campaign_activities.sql similarity index 50% rename from models/intermediate/int_mailchimp__campaign_activities_by_segment.sql rename to macros/agg_campaign_activities.sql index 86b6b194..81a24899 100644 --- a/models/intermediate/int_mailchimp__campaign_activities_by_segment.sql +++ b/macros/agg_campaign_activities.sql @@ -1,4 +1,9 @@ -{{ config(enabled=var('mailchimp_using_segments', True)) }} +{% macro agg_campaign_activities(by) %} + +{% set id_cols = [] %} +{% for col in by %} + {% do id_cols.append(col ~ "_id" ) %} +{% endfor %} with recipients as ( @@ -8,7 +13,9 @@ with recipients as ( ), pivoted as ( select - segment_id, + {% for id_col in id_cols %} + {{ id_col }}, + {% endfor %} count(*) as sends, sum(opens) as opens, sum(clicks) as clicks, @@ -16,10 +23,18 @@ with recipients as ( count(distinct case when was_clicked = True then member_id end) as unique_clicks, count(distinct case when was_unsubscribed = True then member_id end) as unsubscribes from recipients - where segment_id is not null - group by 1 + where + {% for id_col in id_cols %} + {% if not loop.first %}and {% endif %}{{ id_col }} is not null + {% endfor %} + group by + {% for id_col in id_cols %} + {% if not loop.first %}, {% endif %}{{ id_col }} + {% endfor %} ) select * -from pivoted \ No newline at end of file +from pivoted + +{% endmacro %} diff --git a/models/intermediate/int_mailchimp__automation_activities_by_automation.sql b/models/intermediate/int_mailchimp__automation_activities_by_automation.sql deleted file mode 100644 index 57fb5022..00000000 --- a/models/intermediate/int_mailchimp__automation_activities_by_automation.sql +++ /dev/null @@ -1,67 +0,0 @@ -{{ config(enabled=var('mailchimp_using_automations', True)) }} - -with activities as ( - - select * - from {{ ref('mailchimp__automation_activities')}} - -), recipients as ( - - select * - from {{ ref('int_mailchimp__automation_recipients') }} - -), unsubscribes as ( - - select * - from {{ ref('int_mailchimp__automation_unsubscribes') }} - - --- aggregate automation opens and clicks by automation - -), pivoted as ( - - select - automation_id, - sum(case when action_type = 'open' then 1 end) as opens, - sum(case when action_type = 'click' then 1 end) as clicks, - count(distinct case when action_type = 'open' then member_id end) as unique_opens, - count(distinct case when action_type = 'click' then member_id end) as unique_clicks - from activities - group by 1 - -), sends as ( - - select - automation_id, - count(*) as sends - from recipients - group by 1 - -), unsubscribes_xf as ( - - select - automation_id, - count(*) as unsubscribes - from unsubscribes - group by 1 - -), joined as ( - - select - coalesce(pivoted.automation_id, sends.automation_id, unsubscribes_xf.automation_id) as automation_id, - pivoted.opens, - pivoted.clicks, - pivoted.unique_opens, - pivoted.unique_clicks, - sends.sends, - unsubscribes_xf.unsubscribes - from pivoted - full outer join sends - on pivoted.automation_id = sends.automation_id - full outer join unsubscribes_xf - on pivoted.automation_id = unsubscribes_xf.automation_id - -) - -select * -from joined \ No newline at end of file diff --git a/models/intermediate/int_mailchimp__automation_activities_by_email.sql b/models/intermediate/int_mailchimp__automation_activities_by_email.sql deleted file mode 100644 index 5fe0eb02..00000000 --- a/models/intermediate/int_mailchimp__automation_activities_by_email.sql +++ /dev/null @@ -1,67 +0,0 @@ -{{ config(enabled=var('mailchimp_using_automations', True)) }} - -with activities as ( - - select * - from {{ ref('mailchimp__automation_activities')}} - -), recipients as ( - - select * - from {{ ref('int_mailchimp__automation_recipients') }} - -), unsubscribes as ( - - select * - from {{ ref('int_mailchimp__automation_unsubscribes') }} - - --- aggregate automation opens and clicks by email - -), pivoted as ( - - select - automation_email_id, - sum(case when action_type = 'open' then 1 end) as opens, - sum(case when action_type = 'click' then 1 end) as clicks, - count(distinct case when action_type = 'open' then member_id end) as unique_opens, - count(distinct case when action_type = 'click' then member_id end) as unique_clicks - from activities - group by 1 - -), sends as ( - - select - automation_email_id, - count(*) as sends - from recipients - group by 1 - -), unsubscribes_xf as ( - - select - campaign_id as automation_email_id, - count(*) as unsubscribes - from unsubscribes - group by 1 - -), joined as ( - - select - coalesce(pivoted.automation_email_id, sends.automation_email_id, unsubscribes_xf.automation_email_id) as automation_email_id, - pivoted.opens, - pivoted.clicks, - pivoted.unique_opens, - pivoted.unique_clicks, - sends.sends, - unsubscribes_xf.unsubscribes - from pivoted - full outer join sends - on pivoted.automation_email_id = sends.automation_email_id - full outer join unsubscribes_xf - on pivoted.automation_email_id = unsubscribes_xf.automation_email_id - -) - -select * -from joined \ No newline at end of file diff --git a/models/intermediate/int_mailchimp__automation_activities_by_list.sql b/models/intermediate/int_mailchimp__automation_activities_by_list.sql deleted file mode 100644 index a39a3d05..00000000 --- a/models/intermediate/int_mailchimp__automation_activities_by_list.sql +++ /dev/null @@ -1,67 +0,0 @@ -{{ config(enabled=var('mailchimp_using_automations', True)) }} - -with activities as ( - - select * - from {{ ref('mailchimp__automation_activities')}} - -), recipients as ( - - select * - from {{ ref('int_mailchimp__automation_recipients') }} - -), unsubscribes as ( - - select * - from {{ ref('int_mailchimp__automation_unsubscribes') }} - - --- aggregate automation opens and clicks by list - -), pivoted as ( - - select - list_id, - sum(case when action_type = 'open' then 1 end) as opens, - sum(case when action_type = 'click' then 1 end) as clicks, - count(distinct case when action_type = 'open' then member_id end) as unique_opens, - count(distinct case when action_type = 'click' then member_id end) as unique_clicks - from activities - group by 1 - -), sends as ( - - select - list_id, - count(*) as sends - from recipients - group by 1 - -), unsubscribes_xf as ( - - select - list_id, - count(*) as unsubscribes - from unsubscribes - group by 1 - -), joined as ( - - select - coalesce(pivoted.list_id, sends.list_id, unsubscribes_xf.list_id) as list_id, - pivoted.opens, - pivoted.clicks, - pivoted.unique_opens, - pivoted.unique_clicks, - sends.sends, - unsubscribes_xf.unsubscribes - from pivoted - full outer join sends - on pivoted.list_id = sends.list_id - full outer join unsubscribes_xf - on pivoted.list_id = unsubscribes_xf.list_id - -) - -select * -from joined \ No newline at end of file diff --git a/models/intermediate/int_mailchimp__automation_activities_by_member.sql b/models/intermediate/int_mailchimp__automation_activities_by_member.sql deleted file mode 100644 index 3cfa4f13..00000000 --- a/models/intermediate/int_mailchimp__automation_activities_by_member.sql +++ /dev/null @@ -1,67 +0,0 @@ -{{ config(enabled=var('mailchimp_using_automations', True)) }} - -with activities as ( - - select * - from {{ ref('mailchimp__automation_activities')}} - -), recipients as ( - - select * - from {{ ref('int_mailchimp__automation_recipients') }} - -), unsubscribes as ( - - select * - from {{ ref('int_mailchimp__automation_unsubscribes') }} - - --- aggregate automation opens and clicks by member - -), pivoted as ( - - select - member_id, - sum(case when action_type = 'open' then 1 end) as opens, - sum(case when action_type = 'click' then 1 end) as clicks, - count(distinct case when action_type = 'open' then member_id end) as unique_opens, - count(distinct case when action_type = 'click' then member_id end) as unique_clicks - from activities - group by 1 - -), sends as ( - - select - member_id, - count(*) as sends - from recipients - group by 1 - -), unsubscribes_xf as ( - - select - member_id, - count(*) as unsubscribes - from unsubscribes - group by 1 - -), joined as ( - - select - coalesce(pivoted.member_id, sends.member_id, unsubscribes_xf.member_id) as member_id, - pivoted.opens, - pivoted.clicks, - pivoted.unique_opens, - pivoted.unique_clicks, - sends.sends, - unsubscribes_xf.unsubscribes - from pivoted - full outer join sends - on pivoted.member_id = sends.member_id - full outer join unsubscribes_xf - on pivoted.member_id = unsubscribes_xf.member_id - -) - -select * -from joined \ No newline at end of file diff --git a/models/intermediate/int_mailchimp__automation_activities_by_segment.sql b/models/intermediate/int_mailchimp__automation_activities_by_segment.sql deleted file mode 100644 index dddbe36b..00000000 --- a/models/intermediate/int_mailchimp__automation_activities_by_segment.sql +++ /dev/null @@ -1,70 +0,0 @@ -{{ config(enabled=var('mailchimp_using_automations', True)) }} - -with activities as ( - - select * - from {{ ref('mailchimp__automation_activities')}} - -), recipients as ( - - select * - from {{ ref('int_mailchimp__automation_recipients') }} - -), unsubscribes as ( - - select * - from {{ ref('int_mailchimp__automation_unsubscribes') }} - - --- aggregate automation opens and clicks by segment - -), pivoted as ( - - select - segment_id, - sum(case when action_type = 'open' then 1 end) as opens, - sum(case when action_type = 'click' then 1 end) as clicks, - count(distinct case when action_type = 'open' then segment_id end) as unique_opens, - count(distinct case when action_type = 'click' then segment_id end) as unique_clicks - from activities - where segment_id is not null - group by 1 - -), sends as ( - - select - segment_id, - count(*) as sends - from recipients - where segment_id is not null - group by 1 - -), unsubscribes_xf as ( - - select - segment_id, - count(*) as unsubscribes - from unsubscribes - where segment_id is not null - group by 1 - -), joined as ( - - select - coalesce(pivoted.segment_id, sends.segment_id, unsubscribes_xf.segment_id) as segment_id, - pivoted.opens, - pivoted.clicks, - pivoted.unique_opens, - pivoted.unique_clicks, - sends.sends, - unsubscribes_xf.unsubscribes - from pivoted - full outer join sends - on pivoted.segment_id = sends.segment_id - full outer join unsubscribes_xf - on pivoted.segment_id = unsubscribes_xf.segment_id - -) - -select * -from joined \ No newline at end of file diff --git a/models/intermediate/int_mailchimp__automation_unsubscribes.sql b/models/intermediate/int_mailchimp__automation_unsubscribes.sql index 4a0cc903..625eaf80 100644 --- a/models/intermediate/int_mailchimp__automation_unsubscribes.sql +++ b/models/intermediate/int_mailchimp__automation_unsubscribes.sql @@ -20,8 +20,8 @@ with unsubscribes as ( select unsubscribes.*, automations.segment_id, - automations.automation_id - + automations.automation_id, + automation_emails.automation_email_id from unsubscribes left join automation_emails on unsubscribes.campaign_id = automation_emails.automation_email_id diff --git a/models/intermediate/int_mailchimp__campaign_activities_by_campaign.sql b/models/intermediate/int_mailchimp__campaign_activities_by_campaign.sql deleted file mode 100644 index 6b5b48cc..00000000 --- a/models/intermediate/int_mailchimp__campaign_activities_by_campaign.sql +++ /dev/null @@ -1,22 +0,0 @@ -with recipients as ( - - select * - from {{ ref('mailchimp__campaign_recipients')}} - -), pivoted as ( - - select - campaign_id, - count(*) as sends, - sum(opens) as opens, - sum(clicks) as clicks, - count(distinct case when was_opened = True then member_id end) as unique_opens, - count(distinct case when was_clicked = True then member_id end) as unique_clicks, - count(distinct case when was_unsubscribed = True then member_id end) as unsubscribes - from recipients - group by 1 - -) - -select * -from pivoted \ No newline at end of file diff --git a/models/intermediate/int_mailchimp__campaign_activities_by_list.sql b/models/intermediate/int_mailchimp__campaign_activities_by_list.sql deleted file mode 100644 index 4dc39512..00000000 --- a/models/intermediate/int_mailchimp__campaign_activities_by_list.sql +++ /dev/null @@ -1,22 +0,0 @@ -with recipients as ( - - select * - from {{ ref('mailchimp__campaign_recipients')}} - -), pivoted as ( - - select - list_id, - count(*) as sends, - sum(opens) as opens, - sum(clicks) as clicks, - count(distinct case when was_opened = True then member_id end) as unique_opens, - count(distinct case when was_clicked = True then member_id end) as unique_clicks, - count(distinct case when was_unsubscribed = True then member_id end) as unsubscribes - from recipients - group by 1 - -) - -select * -from pivoted \ No newline at end of file diff --git a/models/intermediate/int_mailchimp__campaign_activities_by_member.sql b/models/intermediate/int_mailchimp__campaign_activities_by_member.sql deleted file mode 100644 index d38937a2..00000000 --- a/models/intermediate/int_mailchimp__campaign_activities_by_member.sql +++ /dev/null @@ -1,22 +0,0 @@ -with recipients as ( - - select * - from {{ ref('mailchimp__campaign_recipients')}} - -), pivoted as ( - - select - member_id, - count(*) as sends, - sum(opens) as opens, - sum(clicks) as clicks, - count(distinct case when was_opened = True then member_id end) as unique_opens, - count(distinct case when was_clicked = True then member_id end) as unique_clicks, - count(distinct case when was_unsubscribed = True then member_id end) as unsubscribes - from recipients - group by 1 - -) - -select * -from pivoted \ No newline at end of file diff --git a/models/mailchimp.yml b/models/mailchimp.yml index d580c71d..b6dea380 100644 --- a/models/mailchimp.yml +++ b/models/mailchimp.yml @@ -127,12 +127,14 @@ models: - name: mailchimp__members description: Each record represents a member in Mailchimp, enriched with campaign metrics and (optionally) automation metrics. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - member_id + - list_id columns: - name: member_id description: The ID of the member. - tests: - - unique - - not_null - name: email_address description: Email address for a subscriber. diff --git a/models/mailchimp__automation_emails.sql b/models/mailchimp__automation_emails.sql index 49cdf400..d3bae9ca 100644 --- a/models/mailchimp__automation_emails.sql +++ b/models/mailchimp__automation_emails.sql @@ -5,10 +5,9 @@ with emails as ( select * from {{ var('automation_email') }} -), activities as ( +), automation_activities as ( - select * - from {{ ref('int_mailchimp__automation_activities_by_email') }} + {{ agg_automation_activities(['automation_email']) }} ), automations as ( @@ -29,15 +28,15 @@ with emails as ( select joined.*, - coalesce(activities.sends, 0) as sends, - coalesce(activities.opens, 0) as opens, - coalesce(activities.clicks, 0) as clicks, - coalesce(activities.unique_opens, 0) as unique_opens, - coalesce(activities.unique_clicks, 0) as unique_clicks, - coalesce(activities.unsubscribes, 0) as unsubscribes + coalesce(automation_activities.sends, 0) as sends, + coalesce(automation_activities.opens, 0) as opens, + coalesce(automation_activities.clicks, 0) as clicks, + coalesce(automation_activities.unique_opens, 0) as unique_opens, + coalesce(automation_activities.unique_clicks, 0) as unique_clicks, + coalesce(automation_activities.unsubscribes, 0) as unsubscribes from joined - left join activities - on joined.automation_email_id = activities.automation_email_id + left join automation_activities + on joined.automation_email_id = automation_activities.automation_email_id ) diff --git a/models/mailchimp__automations.sql b/models/mailchimp__automations.sql index 686b2d4d..5d48a013 100644 --- a/models/mailchimp__automations.sql +++ b/models/mailchimp__automations.sql @@ -5,24 +5,23 @@ with automations as ( select * from {{ var('automation')}} -), activities as ( +), automation_activities as ( - select * - from {{ ref('int_mailchimp__automation_activities_by_automation') }} + {{ agg_automation_activities(['automation']) }} ), joined as ( select automations.*, - coalesce(activities.sends,0) as sends, - coalesce(activities.opens,0) as opens, - coalesce(activities.clicks,0) as clicks, - coalesce(activities.unique_opens,0) as unique_opens, - coalesce(activities.unique_clicks,0) as unique_clicks, - coalesce(activities.unsubscribes,0) as unsubscribes + coalesce(automation_activities.sends,0) as sends, + coalesce(automation_activities.opens,0) as opens, + coalesce(automation_activities.clicks,0) as clicks, + coalesce(automation_activities.unique_opens,0) as unique_opens, + coalesce(automation_activities.unique_clicks,0) as unique_clicks, + coalesce(automation_activities.unsubscribes,0) as unsubscribes from automations - left join activities - on automations.automation_id = activities.automation_id + left join automation_activities + on automations.automation_id = automation_activities.automation_id ) diff --git a/models/mailchimp__campaigns.sql b/models/mailchimp__campaigns.sql index c7666352..18521a92 100644 --- a/models/mailchimp__campaigns.sql +++ b/models/mailchimp__campaigns.sql @@ -5,8 +5,7 @@ with campaigns as ( ), activities as ( - select * - from {{ ref('int_mailchimp__campaign_activities_by_campaign') }} + {{ agg_campaign_activities(['campaign']) }} ), joined as ( diff --git a/models/mailchimp__lists.sql b/models/mailchimp__lists.sql index 417dde89..835732bf 100644 --- a/models/mailchimp__lists.sql +++ b/models/mailchimp__lists.sql @@ -5,8 +5,7 @@ with lists as ( ), campaign_activities as ( - select * - from {{ ref('int_mailchimp__campaign_activities_by_list') }} + {{ agg_campaign_activities(['list']) }} ), members as ( @@ -41,8 +40,7 @@ with lists as ( ), automation_activities as ( - select * - from {{ ref('int_mailchimp__automation_activities_by_list') }} + {{ agg_automation_activities(['list']) }} ), metrics_xf as ( diff --git a/models/mailchimp__members.sql b/models/mailchimp__members.sql index 4e70a618..27735256 100644 --- a/models/mailchimp__members.sql +++ b/models/mailchimp__members.sql @@ -5,8 +5,7 @@ with members as ( ), campaign_activities as ( - select * - from {{ ref('int_mailchimp__campaign_activities_by_member') }} + {{ agg_campaign_activities(['member', 'list']) }} ), metrics as ( @@ -21,13 +20,13 @@ with members as ( from members left join campaign_activities on members.member_id = campaign_activities.member_id + and members.list_id = campaign_activities.list_id {% if var('mailchimp_using_automations', True) %} ), automation_activities as ( - select * - from {{ ref('int_mailchimp__automation_activities_by_member') }} + {{ agg_automation_activities(['member', 'list']) }} ), metrics_xf as ( @@ -42,6 +41,7 @@ with members as ( from metrics left join automation_activities on metrics.member_id = automation_activities.member_id + and metrics.list_id = automation_activities.list_id ) diff --git a/models/mailchimp__segments.sql b/models/mailchimp__segments.sql index 5ee924f4..2afdec18 100644 --- a/models/mailchimp__segments.sql +++ b/models/mailchimp__segments.sql @@ -7,8 +7,7 @@ with segments as ( ), campaign_activities as ( - select * - from {{ ref('int_mailchimp__campaign_activities_by_segment') }} + {{ agg_campaign_activities(['segment']) }} ), lists as ( @@ -37,8 +36,7 @@ with segments as ( ), automation_activities as ( - select * - from {{ ref('int_mailchimp__automation_activities_by_segment') }} + {{ agg_automation_activities(['segment']) }} ), metrics_xf as (