feat: Final round of cleanup

macros/format_problem_number_location.sql

@@ -1,6 +1,4 @@
-{% macro format_problem_number_location(
-    object_id_var, display_name_with_location_var
-) %}
+{% macro format_problem_location(object_id_var, display_name_with_location_var) %}
     substring(
         regexpExtract({{ object_id_var }}, '(@problem\+block@[\w\d][^_\/]*)(_\d)?', 2),
         2
@@ -14,6 +12,23 @@
         ),
         ':'
     ) as _problem_number,
-    concat(_problem_number, '_', _problem_id_or_1) as problem_number,
-    concat(problem_number, ' - ', _problem_with_name[2]) as problem_name_location
+    concat(_problem_number, '_', _problem_id_or_1) as problem_location,
+    concat(problem_location, ' - ', _problem_with_name[2]) as problem_name_location
+{% endmacro %}
+
+{% macro format_object_location(display_name_with_location_var) %}
+    arrayStringConcat(
+        arrayMap(
+            x -> (leftPad(x, 2, char(917768))),
+            splitByString(
+                ':', splitByString(' - ', {{ display_name_with_location_var }})[1]
+            )
+        ),
+        ':'
+    ) as object_location,
+    concat(
+        object_location,
+        ' - ',
+        splitByString(' - ', {{ display_name_with_location_var }})[2]
+    ) as object_name_location
 {% endmacro %}

macros/remove_deprecated_models.sql

@@ -30,64 +30,67 @@
 
     -- https://github.com/openedx/aspects-dbt/pull/111/
     {% set models_to_drop = [
-        ("view", reporting, "fact_completions"),
-        ("view", xapi, "fact_forum_interactions"),
-        ("view", reporting, "fact_grades"),
-        ("view", reporting, "learner_summary"),
-        ("view", reporting, "fact_navigation_dropoff"),
-        ("view", reporting, "fact_learner_problem_summary"),
-        ("view", reporting, "fact_problem_engagement"),
-        ("view", reporting, "fact_problem_engagement_per_subsection"),
-        ("view", reporting, "fact_problem_responses_extended"),
-        ("view", reporting, "dim_at_risk_learners"),
-        ("view", reporting, "fact_transcript_usage"),
-        ("view", reporting, "fact_watched_video_segments"),
-        ("view", reporting, "dim_course_blocks_extended"),
-        ("view", reporting, "fact_navigation"),
-        ("view", reporting, "int_pages_per_subsection"),
-        ("view", reporting, "int_problems_per_subsection"),
-        ("view", reporting, "fact_problem_responses"),
-        ("view", reporting, "int_problem_hints"),
-        ("view", reporting, "int_problem_results"),
-        ("view", reporting, "most_recent_course_tags"),
-        ("view", reporting, "fact_student_status"),
-        ("view", reporting, "watched_video_duration"),
-        ("view", reporting, "fact_video_plays"),
-        ("view", reporting, "fact_watched_video_duration"),
-        ("view", reporting, "fact_learner_response_attempts"),
+        ("dictionary", event_sink, "course_block_names"),
+        ("dictionary", event_sink, "course_names"),
+        ("dictionary", event_sink, "course_tags"),
         ("mv", event_sink, "course_enrollment"),
         ("mv", event_sink, "course_relationships"),
-        ("mv", xapi, "forum_events"),
-        ("mv", reporting, "video_transcript_events"),
-        ("mv", reporting, "fact_learner_course_grade"),
-        ("mv", reporting, "fact_learner_course_status"),
         ("mv", event_sink, "most_recent_course_blocks"),
-        ("mv", reporting, "fact_enrollment_status"),
         ("mv", event_sink, "most_recent_object_tags"),
         ("mv", event_sink, "most_recent_tags"),
         ("mv", event_sink, "most_recent_taxonomies"),
+        ("mv", reporting, "dim_learner_first_response"),
+        ("mv", reporting, "dim_learner_first_success_response"),
+        ("mv", reporting, "dim_learner_last_course_visit"),
+        ("mv", reporting, "dim_learner_most_recent_course_grade"),
+        ("mv", reporting, "dim_learner_response_attempt"),
+        ("mv", reporting, "fact_enrollment_status"),
+        ("mv", reporting, "fact_learner_course_grade"),
+        ("mv", reporting, "fact_learner_course_status"),
+        ("mv", reporting, "fact_section_page_engagement"),
+        ("mv", reporting, "fact_section_problem_engagement"),
+        ("mv", reporting, "fact_section_video_engagement"),
+        ("mv", reporting, "fact_subsection_page_engagement"),
+        ("mv", reporting, "fact_subsection_problem_engagement"),
+        ("mv", reporting, "fact_subsection_video_engagement"),
+        ("mv", reporting, "int_videos_per_subsection"),
+        ("mv", reporting, "video_transcript_events"),
+        ("mv", xapi, "fact_instance_enrollments"),
         ("mv", xapi, "fact_learner_last_course_visit"),
+        ("mv", xapi, "forum_events"),
         ("mv", xapi, "responses"),
         ("mv", xapi, "section_page_engagement"),
         ("mv", xapi, "section_problem_engagement"),
         ("mv", xapi, "section_video_engagement"),
         ("mv", xapi, "subsection_page_engagement"),
         ("mv", xapi, "subsection_problem_engagement"),
         ("mv", xapi, "subsection_video_engagement"),
-        ("mv", xapi, "fact_instance_enrollments"),
-        ("mv", reporting, "int_videos_per_subsection"),
-        ("mv", reporting, "fact_section_video_engagement"),
-        ("mv", reporting, "fact_subsection_video_engagement"),
-        ("mv", reporting, "dim_learner_first_response"),
-        ("mv", reporting, "dim_learner_first_success_response"),
-        ("mv", reporting, "dim_learner_response_attempt"),
-        ("mv", reporting, "fact_section_problem_engagement"),
-        ("mv", reporting, "fact_subsection_problem_engagement"),
-        ("mv", reporting, "fact_section_page_engagement"),
-        ("mv", reporting, "fact_subsection_page_engagement"),
-        ("dictionary", event_sink, "course_names"),
-        ("dictionary", event_sink, "course_block_names"),
-        ("dictionary", event_sink, "course_tags"),
+        ("view", reporting, "dim_at_risk_learners"),
+        ("view", reporting, "dim_course_blocks_extended"),
+        ("view", reporting, "dim_user_pii"),
+        ("view", reporting, "fact_completions"),
+        ("view", reporting, "fact_enrollments"),
+        ("view", reporting, "fact_grades"),
+        ("view", reporting, "fact_learner_problem_summary"),
+        ("view", reporting, "fact_learner_response_attempts"),
+        ("view", reporting, "fact_navigation_dropoff"),
+        ("view", reporting, "fact_navigation"),
+        ("view", reporting, "fact_problem_engagement_per_subsection"),
+        ("view", reporting, "fact_problem_responses_extended"),
+        ("view", reporting, "fact_problem_responses"),
+        ("view", reporting, "fact_student_status"),
+        ("view", reporting, "fact_transcript_usage"),
+        ("view", reporting, "fact_video_plays"),
+        ("view", reporting, "fact_watched_video_duration"),
+        ("view", reporting, "fact_watched_video_segments"),
+        ("view", reporting, "int_pages_per_subsection"),
+        ("view", reporting, "int_problem_hints"),
+        ("view", reporting, "int_problem_results"),
+        ("view", reporting, "int_problems_per_subsection"),
+        ("view", reporting, "learner_summary"),
+        ("view", reporting, "most_recent_course_tags"),
+        ("view", reporting, "watched_video_duration"),
+        ("view", xapi, "fact_forum_interactions"),
     ] %}
 
     {% for model in models_to_drop %}

models/_docs/docs.md

@@ -116,10 +116,14 @@ The "id" portion of the xAPI Verb for this event. See: https://xapi.com/statemen
 Number indicating which attempt this was
 {% enddocs %}
 
-{% docs problem_number %}
+{% docs problem_location %}
 The section, subsection, unit, and part number of the problem block. In the format 1:2:3_1
 {% enddocs %}
 
+{% docs problem_name_location %}
+The problem_location followed by the name of the problem
+{% enddocs %}
+
 {% docs success %}
 Boolean indicating whether the responses were correct
 {% enddocs %}

models/courses/dim_course_blocks.sql

@@ -1,3 +1,12 @@
+{{
+    config(
+        materialized="materialized_view",
+        engine=get_engine("ReplacingMergeTree()"),
+        primary_key="(org, course_key, block_id)",
+        order_by="(org, course_key, block_id)",
+    )
+}}
+
 with
     dim_course_blocks as (
         select
@@ -7,10 +16,12 @@ with
             courses.course_run as course_run,
             blocks.location as block_id,
             blocks.block_name as block_name,
-            {{ section_from_display("blocks.display_name_with_location") }}
-            as section_number,
-            {{ subsection_from_display("blocks.display_name_with_location") }}
-            as subsection_number,
+            concat(
+                regexpExtract(blocks.display_name_with_location, '(\d+):'), ':0:0'
+            ) as section_number,
+            concat(
+                regexpExtract(blocks.display_name_with_location, '(.+):'), ':0'
+            ) as subsection_number,
             splitByString(' - ', blocks.display_name_with_location)[
                 1
             ] as hierarchy_location,

models/courses/dim_items_per_subsection.sql

@@ -0,0 +1,58 @@
+{{
+    config(
+        materialized="materialized_view",
+        engine=get_engine("ReplacingMergeTree()"),
+        primary_key="(org, course_key, block_type, section_number, subsection_number)",
+        order_by="(org, course_key, block_type, section_number, subsection_number)",
+    )
+}}
+
+select
+    ips.org as org,
+    ips.course_key as course_key,
+    ips.section_number as section_number,
+    section_blocks.display_name_with_location as section_with_name,
+    ips.subsection_number as subsection_number,
+    subsection_blocks.display_name_with_location as subsection_with_name,
+    count(*) as item_count,
+    subsection_blocks.block_id as subsection_block_id,
+    section_blocks.block_id as section_block_id,
+    subsection_blocks.course_order as subsection_course_order,
+    case
+        when ips.block_id like '%vertical+block%'
+        then 'vertical+block'
+        when ips.block_id like '%problem+block%'
+        then 'problem+block'
+        when ips.block_id like '%video+block%'
+        then 'video+block'
+        else 'none'
+    end as block_type
+from {{ ref("dim_course_blocks") }} ips
+left join
+    {{ ref("dim_course_blocks") }} section_blocks
+    on (
+        ips.section_number = section_blocks.hierarchy_location
+        and ips.org = section_blocks.org
+        and ips.course_key = section_blocks.course_key
+        and section_blocks.block_id like '%@chapter+block@%'
+    )
+left join
+    {{ ref("dim_course_blocks") }} subsection_blocks
+    on (
+        ips.subsection_number = subsection_blocks.hierarchy_location
+        and ips.org = subsection_blocks.org
+        and ips.course_key = subsection_blocks.course_key
+        and subsection_blocks.block_id like '%@sequential+block@%'
+    )
+where block_type <> 'none'
+group by
+    org,
+    course_key,
+    section_number,
+    section_with_name,
+    subsection_number,
+    subsection_with_name,
+    subsection_block_id,
+    section_block_id,
+    subsection_course_order,
+    block_type