From 322205f7431916e60170885c2739b8ea5df883b6 Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Tue, 23 Sep 2025 13:22:35 -0400 Subject: [PATCH 1/2] config: Add sqlfmt as pre-commit hook sqlfmt is a tool similar to Ruff and Black that auto-formats SQL, including Jinja templated SQL for dbt, with strong opinions. This adds this tool to our pre-commit checks to keep our SQL consistently formatted. --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 76527258c..a28c109bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,6 +65,12 @@ repos: - types-pytz - types-pymysql - types-requests +- repo: https://github.com/tconbeer/sqlfmt + rev: v0.27.0 + hooks: + - id: sqlfmt + language_version: python + additional_dependencies: ['.[jinjafmt]'] - repo: https://github.com/sqlfluff/sqlfluff rev: 3.4.2 hooks: From 6ae0956df49d0b2cc73569bd06a71e1f2dd9d9b8 Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Tue, 23 Sep 2025 14:37:57 -0400 Subject: [PATCH 2/2] chore: Apply sqlfmt to all dbt files --- pyproject.toml | 6 +- .../macros/apply_deduplication_query.sql | 34 +- .../macros/apply_grants_macro_override.sql | 8 +- src/ol_dbt/macros/cast_date_to_iso8601.sql | 10 +- .../macros/cast_timestamp_to_iso8601.sql | 12 +- src/ol_dbt/macros/extract_course_id.sql | 74 +- src/ol_dbt/macros/generate_hash_id.sql | 12 +- .../macros/generate_model_yaml_enhanced.sql | 100 +- .../macros/generate_program_readable_id.sql | 31 +- .../macros/starburst_trino_grant_sql.sql | 5 +- .../transform_code_to_readable_values.sql | 345 ++++-- .../macros/transform_studentmodule_data.sql | 143 ++- .../translate_course_id_to_platform.sql | 21 +- .../afact_course_page_engagement.sql | 110 +- .../afact_discussion_engagement.sql | 119 +- .../dimensional/afact_problem_engagement.sql | 152 ++- .../dimensional/afact_video_engagement.sql | 166 +-- .../models/dimensional/dim_course_content.sql | 295 ++--- .../dimensional/dim_discussion_topic.sql | 113 +- .../models/dimensional/dim_platform.sql | 14 +- src/ol_dbt/models/dimensional/dim_problem.sql | 208 ++-- src/ol_dbt/models/dimensional/dim_user.sql | 1089 ++++++++--------- src/ol_dbt/models/dimensional/dim_video.sql | 160 ++- .../tfact_course_navigation_events.sql | 433 ++++--- .../dimensional/tfact_discussion_events.sql | 384 +++--- .../dimensional/tfact_problem_events.sql | 451 ++++--- .../tfact_studentmodule_problems.sql | 260 ++-- .../models/dimensional/tfact_video_events.sql | 392 +++--- ..._mitx__openedx__bigquery__email_opt_in.sql | 11 +- ...penedx__mysql__assessment_aiclassifier.sql | 7 +- ...edx__mysql__assessment_aiclassifierset.sql | 7 +- ...x__mysql__assessment_aigradingworkflow.sql | 7 +- ...__mysql__assessment_aitrainingworkflow.sql | 8 +- ...t_aitrainingworkflow_training_examples.sql | 10 +- ..._openedx__mysql__assessment_assessment.sql | 20 +- ...__mysql__assessment_assessmentfeedback.sql | 44 +- ...essment_assessmentfeedback_assessments.sql | 37 +- ..._assessment_assessmentfeedback_options.sql | 53 +- ...l__assessment_assessmentfeedbackoption.sql | 61 +- ...nedx__mysql__assessment_assessmentpart.sql | 26 +- ...__openedx__mysql__assessment_criterion.sql | 117 +- ...edx__mysql__assessment_criterionoption.sql | 128 +- ...penedx__mysql__assessment_peerworkflow.sql | 8 +- ...dx__mysql__assessment_peerworkflowitem.sql | 8 +- ...itx__openedx__mysql__assessment_rubric.sql | 80 +- ...ql__assessment_studenttrainingworkflow.sql | 8 +- ...assessment_studenttrainingworkflowitem.sql | 8 +- ...edx__mysql__assessment_trainingexample.sql | 56 +- ...sment_trainingexample_options_selected.sql | 93 +- .../irx__mitx__openedx__mysql__auth_user.sql | 56 +- ...mitx__openedx__mysql__auth_userprofile.sql | 48 +- ...sql__certificates_generatedcertificate.sql | 41 +- ..._mysql__course_groups_cohortmembership.sql | 21 +- ...ourseware_studentmodulehistoryextended.sql | 28 +- ...enedx__mysql__credit_crediteligibility.sql | 18 +- ...ysql__django_comment_client_role_users.sql | 22 +- ...x__mysql__grades_persistentcoursegrade.sql | 19 +- ...ysql__grades_persistentsubsectiongrade.sql | 31 +- ...penedx__mysql__student_anonymoususerid.sql | 7 +- ...enedx__mysql__student_courseaccessrole.sql | 8 +- ...enedx__mysql__student_courseenrollment.sql | 8 +- ...dx__mysql__student_languageproficiency.sql | 26 +- ...itx__openedx__mysql__submissions_score.sql | 18 +- ...enedx__mysql__submissions_scoresummary.sql | 17 +- ...penedx__mysql__submissions_studentitem.sql | 8 +- ...openedx__mysql__submissions_submission.sql | 17 +- .../mitx/irx__mitx__openedx__mysql__teams.sql | 7 +- ...mitx__openedx__mysql__teams_membership.sql | 19 +- ...openedx__mysql__user_api_usercoursetag.sql | 7 +- ...irx__mitx__openedx__mysql__user_id_map.sql | 24 +- ...dx__mysql__workflow_assessmentworkflow.sql | 16 +- ...mysql__workflow_assessmentworkflowstep.sql | 19 +- ...nline__openedx__bigquery__email_opt_in.sql | 12 +- ..._openedx__mysql__assessment_assessment.sql | 26 +- ...__mysql__assessment_assessmentfeedback.sql | 52 +- ...essment_assessmentfeedback_assessments.sql | 44 +- ..._assessment_assessmentfeedback_options.sql | 66 +- ...l__assessment_assessmentfeedbackoption.sql | 79 +- ...nedx__mysql__assessment_assessmentpart.sql | 35 +- ...penedx__mysql__assessment_peerworkflow.sql | 8 +- ...dx__mysql__assessment_peerworkflowitem.sql | 8 +- ...ql__assessment_studenttrainingworkflow.sql | 8 +- ...assessment_studenttrainingworkflowitem.sql | 8 +- ..._mitxonline__openedx__mysql__auth_user.sql | 56 +- ...line__openedx__mysql__auth_userprofile.sql | 48 +- ...sql__certificates_generatedcertificate.sql | 41 +- ..._mysql__course_groups_cohortmembership.sql | 23 +- ...enedx__mysql__credit_crediteligibility.sql | 20 +- ...ysql__django_comment_client_role_users.sql | 23 +- ...x__mysql__grades_persistentcoursegrade.sql | 20 +- ...ysql__grades_persistentsubsectiongrade.sql | 31 +- ...penedx__mysql__student_anonymoususerid.sql | 7 +- ...enedx__mysql__student_courseaccessrole.sql | 8 +- ...enedx__mysql__student_courseenrollment.sql | 8 +- ...dx__mysql__student_languageproficiency.sql | 27 +- ...ine__openedx__mysql__submissions_score.sql | 20 +- ...enedx__mysql__submissions_scoresummary.sql | 19 +- ...penedx__mysql__submissions_studentitem.sql | 8 +- ...openedx__mysql__submissions_submission.sql | 19 +- ...irx__mitxonline__openedx__mysql__teams.sql | 7 +- ...line__openedx__mysql__teams_membership.sql | 20 +- ...openedx__mysql__user_api_usercoursetag.sql | 7 +- ...itxonline__openedx__mysql__user_id_map.sql | 24 +- ...dx__mysql__workflow_assessmentworkflow.sql | 17 +- ...mysql__workflow_assessmentworkflowstep.sql | 19 +- ..._xpro__openedx__bigquery__email_opt_in.sql | 11 +- ..._openedx__mysql__assessment_assessment.sql | 20 +- ...__mysql__assessment_assessmentfeedback.sql | 44 +- ...essment_assessmentfeedback_assessments.sql | 37 +- ..._assessment_assessmentfeedback_options.sql | 53 +- ...l__assessment_assessmentfeedbackoption.sql | 61 +- ...nedx__mysql__assessment_assessmentpart.sql | 26 +- ...penedx__mysql__assessment_peerworkflow.sql | 8 +- ...dx__mysql__assessment_peerworkflowitem.sql | 8 +- ...ql__assessment_studenttrainingworkflow.sql | 8 +- ...assessment_studenttrainingworkflowitem.sql | 8 +- .../irx__xpro__openedx__mysql__auth_user.sql | 56 +- ...xpro__openedx__mysql__auth_userprofile.sql | 48 +- ...sql__certificates_generatedcertificate.sql | 41 +- ..._mysql__course_groups_cohortmembership.sql | 21 +- ...enedx__mysql__credit_crediteligibility.sql | 18 +- ...ysql__django_comment_client_role_users.sql | 22 +- ...x__mysql__grades_persistentcoursegrade.sql | 19 +- ...ysql__grades_persistentsubsectiongrade.sql | 31 +- ...penedx__mysql__student_anonymoususerid.sql | 7 +- ...enedx__mysql__student_courseaccessrole.sql | 8 +- ...enedx__mysql__student_courseenrollment.sql | 8 +- ...dx__mysql__student_languageproficiency.sql | 26 +- ...pro__openedx__mysql__submissions_score.sql | 18 +- ...enedx__mysql__submissions_scoresummary.sql | 17 +- ...penedx__mysql__submissions_studentitem.sql | 8 +- ...openedx__mysql__submissions_submission.sql | 17 +- .../xpro/irx__xpro__openedx__mysql__teams.sql | 7 +- ...xpro__openedx__mysql__teams_membership.sql | 19 +- ...openedx__mysql__user_api_usercoursetag.sql | 7 +- ...irx__xpro__openedx__mysql__user_id_map.sql | 24 +- ...dx__mysql__workflow_assessmentworkflow.sql | 16 +- ...mysql__workflow_assessmentworkflowstep.sql | 19 +- .../int__bootcamps__applications.sql | 159 +-- .../bootcamps/int__bootcamps__course_runs.sql | 14 +- ...int__bootcamps__courserun_certificates.sql | 60 +- .../int__bootcamps__courserunenrollments.sql | 58 +- .../bootcamps/int__bootcamps__courses.sql | 11 +- .../int__bootcamps__ecommerce_order.sql | 73 +- .../int__bootcamps__ecommerce_receipt.sql | 57 +- ...otcamps__ecommerce_wiretransferreceipt.sql | 15 +- .../bootcamps/int__bootcamps__users.sql | 60 +- .../combined/int__combined__course_runs.sql | 491 ++++---- .../int__combined__course_structure.sql | 154 ++- .../combined/int__combined__course_videos.sql | 104 +- .../int__combined__courserun_certificates.sql | 111 +- .../int__combined__courserun_enrollments.sql | 450 ++++--- .../int__combined__user_course_roles.sql | 273 ++--- .../combined/int__combined__users.sql | 485 ++++---- .../int__edxorg__mitx_course_structure.sql | 112 +- ...t__edxorg__mitx_courserun_certificates.sql | 309 +++-- ...nt__edxorg__mitx_courserun_enrollments.sql | 206 ++-- .../int__edxorg__mitx_courserun_grades.sql | 81 +- .../edxorg/int__edxorg__mitx_courseruns.sql | 240 ++-- .../edxorg/int__edxorg__mitx_product.sql | 20 +- ...int__edxorg__mitx_program_certificates.sql | 64 +- .../int__edxorg__mitx_program_courses.sql | 38 +- .../int__edxorg__mitx_program_enrollments.sql | 54 +- .../int__edxorg__mitx_user_activity.sql | 26 +- ...nt__edxorg__mitx_user_courseactivities.sql | 119 +- ...xorg__mitx_user_courseactivities_daily.sql | 30 +- ...g__mitx_user_courseactivity_discussion.sql | 34 +- ..._mitx_user_courseactivity_problemcheck.sql | 39 +- ...x_user_courseactivity_problemsubmitted.sql | 32 +- ...edxorg__mitx_user_courseactivity_video.sql | 43 +- .../edxorg/int__edxorg__mitx_users.sql | 148 +-- .../learn-ai/int__learn_ai__chatbot.sql | 120 +- .../learn-ai/int__learn_ai__tutorbot.sql | 48 +- ...int__micromasters__course_certificates.sql | 309 +++-- .../int__micromasters__course_enrollments.sql | 53 +- .../int__micromasters__course_grades.sql | 300 +++-- ...cromasters__dedp_proctored_exam_grades.sql | 73 +- .../int__micromasters__orders.sql | 138 +-- ...nt__micromasters__program_certificates.sql | 250 ++-- ...int__micromasters__program_enrollments.sql | 338 +++-- ...nt__micromasters__program_requirements.sql | 128 +- .../int__micromasters__programs.sql | 12 +- .../micromasters/int__micromasters__users.sql | 137 +-- .../subqueries/__micromasters__users.sql | 155 ++- ...se_certificates_dedp_from_micromasters.sql | 111 +- ...urse_certificates_dedp_from_mitxonline.sql | 61 +- ...urse_certificates_non_dedp_from_edxorg.sql | 41 +- ...s_course_grades_dedp_from_micromasters.sql | 116 +- ...ers_course_grades_dedp_from_mitxonline.sql | 52 +- ...ers_course_grades_non_dedp_from_edxorg.sql | 37 +- ...am_certificates_dedp_from_micromasters.sql | 29 +- ...gram_certificates_dedp_from_mitxonline.sql | 32 +- ...omasters_program_certificates_non_dedp.sql | 127 +- .../int__mitx__courserun_certificates.sql | 168 ++- .../mitx/int__mitx__courserun_enrollments.sql | 125 +- ...x__courserun_enrollments_with_programs.sql | 131 +- .../mitx/int__mitx__courserun_grades.sql | 150 ++- .../intermediate/mitx/int__mitx__courses.sql | 82 +- .../mitx/int__mitx__program_certificates.sql | 77 +- .../mitx/int__mitx__program_requirements.sql | 85 +- .../intermediate/mitx/int__mitx__programs.sql | 37 +- .../intermediate/mitx/int__mitx__users.sql | 413 +++---- .../int__mitxonline__bulk_email_optin.sql | 39 +- ...t__mitxonline__course_blockedcountries.sql | 10 +- .../int__mitxonline__course_instructors.sql | 25 +- .../int__mitxonline__course_runs.sql | 44 +- .../int__mitxonline__course_structure.sql | 114 +- ...int__mitxonline__course_to_departments.sql | 17 +- .../int__mitxonline__course_to_topics.sql | 27 +- ...nt__mitxonline__courserun_certificates.sql | 68 +- .../int__mitxonline__courserun_grades.sql | 68 +- ...itxonline__courserun_subsection_grades.sql | 106 +- .../int__mitxonline__courserun_videos.sql | 41 +- .../int__mitxonline__courserunenrollments.sql | 215 ++-- ...ne__courserunenrollments_with_programs.sql | 152 ++- .../mitxonline/int__mitxonline__courses.sql | 123 +- .../int__mitxonline__ecommerce_basket.sql | 11 +- ...__mitxonline__ecommerce_basketdiscount.sql | 19 +- .../int__mitxonline__ecommerce_basketitem.sql | 13 +- .../int__mitxonline__ecommerce_discount.sql | 26 +- ..._mitxonline__ecommerce_discountproduct.sql | 11 +- ...txonline__ecommerce_discountredemption.sql | 11 +- .../int__mitxonline__ecommerce_order.sql | 165 +-- .../int__mitxonline__ecommerce_product.sql | 83 +- ...int__mitxonline__ecommerce_transaction.sql | 72 +- ...nt__mitxonline__ecommerce_userdiscount.sql | 11 +- ...flexiblepricing_countryincomethreshold.sql | 17 +- ...__flexiblepricing_currencyexchangerate.sql | 16 +- ...exiblepricing_flexiblepriceapplication.sql | 90 +- ...ine__flexiblepricing_flexiblepricetier.sql | 20 +- ...int__mitxonline__proctored_exam_grades.sql | 122 +- .../int__mitxonline__program_certificates.sql | 74 +- .../int__mitxonline__program_instructors.sql | 25 +- .../int__mitxonline__program_requirements.sql | 199 ++- .../int__mitxonline__programenrollments.sql | 59 +- .../mitxonline/int__mitxonline__programs.sql | 109 +- ...online__uai_customer_courserun_mapping.sql | 12 +- ...int__mitxonline__user_courseactivities.sql | 182 ++- ...itxonline__user_courseactivities_daily.sql | 35 +- ...online__user_courseactivity_discussion.sql | 40 +- ...line__user_courseactivity_problemcheck.sql | 45 +- ...__user_courseactivity_problemsubmitted.sql | 38 +- ...online__user_courseactivity_showanswer.sql | 24 +- ..._mitxonline__user_courseactivity_video.sql | 45 +- .../mitxonline/int__mitxonline__users.sql | 134 +- ..._good_economics_for_hard_times_program.sql | 75 +- .../int__mitxpro__b2becommerce_b2bcoupon.sql | 29 +- ...xpro__b2becommerce_b2bcouponredemption.sql | 12 +- .../int__mitxpro__b2becommerce_b2border.sql | 89 +- .../int__mitxpro__b2becommerce_b2breceipt.sql | 29 +- .../mitxpro/int__mitxpro__course_runs.sql | 52 +- .../int__mitxpro__course_structure.sql | 114 +- .../int__mitxpro__courserun_certificates.sql | 62 +- .../int__mitxpro__courserun_grades.sql | 60 +- .../int__mitxpro__courserun_videos.sql | 41 +- .../int__mitxpro__courserunenrollments.sql | 84 +- .../mitxpro/int__mitxpro__courses.sql | 123 +- .../int__mitxpro__courses_to_topics.sql | 31 +- .../mitxpro/int__mitxpro__coursesfaculty.sql | 75 +- .../int__mitxpro__coursesinprogram.sql | 75 +- .../mitxpro/int__mitxpro__coursetopic.sql | 11 +- .../int__mitxpro__ecommerce_allcoupons.sql | 165 ++- .../int__mitxpro__ecommerce_allorders.sql | 387 +++--- .../int__mitxpro__ecommerce_basket.sql | 19 +- .../int__mitxpro__ecommerce_basketitem.sql | 28 +- ..._mitxpro__ecommerce_basketrunselection.sql | 24 +- .../int__mitxpro__ecommerce_company.sql | 11 +- .../int__mitxpro__ecommerce_coupon.sql | 28 +- ...itxpro__ecommerce_couponpaymentversion.sql | 48 +- .../int__mitxpro__ecommerce_couponproduct.sql | 13 +- ...t__mitxpro__ecommerce_couponredemption.sql | 12 +- .../int__mitxpro__ecommerce_couponversion.sql | 45 +- .../mitxpro/int__mitxpro__ecommerce_line.sql | 50 +- ...t__mitxpro__ecommerce_linerunselection.sql | 12 +- .../mitxpro/int__mitxpro__ecommerce_order.sql | 94 +- .../int__mitxpro__ecommerce_product.sql | 109 +- ...pro__ecommerce_productcouponassignment.sql | 36 +- ...int__mitxpro__ecommerce_productversion.sql | 21 +- .../int__mitxpro__ecommerce_receipt.sql | 57 +- .../mitxpro/int__mitxpro__platforms.sql | 9 +- .../int__mitxpro__program_certificates.sql | 64 +- .../mitxpro/int__mitxpro__program_runs.sql | 27 +- .../int__mitxpro__programenrollments.sql | 61 +- .../mitxpro/int__mitxpro__programs.sql | 131 +- .../mitxpro/int__mitxpro__programsfaculty.sql | 79 +- .../int__mitxpro__user_courseactivities.sql | 182 ++- ...__mitxpro__user_courseactivities_daily.sql | 35 +- ...itxpro__user_courseactivity_discussion.sql | 40 +- ...xpro__user_courseactivity_problemcheck.sql | 45 +- ...__user_courseactivity_problemsubmitted.sql | 38 +- ...itxpro__user_courseactivity_showanswer.sql | 24 +- ...nt__mitxpro__user_courseactivity_video.sql | 45 +- .../mitxpro/int__mitxpro__users.sql | 114 +- ...int__mitxresidential__course_structure.sql | 114 +- ...mitxresidential__courserun_enrollments.sql | 51 +- ...int__mitxresidential__courserun_grades.sql | 51 +- ...int__mitxresidential__courserun_videos.sql | 22 +- .../int__mitxresidential__courseruns.sql | 24 +- ...mitxresidential__user_courseactivities.sql | 124 +- ...sidential__user_courseactivities_daily.sql | 35 +- ...ential__user_courseactivity_discussion.sql | 40 +- ...tial__user_courseactivity_problemcheck.sql | 45 +- ...__user_courseactivity_problemsubmitted.sql | 38 +- ...ential__user_courseactivity_showanswer.sql | 24 +- ...residential__user_courseactivity_video.sql | 47 +- .../int__mitxresidential__users.sql | 38 +- .../ocw/int__ocw__course_departments.sql | 12 +- .../ocw/int__ocw__course_instructors.sql | 61 +- .../ocw/int__ocw__course_topics.sql | 16 +- .../intermediate/ocw/int__ocw__courses.sql | 73 +- .../intermediate/ocw/int__ocw__resources.sql | 228 ++-- .../intermediate/ovs/int__ovs__videos.sql | 66 +- .../int__salesforce__opportunity.sql | 52 +- .../int__salesforce__opportunitylineitem.sql | 30 +- .../zendesk/int__zendesk__ticket.sql | 197 ++- .../zendesk/int__zendesk__ticket_comment.sql | 44 +- .../combined/marts__combined__orders.sql | 872 +++++++------ .../combined/marts__combined__products.sql | 485 ++++---- .../marts/combined/marts__combined__users.sql | 485 ++++---- .../marts__combined_course_engagements.sql | 443 +++---- ...rts__combined_course_enrollment_detail.sql | 876 +++++++------ .../marts__combined_coursesinprogram.sql | 109 +- .../combined/marts__combined_discounts.sql | 171 ++- .../marts__combined_problem_submissions.sql | 186 ++- ...ts__combined_program_enrollment_detail.sql | 624 +++++----- ...rts__combined_total_course_engagements.sql | 527 ++++---- .../marts__combined_video_engagements.sql | 216 ++-- ...arts__micromasters_course_certificates.sql | 60 +- .../marts__micromasters_dedp_exam_grades.sql | 71 +- ...rts__micromasters_program_certificates.sql | 5 +- .../marts__micromasters_summary.sql | 167 ++- ...marts__micromasters_summary_timeseries.sql | 407 +++--- .../marts__mitxonline_course_certificates.sql | 34 +- ...s__mitxonline_course_engagements_daily.sql | 167 +-- .../marts__mitxonline_course_enrollments.sql | 47 +- .../marts__mitxonline_discussions.sql | 52 +- .../marts__mitxonline_problem_submissions.sql | 71 +- .../marts__mitxonline_problem_summary.sql | 117 +- .../marts__mitxonline_user_profiles.sql | 59 +- .../marts__mitxonline_video_engagements.sql | 98 +- .../mitxpro/marts__mitxpro_all_coupons.sql | 388 +++--- .../marts__mitxpro_ecommerce_productlist.sql | 225 ++-- .../models/marts/ocw/marts__ocw_courses.sql | 54 +- .../edxorg_to_mitxonline_course_runs.sql | 150 +-- .../models/reporting/chatbot_usage_report.sql | 139 +-- .../reporting/cheating_detection_report.sql | 229 ++-- .../reporting/enrollment_detail_report.sql | 160 ++- .../reporting/learner_engagement_report.sql | 522 ++++---- .../reporting/program_summary_report.sql | 115 +- .../reporting/video_engagement_report.sql | 59 +- ...postgres__applications_applicationstep.sql | 28 +- ...pplications_applicationstep_submission.sql | 49 +- ...es__applications_courserun_application.sql | 39 +- ...applications_courserun_applicationstep.sql | 34 +- ...g__bootcamps__app__postgres__auth_user.sql | 29 +- ...ps__app__postgres__courserunenrollment.sql | 39 +- ...otcamps__app__postgres__courses_course.sql | 18 +- ...amps__app__postgres__courses_courserun.sql | 32 +- ...postgres__courses_courseruncertificate.sql | 40 +- ...ps__app__postgres__courses_installment.sql | 27 +- ...__app__postgres__courses_personalprice.sql | 27 +- ...mps__app__postgres__django_contenttype.sql | 24 +- ...otcamps__app__postgres__ecommerce_line.sql | 33 +- ...tcamps__app__postgres__ecommerce_order.sql | 37 +- ...s__app__postgres__ecommerce_orderaudit.sql | 33 +- ...amps__app__postgres__ecommerce_receipt.sql | 33 +- ...ostgres__ecommerce_wiretransferreceipt.sql | 32 +- ...__app__postgres__profiles_legaladdress.sql | 51 +- ...camps__app__postgres__profiles_profile.sql | 91 +- .../edxorg/stg__edxorg__api__course.sql | 50 +- .../edxorg/stg__edxorg__api__courserun.sql | 97 +- .../stg__edxorg__bigquery__mitx_courserun.sql | 79 +- ...__edxorg__bigquery__mitx_person_course.sql | 128 +- ...xorg__bigquery__mitx_user_email_opt_in.sql | 34 +- ...edxorg__bigquery__mitx_user_info_combo.sql | 101 +- ...xorg__s3__course_certificate_signatory.sql | 38 +- .../edxorg/stg__edxorg__s3__course_policy.sql | 30 +- .../stg__edxorg__s3__course_structure.sql | 69 +- .../edxorg/stg__edxorg__s3__course_video.sql | 28 +- .../edxorg/stg__edxorg__s3__courserun.sql | 66 +- ...stg__edxorg__s3__courserun_certificate.sql | 63 +- .../stg__edxorg__s3__courserun_enrollment.sql | 32 +- .../stg__edxorg__s3__courserun_grade.sql | 68 +- ...__edxorg__s3__courseware_studentmodule.sql | 38 +- .../stg__edxorg__s3__program_courses.sql | 28 +- ...tg__edxorg__s3__program_learner_report.sql | 231 ++-- .../edxorg/stg__edxorg__s3__programs.sql | 32 +- ...xorg__s3__tracking_logs__user_activity.sql | 106 +- .../staging/edxorg/stg__edxorg__s3__user.sql | 56 +- ...stg__edxorg__s3__user_courseaccessrole.sql | 28 +- .../edxorg/stg__edxorg__s3__user_profile.sql | 80 +- ...p__postgres__chatbots_djangocheckpoint.sql | 47 +- ...app__postgres__chatbots_tutorbotoutput.sql | 23 +- ...pp__postgres__chatbots_userchatsession.sql | 38 +- ...g__learn_ai__app__postgres__users_user.sql | 40 +- ...micromasters__app__postgres__auth_user.sql | 30 +- ...rs__app__postgres__auth_usersocialauth.sql | 25 +- ...masters__app__postgres__courses_course.sql | 38 +- ...ters__app__postgres__courses_courserun.sql | 70 +- ...rs__app__postgres__courses_electiveset.sql | 25 +- ...ostgres__courses_electiveset_to_course.sql | 20 +- ...asters__app__postgres__courses_program.sql | 40 +- ..._postgres__dashboard_programenrollment.sql | 21 +- ...ers__app__postgres__django_contenttype.sql | 24 +- ...sters__app__postgres__ecommerce_coupon.sql | 55 +- ...app__postgres__ecommerce_couponinvoice.sql | 22 +- ...masters__app__postgres__ecommerce_line.sql | 35 +- ...asters__app__postgres__ecommerce_order.sql | 33 +- ...ters__app__postgres__ecommerce_receipt.sql | 71 +- ...pp__postgres__ecommerce_redeemedcoupon.sql | 30 +- ...s__app__postgres__ecommerce_usercoupon.sql | 29 +- ...omasters__app__postgres__exams_examrun.sql | 39 +- ...__postgres__grades_combinedcoursegrade.sql | 34 +- ...pp__postgres__grades_coursecertificate.sql | 41 +- ...__app__postgres__grades_courserungrade.sql | 39 +- ...p__postgres__grades_proctoredexamgrade.sql | 46 +- ...p__postgres__grades_programcertificate.sql | 39 +- ...ers__app__postgres__profiles_education.sql | 40 +- ...rs__app__postgres__profiles_employment.sql | 40 +- ...sters__app__postgres__profiles_profile.sql | 135 +- ...user_program_certificate_override_list.sql | 9 +- ...arning_resources_learningresourcetopic.sql | 34 +- ...arning_resources_search_percolatequery.sql | 40 +- ..._resources_search_percolatequery_users.sql | 28 +- ..._postgres__learning_resources_userlist.sql | 34 +- ...es__learning_resources_userlist_topics.sql | 21 +- ...earning_resources_userlistrelationship.sql | 38 +- ...learn__app__postgres__profiles_profile.sql | 60 +- ...gres__profiles_profile_topic_interests.sql | 21 +- ...g__mitlearn__app__postgres__users_user.sql | 50 +- ...xonline__app__postgres__cms_coursepage.sql | 39 +- ...__app__postgres__cms_coursepage_topics.sql | 11 +- ...ine__app__postgres__cms_instructorpage.sql | 25 +- ..._app__postgres__cms_instructorpagelink.sql | 18 +- ...online__app__postgres__cms_programpage.sql | 39 +- ...nline__app__postgres__cms_wagtail_page.sql | 51 +- ..._app__postgres__courses_blockedcountry.sql | 30 +- ...xonline__app__postgres__courses_course.sql | 34 +- ...postgres__courses_course_to_department.sql | 20 +- ...line__app__postgres__courses_courserun.sql | 62 +- ...postgres__courses_courseruncertificate.sql | 42 +- ..._postgres__courses_courserunenrollment.sql | 46 +- ..._app__postgres__courses_courserungrade.sql | 38 +- ...ne__app__postgres__courses_coursetopic.sql | 29 +- ...ine__app__postgres__courses_department.sql | 28 +- ...online__app__postgres__courses_program.sql | 47 +- ...__postgres__courses_programcertificate.sql | 36 +- ...p__postgres__courses_programenrollment.sql | 36 +- ...__postgres__courses_programrequirement.sql | 41 +- ...ine__app__postgres__courses_programrun.sql | 33 +- ...ine__app__postgres__django_contenttype.sql | 24 +- ...nline__app__postgres__ecommerce_basket.sql | 28 +- ...pp__postgres__ecommerce_basketdiscount.sql | 33 +- ...e__app__postgres__ecommerce_basketitem.sql | 31 +- ...ine__app__postgres__ecommerce_discount.sql | 51 +- ...p__postgres__ecommerce_discountproduct.sql | 29 +- ...postgres__ecommerce_discountredemption.sql | 30 +- ...xonline__app__postgres__ecommerce_line.sql | 33 +- ...online__app__postgres__ecommerce_order.sql | 33 +- ...line__app__postgres__ecommerce_product.sql | 35 +- ...__app__postgres__ecommerce_transaction.sql | 35 +- ..._app__postgres__ecommerce_userdiscount.sql | 29 +- ...flexiblepricing_countryincomethreshold.sql | 36 +- ...__flexiblepricing_currencyexchangerate.sql | 39 +- ...exiblepricing_flexiblepriceapplication.sql | 54 +- ...res__flexiblepricing_flexiblepricetier.sql | 37 +- ...ne__app__postgres__openedx_openedxuser.sql | 35 +- ...ine__app__postgres__reversion_revision.sql | 27 +- ...line__app__postgres__reversion_version.sql | 29 +- ...ine__app__postgres__users_legaladdress.sql | 32 +- ..._mitxonline__app__postgres__users_user.sql | 40 +- ...line__app__postgres__users_userprofile.sql | 55 +- ...online__openedx__api__course_structure.sql | 78 +- ...__mitxonline__openedx__blockcompletion.sql | 62 +- ...ourseware_studentmodulehistoryextended.sql | 64 +- ..._mitxonline__openedx__mysql__auth_user.sql | 30 +- ...ine__openedx__mysql__bulk_email_optout.sql | 22 +- ...enedx__mysql__courseware_studentmodule.sql | 40 +- ...ne__openedx__mysql__edxval_coursevideo.sql | 23 +- ...txonline__openedx__mysql__edxval_video.sql | 29 +- ...openedx__mysql__grades_subsectiongrade.sql | 46 +- ..._mysql__grades_subsectiongradeoverride.sql | 49 +- ...__openedx__mysql__grades_visibleblocks.sql | 27 +- ..._openedx__mysql__user_courseaccessrole.sql | 29 +- ..._openedx__tracking_logs__user_activity.sql | 110 +- ...ritus__api__bigquery__user_enrollments.sql | 113 +- ...lumni__api__bigquery__user_enrollments.sql | 94 +- ..._app__postgres__b2becommerce_b2bcoupon.sql | 43 +- ..._postgres__b2becommerce_b2bcouponaudit.sql | 33 +- ...gres__b2becommerce_b2bcouponredemption.sql | 30 +- ...__app__postgres__b2becommerce_b2border.sql | 49 +- ...__postgres__b2becommerce_b2borderaudit.sql | 33 +- ...app__postgres__b2becommerce_b2breceipt.sql | 43 +- ...ro__app__postgres__cms_certificatepage.sql | 31 +- ...mitxpro__app__postgres__cms_coursepage.sql | 89 +- ...__app__postgres__cms_coursepage_topics.sql | 22 +- ...pp__postgres__cms_coursesinprogrampage.sql | 27 +- ..._app__postgres__cms_facultymemberspage.sql | 25 +- ...itxpro__app__postgres__cms_programpage.sql | 59 +- ...xpro__app__postgres__cms_signatorypage.sql | 27 +- ...mitxpro__app__postgres__courses_course.sql | 51 +- ...xpro__app__postgres__courses_courserun.sql | 50 +- ...postgres__courses_courseruncertificate.sql | 38 +- ..._postgres__courses_courserunenrollment.sql | 40 +- ..._app__postgres__courses_courserungrade.sql | 36 +- ...ro__app__postgres__courses_coursetopic.sql | 28 +- ...txpro__app__postgres__courses_platform.sql | 19 +- ...itxpro__app__postgres__courses_program.sql | 49 +- ...__postgres__courses_programcertificate.sql | 36 +- ...p__postgres__courses_programenrollment.sql | 38 +- ...pro__app__postgres__courses_programrun.sql | 32 +- ...pro__app__postgres__django_contenttype.sql | 24 +- ...txpro__app__postgres__ecommerce_basket.sql | 28 +- ...o__app__postgres__ecommerce_basketitem.sql | 33 +- ...postgres__ecommerce_basketrunselection.sql | 29 +- ...stgres__ecommerce_bulkcouponassignment.sql | 37 +- ...xpro__app__postgres__ecommerce_company.sql | 29 +- ...txpro__app__postgres__ecommerce_coupon.sql | 35 +- ..._app__postgres__ecommerce_couponbasket.sql | 30 +- ...app__postgres__ecommerce_couponpayment.sql | 27 +- ...stgres__ecommerce_couponpaymentversion.sql | 61 +- ...app__postgres__ecommerce_couponproduct.sql | 31 +- ...__postgres__ecommerce_couponredemption.sql | 29 +- ...app__postgres__ecommerce_couponversion.sql | 29 +- ...mitxpro__app__postgres__ecommerce_line.sql | 29 +- ...__postgres__ecommerce_linerunselection.sql | 29 +- ...itxpro__app__postgres__ecommerce_order.sql | 46 +- ...o__app__postgres__ecommerce_orderaudit.sql | 33 +- ...xpro__app__postgres__ecommerce_product.sql | 33 +- ...res__ecommerce_productcouponassignment.sql | 40 +- ...pp__postgres__ecommerce_productversion.sql | 35 +- ...pp__postgres__ecommerce_programrunline.sql | 29 +- ...xpro__app__postgres__ecommerce_receipt.sql | 29 +- ...pro__app__postgres__users_legaladdress.sql | 52 +- ..._mitxpro__app__postgres__users_profile.sql | 42 +- ...tg__mitxpro__app__postgres__users_user.sql | 32 +- ...__mitxpro__app__postgres__wagtail_page.sql | 52 +- ...itxpro__openedx__api__course_structure.sql | 70 +- ...stg__mitxpro__openedx__blockcompletion.sql | 62 +- ...ourseware_studentmodulehistoryextended.sql | 64 +- ...tg__mitxpro__openedx__mysql__auth_user.sql | 29 +- ...__openedx__mysql__courserun_enrollment.sql | 29 +- ...enedx__mysql__courseware_studentmodule.sql | 40 +- ...ro__openedx__mysql__edxval_coursevideo.sql | 21 +- ..._mitxpro__openedx__mysql__edxval_video.sql | 29 +- ..._openedx__mysql__user_courseaccessrole.sql | 29 +- ..._openedx__tracking_logs__user_activity.sql | 110 +- ...ential__openedx__api__course_structure.sql | 70 +- ...g__mitxresidential__openedx__auth_user.sql | 48 +- ...residential__openedx__auth_userprofile.sql | 33 +- ...xresidential__openedx__blockcompletion.sql | 62 +- ...g__mitxresidential__openedx__courserun.sql | 52 +- ...dential__openedx__courserun_enrollment.sql | 31 +- ...xresidential__openedx__courserun_grade.sql | 35 +- ...ial__openedx__courseware_studentmodule.sql | 40 +- ...ourseware_studentmodulehistoryextended.sql | 73 +- ...sidential__openedx__edxval_coursevideo.sql | 21 +- ...mitxresidential__openedx__edxval_video.sql | 29 +- ..._openedx__tracking_logs__user_activity.sql | 101 +- ...ential__openedx__user_courseaccessrole.sql | 29 +- ...cw__studio__postgres__websites_website.sql | 78 +- ...dio__postgres__websites_websitecontent.sql | 129 +- ...dio__postgres__websites_websitestarter.sql | 35 +- ...__ovs__studio__postgres__ui_collection.sql | 43 +- ...io__postgres__ui_collectionedxendpoint.sql | 21 +- ..._ovs__studio__postgres__ui_edxendpoint.sql | 31 +- ...g__ovs__studio__postgres__ui_encodejob.sql | 121 +- .../stg__ovs__studio__postgres__ui_video.sql | 45 +- .../stg__salesforce__opportunity.sql | 72 +- .../stg__salesforce__opportunitylineitem.sql | 50 +- .../staging/zendesk/stg__zendesk__brand.sql | 42 +- .../staging/zendesk/stg__zendesk__group.sql | 36 +- .../zendesk/stg__zendesk__organization.sql | 42 +- .../staging/zendesk/stg__zendesk__ticket.sql | 111 +- .../zendesk/stg__zendesk__ticket_comment.sql | 52 +- .../zendesk/stg__zendesk__ticket_field.sql | 32 +- .../staging/zendesk/stg__zendesk__user.sql | 104 +- 577 files changed, 19206 insertions(+), 22441 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6bf7b1e22..808e95871 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,11 @@ build-backend = "hatchling.build" [tool.sqlfmt] line_length = 120 -exclude=["target/**/*", "dbt_packages/**/*"] +exclude=[ + "target/**/*", + "dbt_packages/**/*", + "src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_encodejob.sql" +] [tool.sqlfluff.core] templater = "jinja" diff --git a/src/ol_dbt/macros/apply_deduplication_query.sql b/src/ol_dbt/macros/apply_deduplication_query.sql index 05fa2a59a..6b8fec36e 100644 --- a/src/ol_dbt/macros/apply_deduplication_query.sql +++ b/src/ol_dbt/macros/apply_deduplication_query.sql @@ -1,4 +1,5 @@ -{% macro deduplicate_query(cte_name1='source', cte_name2='most_recent_source', partition_columns='id') %} +{% macro deduplicate_query(cte_name1="source", cte_name2="most_recent_source", partition_columns="id") %} + , /* Add additional queries to handle duplicated data introduced by airbyte sync mode "Incremental Sync - Append" where cursor is set to a timestamp field. The deduplication logic works like this: @@ -6,34 +7,23 @@ recent sync in the source table. - If there is only one record for the same partition_columns, it will just load that record as it is. */ - , source_sorted as ( - select - * - , row_number() over ( partition by {{ partition_columns }} order by _airbyte_emitted_at desc) as row_num + source_sorted as ( + select *, row_number() over (partition by {{ partition_columns }} order by _airbyte_emitted_at desc) as row_num from {{ cte_name1 }} - ) - - , {{ cte_name2 }} as ( - select * - from source_sorted - where row_num = 1 - ) + ), + {{ cte_name2 }} as (select * from source_sorted where row_num = 1) {% endmacro %} -{% macro deduplicate_raw_table(order_by='_airbyte_extracted_at', partition_columns='id') %} +{% macro deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") %} + , /* This dedupe applied to the raw tables via Incremental Sync from Airbyte. It will deduplicate the data based on the partition_columns and order by columns. */ - , source_sorted as ( - select - * - , row_number() over ( partition by {{ partition_columns }} order by {{ order_by }} desc) as row_num + source_sorted as ( + select *, row_number() over (partition by {{ partition_columns }} order by {{ order_by }} desc) as row_num from source - ) - , most_recent_source as ( - select * from source_sorted - where row_num = 1 - ) + ), + most_recent_source as (select * from source_sorted where row_num = 1) {% endmacro %} diff --git a/src/ol_dbt/macros/apply_grants_macro_override.sql b/src/ol_dbt/macros/apply_grants_macro_override.sql index 3d3ee4c64..43107aa9a 100644 --- a/src/ol_dbt/macros/apply_grants_macro_override.sql +++ b/src/ol_dbt/macros/apply_grants_macro_override.sql @@ -1,9 +1,9 @@ {% macro apply_grants(relation, grant_config, should_revoke) %} - {% if target.name == 'production' %} - {#-- Only apply grants in production schema --#} + {% if target.name == "production" %} + {#- - Only apply grants in production schema --#} {{ return(adapter.dispatch("apply_grants", "dbt")(relation, grant_config, should_revoke)) }} {% else %} - {#-- Bypass code-based grant application in non-prod targets. --#} - {#-- This is so that we don't accidentally expose test tables to end-consumers of data --#} + {#- - Bypass code-based grant application in non-prod targets. --#} + {#- - This is so that we don't accidentally expose test tables to end-consumers of data --#} {% endif %} {% endmacro %} diff --git a/src/ol_dbt/macros/cast_date_to_iso8601.sql b/src/ol_dbt/macros/cast_date_to_iso8601.sql index e79a75574..57f3da70c 100644 --- a/src/ol_dbt/macros/cast_date_to_iso8601.sql +++ b/src/ol_dbt/macros/cast_date_to_iso8601.sql @@ -1,8 +1,8 @@ {% macro cast_date_to_iso8601(column_name) %} - case - when try_cast({{ column_name }} AS date) is not null -- Use try_cast to check if the column is already a date - then to_iso8601(try_cast({{ column_name }} AS date)) + case + when try_cast({{ column_name }} as date) is not null -- Use try_cast to check if the column is already a date + then to_iso8601(try_cast({{ column_name }} as date)) - else to_iso8601(from_iso8601_date(try_cast({{ column_name }} AS varchar))) -- Convert to ISO 8601 from a string -end + else to_iso8601(from_iso8601_date(try_cast({{ column_name }} as varchar))) -- Convert to ISO 8601 from a string + end {% endmacro %} diff --git a/src/ol_dbt/macros/cast_timestamp_to_iso8601.sql b/src/ol_dbt/macros/cast_timestamp_to_iso8601.sql index dc09dd37c..9ce488a74 100644 --- a/src/ol_dbt/macros/cast_timestamp_to_iso8601.sql +++ b/src/ol_dbt/macros/cast_timestamp_to_iso8601.sql @@ -1,9 +1,9 @@ {% macro cast_timestamp_to_iso8601(column_name) %} - case - -- If the column is already a timestamp, convert to ISO 8601 - when try_cast({{ column_name }} AS timestamp) is not null - then to_iso8601(try_cast({{ column_name }} AS timestamp)) + case + -- If the column is already a timestamp, convert to ISO 8601 + when try_cast({{ column_name }} as timestamp) is not null + then to_iso8601(try_cast({{ column_name }} as timestamp)) -- otherwise, convert to ISO 8601 from a string - else to_iso8601(from_iso8601_timestamp(try_cast({{ column_name }} AS varchar))) - end + else to_iso8601(from_iso8601_timestamp(try_cast({{ column_name }} as varchar))) + end {% endmacro %} diff --git a/src/ol_dbt/macros/extract_course_id.sql b/src/ol_dbt/macros/extract_course_id.sql index f64444102..968e82947 100644 --- a/src/ol_dbt/macros/extract_course_id.sql +++ b/src/ol_dbt/macros/extract_course_id.sql @@ -1,49 +1,47 @@ {% macro extract_course_id_from_tracking_log(course_id_has_old_format=false) %} - ---course ID format: {key type}:{org}+{course number}+{run tag} for courses created since Fall 2014 - ---course ID format: {org}/{course number}/{run tag} for courses created before Fall 2014 - ---Course number and run tag can be letters, numbers, period, dashes, underscores + -- -course ID format: {key type}:{org}+{course number}+{run tag} for courses created since Fall 2014 + -- -course ID format: {org}/{course number}/{run tag} for courses created before Fall 2014 + -- -Course number and run tag can be letters, numbers, period, dashes, underscores {% if course_id_has_old_format %} - {% set course_id_regex = '(([\w\.\-\_]+):([\w\.\-\_]+)[+]([\w\.\-\_]+)[+]([\w\.\-\_]+))|(([\w\.\-\_]+)[/]([\w\.\-\_]+)[/]([\w\.\-\_]+))' %} - {% else %} - {% set course_id_regex = 'course-v(\d{1}):([\w\.\-\_]+)\+([\w\.\-\_]+)\+([\w\.\-\_]+)' %} -{% endif %} + {% set course_id_regex = "(([\w\.\-\_]+):([\w\.\-\_]+)[+]([\w\.\-\_]+)[+]([\w\.\-\_]+))|(([\w\.\-\_]+)[/]([\w\.\-\_]+)[/]([\w\.\-\_]+))" %} + {% else %} {% set course_id_regex = "course-v(\d{1}):([\w\.\-\_]+)\+([\w\.\-\_]+)\+([\w\.\-\_]+)" %} + {% endif %} - case - when regexp_extract(json_query(context, 'lax $.course_id' omit quotes), '{{ course_id_regex }}') is not null - then json_query(context, 'lax $.course_id' omit quotes) - when regexp_extract(json_query(context, 'lax $.path' omit quotes), '{{ course_id_regex }}') is not null - then regexp_extract(json_query(context, 'lax $.path' omit quotes), '{{ course_id_regex }}') - when regexp_extract(event_type, '{{ course_id_regex }}') is not null - then regexp_extract(event_type, '{{ course_id_regex }}') - when regexp_extract(page, '{{ course_id_regex }}') is not null - then regexp_extract(page, '{{ course_id_regex }}') - end + case + when regexp_extract(json_query(context, 'lax $.course_id' omit quotes), '{{ course_id_regex }}') is not null + then json_query(context, 'lax $.course_id' omit quotes) + when regexp_extract(json_query(context, 'lax $.path' omit quotes), '{{ course_id_regex }}') is not null + then regexp_extract(json_query(context, 'lax $.path' omit quotes), '{{ course_id_regex }}') + when regexp_extract(event_type, '{{ course_id_regex }}') is not null + then regexp_extract(event_type, '{{ course_id_regex }}') + when regexp_extract(page, '{{ course_id_regex }}') is not null + then regexp_extract(page, '{{ course_id_regex }}') + end {% endmacro %} {% macro extract_course_readable_id(courserun_readable_id) %} - ---Output: course_readable_id in course-v1:{org}+{course number} format - ---Input: courserun_readable_id in course-v1:{org}+{course number}+{run tag} for courses created since Fall 2014, - --- {org}/{course number}/{run tag} for courses created before Fall 2014 - case - when position('course-v' in {{ courserun_readable_id }} ) > 0 - then regexp_extract({{ courserun_readable_id }}, 'course-v(\d{1}):([\w\.\-]+)\+([a-zA-Z0-9.-]+)') - else - concat( - 'course-v1:' - , replace(regexp_extract({{ courserun_readable_id }}, '([\w]+)/([a-zA-Z0-9.-]+)'), '/', '+') - ) - end + -- -Output: course_readable_id in course-v1:{org}+{course number} format + -- -Input: courserun_readable_id in course-v1:{org}+{course number}+{run tag} for courses created since Fall 2014, + -- - {org}/{course number}/{run tag} for courses created before Fall 2014 + case + when position('course-v' in {{ courserun_readable_id }}) > 0 + then regexp_extract({{ courserun_readable_id }}, 'course-v(\d{1}):([\w\.\-]+)\+([a-zA-Z0-9.-]+)') + else + concat( + 'course-v1:', replace(regexp_extract({{ courserun_readable_id }}, '([\w]+)/([a-zA-Z0-9.-]+)'), '/', '+') + ) + end {% endmacro %} ---- course IDs come in two formats from different sources. This ensures that course IDs are consistently converted in --- all the downstream models. -{% macro format_course_id(column_name='courserun_readable_id', convert_to_old_format=true) %} +-- - course IDs come in two formats from different sources. This ensures that course IDs are consistently converted in +-- all the downstream models. +{% macro format_course_id(column_name="courserun_readable_id", convert_to_old_format=true) %} {% if convert_to_old_format %} - -- format as {org}/{course number}/{run} - replace(replace({{ column_name }}, 'course-v1:', ''), '+', '/') - {% else %} - -- format as course-v1:{org}+{course}+{run} - 'course-v1:' || replace({{ column_name }}, '/', '+') - {% endif %} + -- format as {org}/{course number}/{run} + replace(replace({{ column_name }}, 'course-v1:', ''), '+', '/') + {% else %} + -- format as course-v1:{org}+{course}+{run} + 'course-v1:' || replace({{ column_name }}, '/', '+') + {% endif %} {% endmacro %} diff --git a/src/ol_dbt/macros/generate_hash_id.sql b/src/ol_dbt/macros/generate_hash_id.sql index b5587ed6a..21b504202 100644 --- a/src/ol_dbt/macros/generate_hash_id.sql +++ b/src/ol_dbt/macros/generate_hash_id.sql @@ -1,10 +1,10 @@ {% macro generate_hash_id(string) %} -- Be cautious about changing the hash function as it will impact the primary key used by Hightouch - lower( - to_hex( - sha256( - cast({{ string }} as varbinary) --noqa - ) - ) + lower( + to_hex( + sha256( + cast({{ string }} as varbinary) -- noqa + ) + ) ) {% endmacro %} diff --git a/src/ol_dbt/macros/generate_model_yaml_enhanced.sql b/src/ol_dbt/macros/generate_model_yaml_enhanced.sql index 0b0a61935..a6413d77d 100644 --- a/src/ol_dbt/macros/generate_model_yaml_enhanced.sql +++ b/src/ol_dbt/macros/generate_model_yaml_enhanced.sql @@ -1,25 +1,34 @@ -- Enhanced version of generate_model_yaml that works with our enhanced staging models -- This macro compiles models first to get their actual column structure - {% macro generate_model_yaml_enhanced(model_names=[], upstream_descriptions=False, include_data_types=True) -%} - {{ return(adapter.dispatch('generate_model_yaml_enhanced', 'codegen')(model_names, upstream_descriptions, include_data_types)) }} + {{ + return( + adapter.dispatch("generate_model_yaml_enhanced", "codegen")( + model_names, upstream_descriptions, include_data_types + ) + ) + }} {%- endmacro %} {% macro default__generate_model_yaml_enhanced(model_names, upstream_descriptions, include_data_types) %} - {% set model_yaml=[] %} + {% set model_yaml = [] %} - {% do model_yaml.append('version: 2') %} - {% do model_yaml.append('') %} - {% do model_yaml.append('models:') %} + {% do model_yaml.append("version: 2") %} + {% do model_yaml.append("") %} + {% do model_yaml.append("models:") %} {% if model_names is string %} - {{ exceptions.raise_compiler_error("The `model_names` argument must always be a list, even if there is only one model.") }} + {{ + exceptions.raise_compiler_error( + "The `model_names` argument must always be a list, even if there is only one model." + ) + }} {% else %} {% for model in model_names %} - {% do model_yaml.append(' - name: ' ~ model | lower) %} + {% do model_yaml.append(" - name: " ~ model | lower) %} {% do model_yaml.append(' description: ""') %} - {% do model_yaml.append(' columns:') %} + {% do model_yaml.append(" columns:") %} {% set columns = [] %} {% set column_desc_dict = {} %} @@ -36,22 +45,18 @@ {# Generate column YAML manually since we have simple column info #} {% for column in columns %} - {% do model_yaml.append(' - name: ' ~ column.name | lower) %} + {% do model_yaml.append(" - name: " ~ column.name | lower) %} {% if include_data_types and column.data_type %} - {% do model_yaml.append(' data_type: ' ~ column.data_type) %} + {% do model_yaml.append(" data_type: " ~ column.data_type) %} {% endif %} - {% set desc = column_desc_dict.get(column.name | lower, '') %} + {% set desc = column_desc_dict.get(column.name | lower, "") %} {% do model_yaml.append(' description: "' ~ desc ~ '"') %} - {% do model_yaml.append('') %} + {% do model_yaml.append("") %} {% endfor %} {% endfor %} {% endif %} - {% if execute %} - {% set joined = model_yaml | join ('\n') %} - {{ print(joined) }} - {% do return(joined) %} - {% endif %} + {% if execute %} {% set joined = model_yaml | join("\n") %} {{ print(joined) }} {% do return(joined) %} {% endif %} {% endmacro %} @@ -77,10 +82,7 @@ {# Create column objects with proper data types #} {% for col_name in column_names %} {% set source_col_info = get_source_column_type(col_name, source_columns_map, sql) %} - {% set col_obj = { - 'name': col_name, - 'data_type': source_col_info.data_type - } %} + {% set col_obj = {"name": col_name, "data_type": source_col_info.data_type} %} {% do ns.columns.append(col_obj) %} {% endfor %} {% endif %} @@ -98,10 +100,10 @@ {# Look for source('source_name', 'table_name') pattern #} {% set source_pattern = "source\\s*\\(\\s*['\"]([^'\"]+)['\"]\\s*,\\s*['\"]([^'\"]+)['\"]\\s*\\)" %} {% set matches = modules.re.findall(source_pattern, sql) %} - {% if matches and matches|length > 0 %} - {% do return(matches[0][1]) %} {# Return the table name #} + {% if matches and matches | length > 0 %} + {% do return(matches[0][1]) %} {# Return the table name #} {% endif %} - {% do return('') %} + {% do return("") %} {% endmacro %} {# Get source columns mapping from the graph #} @@ -125,17 +127,16 @@ {# Get the data type for a column by mapping it back to source #} {% macro get_source_column_type(staging_col_name, source_columns_map, sql) %} - {% set ns = namespace(data_type='text') %} + {% set ns = namespace(data_type="text") %} {# Try to find the source column mapping by looking at the SQL transformations #} {% set source_col_name = map_staging_to_source_column(staging_col_name, sql) %} {% if source_col_name and source_col_name in source_columns_map %} - {% set source_column = source_columns_map[source_col_name] %} - {% set ns.data_type = source_column.data_type %} + {% set source_column = source_columns_map[source_col_name] %} {% set ns.data_type = source_column.data_type %} {% endif %} - {% do return({'data_type': ns.data_type}) %} + {% do return({"data_type": ns.data_type}) %} {% endmacro %} {# Map staging column names back to source column names by parsing the SELECT clause #} @@ -145,30 +146,27 @@ - "function(source_col) as staging_col" - "source_col," (same name) #} - {# Extract the cleaned CTE part of the SQL #} - {% set cleaned_start = sql.lower().find(', cleaned as (') %} - {% set cleaned_end = sql.lower().find(')', cleaned_start) %} + {% set cleaned_start = sql.lower().find(", cleaned as (") %} + {% set cleaned_end = sql.lower().find(")", cleaned_start) %} {% if cleaned_start >= 0 and cleaned_end >= 0 %} {% set cleaned_sql = sql[cleaned_start:cleaned_end] %} {# Look for the staging column in the cleaned SELECT #} - {% set lines = cleaned_sql.split('\n') %} + {% set lines = cleaned_sql.split("\n") %} {% for line in lines %} - {% set clean_line = line.strip().rstrip(',') %} - {% if ' as ' ~ staging_col_name in clean_line.lower() %} + {% set clean_line = line.strip().rstrip(",") %} + {% if " as " ~ staging_col_name in clean_line.lower() %} {# Extract the source column name before 'as' #} - {% set as_pos = clean_line.lower().find(' as ' ~ staging_col_name.lower()) %} + {% set as_pos = clean_line.lower().find(" as " ~ staging_col_name.lower()) %} {% set source_part = clean_line[:as_pos].strip() %} {# Handle different patterns #} - {% if source_part.startswith('{{') %} + {% if source_part.startswith("{{") %} {# Macro call like {{ cast_timestamp_to_iso8601('last_login') }} #} {% set macro_match = modules.re.search("\\('([^']+)'\\)", source_part) %} - {% if macro_match %} - {% do return(macro_match.group(1)) %} - {% endif %} + {% if macro_match %} {% do return(macro_match.group(1)) %} {% endif %} {% else %} {# Simple column reference #} {% do return(source_part) %} @@ -181,10 +179,8 @@ {% endif %} {# Fallback: try common name mappings #} - {% if staging_col_name.startswith('user_') %} - {% do return(staging_col_name[5:]) %} {# Remove user_ prefix #} - {% elif staging_col_name.startswith('item_') %} - {% do return(staging_col_name[5:]) %} {# Remove item_ prefix #} + {% if staging_col_name.startswith("user_") %} {% do return(staging_col_name[5:]) %} {# Remove user_ prefix #} + {% elif staging_col_name.startswith("item_") %} {% do return(staging_col_name[5:]) %} {# Remove item_ prefix #} {% endif %} {% do return(staging_col_name) %} @@ -196,22 +192,22 @@ {% if sql %} {# Find the last occurrence of 'select' before 'from cleaned' #} - {% set select_pos = sql.lower().rfind('select') %} - {% set from_cleaned_pos = sql.lower().find('from cleaned') %} + {% set select_pos = sql.lower().rfind("select") %} + {% set from_cleaned_pos = sql.lower().find("from cleaned") %} {% if select_pos >= 0 and from_cleaned_pos >= 0 and select_pos < from_cleaned_pos %} {# Extract the portion between select and from cleaned #} {% set select_part = sql[select_pos:from_cleaned_pos] %} {# Split by lines and extract column names #} - {% set lines = select_part.split('\n') %} + {% set lines = select_part.split("\n") %} {% for line in lines %} {% set clean_line = line.strip() %} - {% if clean_line and not clean_line.lower().startswith('select') and not clean_line.startswith('--') %} - {% set col_clean = clean_line.rstrip(',').strip() %} - {% if col_clean %} - {% do ns.columns.append(col_clean) %} - {% endif %} + {% if clean_line and not clean_line.lower().startswith("select") and not clean_line.startswith( + "--" + ) %} + {% set col_clean = clean_line.rstrip(",").strip() %} + {% if col_clean %} {% do ns.columns.append(col_clean) %} {% endif %} {% endif %} {% endfor %} {% endif %} diff --git a/src/ol_dbt/macros/generate_program_readable_id.sql b/src/ol_dbt/macros/generate_program_readable_id.sql index 6faa25ec1..042fd613b 100644 --- a/src/ol_dbt/macros/generate_program_readable_id.sql +++ b/src/ol_dbt/macros/generate_program_readable_id.sql @@ -1,19 +1,24 @@ {% macro generate_micromasters_program_readable_id(micromasters_program_id, program_title) %} - -- Generate readable IDs for MicroMasters programs that runs on the edx platform - --- This is needed for programs that have not migrated to MITx Online yet + -- Generate readable IDs for MicroMasters programs that runs on the edx platform + -- - This is needed for programs that have not migrated to MITx Online yet case when {{ micromasters_program_id }} = 4 - --- Statistics and Data Science has multiple tracks, we need to extract the track name and add it to - --- the readable id - then 'program-v1:MITx+SDS+' || replace(regexp_extract({{ program_title }}, '\((.*?)\)', 1),' ', '-') + -- - Statistics and Data Science has multiple tracks, we need to extract the track name and add it to + -- - the readable id + then 'program-v1:MITx+SDS+' || replace(regexp_extract({{ program_title }}, '\((.*?)\)', 1), ' ', '-') when {{ micromasters_program_id }} = 5 - --- Finance (active) and MIT Finance (retired) has the same readable ID - then 'program-v1:MITx+FIN' - when {{ micromasters_program_id }} is not null then - 'program-v1:MITx+' || upper(array_join( - transform(split({{ program_title }}, ' '), -- noqa: PRS - x -> substring(x, 1, 1) - ), '' - )) + -- - Finance (active) and MIT Finance (retired) has the same readable ID + then 'program-v1:MITx+FIN' + when {{ micromasters_program_id }} is not null + then + 'program-v1:MITx+' || upper( + array_join( + transform( + split({{ program_title }}, ' '), -- noqa: PRS + x -> substring(x, 1, 1) + ), + '' + ) + ) end {% endmacro %} diff --git a/src/ol_dbt/macros/starburst_trino_grant_sql.sql b/src/ol_dbt/macros/starburst_trino_grant_sql.sql index 515075497..829dff006 100644 --- a/src/ol_dbt/macros/starburst_trino_grant_sql.sql +++ b/src/ol_dbt/macros/starburst_trino_grant_sql.sql @@ -1,12 +1,11 @@ -- This overrides the grant SQL to apply only to ROLEs since that is the requirement for -- Starburst Galaxy's permissions system. - {%- macro trino__get_grant_sql(relation, privilege, grantees) -%} - GRANT {{ privilege }} ON {{ relation }} TO ROLE {{ adapter.quote(grantees[0]) }} + grant {{ privilege }} on {{ relation }} to role {{ adapter.quote(grantees[0]) }} {%- endmacro %} -- This overrides the revoke SQL to apply only to ROLEs since that is the requirement for -- Starburst Galaxy's permissions system.' {%- macro trino__get_revoke_sql(relation, privilege, grantees) -%} - REVOKE {{ privilege }} ON {{ relation }} FROM ROLE {{ adapter.quote(grantees[0]) }} + revoke {{ privilege }} on {{ relation }} from role {{ adapter.quote(grantees[0]) }} {%- endmacro %} diff --git a/src/ol_dbt/macros/transform_code_to_readable_values.sql b/src/ol_dbt/macros/transform_code_to_readable_values.sql index 825e94b74..c861a0af9 100644 --- a/src/ol_dbt/macros/transform_code_to_readable_values.sql +++ b/src/ol_dbt/macros/transform_code_to_readable_values.sql @@ -1,144 +1,249 @@ {% macro transform_gender_value(column_name) %} - case - when {{ column_name }} = 'm' then 'Male' - when {{ column_name }} = 'f' then 'Female' - when {{ column_name }} = 't' then 'Transgender' - when {{ column_name }} = 'b' then 'Binary' - when {{ column_name }} = 'nb' then 'Non-binary/non-conforming' - when {{ column_name }} = 'o' then 'Other/Prefer Not to Say' - else null - end + case + when {{ column_name }} = 'm' + then 'Male' + when {{ column_name }} = 'f' + then 'Female' + when {{ column_name }} = 't' + then 'Transgender' + when {{ column_name }} = 'b' + then 'Binary' + when {{ column_name }} = 'nb' + then 'Non-binary/non-conforming' + when {{ column_name }} = 'o' + then 'Other/Prefer Not to Say' + else null + end {% endmacro %} {% macro transform_education_value(column_name) %} case - when {{ column_name }} = 'p' then 'Doctorate' - when {{ column_name }} = 'm' then 'Master''s or professional degree' - when {{ column_name }} = 'b' then 'Bachelor''s degree' - when {{ column_name }} = 'a' then 'Associate degree' - when {{ column_name }} = 'hs' then 'Secondary/high school' - when {{ column_name }} = 'jhs' then 'Junior secondary/junior high/middle school' - when {{ column_name }} = 'el' then 'Elementary/primary school' - when {{ column_name }} = 'none' then 'No formal education' - when {{ column_name }} = 'other' or {{ column_name }} = 'o' then 'Other education' - --- the following two are no longer used, but there are still users' profiles with these values - when {{ column_name }} = 'p_se' then 'Doctorate in science or engineering' - when {{ column_name }} = 'p_oth' then 'Doctorate in another field' + when {{ column_name }} = 'p' + then 'Doctorate' + when {{ column_name }} = 'm' + then 'Master''s or professional degree' + when {{ column_name }} = 'b' + then 'Bachelor''s degree' + when {{ column_name }} = 'a' + then 'Associate degree' + when {{ column_name }} = 'hs' + then 'Secondary/high school' + when {{ column_name }} = 'jhs' + then 'Junior secondary/junior high/middle school' + when {{ column_name }} = 'el' + then 'Elementary/primary school' + when {{ column_name }} = 'none' + then 'No formal education' + when {{ column_name }} = 'other' or {{ column_name }} = 'o' + then 'Other education' + -- - the following two are no longer used, but there are still users' profiles with these values + when {{ column_name }} = 'p_se' + then 'Doctorate in science or engineering' + when {{ column_name }} = 'p_oth' + then 'Doctorate in another field' else null end {% endmacro %} -{% macro transform_company_size_value(column_name='company_size') %} +{% macro transform_company_size_value(column_name="company_size") %} case - when {{ column_name }} = 1 then 'Small/Start-up (1+ employees)' - when {{ column_name }} = 9 then 'Small/Home office (1-9 employees)' - when {{ column_name }} = 99 then 'Small (10-99 employees)' - when {{ column_name }} = 999 then 'Small to medium-sized (100-999 employees)' - when {{ column_name }} = 9999 then 'Medium-sized (1000-9999 employees)' - when {{ column_name }} = 10000 then 'Large Enterprise (10,000+ employees)' - when {{ column_name }} = 0 then 'Other (N/A or Don''t know)' + when {{ column_name }} = 1 + then 'Small/Start-up (1+ employees)' + when {{ column_name }} = 9 + then 'Small/Home office (1-9 employees)' + when {{ column_name }} = 99 + then 'Small (10-99 employees)' + when {{ column_name }} = 999 + then 'Small to medium-sized (100-999 employees)' + when {{ column_name }} = 9999 + then 'Medium-sized (1000-9999 employees)' + when {{ column_name }} = 10000 + then 'Large Enterprise (10,000+ employees)' + when {{ column_name }} = 0 + then 'Other (N/A or Don''t know)' else cast({{ column_name }} as varchar) end {% endmacro %} -{% macro transform_years_experience_value(column_name='years_experience') %} - case - when {{ column_name }} = 2 then 'Less than 2 years' - when {{ column_name }} = 5 then '2-5 years' - when {{ column_name }} = 10 then '6 - 10 years' - when {{ column_name }} = 15 then '11 - 15 years' - when {{ column_name }} = 20 then '16 - 20 years' - when {{ column_name }} = 21 then 'More than 20 years' - when {{ column_name }} = 0 then 'Prefer not to say' +{% macro transform_years_experience_value(column_name="years_experience") %} + case + when {{ column_name }} = 2 + then 'Less than 2 years' + when {{ column_name }} = 5 + then '2-5 years' + when {{ column_name }} = 10 + then '6 - 10 years' + when {{ column_name }} = 15 + then '11 - 15 years' + when {{ column_name }} = 20 + then '16 - 20 years' + when {{ column_name }} = 21 + then 'More than 20 years' + when {{ column_name }} = 0 + then 'Prefer not to say' else cast({{ column_name }} as varchar) end {% endmacro %} ----- https://github.com/mitodl/ocw-studio/blob/master/static/js/resources/departments.json -{% macro transform_ocw_department_number(column_name='department_number') %} - case - when {{ column_name }} = '5' then 'Chemistry' - when {{ column_name }} = '20' then 'Biological Engineering' - when {{ column_name }} = '16' then 'Aeronautics and Astronautics' - when {{ column_name }} = '21G' then 'Global Studies and Languages' - when {{ column_name }} = '18' then 'Mathematics' - when {{ column_name }} = 'HST' then 'Health Sciences and Technology' - when {{ column_name }} = 'EC' then 'Edgerton Center' - when {{ column_name }} = 'WGS' then 'Women''s and Gender Studies' - when {{ column_name }} = '11' then 'Urban Studies and Planning' - when {{ column_name }} = '15' then 'Sloan School of Management' - when {{ column_name }} = '21A' then 'Anthropology' - when {{ column_name }} = 'ESD' then 'Engineering Systems Division' - when {{ column_name }} = '10' then 'Chemical Engineering' - when {{ column_name }} = '12' then 'Earth, Atmospheric, and Planetary Sciences' - when {{ column_name }} = '21M' then 'Music and Theater Arts' - when {{ column_name }} = 'PE' then 'Athletics, Physical Education and Recreation' - when {{ column_name }} = '4' then 'Architecture' - when {{ column_name }} = 'ES' then 'Experimental Study Group' - when {{ column_name }} = '21L' then 'Literature' - when {{ column_name }} = '2' then 'Mechanical Engineering' - when {{ column_name }} = '6' then 'Electrical Engineering and Computer Science' - when {{ column_name }} = '3' then 'Materials Science and Engineering' - when {{ column_name }} = '21H' then 'History' - when {{ column_name }} = '24' then 'Linguistics and Philosophy' - when {{ column_name }} = 'IDS' then 'Institute for Data, Systems, and Society' - when {{ column_name }} = '7' then 'Biology' - when {{ column_name }} = '1' then 'Civil and Environmental Engineering' - when {{ column_name }} = 'CMS-W' then 'Comparative Media Studies/Writing' - when {{ column_name }} = '22' then 'Nuclear Science and Engineering' - when {{ column_name }} = '14' then 'Economics' - when {{ column_name }} = 'CC' then 'Concourse' - when {{ column_name }} = '9' then 'Brain and Cognitive Sciences' - when {{ column_name }} = 'STS' then 'Science, Technology, and Society' - when {{ column_name }} = '17' then 'Political Science' - when {{ column_name }} = 'MAS' then 'Media Arts and Sciences' - when {{ column_name }} = '8' then 'Physics' - when {{ column_name }} = 'RES' then 'Supplemental Resources' +-- -- https://github.com/mitodl/ocw-studio/blob/master/static/js/resources/departments.json +{% macro transform_ocw_department_number(column_name="department_number") %} + case + when {{ column_name }} = '5' + then 'Chemistry' + when {{ column_name }} = '20' + then 'Biological Engineering' + when {{ column_name }} = '16' + then 'Aeronautics and Astronautics' + when {{ column_name }} = '21G' + then 'Global Studies and Languages' + when {{ column_name }} = '18' + then 'Mathematics' + when {{ column_name }} = 'HST' + then 'Health Sciences and Technology' + when {{ column_name }} = 'EC' + then 'Edgerton Center' + when {{ column_name }} = 'WGS' + then 'Women''s and Gender Studies' + when {{ column_name }} = '11' + then 'Urban Studies and Planning' + when {{ column_name }} = '15' + then 'Sloan School of Management' + when {{ column_name }} = '21A' + then 'Anthropology' + when {{ column_name }} = 'ESD' + then 'Engineering Systems Division' + when {{ column_name }} = '10' + then 'Chemical Engineering' + when {{ column_name }} = '12' + then 'Earth, Atmospheric, and Planetary Sciences' + when {{ column_name }} = '21M' + then 'Music and Theater Arts' + when {{ column_name }} = 'PE' + then 'Athletics, Physical Education and Recreation' + when {{ column_name }} = '4' + then 'Architecture' + when {{ column_name }} = 'ES' + then 'Experimental Study Group' + when {{ column_name }} = '21L' + then 'Literature' + when {{ column_name }} = '2' + then 'Mechanical Engineering' + when {{ column_name }} = '6' + then 'Electrical Engineering and Computer Science' + when {{ column_name }} = '3' + then 'Materials Science and Engineering' + when {{ column_name }} = '21H' + then 'History' + when {{ column_name }} = '24' + then 'Linguistics and Philosophy' + when {{ column_name }} = 'IDS' + then 'Institute for Data, Systems, and Society' + when {{ column_name }} = '7' + then 'Biology' + when {{ column_name }} = '1' + then 'Civil and Environmental Engineering' + when {{ column_name }} = 'CMS-W' + then 'Comparative Media Studies/Writing' + when {{ column_name }} = '22' + then 'Nuclear Science and Engineering' + when {{ column_name }} = '14' + then 'Economics' + when {{ column_name }} = 'CC' + then 'Concourse' + when {{ column_name }} = '9' + then 'Brain and Cognitive Sciences' + when {{ column_name }} = 'STS' + then 'Science, Technology, and Society' + when {{ column_name }} = '17' + then 'Political Science' + when {{ column_name }} = 'MAS' + then 'Media Arts and Sciences' + when {{ column_name }} = '8' + then 'Physics' + when {{ column_name }} = 'RES' + then 'Supplemental Resources' else {{ column_name }} end {% endmacro %} ---- https://github.com/mitodl/mit-learn/blob/main/learning_resources/constants.py#L189-L227 -{% macro transform_edx_department_number(column_name='department_number') %} - case - when {{ column_name }} = '1' then 'Civil and Environmental Engineering' - when {{ column_name }} = '2' then 'Mechanical Engineering' - when {{ column_name }} = '3' then 'Materials Science and Engineering' - when {{ column_name }} = '4' then 'Architecture' - when {{ column_name }} = '5' then 'Chemistry' - when {{ column_name }} = '6' then 'Electrical Engineering and Computer Science' - when {{ column_name }} = '7' then 'Biology' - when {{ column_name }} = '8' then 'Physics' - when {{ column_name }} = '9' then 'Brain and Cognitive Sciences' - when {{ column_name }} = '10' then 'Chemical Engineering' - when {{ column_name }} = '11' then 'Urban Studies and Planning' - when {{ column_name }} = '12' then 'Earth, Atmospheric, and Planetary Sciences' - when {{ column_name }} = '14' then 'Economics' - when {{ column_name }} = '15' then 'Management' - when {{ column_name }} = '16' then 'Aeronautics and Astronautics' - when {{ column_name }} = '17' then 'Political Science' - when {{ column_name }} = '18' then 'Mathematics' - when {{ column_name }} = '20' then 'Biological Engineering' - when {{ column_name }} = '21A' then 'Anthropology' - when {{ column_name }} = '21G' then 'Global Languages' - when {{ column_name }} = '21H' then 'History' - when {{ column_name }} = '21L' then 'Literature' - when {{ column_name }} = '21M' then 'Music and Theater Arts' - when {{ column_name }} = '22' then 'Nuclear Science and Engineering' - when {{ column_name }} = '24' then 'Linguistics and Philosophy' - when {{ column_name }} = 'CC' then 'Concourse' - when {{ column_name }} = 'CMS-W' then 'Comparative Media Studies/Writing' - when {{ column_name }} = 'EC' then 'Edgerton Center' - when {{ column_name }} = 'ES' then 'Experimental Study Group' - when {{ column_name }} = 'ESD' then 'Engineering Systems Division' - when {{ column_name }} = 'HST' then 'Medical Engineering and Science' - when {{ column_name }} = 'IDS' then 'Data, Systems, and Society' - when {{ column_name }} = 'MAS' then 'Media Arts and Sciences' - when {{ column_name }} = 'PE' then 'Athletics, Physical Education and Recreation' - when {{ column_name }} = 'SP' then 'Special Programs' - when {{ column_name }} = 'STS' then 'Science, Technology, and Society' - when {{ column_name }} = 'WGS' then 'Women''s and Gender Studies' - else null - end +-- - https://github.com/mitodl/mit-learn/blob/main/learning_resources/constants.py#L189-L227 +{% macro transform_edx_department_number(column_name="department_number") %} + case + when {{ column_name }} = '1' + then 'Civil and Environmental Engineering' + when {{ column_name }} = '2' + then 'Mechanical Engineering' + when {{ column_name }} = '3' + then 'Materials Science and Engineering' + when {{ column_name }} = '4' + then 'Architecture' + when {{ column_name }} = '5' + then 'Chemistry' + when {{ column_name }} = '6' + then 'Electrical Engineering and Computer Science' + when {{ column_name }} = '7' + then 'Biology' + when {{ column_name }} = '8' + then 'Physics' + when {{ column_name }} = '9' + then 'Brain and Cognitive Sciences' + when {{ column_name }} = '10' + then 'Chemical Engineering' + when {{ column_name }} = '11' + then 'Urban Studies and Planning' + when {{ column_name }} = '12' + then 'Earth, Atmospheric, and Planetary Sciences' + when {{ column_name }} = '14' + then 'Economics' + when {{ column_name }} = '15' + then 'Management' + when {{ column_name }} = '16' + then 'Aeronautics and Astronautics' + when {{ column_name }} = '17' + then 'Political Science' + when {{ column_name }} = '18' + then 'Mathematics' + when {{ column_name }} = '20' + then 'Biological Engineering' + when {{ column_name }} = '21A' + then 'Anthropology' + when {{ column_name }} = '21G' + then 'Global Languages' + when {{ column_name }} = '21H' + then 'History' + when {{ column_name }} = '21L' + then 'Literature' + when {{ column_name }} = '21M' + then 'Music and Theater Arts' + when {{ column_name }} = '22' + then 'Nuclear Science and Engineering' + when {{ column_name }} = '24' + then 'Linguistics and Philosophy' + when {{ column_name }} = 'CC' + then 'Concourse' + when {{ column_name }} = 'CMS-W' + then 'Comparative Media Studies/Writing' + when {{ column_name }} = 'EC' + then 'Edgerton Center' + when {{ column_name }} = 'ES' + then 'Experimental Study Group' + when {{ column_name }} = 'ESD' + then 'Engineering Systems Division' + when {{ column_name }} = 'HST' + then 'Medical Engineering and Science' + when {{ column_name }} = 'IDS' + then 'Data, Systems, and Society' + when {{ column_name }} = 'MAS' + then 'Media Arts and Sciences' + when {{ column_name }} = 'PE' + then 'Athletics, Physical Education and Recreation' + when {{ column_name }} = 'SP' + then 'Special Programs' + when {{ column_name }} = 'STS' + then 'Science, Technology, and Society' + when {{ column_name }} = 'WGS' + then 'Women''s and Gender Studies' + else null + end {% endmacro %} diff --git a/src/ol_dbt/macros/transform_studentmodule_data.sql b/src/ol_dbt/macros/transform_studentmodule_data.sql index c6b086016..cc1788030 100644 --- a/src/ol_dbt/macros/transform_studentmodule_data.sql +++ b/src/ol_dbt/macros/transform_studentmodule_data.sql @@ -1,77 +1,74 @@ {% macro generate_studentmodule_problem_events(studentmodule_table, studentmodulehistory_table, user_id_field) %} - with studentmodule as ( - select - studentmodule_id - , courserun_readable_id - , coursestructure_block_id - , coursestructure_block_category - , {{ user_id_field }} as user_id - , studentmodule_state_data - , cast(studentmodule_problem_grade as varchar) as grade - , cast(studentmodule_problem_max_grade as varchar) as max_grade - , from_iso8601_timestamp_nanos(studentmodule_created_on) as studentmodule_created_on - , from_iso8601_timestamp_nanos(studentmodule_updated_on) as studentmodule_updated_on - , cast(json_query(studentmodule_state_data, 'lax $.attempts' omit quotes) as int) as attempt - from {{ studentmodule_table }} - where coursestructure_block_category = 'problem' - ) - - , studentmodulehistoryextended as ( - select - studentmodulehistoryextended_id - , studentmodule_id - , studentmodule_state_data - , cast(studentmodule_problem_grade as varchar) as grade - , cast(studentmodule_problem_max_grade as varchar) as max_grade - , from_iso8601_timestamp_nanos(to_iso8601(studentmodule_created_on)) as studentmodule_created_on - , cast(json_query(studentmodule_state_data, 'lax $.attempts' omit quotes) as int) as attempt - from {{ studentmodulehistory_table }} - ) + with + studentmodule as ( + select + studentmodule_id, + courserun_readable_id, + coursestructure_block_id, + coursestructure_block_category, + {{ user_id_field }} as user_id, + studentmodule_state_data, + cast(studentmodule_problem_grade as varchar) as grade, + cast(studentmodule_problem_max_grade as varchar) as max_grade, + from_iso8601_timestamp_nanos(studentmodule_created_on) as studentmodule_created_on, + from_iso8601_timestamp_nanos(studentmodule_updated_on) as studentmodule_updated_on, + cast(json_query(studentmodule_state_data, 'lax $.attempts' omit quotes) as int) as attempt + from {{ studentmodule_table }} + where coursestructure_block_category = 'problem' + ), + studentmodulehistoryextended as ( + select + studentmodulehistoryextended_id, + studentmodule_id, + studentmodule_state_data, + cast(studentmodule_problem_grade as varchar) as grade, + cast(studentmodule_problem_max_grade as varchar) as max_grade, + from_iso8601_timestamp_nanos(to_iso8601(studentmodule_created_on)) as studentmodule_created_on, + cast(json_query(studentmodule_state_data, 'lax $.attempts' omit quotes) as int) as attempt + from {{ studentmodulehistory_table }} + ), + -- Pull out arrays from the state data + -- Exclude rows without an attempt number, as these are not valid problem events + -- Records from historyextended that join to studentmodule with fallback/default logic + history_joined as ( + select + sm.user_id, + sm.courserun_readable_id, + sm.studentmodule_id, + sm.coursestructure_block_id, + coalesce(smhe.studentmodule_state_data, sm.studentmodule_state_data) as studentmodule_state_data, + coalesce(smhe.grade, sm.grade) as grade, + coalesce(smhe.max_grade, sm.max_grade) as max_grade, + coalesce(smhe.studentmodule_created_on, sm.studentmodule_updated_on) as event_timestamp, + cast(smhe.attempt as varchar) as attempt, + if( + coalesce(smhe.grade, sm.grade) is not null + and coalesce(smhe.max_grade, sm.max_grade) is not null + and coalesce(smhe.grade, sm.grade) = coalesce(smhe.max_grade, sm.max_grade), + 'correct', + 'incorrect' + ) as success, + cast( + json_query( + coalesce(smhe.studentmodule_state_data, sm.studentmodule_state_data), 'lax $.student_answers' + ) as varchar + ) as answers + from studentmodule sm + left join studentmodulehistoryextended smhe on sm.studentmodule_id = smhe.studentmodule_id + where smhe.attempt is not null + ) - -- Pull out arrays from the state data - -- Exclude rows without an attempt number, as these are not valid problem events - -- Records from historyextended that join to studentmodule with fallback/default logic - , history_joined as ( select - sm.user_id - , sm.courserun_readable_id - , sm.studentmodule_id - , sm.coursestructure_block_id - , coalesce(smhe.studentmodule_state_data, sm.studentmodule_state_data) as studentmodule_state_data - , coalesce(smhe.grade, sm.grade) as grade - , coalesce(smhe.max_grade, sm.max_grade) as max_grade - , coalesce(smhe.studentmodule_created_on, sm.studentmodule_updated_on) as event_timestamp - , cast(smhe.attempt as varchar) as attempt - , if( - coalesce(smhe.grade, sm.grade) is not null - and coalesce(smhe.max_grade, sm.max_grade) is not null - and coalesce(smhe.grade, sm.grade) = coalesce(smhe.max_grade, sm.max_grade) - , 'correct' - , 'incorrect' - ) as success - , cast( - json_query( - coalesce(smhe.studentmodule_state_data, sm.studentmodule_state_data), - 'lax $.student_answers') - as varchar - ) as answers - from studentmodule sm - left join studentmodulehistoryextended smhe - on sm.studentmodule_id = smhe.studentmodule_id - where smhe.attempt is not null - ) - - select - user_id - , courserun_readable_id - , studentmodule_id - , coursestructure_block_id - , studentmodule_state_data - , grade - , max_grade - , event_timestamp - , attempt - , success - , answers - from history_joined + user_id, + courserun_readable_id, + studentmodule_id, + coursestructure_block_id, + studentmodule_state_data, + grade, + max_grade, + event_timestamp, + attempt, + success, + answers + from history_joined {% endmacro %} diff --git a/src/ol_dbt/macros/translate_course_id_to_platform.sql b/src/ol_dbt/macros/translate_course_id_to_platform.sql index aa07b30f0..e238040bd 100644 --- a/src/ol_dbt/macros/translate_course_id_to_platform.sql +++ b/src/ol_dbt/macros/translate_course_id_to_platform.sql @@ -1,11 +1,14 @@ {% macro translate_course_id_to_platform(course_id) %} - case - when {{ course_id }} like 'MITxT%' then '{{ var("mitxonline") }}' - ---- only course_id starts with xPRO are from xPro open edx platform - when {{ course_id }} like 'xPRO%' then '{{ var("mitxpro") }}' - --- Some runs from course - VJx Visualizing Japan (1850s-1930s) that run on edx don't start with 'MITx/` - --- e.g. VJx/VJx_S/3T2015, VJx/VJx/3T2014, VJx/VJx_2/3T2016 - when {{ course_id }} like 'MITx/%' or {{ course_id }} like 'VJx%' then '{{ var("edxorg") }}' - else null - end + case + when {{ course_id }} like 'MITxT%' + then '{{ var("mitxonline") }}' + -- -- only course_id starts with xPRO are from xPro open edx platform + when {{ course_id }} like 'xPRO%' + then '{{ var("mitxpro") }}' + -- - Some runs from course - VJx Visualizing Japan (1850s-1930s) that run on edx don't start with 'MITx/` + -- - e.g. VJx/VJx_S/3T2015, VJx/VJx/3T2014, VJx/VJx_2/3T2016 + when {{ course_id }} like 'MITx/%' or {{ course_id }} like 'VJx%' + then '{{ var("edxorg") }}' + else null + end {% endmacro %} diff --git a/src/ol_dbt/models/dimensional/afact_course_page_engagement.sql b/src/ol_dbt/models/dimensional/afact_course_page_engagement.sql index 69f15f333..6f627fc03 100644 --- a/src/ol_dbt/models/dimensional/afact_course_page_engagement.sql +++ b/src/ol_dbt/models/dimensional/afact_course_page_engagement.sql @@ -1,60 +1,56 @@ -with vertical_structure as ( - select - courserun_readable_id - , block_id - , block_index - , sequential_block_id as sequential_block_fk - , chapter_block_id as chapter_block_fk - from {{ ref('dim_course_content') }} - where - block_category = 'vertical' - and is_latest = true -) - -, page_navigation_events as ( - select - platform - , openedx_user_id - , courserun_readable_id - , block_fk - , arbitrary(user_username) as user_username - , arbitrary(platform_fk) as platform_fk - , arbitrary(user_fk) as user_fk - , count(*) as num_of_views - , max(event_timestamp) as last_view_timestamp - from {{ ref('tfact_course_navigation_events') }} - where block_fk is not null - group by platform, courserun_readable_id, openedx_user_id, block_fk -) - -, combined as ( - select - page_navigation_events.platform_fk - , page_navigation_events.user_fk - , page_navigation_events.platform - , page_navigation_events.openedx_user_id - , page_navigation_events.user_username - , page_navigation_events.courserun_readable_id - , page_navigation_events.block_fk - , page_navigation_events.num_of_views - , page_navigation_events.last_view_timestamp - , vertical_structure.sequential_block_fk - , vertical_structure.chapter_block_fk - from page_navigation_events - left join vertical_structure - on page_navigation_events.block_fk = vertical_structure.block_id -) +with + vertical_structure as ( + select + courserun_readable_id, + block_id, + block_index, + sequential_block_id as sequential_block_fk, + chapter_block_id as chapter_block_fk + from {{ ref("dim_course_content") }} + where block_category = 'vertical' and is_latest = true + ), + page_navigation_events as ( + select + platform, + openedx_user_id, + courserun_readable_id, + block_fk, + arbitrary(user_username) as user_username, + arbitrary(platform_fk) as platform_fk, + arbitrary(user_fk) as user_fk, + count(*) as num_of_views, + max(event_timestamp) as last_view_timestamp + from {{ ref("tfact_course_navigation_events") }} + where block_fk is not null + group by platform, courserun_readable_id, openedx_user_id, block_fk + ), + combined as ( + select + page_navigation_events.platform_fk, + page_navigation_events.user_fk, + page_navigation_events.platform, + page_navigation_events.openedx_user_id, + page_navigation_events.user_username, + page_navigation_events.courserun_readable_id, + page_navigation_events.block_fk, + page_navigation_events.num_of_views, + page_navigation_events.last_view_timestamp, + vertical_structure.sequential_block_fk, + vertical_structure.chapter_block_fk + from page_navigation_events + left join vertical_structure on page_navigation_events.block_fk = vertical_structure.block_id + ) select - platform_fk - , user_fk - , platform - , openedx_user_id - , user_username - , courserun_readable_id - , block_fk - , num_of_views - , sequential_block_fk - , chapter_block_fk - , last_view_timestamp + platform_fk, + user_fk, + platform, + openedx_user_id, + user_username, + courserun_readable_id, + block_fk, + num_of_views, + sequential_block_fk, + chapter_block_fk, + last_view_timestamp from combined diff --git a/src/ol_dbt/models/dimensional/afact_discussion_engagement.sql b/src/ol_dbt/models/dimensional/afact_discussion_engagement.sql index 3f2cb797f..ea1f923cd 100644 --- a/src/ol_dbt/models/dimensional/afact_discussion_engagement.sql +++ b/src/ol_dbt/models/dimensional/afact_discussion_engagement.sql @@ -1,67 +1,58 @@ -with discussion_events as ( - select * from {{ ref('tfact_discussion_events') }} -) - -, discussion_topics as ( - select * from {{ ref('dim_discussion_topic') }} -) - -, course_content as ( - select * from {{ ref('dim_course_content') }} -) - -, aggregated as ( - select - discussion_events.platform_fk - , discussion_events.courserun_readable_id - , discussion_events.openedx_user_id - , discussion_events.commentable_id - , discussion_events.post_id - , arbitrary(discussion_events.user_username) as user_username - , arbitrary(discussion_events.platform) as platform - , arbitrary(discussion_events.user_fk) as user_fk - , arbitrary(discussion_events.post_title) as post_title - , count_if(discussion_events.event_type = 'edx.forum.thread.created') as post_created - , count_if(discussion_events.event_type = 'edx.forum.thread.viewed') as post_viewed - , count_if(discussion_events.event_type = 'edx.forum.thread.voted') as post_voted - , count_if(discussion_events.event_type = 'edx.forum.response.created') as post_replied - , count_if(discussion_events.event_type = 'edx.forum.response.voted') as response_voted - , count_if(discussion_events.event_type = 'edx.forum.comment.created') as post_commented - from discussion_events - where discussion_events.commentable_id is not null - group by - discussion_events.platform_fk - , discussion_events.openedx_user_id - , discussion_events.courserun_readable_id - , discussion_events.commentable_id - , discussion_events.post_id -) +with + discussion_events as (select * from {{ ref("tfact_discussion_events") }}), + discussion_topics as (select * from {{ ref("dim_discussion_topic") }}), + course_content as (select * from {{ ref("dim_course_content") }}), + aggregated as ( + select + discussion_events.platform_fk, + discussion_events.courserun_readable_id, + discussion_events.openedx_user_id, + discussion_events.commentable_id, + discussion_events.post_id, + arbitrary(discussion_events.user_username) as user_username, + arbitrary(discussion_events.platform) as platform, + arbitrary(discussion_events.user_fk) as user_fk, + arbitrary(discussion_events.post_title) as post_title, + count_if(discussion_events.event_type = 'edx.forum.thread.created') as post_created, + count_if(discussion_events.event_type = 'edx.forum.thread.viewed') as post_viewed, + count_if(discussion_events.event_type = 'edx.forum.thread.voted') as post_voted, + count_if(discussion_events.event_type = 'edx.forum.response.created') as post_replied, + count_if(discussion_events.event_type = 'edx.forum.response.voted') as response_voted, + count_if(discussion_events.event_type = 'edx.forum.comment.created') as post_commented + from discussion_events + where discussion_events.commentable_id is not null + group by + discussion_events.platform_fk, + discussion_events.openedx_user_id, + discussion_events.courserun_readable_id, + discussion_events.commentable_id, + discussion_events.post_id + ) select - aggregated.platform_fk - , aggregated.courserun_readable_id - , aggregated.openedx_user_id - , aggregated.user_username - , aggregated.commentable_id - , aggregated.post_id - , aggregated.platform - , aggregated.user_fk - , aggregated.post_title - , aggregated.post_created - , aggregated.post_viewed - , aggregated.post_voted - , aggregated.post_replied - , aggregated.response_voted - , aggregated.post_commented - , discussion_topics.discussion_topic_pk as discussion_topic_fk - , discussion_topics.discussion_type - , discussion_topics.content_block_fk - , course_content.sequential_block_id as sequential_block_fk - , course_content.chapter_block_id as chapter_block_fk + aggregated.platform_fk, + aggregated.courserun_readable_id, + aggregated.openedx_user_id, + aggregated.user_username, + aggregated.commentable_id, + aggregated.post_id, + aggregated.platform, + aggregated.user_fk, + aggregated.post_title, + aggregated.post_created, + aggregated.post_viewed, + aggregated.post_voted, + aggregated.post_replied, + aggregated.response_voted, + aggregated.post_commented, + discussion_topics.discussion_topic_pk as discussion_topic_fk, + discussion_topics.discussion_type, + discussion_topics.content_block_fk, + course_content.sequential_block_id as sequential_block_fk, + course_content.chapter_block_id as chapter_block_fk from aggregated -left join discussion_topics - on - aggregated.commentable_id = discussion_topics.commentable_id - and aggregated.courserun_readable_id = discussion_topics.courserun_readable_id -left join course_content - on discussion_topics.content_block_fk = course_content.content_block_pk +left join + discussion_topics + on aggregated.commentable_id = discussion_topics.commentable_id + and aggregated.courserun_readable_id = discussion_topics.courserun_readable_id +left join course_content on discussion_topics.content_block_fk = course_content.content_block_pk diff --git a/src/ol_dbt/models/dimensional/afact_problem_engagement.sql b/src/ol_dbt/models/dimensional/afact_problem_engagement.sql index bfce1518f..76401bb00 100644 --- a/src/ol_dbt/models/dimensional/afact_problem_engagement.sql +++ b/src/ol_dbt/models/dimensional/afact_problem_engagement.sql @@ -1,85 +1,77 @@ -with problem_structure as ( - select - courserun_readable_id - , block_id as problem_block_fk - , sequential_block_id as sequential_block_fk - , chapter_block_id as chapter_block_fk - from {{ ref('dim_course_content') }} - where - block_category = 'problem' - and is_latest = true -) - -, pre_problem_attempt_aggregated as ( - select - platform_fk - , user_fk - , platform - , openedx_user_id - , user_username - , problem_block_fk - , courserun_readable_id - , cast(attempt as int) as attempt - , event_timestamp - , success - , row_number() - over - ( +with + problem_structure as ( + select + courserun_readable_id, + block_id as problem_block_fk, + sequential_block_id as sequential_block_fk, + chapter_block_id as chapter_block_fk + from {{ ref("dim_course_content") }} + where block_category = 'problem' and is_latest = true + ), + pre_problem_attempt_aggregated as ( + select + platform_fk, + user_fk, + platform, + openedx_user_id, + user_username, + problem_block_fk, + courserun_readable_id, + cast(attempt as int) as attempt, + event_timestamp, + success, + row_number() over ( partition by platform, openedx_user_id, courserun_readable_id, problem_block_fk, attempt order by event_timestamp desc - ) - as rn - from {{ ref('tfact_problem_events') }} - where event_type = 'problem_check' -) - -, problem_attempt_aggregated as ( - select - platform - , openedx_user_id - , problem_block_fk - , courserun_readable_id - , arbitrary(user_username) as user_username - , arbitrary(platform_fk) as platform_fk - , arbitrary(user_fk) as user_fk - , max(attempt) as num_of_attempts - , max(event_timestamp) as last_attempt_timestamp - , count(case when success = 'correct' then 1 end) as num_of_correct_attempts - from pre_problem_attempt_aggregated - where rn = 1 - group by platform, courserun_readable_id, problem_block_fk, openedx_user_id -) - -, combined as ( - select - problem_attempt_aggregated.platform_fk - , problem_attempt_aggregated.user_fk - , problem_attempt_aggregated.platform - , problem_attempt_aggregated.openedx_user_id - , problem_attempt_aggregated.user_username - , problem_attempt_aggregated.courserun_readable_id - , problem_attempt_aggregated.problem_block_fk - , problem_attempt_aggregated.num_of_attempts - , problem_attempt_aggregated.num_of_correct_attempts - , problem_attempt_aggregated.last_attempt_timestamp - , problem_structure.sequential_block_fk - , problem_structure.chapter_block_fk - from problem_attempt_aggregated - left join problem_structure - on problem_attempt_aggregated.problem_block_fk = problem_structure.problem_block_fk -) + ) as rn + from {{ ref("tfact_problem_events") }} + where event_type = 'problem_check' + ), + problem_attempt_aggregated as ( + select + platform, + openedx_user_id, + problem_block_fk, + courserun_readable_id, + arbitrary(user_username) as user_username, + arbitrary(platform_fk) as platform_fk, + arbitrary(user_fk) as user_fk, + max(attempt) as num_of_attempts, + max(event_timestamp) as last_attempt_timestamp, + count(case when success = 'correct' then 1 end) as num_of_correct_attempts + from pre_problem_attempt_aggregated + where rn = 1 + group by platform, courserun_readable_id, problem_block_fk, openedx_user_id + ), + combined as ( + select + problem_attempt_aggregated.platform_fk, + problem_attempt_aggregated.user_fk, + problem_attempt_aggregated.platform, + problem_attempt_aggregated.openedx_user_id, + problem_attempt_aggregated.user_username, + problem_attempt_aggregated.courserun_readable_id, + problem_attempt_aggregated.problem_block_fk, + problem_attempt_aggregated.num_of_attempts, + problem_attempt_aggregated.num_of_correct_attempts, + problem_attempt_aggregated.last_attempt_timestamp, + problem_structure.sequential_block_fk, + problem_structure.chapter_block_fk + from problem_attempt_aggregated + left join problem_structure on problem_attempt_aggregated.problem_block_fk = problem_structure.problem_block_fk + ) select - platform_fk - , user_fk - , platform - , openedx_user_id - , user_username - , courserun_readable_id - , problem_block_fk - , num_of_attempts - , num_of_correct_attempts - , sequential_block_fk - , chapter_block_fk - , last_attempt_timestamp + platform_fk, + user_fk, + platform, + openedx_user_id, + user_username, + courserun_readable_id, + problem_block_fk, + num_of_attempts, + num_of_correct_attempts, + sequential_block_fk, + chapter_block_fk, + last_attempt_timestamp from combined diff --git a/src/ol_dbt/models/dimensional/afact_video_engagement.sql b/src/ol_dbt/models/dimensional/afact_video_engagement.sql index aaa51d864..fb79e95e7 100644 --- a/src/ol_dbt/models/dimensional/afact_video_engagement.sql +++ b/src/ol_dbt/models/dimensional/afact_video_engagement.sql @@ -1,104 +1,70 @@ -with course_content as ( - select * from {{ ref('dim_course_content') }} -) - -, tfact_video_events as ( - select * from {{ ref('tfact_video_events') }} -) - -, d_video as ( - select * from {{ ref('dim_video') }} -) - -, start_and_end_times as ( - select - openedx_user_id - , courserun_readable_id - , video_block_fk - , arbitrary(user_username) as user_username - , arbitrary(platform_fk) as platform_fk - , arbitrary(user_fk) as user_fk - , max(case when video_position = 'null' then '0' end) as end_time - , min(case when event_type = 'play_video' then (case when video_position = 'null' then '0' end) end) - as start_time - from tfact_video_events - where - event_type in ( - 'play_video' - , 'seek_video' - , 'pause_video' - , 'stop_video' - , 'complete_video' - ) - group by - openedx_user_id - , courserun_readable_id - , video_block_fk -) +with + course_content as (select * from {{ ref("dim_course_content") }}), + tfact_video_events as (select * from {{ ref("tfact_video_events") }}), + d_video as (select * from {{ ref("dim_video") }}), + start_and_end_times as ( + select + openedx_user_id, + courserun_readable_id, + video_block_fk, + arbitrary(user_username) as user_username, + arbitrary(platform_fk) as platform_fk, + arbitrary(user_fk) as user_fk, + max(case when video_position = 'null' then '0' end) as end_time, + min( + case when event_type = 'play_video' then (case when video_position = 'null' then '0' end) end + ) as start_time + from tfact_video_events + where event_type in ('play_video', 'seek_video', 'pause_video', 'stop_video', 'complete_video') + group by openedx_user_id, courserun_readable_id, video_block_fk + ) select - tfact_video_events.platform - , tfact_video_events.video_block_fk as video_id - , tfact_video_events.openedx_user_id - , tfact_video_events.courserun_readable_id - , c.block_title as video_title - , d.block_title as unit_title - , d.content_block_pk as unit_content_fk - , f.block_title as subsection_title - , f.content_block_pk as subsection_content_fk - , g.block_title as section_title - , g.content_block_pk as section_content_fk - , arbitrary(tfact_video_events.user_username) as user_username - , arbitrary(tfact_video_events.platform_fk) as platform_fk - , arbitrary(tfact_video_events.user_fk) as user_fk - , ( - cast(start_and_end_times.end_time as decimal(30, 10)) - - cast(start_and_end_times.start_time as decimal(30, 10)) - ) - as estimated_time_played - , max(tfact_video_events.event_timestamp) as latest_activity_timestamp - , sum(case when tfact_video_events.event_type = 'play_video' then 1 else 0 end) as video_played_count - , sum(case when tfact_video_events.event_type = 'complete_video' then 1 else 0 end) as video_completed_count + tfact_video_events.platform, + tfact_video_events.video_block_fk as video_id, + tfact_video_events.openedx_user_id, + tfact_video_events.courserun_readable_id, + c.block_title as video_title, + d.block_title as unit_title, + d.content_block_pk as unit_content_fk, + f.block_title as subsection_title, + f.content_block_pk as subsection_content_fk, + g.block_title as section_title, + g.content_block_pk as section_content_fk, + arbitrary(tfact_video_events.user_username) as user_username, + arbitrary(tfact_video_events.platform_fk) as platform_fk, + arbitrary(tfact_video_events.user_fk) as user_fk, + ( + cast(start_and_end_times.end_time as decimal(30, 10)) - cast(start_and_end_times.start_time as decimal(30, 10)) + ) as estimated_time_played, + max(tfact_video_events.event_timestamp) as latest_activity_timestamp, + sum(case when tfact_video_events.event_type = 'play_video' then 1 else 0 end) as video_played_count, + sum(case when tfact_video_events.event_type = 'complete_video' then 1 else 0 end) as video_completed_count from tfact_video_events -inner join d_video - on - tfact_video_events.video_block_fk - = substring(d_video.video_block_pk, regexp_position(d_video.video_block_pk, 'block@') + 6) - and tfact_video_events.courserun_readable_id = d_video.courserun_readable_id -inner join course_content as c - on - d_video.content_block_fk = c.content_block_pk - and c.is_latest = true -left join course_content as d - on - c.parent_block_id = d.block_id - and d.is_latest = true -left join course_content as f - on - d.parent_block_id = f.block_id - and f.is_latest = true -left join course_content as g - on - f.parent_block_id = g.block_id - and g.is_latest = true -left join start_and_end_times - on - tfact_video_events.video_block_fk = start_and_end_times.video_block_fk - and tfact_video_events.courserun_readable_id = start_and_end_times.courserun_readable_id - and tfact_video_events.openedx_user_id = start_and_end_times.openedx_user_id +inner join + d_video + on tfact_video_events.video_block_fk + = substring(d_video.video_block_pk, regexp_position(d_video.video_block_pk, 'block@') + 6) + and tfact_video_events.courserun_readable_id = d_video.courserun_readable_id +inner join course_content as c on d_video.content_block_fk = c.content_block_pk and c.is_latest = true +left join course_content as d on c.parent_block_id = d.block_id and d.is_latest = true +left join course_content as f on d.parent_block_id = f.block_id and f.is_latest = true +left join course_content as g on f.parent_block_id = g.block_id and g.is_latest = true +left join + start_and_end_times + on tfact_video_events.video_block_fk = start_and_end_times.video_block_fk + and tfact_video_events.courserun_readable_id = start_and_end_times.courserun_readable_id + and tfact_video_events.openedx_user_id = start_and_end_times.openedx_user_id group by - tfact_video_events.platform - , tfact_video_events.video_block_fk - , tfact_video_events.openedx_user_id - , tfact_video_events.courserun_readable_id - , c.block_title - , d.block_title - , d.content_block_pk - , f.block_title - , f.content_block_pk - , g.block_title - , g.content_block_pk - , ( - cast(start_and_end_times.end_time as decimal(30, 10)) - - cast(start_and_end_times.start_time as decimal(30, 10)) - ) + tfact_video_events.platform, + tfact_video_events.video_block_fk, + tfact_video_events.openedx_user_id, + tfact_video_events.courserun_readable_id, + c.block_title, + d.block_title, + d.content_block_pk, + f.block_title, + f.content_block_pk, + g.block_title, + g.content_block_pk, + (cast(start_and_end_times.end_time as decimal(30, 10)) - cast(start_and_end_times.start_time as decimal(30, 10))) diff --git a/src/ol_dbt/models/dimensional/dim_course_content.sql b/src/ol_dbt/models/dimensional/dim_course_content.sql index 287a875d4..1ddaed870 100644 --- a/src/ol_dbt/models/dimensional/dim_course_content.sql +++ b/src/ol_dbt/models/dimensional/dim_course_content.sql @@ -1,174 +1,149 @@ -with mitxonline_course_structure as ( - select * from {{ ref('stg__mitxonline__openedx__api__course_structure') }} -) - -, edxorg_course_structure as ( - select * from {{ ref('stg__edxorg__s3__course_structure') }} -) - -, xpro_course_structure as ( - select * from {{ ref('stg__mitxpro__openedx__api__course_structure') }} -) - -, residential_course_structure as ( - select * from {{ ref('stg__mitxresidential__openedx__api__course_structure') }} -) - -, combined as ( - select - courserun_readable_id - , coursestructure_block_index - , coursestructure_block_id - , coursestructure_parent_block_id - , coursestructure_block_category - , coursestructure_block_title - , coursestructure_block_metadata - , coursestructure_retrieved_at - from mitxonline_course_structure - - union distinct - - select - courserun_readable_id - , coursestructure_block_index - , coursestructure_block_id - , coursestructure_parent_block_id - , coursestructure_block_category - , coursestructure_block_title - , coursestructure_block_metadata - , coursestructure_retrieved_at - from edxorg_course_structure - - union distinct - - select - courserun_readable_id - , coursestructure_block_index - , coursestructure_block_id - , coursestructure_parent_block_id - , coursestructure_block_category - , coursestructure_block_title - , coursestructure_block_metadata - , coursestructure_retrieved_at - from xpro_course_structure - - union distinct - - select - courserun_readable_id - , coursestructure_block_index - , coursestructure_block_id - , coursestructure_parent_block_id - , coursestructure_block_category - , coursestructure_block_title - , coursestructure_block_metadata - , coursestructure_retrieved_at - from residential_course_structure -) - -, latest_course_structure as ( - select - courserun_readable_id - , max(coursestructure_retrieved_at) as max_retrieved_date - from combined - group by courserun_readable_id -) - -, sequentials as ( - select * from combined - where coursestructure_block_category = 'sequential' -) - -, combined_with_sequentials as ( - select - combined.* - , sequentials.coursestructure_block_id as sequential_block_id - , sequentials.coursestructure_block_title as sequential_title - , row_number() over ( - partition by - combined.courserun_readable_id - , combined.coursestructure_block_index - , combined.coursestructure_retrieved_at - order by sequentials.coursestructure_block_index desc - ) as row_num - from combined - inner join sequentials - on - combined.courserun_readable_id = sequentials.courserun_readable_id +with + mitxonline_course_structure as (select * from {{ ref("stg__mitxonline__openedx__api__course_structure") }}), + edxorg_course_structure as (select * from {{ ref("stg__edxorg__s3__course_structure") }}), + xpro_course_structure as (select * from {{ ref("stg__mitxpro__openedx__api__course_structure") }}), + residential_course_structure as (select * from {{ ref("stg__mitxresidential__openedx__api__course_structure") }}), + combined as ( + select + courserun_readable_id, + coursestructure_block_index, + coursestructure_block_id, + coursestructure_parent_block_id, + coursestructure_block_category, + coursestructure_block_title, + coursestructure_block_metadata, + coursestructure_retrieved_at + from mitxonline_course_structure + + union distinct + + select + courserun_readable_id, + coursestructure_block_index, + coursestructure_block_id, + coursestructure_parent_block_id, + coursestructure_block_category, + coursestructure_block_title, + coursestructure_block_metadata, + coursestructure_retrieved_at + from edxorg_course_structure + + union distinct + + select + courserun_readable_id, + coursestructure_block_index, + coursestructure_block_id, + coursestructure_parent_block_id, + coursestructure_block_category, + coursestructure_block_title, + coursestructure_block_metadata, + coursestructure_retrieved_at + from xpro_course_structure + + union distinct + + select + courserun_readable_id, + coursestructure_block_index, + coursestructure_block_id, + coursestructure_parent_block_id, + coursestructure_block_category, + coursestructure_block_title, + coursestructure_block_metadata, + coursestructure_retrieved_at + from residential_course_structure + ), + latest_course_structure as ( + select courserun_readable_id, max(coursestructure_retrieved_at) as max_retrieved_date + from combined + group by courserun_readable_id + ), + sequentials as (select * from combined where coursestructure_block_category = 'sequential'), + combined_with_sequentials as ( + select + combined.*, + sequentials.coursestructure_block_id as sequential_block_id, + sequentials.coursestructure_block_title as sequential_title, + row_number() over ( + partition by + combined.courserun_readable_id, + combined.coursestructure_block_index, + combined.coursestructure_retrieved_at + order by sequentials.coursestructure_block_index desc + ) as row_num + from combined + inner join + sequentials + on combined.courserun_readable_id = sequentials.courserun_readable_id and combined.coursestructure_retrieved_at = sequentials.coursestructure_retrieved_at and combined.coursestructure_block_index >= sequentials.coursestructure_block_index -) - -, chapters as ( - select * from combined - where coursestructure_block_category = 'chapter' -) - -, combined_with_chapters as ( - select - combined.* - , chapters.coursestructure_block_id as chapter_block_id - , chapters.coursestructure_block_title as chapter_title - , row_number() over ( - partition by - combined.courserun_readable_id - , combined.coursestructure_block_index - , combined.coursestructure_retrieved_at - order by chapters.coursestructure_block_index desc - ) as row_num - from combined - inner join chapters - on - combined.courserun_readable_id = chapters.courserun_readable_id + ), + chapters as (select * from combined where coursestructure_block_category = 'chapter'), + combined_with_chapters as ( + select + combined.*, + chapters.coursestructure_block_id as chapter_block_id, + chapters.coursestructure_block_title as chapter_title, + row_number() over ( + partition by + combined.courserun_readable_id, + combined.coursestructure_block_index, + combined.coursestructure_retrieved_at + order by chapters.coursestructure_block_index desc + ) as row_num + from combined + inner join + chapters + on combined.courserun_readable_id = chapters.courserun_readable_id and combined.coursestructure_retrieved_at = chapters.coursestructure_retrieved_at and combined.coursestructure_block_index >= chapters.coursestructure_block_index -) - -, combined_course_content as ( - select - combined.courserun_readable_id - , combined.coursestructure_block_index as block_index - , combined.coursestructure_block_id as block_id - , combined.coursestructure_parent_block_id as parent_block_id - , combined.coursestructure_block_category as block_category - , combined.coursestructure_block_title as block_title - , combined.coursestructure_block_metadata as block_metadata - , combined.coursestructure_retrieved_at as retrieved_at - , combined_with_chapters.chapter_block_id - , combined_with_chapters.chapter_title - , combined_with_sequentials.sequential_block_id - , combined_with_sequentials.sequential_title - , if(latest_course_structure.max_retrieved_date is not null, true, false) as is_latest - from combined - left join latest_course_structure - on - combined.courserun_readable_id = latest_course_structure.courserun_readable_id + ), + combined_course_content as ( + select + combined.courserun_readable_id, + combined.coursestructure_block_index as block_index, + combined.coursestructure_block_id as block_id, + combined.coursestructure_parent_block_id as parent_block_id, + combined.coursestructure_block_category as block_category, + combined.coursestructure_block_title as block_title, + combined.coursestructure_block_metadata as block_metadata, + combined.coursestructure_retrieved_at as retrieved_at, + combined_with_chapters.chapter_block_id, + combined_with_chapters.chapter_title, + combined_with_sequentials.sequential_block_id, + combined_with_sequentials.sequential_title, + if(latest_course_structure.max_retrieved_date is not null, true, false) as is_latest + from combined + left join + latest_course_structure + on combined.courserun_readable_id = latest_course_structure.courserun_readable_id and combined.coursestructure_retrieved_at = latest_course_structure.max_retrieved_date - left join combined_with_chapters - on - combined.courserun_readable_id = combined_with_chapters.courserun_readable_id + left join + combined_with_chapters + on combined.courserun_readable_id = combined_with_chapters.courserun_readable_id and combined.coursestructure_block_id = combined_with_chapters.coursestructure_block_id and combined.coursestructure_retrieved_at = combined_with_chapters.coursestructure_retrieved_at and combined_with_chapters.row_num = 1 - left join combined_with_sequentials - on - combined.courserun_readable_id = combined_with_sequentials.courserun_readable_id + left join + combined_with_sequentials + on combined.courserun_readable_id = combined_with_sequentials.courserun_readable_id and combined.coursestructure_block_id = combined_with_sequentials.coursestructure_block_id and combined.coursestructure_retrieved_at = combined_with_sequentials.coursestructure_retrieved_at and combined_with_sequentials.row_num = 1 -) + ) select - {{ dbt_utils.generate_surrogate_key(['block_id','retrieved_at']) }} as content_block_pk - , courserun_readable_id - , block_index - , block_id - , parent_block_id - , sequential_block_id - , chapter_block_id - , block_category - , block_title - , block_metadata - , retrieved_at - , is_latest + {{ dbt_utils.generate_surrogate_key(["block_id", "retrieved_at"]) }} as content_block_pk, + courserun_readable_id, + block_index, + block_id, + parent_block_id, + sequential_block_id, + chapter_block_id, + block_category, + block_title, + block_metadata, + retrieved_at, + is_latest from combined_course_content diff --git a/src/ol_dbt/models/dimensional/dim_discussion_topic.sql b/src/ol_dbt/models/dimensional/dim_discussion_topic.sql index b0ec98de3..6ed1a1ca2 100644 --- a/src/ol_dbt/models/dimensional/dim_discussion_topic.sql +++ b/src/ol_dbt/models/dimensional/dim_discussion_topic.sql @@ -1,63 +1,62 @@ -with discussion_component_topics as ( - select - * - , json_query(block_metadata, 'lax $.discussion_category' omit quotes) as discussion_component_category - , json_query(block_metadata, 'lax $.discussion_target' omit quotes) as discussion_component_name - , json_query(block_metadata, 'lax $.discussion_id' omit quotes) as discussion_component_id - from {{ ref('dim_course_content') }} - where - block_category = 'discussion' - and is_latest = true -) +with + discussion_component_topics as ( + select + *, + json_query(block_metadata, 'lax $.discussion_category' omit quotes) as discussion_component_category, + json_query(block_metadata, 'lax $.discussion_target' omit quotes) as discussion_component_name, + json_query(block_metadata, 'lax $.discussion_id' omit quotes) as discussion_component_id + from {{ ref("dim_course_content") }} + where block_category = 'discussion' and is_latest = true + ), + course_level_discussion_topics as ( + select + course.*, + t.key as topic_name, -- noqa + json_extract_scalar(t.topic, '$.id') as topic_id, -- noqa + row_number() over ( + partition by course.block_id, json_extract_scalar(t.topic, '$.id') -- noqa + order by json_extract_scalar(t.topic, '$.sort_key') asc -- noqa + ) as row_num + from {{ ref("dim_course_content") }} as course + cross join + unnest(cast(json_extract(course.block_metadata, '$.discussion_topics') as map(varchar, json))) as t( + key, topic + ) -- noqa + where course.block_category = 'course' and course.is_latest = true -, course_level_discussion_topics as ( - select - course.* - , t.key as topic_name -- noqa - , json_extract_scalar(t.topic, '$.id') as topic_id -- noqa - , row_number() over ( - partition by course.block_id, json_extract_scalar(t.topic, '$.id') -- noqa - order by json_extract_scalar(t.topic, '$.sort_key') asc -- noqa - ) as row_num - from {{ ref('dim_course_content') }} as course - cross join unnest(cast(json_extract(course.block_metadata, '$.discussion_topics') as map(varchar, json))) AS t(key, topic) -- noqa - where course.block_category = 'course' - and course.is_latest = true + ), + combined as ( + select + content_block_pk as content_block_fk, + block_id as discussion_block_pk, + courserun_readable_id, + discussion_component_name as topic_name, + discussion_component_id as commentable_id, + discussion_component_category as category_name, + 'discussion component' as discussion_type + from discussion_component_topics -) + union all -, combined as ( - select - content_block_pk as content_block_fk - , block_id as discussion_block_pk - , courserun_readable_id - , discussion_component_name as topic_name - , discussion_component_id as commentable_id - , discussion_component_category as category_name - , 'discussion component' as discussion_type - from discussion_component_topics - - union all - - select - content_block_pk as content_block_fk - , block_id as discussion_block_pk - , courserun_readable_id - , topic_name - , topic_id as commentable_id - , topic_name as category_name - , 'course-wide discussion' as discussion_type - from course_level_discussion_topics - where row_num = 1 -) + select + content_block_pk as content_block_fk, + block_id as discussion_block_pk, + courserun_readable_id, + topic_name, + topic_id as commentable_id, + topic_name as category_name, + 'course-wide discussion' as discussion_type + from course_level_discussion_topics + where row_num = 1 + ) select - {{ dbt_utils.generate_surrogate_key(['discussion_block_pk', 'commentable_id']) }} as discussion_topic_pk - , content_block_fk - , discussion_block_pk - , courserun_readable_id - , commentable_id - , topic_name - , category_name - , discussion_type + {{ dbt_utils.generate_surrogate_key(["discussion_block_pk", "commentable_id"]) }} as discussion_topic_pk, + content_block_fk, + discussion_block_pk, + courserun_readable_id, + commentable_id, + topic_name, + category_name, + discussion_type from combined diff --git a/src/ol_dbt/models/dimensional/dim_platform.sql b/src/ol_dbt/models/dimensional/dim_platform.sql index f05747a8c..e1e8f2c29 100644 --- a/src/ol_dbt/models/dimensional/dim_platform.sql +++ b/src/ol_dbt/models/dimensional/dim_platform.sql @@ -1,11 +1,9 @@ -with platforms as ( - select * from {{ ref('platforms') }} -) +with platforms as (select * from {{ ref("platforms") }}) select - {{ dbt_utils.generate_surrogate_key(['id']) }} as platform_pk - , id as platform_readable_id - , platform_name - , platform_description - , platform_domain + {{ dbt_utils.generate_surrogate_key(["id"]) }} as platform_pk, + id as platform_readable_id, + platform_name, + platform_description, + platform_domain from platforms diff --git a/src/ol_dbt/models/dimensional/dim_problem.sql b/src/ol_dbt/models/dimensional/dim_problem.sql index 28c6572b2..5d0695781 100644 --- a/src/ol_dbt/models/dimensional/dim_problem.sql +++ b/src/ol_dbt/models/dimensional/dim_problem.sql @@ -1,118 +1,108 @@ -with problems as ( - select - content_block_pk - , block_id - , block_title - , courserun_readable_id - , nullif(json_query(block_metadata, 'lax $.markdown' omit quotes), 'null') as markdown - , nullif(json_query(block_metadata, 'lax $.max_attempts' omit quotes), 'null') as max_attempts - , nullif(json_query(block_metadata, 'lax $.start' omit quotes), 'null') as start_date - , nullif(json_query(block_metadata, 'lax $.due' omit quotes), 'null') as due_date - , nullif(json_query(block_metadata, 'lax $.weight' omit quotes), 'null') as weight - , row_number() over ( - partition by block_id - order by is_latest desc, retrieved_at desc - ) as row_num - from {{ ref('dim_course_content') }} - where block_category = 'problem' -) +with + problems as ( + select + content_block_pk, + block_id, + block_title, + courserun_readable_id, + nullif(json_query(block_metadata, 'lax $.markdown' omit quotes), 'null') as markdown, + nullif(json_query(block_metadata, 'lax $.max_attempts' omit quotes), 'null') as max_attempts, + nullif(json_query(block_metadata, 'lax $.start' omit quotes), 'null') as start_date, + nullif(json_query(block_metadata, 'lax $.due' omit quotes), 'null') as due_date, + nullif(json_query(block_metadata, 'lax $.weight' omit quotes), 'null') as weight, + row_number() over (partition by block_id order by is_latest desc, retrieved_at desc) as row_num + from {{ ref("dim_course_content") }} + where block_category = 'problem' + ), + problem_events as ( + select distinct + courserun_readable_id, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name, + json_extract(useractivity_event_object, '$.submission') as submission + from {{ ref("stg__mitxonline__openedx__tracking_logs__user_activity") }} + where + courserun_readable_id is not null + and useractivity_event_type = 'problem_check' + and json_query(useractivity_event_object, 'lax $.submission' omit quotes) is not null -, problem_events as ( - select distinct - courserun_readable_id - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name - , json_extract(useractivity_event_object, '$.submission') as submission - from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type = 'problem_check' - and json_query(useractivity_event_object, 'lax $.submission' omit quotes) is not null + union all - union all + select distinct + courserun_readable_id, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name, + json_extract(useractivity_event_object, '$.submission') as submission + from {{ ref("stg__mitxpro__openedx__tracking_logs__user_activity") }} + where + courserun_readable_id is not null + and useractivity_event_type = 'problem_check' + and json_query(useractivity_event_object, 'lax $.submission' omit quotes) is not null - select distinct - courserun_readable_id - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name - , json_extract(useractivity_event_object, '$.submission') as submission - from {{ ref('stg__mitxpro__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type = 'problem_check' - and json_query(useractivity_event_object, 'lax $.submission' omit quotes) is not null + union all - union all + select distinct + courserun_readable_id, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name, + json_extract(useractivity_event_object, '$.submission') as submission + from {{ ref("stg__mitxresidential__openedx__tracking_logs__user_activity") }} + where + courserun_readable_id is not null + and useractivity_event_type = 'problem_check' + and json_query(useractivity_event_object, 'lax $.submission' omit quotes) is not null - select distinct - courserun_readable_id - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name - , json_extract(useractivity_event_object, '$.submission') as submission - from {{ ref('stg__mitxresidential__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type = 'problem_check' - and json_query(useractivity_event_object, 'lax $.submission' omit quotes) is not null + union all - union all - - select distinct - courserun_readable_id - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name - , json_extract(useractivity_event_object, '$.submission') as submission - from {{ ref('stg__edxorg__s3__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type = 'problem_check' - and json_query(useractivity_event_object, 'lax $.submission' omit quotes) is not null -) - -, problem_type_metadata as ( - select - problem_events.courserun_readable_id - , problem_events.problem_block_id - , arbitrary(problem_events.problem_name) as problem_name - , array_agg( - distinct - json_extract_scalar(t.submission_data, '$.response_type') - ) as problem_types - from problem_events - , unnest(cast(problem_events.submission as map )) as t (key, submission_data) - where - json_extract_scalar(t.submission_data, '$.response_type') is not null - and json_extract_scalar(t.submission_data, '$.response_type') <> '' - group by problem_events.courserun_readable_id, problem_events.problem_block_id -) - -, combined as ( - select - problems.content_block_pk as content_block_fk - , problems.markdown - , problems.max_attempts - , problems.start_date - , problems.due_date - , problems.weight - , problems.block_id as problem_block_pk - , problems.courserun_readable_id - , problems.block_title as problem_name - , problem_type_metadata.problem_types - from problems - left join problem_type_metadata - on problems.block_id = problem_type_metadata.problem_block_id - where problems.row_num = 1 -) + select distinct + courserun_readable_id, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name, + json_extract(useractivity_event_object, '$.submission') as submission + from {{ ref("stg__edxorg__s3__tracking_logs__user_activity") }} + where + courserun_readable_id is not null + and useractivity_event_type = 'problem_check' + and json_query(useractivity_event_object, 'lax $.submission' omit quotes) is not null + ), + problem_type_metadata as ( + select + problem_events.courserun_readable_id, + problem_events.problem_block_id, + arbitrary(problem_events.problem_name) as problem_name, + array_agg(distinct json_extract_scalar(t.submission_data, '$.response_type')) as problem_types + from problem_events, unnest(cast(problem_events.submission as map)) as t(key, submission_data) + where + json_extract_scalar(t.submission_data, '$.response_type') is not null + and json_extract_scalar(t.submission_data, '$.response_type') <> '' + group by problem_events.courserun_readable_id, problem_events.problem_block_id + ), + combined as ( + select + problems.content_block_pk as content_block_fk, + problems.markdown, + problems.max_attempts, + problems.start_date, + problems.due_date, + problems.weight, + problems.block_id as problem_block_pk, + problems.courserun_readable_id, + problems.block_title as problem_name, + problem_type_metadata.problem_types + from problems + left join problem_type_metadata on problems.block_id = problem_type_metadata.problem_block_id + where problems.row_num = 1 + ) select - problem_block_pk - , content_block_fk - , courserun_readable_id - , problem_name - , markdown - , max_attempts - , start_date - , due_date - , weight - , problem_types + problem_block_pk, + content_block_fk, + courserun_readable_id, + problem_name, + markdown, + max_attempts, + start_date, + due_date, + weight, + problem_types from combined diff --git a/src/ol_dbt/models/dimensional/dim_user.sql b/src/ol_dbt/models/dimensional/dim_user.sql index bb02b1953..aa5464666 100644 --- a/src/ol_dbt/models/dimensional/dim_user.sql +++ b/src/ol_dbt/models/dimensional/dim_user.sql @@ -1,590 +1,533 @@ -- MITx users from MITx Online and edX with dedup -- For users exist on both MITx Online and edX.org -with mitx_users as ( - select - user_mitxonline_id as mitxonline_application_user_id - , user_mitxonline_username - , user_global_id - , user_edxorg_id as edxorg_openedx_user_id - , user_edxorg_username - , user_mitxonline_email - , user_edxorg_email - , user_full_name as full_name - , user_address_country as address_country - , user_highest_education as highest_education - , user_gender as gender - , user_birth_year as birth_year - , user_company as company - , user_job_title as job_title - , user_industry as industry - , user_is_active_on_mitxonline - , user_is_active_on_edxorg - , user_joined_on_mitxonline - , user_joined_on_edxorg - , case - when user_is_active_on_mitxonline and user_joined_on_mitxonline > user_joined_on_edxorg +with + mitx_users as ( + select + user_mitxonline_id as mitxonline_application_user_id, + user_mitxonline_username, + user_global_id, + user_edxorg_id as edxorg_openedx_user_id, + user_edxorg_username, + user_mitxonline_email, + user_edxorg_email, + user_full_name as full_name, + user_address_country as address_country, + user_highest_education as highest_education, + user_gender as gender, + user_birth_year as birth_year, + user_company as company, + user_job_title as job_title, + user_industry as industry, + user_is_active_on_mitxonline, + user_is_active_on_edxorg, + user_joined_on_mitxonline, + user_joined_on_edxorg, + case + when user_is_active_on_mitxonline and user_joined_on_mitxonline > user_joined_on_edxorg then user_mitxonline_email - else coalesce(user_edxorg_email, user_mitxonline_email, user_micromasters_email) - end as user_email - from {{ ref('int__mitx__users') }} -) - -, mitxonline_openedx_users as ( - select - openedx_user_id - , user_username - , user_email - from {{ ref('stg__mitxonline__openedx__mysql__auth_user') }} -) - -, mitxonline_app_openedxuser_mapping as ( - select * from {{ ref('stg__mitxonline__app__postgres__openedx_openedxuser') }} -) - --- MITx Pro Users -, mitxpro_users as ( - select - user_id - , user_username - , user_email - , user_full_name - , user_is_active - , user_joined_on - from {{ ref('stg__mitxpro__app__postgres__users_user') }} -) - -, mitxpro_legaladdress as ( - select - user_id - , user_address_country - from {{ ref('stg__mitxpro__app__postgres__users_legaladdress') }} -) - -, mitxpro_profile as ( - select - user_id - , user_highest_education - , user_gender - , user_birth_year - , user_company - , user_job_title - , user_industry - from {{ ref('stg__mitxpro__app__postgres__users_profile') }} -) - -, mitxpro_openedx_users as ( - select - openedx_user_id - , user_username - , user_email - from {{ ref('stg__mitxpro__openedx__mysql__auth_user') }} -) - -, mitxpro_user_view as ( - select - mitxpro_users.user_username - , mitxpro_users.user_email - , mitxpro_users.user_full_name - , mitxpro_legaladdress.user_address_country - , mitxpro_profile.user_highest_education - , mitxpro_profile.user_gender - , mitxpro_profile.user_birth_year - , mitxpro_profile.user_company - , mitxpro_profile.user_job_title - , mitxpro_profile.user_industry - , 'mitxpro' as platform - , mitxpro_users.user_id - , mitxpro_users.user_is_active - , mitxpro_users.user_joined_on - from mitxpro_users - left join mitxpro_legaladdress on mitxpro_users.user_id = mitxpro_legaladdress.user_id - left join mitxpro_profile on mitxpro_users.user_id = mitxpro_profile.user_id -) - -, emeritus_users as ( - select * from ( + else coalesce(user_edxorg_email, user_mitxonline_email, user_micromasters_email) + end as user_email + from {{ ref("int__mitx__users") }} + ), + mitxonline_openedx_users as ( + select openedx_user_id, user_username, user_email from {{ ref("stg__mitxonline__openedx__mysql__auth_user") }} + ), + mitxonline_app_openedxuser_mapping as ( + select * from {{ ref("stg__mitxonline__app__postgres__openedx_openedxuser") }} + ), + -- MITx Pro Users + mitxpro_users as ( + select user_id, user_username, user_email, user_full_name, user_is_active, user_joined_on + from {{ ref("stg__mitxpro__app__postgres__users_user") }} + ), + mitxpro_legaladdress as ( + select user_id, user_address_country from {{ ref("stg__mitxpro__app__postgres__users_legaladdress") }} + ), + mitxpro_profile as ( select - user_id - , user_email - , user_full_name - , user_address_country - , user_gender - , user_company - , user_job_title - , user_industry - , row_number() over ( - partition by coalesce(user_id, user_email, user_full_name) - order by user_gdpr_consent_date desc, enrollment_created_on desc - ) as row_num - from {{ ref('stg__emeritus__api__bigquery__user_enrollments') }} - ) - where row_num = 1 -) - -, global_alumni_users as ( - select * from ( + user_id, user_highest_education, user_gender, user_birth_year, user_company, user_job_title, user_industry + from {{ ref("stg__mitxpro__app__postgres__users_profile") }} + ), + mitxpro_openedx_users as ( + select openedx_user_id, user_username, user_email from {{ ref("stg__mitxpro__openedx__mysql__auth_user") }} + ), + mitxpro_user_view as ( select - user_id - , user_email - , user_full_name - , user_address_country - , user_gender - , user_company - , user_job_title - , user_industry - , row_number() over ( - partition by user_email - order by user_gdpr_consent_date desc, courserun_start_on desc - ) as row_num - from {{ ref('stg__global_alumni__api__bigquery__user_enrollments') }} - ) - where row_num = 1 -) - --- Residential Users -, mitxresidential_openedx_users as ( - select - user_username - , user_email - , user_full_name - , user_is_active - , user_id - , user_joined_on - from {{ ref('stg__mitxresidential__openedx__auth_user') }} -) + mitxpro_users.user_username, + mitxpro_users.user_email, + mitxpro_users.user_full_name, + mitxpro_legaladdress.user_address_country, + mitxpro_profile.user_highest_education, + mitxpro_profile.user_gender, + mitxpro_profile.user_birth_year, + mitxpro_profile.user_company, + mitxpro_profile.user_job_title, + mitxpro_profile.user_industry, + 'mitxpro' as platform, + mitxpro_users.user_id, + mitxpro_users.user_is_active, + mitxpro_users.user_joined_on + from mitxpro_users + left join mitxpro_legaladdress on mitxpro_users.user_id = mitxpro_legaladdress.user_id + left join mitxpro_profile on mitxpro_users.user_id = mitxpro_profile.user_id + ), + emeritus_users as ( + select * + from + ( + select + user_id, + user_email, + user_full_name, + user_address_country, + user_gender, + user_company, + user_job_title, + user_industry, + row_number() over ( + partition by coalesce(user_id, user_email, user_full_name) + order by user_gdpr_consent_date desc, enrollment_created_on desc + ) as row_num + from {{ ref("stg__emeritus__api__bigquery__user_enrollments") }} + ) + where row_num = 1 + ), + global_alumni_users as ( + select * + from + ( + select + user_id, + user_email, + user_full_name, + user_address_country, + user_gender, + user_company, + user_job_title, + user_industry, + row_number() over ( + partition by user_email order by user_gdpr_consent_date desc, courserun_start_on desc + ) as row_num + from {{ ref("stg__global_alumni__api__bigquery__user_enrollments") }} + ) + where row_num = 1 + ), + -- Residential Users + mitxresidential_openedx_users as ( + select user_username, user_email, user_full_name, user_is_active, user_id, user_joined_on + from {{ ref("stg__mitxresidential__openedx__auth_user") }} + ), + mitxresidential_profile as ( + select user_address_country, user_highest_education, user_gender, user_birth_year, user_id + from {{ ref("stg__mitxresidential__openedx__auth_userprofile") }} + ), + mitxresidential_user_view as ( + select + mitxresidential_openedx_users.user_username, + mitxresidential_openedx_users.user_email, + mitxresidential_openedx_users.user_full_name, + mitxresidential_profile.user_address_country, + mitxresidential_profile.user_highest_education, + mitxresidential_profile.user_gender, + mitxresidential_profile.user_birth_year, + 'residential' as platform, + mitxresidential_openedx_users.user_id, + mitxresidential_openedx_users.user_is_active, + mitxresidential_openedx_users.user_joined_on + from mitxresidential_openedx_users + left join mitxresidential_profile on mitxresidential_openedx_users.user_id = mitxresidential_profile.user_id + ), + mitx_users_view as ( + select + mitx_users.user_global_id, + mitxonline_openedx_users.openedx_user_id as mitxonline_openedx_user_id, + mitx_users.mitxonline_application_user_id, + coalesce( + mitxonline_app_openedxuser_mapping.openedxuser_username, mitx_users.user_mitxonline_username + ) as user_mitxonline_username, + mitx_users.edxorg_openedx_user_id, + mitx_users.user_edxorg_username, + mitx_users.user_email as email, + mitx_users.full_name, + mitx_users.address_country, + mitx_users.highest_education, + mitx_users.gender, + mitx_users.birth_year, + mitx_users.company, + mitx_users.job_title, + mitx_users.industry, + mitx_users.user_is_active_on_mitxonline, + mitx_users.user_joined_on_mitxonline, + mitx_users.user_is_active_on_edxorg, + mitx_users.user_joined_on_edxorg + from mitx_users + left join + mitxonline_app_openedxuser_mapping + on mitx_users.mitxonline_application_user_id = mitxonline_app_openedxuser_mapping.user_id + left join + mitxonline_openedx_users + on mitxonline_app_openedxuser_mapping.openedxuser_username = mitxonline_openedx_users.user_username + ), + learn_user as ( + select * + from + ( + select *, row_number() over (partition by user_email order by user_created_on desc) as row_num + from {{ ref("stg__mitlearn__app__postgres__users_user") }} + ) + where row_num = 1 + ), + learn_profile as (select * from {{ ref("stg__mitlearn__app__postgres__profiles_profile") }}), + learn_user_topic_interests as ( + select profile_topic_interests.profile_id, array_agg(topic.learningresourcetopic_name) as topic_interests + from {{ ref("stg__mitlearn__app__postgres__profiles_profile_topic_interests") }} as profile_topic_interests + join + {{ ref("stg__mitlearn__app__postgres__learning_resources_learningresourcetopic") }} as topic + on profile_topic_interests.learningresourcetopic_id = topic.learningresourcetopic_id + group by profile_topic_interests.profile_id + + ), + learn_user_view as ( + select + learn_user.user_global_id, + learn_user.user_id as mitlearn_user_id, + learn_user.user_email as email, + concat(learn_user.user_first_name, ' ', learn_user.user_last_name) as full_name, + learn_profile.user_current_education as highest_education, + learn_user.user_is_active as user_is_active_on_mitlearn, + learn_user.user_joined_on as user_joined_on_mitlearn + from learn_user + left join learn_profile on learn_user.user_id = learn_profile.user_id + ), + users_with_global_id as ( + select + learn_user_view.mitlearn_user_id, + mitx_users_view.mitxonline_openedx_user_id, + mitx_users_view.mitxonline_application_user_id, + mitx_users_view.user_mitxonline_username, + mitx_users_view.edxorg_openedx_user_id, + mitx_users_view.user_edxorg_username, + mitx_users_view.address_country, + mitx_users_view.gender, + mitx_users_view.birth_year, + mitx_users_view.company, + mitx_users_view.job_title, + mitx_users_view.industry, + learn_user_view.user_is_active_on_mitlearn, + learn_user_view.user_joined_on_mitlearn, + mitx_users_view.user_is_active_on_mitxonline, + mitx_users_view.user_joined_on_mitxonline, + mitx_users_view.user_is_active_on_edxorg, + mitx_users_view.user_joined_on_edxorg, + coalesce(learn_user_view.full_name, mitx_users_view.full_name) as full_name, + coalesce(learn_user_view.user_global_id, mitx_users_view.user_global_id) as user_global_id, + coalesce(learn_user_view.highest_education, mitx_users_view.highest_education) as highest_education, + coalesce( + case + when + mitx_users_view.user_is_active_on_mitxonline + and mitx_users_view.user_joined_on_mitxonline > learn_user_view.user_joined_on_mitlearn + then mitx_users_view.email + end, + learn_user_view.email, + mitx_users_view.email + ) as email + from mitx_users_view + full outer join learn_user_view on mitx_users_view.user_global_id = learn_user_view.user_global_id + ), + combined_users as ( + select + {{ dbt_utils.generate_surrogate_key(["email"]) }} as user_pk, + user_global_id, + mitlearn_user_id, + mitxonline_openedx_user_id, + mitxonline_application_user_id, + user_mitxonline_username, + null as mitxpro_openedx_user_id, + null as mitxpro_application_user_id, + null as user_mitxpro_username, + null as residential_openedx_user_id, + null as user_residential_username, + edxorg_openedx_user_id, + user_edxorg_username, + null as emeritus_user_id, + null as global_alumni_user_id, + email, + full_name, + address_country, + highest_education, + gender, + birth_year, + company, + job_title, + industry, + user_is_active_on_mitlearn, + user_joined_on_mitlearn, + user_is_active_on_mitxonline, + user_joined_on_mitxonline, + user_is_active_on_edxorg, + user_joined_on_edxorg, + null as user_is_active_on_mitxpro, + null as user_joined_on_mitxpro, + null as user_is_active_on_residential, + null as user_joined_on_residential + from users_with_global_id + + union all -, mitxresidential_profile as ( - select - user_address_country - , user_highest_education - , user_gender - , user_birth_year - , user_id - from {{ ref('stg__mitxresidential__openedx__auth_userprofile') }} -) + select + {{ dbt_utils.generate_surrogate_key(["mitxpro_user_view.user_email"]) }} as user_pk, + null as user_global_id, + null as mitlearn_user_id, + null as mitxonline_openedx_user_id, + null as mitxonline_application_user_id, + null as user_mitxonline_username, + coalesce( + openedx_users_username.openedx_user_id, openedx_users_email.openedx_user_id + ) as mitxpro_openedx_user_id, + mitxpro_user_view.user_id as mitxpro_application_user_id, + mitxpro_user_view.user_username as user_mitxpro_username, + null as residential_openedx_user_id, + null as user_residential_username, + null as edxorg_openedx_user_id, + null as user_edxorg_username, + null as emeritus_user_id, + null as global_alumni_user_id, + mitxpro_user_view.user_email as email, + mitxpro_user_view.user_full_name as full_name, + mitxpro_user_view.user_address_country as address_country, + mitxpro_user_view.user_highest_education as highest_education, + mitxpro_user_view.user_gender as gender, + mitxpro_user_view.user_birth_year as birth_year, + mitxpro_user_view.user_company as company, + mitxpro_user_view.user_job_title as job_title, + mitxpro_user_view.user_industry as industry, + null as user_is_active_on_mitlearn, + null as user_joined_on_mitlearn, + null as user_is_active_on_mitxonline, + null as user_joined_on_mitxonline, + null as user_is_active_on_edxorg, + null as user_joined_on_edxorg, + mitxpro_user_view.user_is_active as user_is_active_on_mitxpro, + mitxpro_user_view.user_joined_on as user_joined_on_mitxpro, + null as user_is_active_on_residential, + null as user_joined_on_residential + from mitxpro_user_view + left join + mitxpro_openedx_users as openedx_users_username + on mitxpro_user_view.user_username = openedx_users_username.user_username + left join + mitxpro_openedx_users as openedx_users_email + on mitxpro_user_view.user_email = openedx_users_email.user_email + + union all -, mitxresidential_user_view as ( - select - mitxresidential_openedx_users.user_username - , mitxresidential_openedx_users.user_email - , mitxresidential_openedx_users.user_full_name - , mitxresidential_profile.user_address_country - , mitxresidential_profile.user_highest_education - , mitxresidential_profile.user_gender - , mitxresidential_profile.user_birth_year - , 'residential' as platform - , mitxresidential_openedx_users.user_id - , mitxresidential_openedx_users.user_is_active - , mitxresidential_openedx_users.user_joined_on - from mitxresidential_openedx_users - left join mitxresidential_profile on mitxresidential_openedx_users.user_id = mitxresidential_profile.user_id -) + select + {{ dbt_utils.generate_surrogate_key(["user_email"]) }} as user_pk, + null as user_global_id, + null as mitlearn_user_id, + null as mitxonline_openedx_user_id, + null as mitxonline_application_user_id, + null as user_mitxonline_username, + null as mitxpro_openedx_user_id, + null as mitxpro_application_user_id, + null as user_mitxpro_username, + null as residential_openedx_user_id, + null as user_residential_username, + null as edxorg_openedx_user_id, + null as user_edxorg_username, + user_id as emeritus_user_id, + null as global_alumni_user_id, + user_email as email, + user_full_name as full_name, + user_address_country as address_country, + null as highest_education, + user_gender as gender, + null as birth_year, + user_company as company, + user_job_title as job_title, + user_industry as industry, + null as user_is_active_on_mitlearn, + null as user_joined_on_mitlearn, + null as user_is_active_on_mitxonline, + null as user_joined_on_mitxonline, + null as user_is_active_on_edxorg, + null as user_joined_on_edxorg, + null as user_is_active_on_mitxpro, + null as user_joined_on_mitxpro, + null as user_is_active_on_residential, + null as user_joined_on_residential + from emeritus_users + where user_email is not null + + union all -, mitx_users_view as ( - select - mitx_users.user_global_id - , mitxonline_openedx_users.openedx_user_id as mitxonline_openedx_user_id - , mitx_users.mitxonline_application_user_id - , coalesce( - mitxonline_app_openedxuser_mapping.openedxuser_username, mitx_users.user_mitxonline_username - ) as user_mitxonline_username - , mitx_users.edxorg_openedx_user_id - , mitx_users.user_edxorg_username - , mitx_users.user_email as email - , mitx_users.full_name - , mitx_users.address_country - , mitx_users.highest_education - , mitx_users.gender - , mitx_users.birth_year - , mitx_users.company - , mitx_users.job_title - , mitx_users.industry - , mitx_users.user_is_active_on_mitxonline - , mitx_users.user_joined_on_mitxonline - , mitx_users.user_is_active_on_edxorg - , mitx_users.user_joined_on_edxorg - from mitx_users - left join mitxonline_app_openedxuser_mapping - on mitx_users.mitxonline_application_user_id = mitxonline_app_openedxuser_mapping.user_id - left join mitxonline_openedx_users - on mitxonline_app_openedxuser_mapping.openedxuser_username = mitxonline_openedx_users.user_username -) + select + {{ dbt_utils.generate_surrogate_key(["user_email"]) }} as user_pk, + null as user_global_id, + null as mitlearn_user_id, + null as mitxonline_openedx_user_id, + null as mitxonline_application_user_id, + null as user_mitxonline_username, + null as mitxpro_openedx_user_id, + null as mitxpro_application_user_id, + null as user_mitxpro_username, + null as residential_openedx_user_id, + null as user_residential_username, + null as edxorg_openedx_user_id, + null as user_edxorg_username, + null as emeritus_user_id, + user_id as global_alumni_user_id, + user_email as email, + user_full_name as full_name, + user_address_country as address_country, + null as highest_education, + user_gender as gender, + null as birth_year, + user_company as company, + user_job_title as job_title, + user_industry as industry, + null as user_is_active_on_mitlearn, + null as user_joined_on_mitlearn, + null as user_is_active_on_mitxonline, + null as user_joined_on_mitxonline, + null as user_is_active_on_edxorg, + null as user_joined_on_edxorg, + null as user_is_active_on_mitxpro, + null as user_joined_on_mitxpro, + null as user_is_active_on_residential, + null as user_joined_on_residential + from global_alumni_users + where user_email is not null + + union all -, learn_user as ( - select * from ( select - * - , row_number() over ( - partition by user_email - order by user_created_on desc + {{ dbt_utils.generate_surrogate_key(["mitxresidential_user_view.user_email"]) }} as user_pk, + null as user_global_id, + null as mitlearn_user_id, + null as mitxonline_openedx_user_id, + null as mitxonline_application_user_id, + null as user_mitxonline_username, + null as mitxpro_openedx_user_id, + null as mitxpro_application_user_id, + null as user_mitxpro_username, + mitxresidential_user_view.user_id as residential_openedx_user_id, + mitxresidential_user_view.user_username as user_residential_username, + null as edxorg_openedx_user_id, + null as user_edxorg_username, + null as emeritus_user_id, + null as global_alumni_user_id, + mitxresidential_user_view.user_email as email, + mitxresidential_user_view.user_full_name as full_name, + mitxresidential_user_view.user_address_country as address_country, + mitxresidential_user_view.user_highest_education as highest_education, + mitxresidential_user_view.user_gender as gender, + mitxresidential_user_view.user_birth_year as birth_year, + null as company, + null as job_title, + null as industry, + null as user_is_active_on_mitlearn, + null as user_joined_on_mitlearn, + null as user_is_active_on_mitxonline, + null as user_joined_on_mitxonline, + null as user_is_active_on_edxorg, + null as user_joined_on_edxorg, + null as user_is_active_on_mitxpro, + null as user_joined_on_mitxpro, + mitxresidential_user_view.user_is_active as user_is_active_on_residential, + mitxresidential_user_view.user_joined_on as user_joined_on_residential + from mitxresidential_user_view + + ), + ranked_users as ( + select + *, + row_number() over ( + partition by user_pk + order by + greatest( + user_joined_on_mitlearn, + user_joined_on_mitxonline, + user_joined_on_edxorg, + user_joined_on_mitxpro, + user_joined_on_residential + ) desc ) as row_num - from {{ ref('stg__mitlearn__app__postgres__users_user') }} + from combined_users + ), + base_info as (select * from ranked_users where row_num = 1), + agg_view as ( + select + user_pk, + max(user_global_id) as user_global_id, + max(mitlearn_user_id) as mitlearn_user_id, + max(mitxonline_openedx_user_id) as mitxonline_openedx_user_id, + max(mitxonline_application_user_id) as mitxonline_application_user_id, + max(user_mitxonline_username) as user_mitxonline_username, + max(mitxpro_openedx_user_id) as mitxpro_openedx_user_id, + max(mitxpro_application_user_id) as mitxpro_application_user_id, + max(user_mitxpro_username) as user_mitxpro_username, + max(residential_openedx_user_id) as residential_openedx_user_id, + max(user_residential_username) as user_residential_username, + max(edxorg_openedx_user_id) as edxorg_openedx_user_id, + max(emeritus_user_id) as emeritus_user_id, + max(global_alumni_user_id) as global_alumni_user_id, + max(user_edxorg_username) as user_edxorg_username, + max(user_is_active_on_mitlearn) as user_is_active_on_mitlearn, + max(user_joined_on_mitlearn) as user_joined_on_mitlearn, + max(user_is_active_on_mitxonline) as user_is_active_on_mitxonline, + max(user_joined_on_mitxonline) as user_joined_on_mitxonline, + max(user_is_active_on_edxorg) as user_is_active_on_edxorg, + max(user_joined_on_edxorg) as user_joined_on_edxorg, + max(user_is_active_on_mitxpro) as user_is_active_on_mitxpro, + max(user_joined_on_mitxpro) as user_joined_on_mitxpro, + max(user_is_active_on_residential) as user_is_active_on_residential, + max(user_joined_on_residential) as user_joined_on_residential + from combined_users + group by user_pk ) - where row_num = 1 -) - -, learn_profile as ( - select * from {{ ref('stg__mitlearn__app__postgres__profiles_profile') }} -) - -, learn_user_topic_interests as ( - select - profile_topic_interests.profile_id - , array_agg(topic.learningresourcetopic_name) as topic_interests - from {{ ref('stg__mitlearn__app__postgres__profiles_profile_topic_interests') }} as profile_topic_interests - join {{ ref('stg__mitlearn__app__postgres__learning_resources_learningresourcetopic') }} as topic - on profile_topic_interests.learningresourcetopic_id = topic.learningresourcetopic_id - group by profile_topic_interests.profile_id - -) - -, learn_user_view as( - select - learn_user.user_global_id - , learn_user.user_id as mitlearn_user_id - , learn_user.user_email as email - , concat(learn_user.user_first_name, ' ', learn_user.user_last_name) as full_name - , learn_profile.user_current_education as highest_education - , learn_user.user_is_active as user_is_active_on_mitlearn - , learn_user.user_joined_on as user_joined_on_mitlearn - from learn_user - left join learn_profile on learn_user.user_id = learn_profile.user_id -) - -, users_with_global_id as ( - select - learn_user_view.mitlearn_user_id - , mitx_users_view.mitxonline_openedx_user_id - , mitx_users_view.mitxonline_application_user_id - , mitx_users_view.user_mitxonline_username - , mitx_users_view.edxorg_openedx_user_id - , mitx_users_view.user_edxorg_username - , mitx_users_view.address_country - , mitx_users_view.gender - , mitx_users_view.birth_year - , mitx_users_view.company - , mitx_users_view.job_title - , mitx_users_view.industry - , learn_user_view.user_is_active_on_mitlearn - , learn_user_view.user_joined_on_mitlearn - , mitx_users_view.user_is_active_on_mitxonline - , mitx_users_view.user_joined_on_mitxonline - , mitx_users_view.user_is_active_on_edxorg - , mitx_users_view.user_joined_on_edxorg - , coalesce(learn_user_view.full_name, mitx_users_view.full_name) as full_name - , coalesce(learn_user_view.user_global_id, mitx_users_view.user_global_id) as user_global_id - , coalesce(learn_user_view.highest_education, mitx_users_view.highest_education) as highest_education - , coalesce( - case - when mitx_users_view.user_is_active_on_mitxonline - and mitx_users_view.user_joined_on_mitxonline > learn_user_view.user_joined_on_mitlearn - then mitx_users_view.email - end, - learn_user_view.email, - mitx_users_view.email - ) as email - from mitx_users_view - full outer join learn_user_view on mitx_users_view.user_global_id = learn_user_view.user_global_id -) - -, combined_users as ( - select - {{ dbt_utils.generate_surrogate_key(['email']) }} as user_pk - , user_global_id - , mitlearn_user_id - , mitxonline_openedx_user_id - , mitxonline_application_user_id - , user_mitxonline_username - , null as mitxpro_openedx_user_id - , null as mitxpro_application_user_id - , null as user_mitxpro_username - , null as residential_openedx_user_id - , null as user_residential_username - , edxorg_openedx_user_id - , user_edxorg_username - , null as emeritus_user_id - , null as global_alumni_user_id - , email - , full_name - , address_country - , highest_education - , gender - , birth_year - , company - , job_title - , industry - , user_is_active_on_mitlearn - , user_joined_on_mitlearn - , user_is_active_on_mitxonline - , user_joined_on_mitxonline - , user_is_active_on_edxorg - , user_joined_on_edxorg - , null as user_is_active_on_mitxpro - , null as user_joined_on_mitxpro - , null as user_is_active_on_residential - , null as user_joined_on_residential - from users_with_global_id - - union all - - select - {{ dbt_utils.generate_surrogate_key(['mitxpro_user_view.user_email']) }} as user_pk - , null as user_global_id - , null as mitlearn_user_id - , null as mitxonline_openedx_user_id - , null as mitxonline_application_user_id - , null as user_mitxonline_username - , coalesce( - openedx_users_username.openedx_user_id, openedx_users_email.openedx_user_id - ) as mitxpro_openedx_user_id - , mitxpro_user_view.user_id as mitxpro_application_user_id - , mitxpro_user_view.user_username as user_mitxpro_username - , null as residential_openedx_user_id - , null as user_residential_username - , null as edxorg_openedx_user_id - , null as user_edxorg_username - , null as emeritus_user_id - , null as global_alumni_user_id - , mitxpro_user_view.user_email as email - , mitxpro_user_view.user_full_name as full_name - , mitxpro_user_view.user_address_country as address_country - , mitxpro_user_view.user_highest_education as highest_education - , mitxpro_user_view.user_gender as gender - , mitxpro_user_view.user_birth_year as birth_year - , mitxpro_user_view.user_company as company - , mitxpro_user_view.user_job_title as job_title - , mitxpro_user_view.user_industry as industry - , null as user_is_active_on_mitlearn - , null as user_joined_on_mitlearn - , null as user_is_active_on_mitxonline - , null as user_joined_on_mitxonline - , null as user_is_active_on_edxorg - , null as user_joined_on_edxorg - , mitxpro_user_view.user_is_active as user_is_active_on_mitxpro - , mitxpro_user_view.user_joined_on as user_joined_on_mitxpro - , null as user_is_active_on_residential - , null as user_joined_on_residential - from mitxpro_user_view - left join mitxpro_openedx_users as openedx_users_username - on mitxpro_user_view.user_username = openedx_users_username.user_username - left join mitxpro_openedx_users as openedx_users_email - on mitxpro_user_view.user_email = openedx_users_email.user_email - - union all - - select - {{ dbt_utils.generate_surrogate_key(['user_email']) }} as user_pk - , null as user_global_id - , null as mitlearn_user_id - , null as mitxonline_openedx_user_id - , null as mitxonline_application_user_id - , null as user_mitxonline_username - , null as mitxpro_openedx_user_id - , null as mitxpro_application_user_id - , null as user_mitxpro_username - , null as residential_openedx_user_id - , null as user_residential_username - , null as edxorg_openedx_user_id - , null as user_edxorg_username - , user_id as emeritus_user_id - , null as global_alumni_user_id - , user_email as email - , user_full_name as full_name - , user_address_country as address_country - , null as highest_education - , user_gender as gender - , null as birth_year - , user_company as company - , user_job_title as job_title - , user_industry as industry - , null as user_is_active_on_mitlearn - , null as user_joined_on_mitlearn - , null as user_is_active_on_mitxonline - , null as user_joined_on_mitxonline - , null as user_is_active_on_edxorg - , null as user_joined_on_edxorg - , null as user_is_active_on_mitxpro - , null as user_joined_on_mitxpro - , null as user_is_active_on_residential - , null as user_joined_on_residential - from emeritus_users - where user_email is not null - - union all - - select - {{ dbt_utils.generate_surrogate_key(['user_email']) }} as user_pk - , null as user_global_id - , null as mitlearn_user_id - , null as mitxonline_openedx_user_id - , null as mitxonline_application_user_id - , null as user_mitxonline_username - , null as mitxpro_openedx_user_id - , null as mitxpro_application_user_id - , null as user_mitxpro_username - , null as residential_openedx_user_id - , null as user_residential_username - , null as edxorg_openedx_user_id - , null as user_edxorg_username - , null as emeritus_user_id - , user_id as global_alumni_user_id - , user_email as email - , user_full_name as full_name - , user_address_country as address_country - , null as highest_education - , user_gender as gender - , null as birth_year - , user_company as company - , user_job_title as job_title - , user_industry as industry - , null as user_is_active_on_mitlearn - , null as user_joined_on_mitlearn - , null as user_is_active_on_mitxonline - , null as user_joined_on_mitxonline - , null as user_is_active_on_edxorg - , null as user_joined_on_edxorg - , null as user_is_active_on_mitxpro - , null as user_joined_on_mitxpro - , null as user_is_active_on_residential - , null as user_joined_on_residential - from global_alumni_users - where user_email is not null - - union all - - select - {{ dbt_utils.generate_surrogate_key(['mitxresidential_user_view.user_email']) }} as user_pk - , null as user_global_id - , null as mitlearn_user_id - , null as mitxonline_openedx_user_id - , null as mitxonline_application_user_id - , null as user_mitxonline_username - , null as mitxpro_openedx_user_id - , null as mitxpro_application_user_id - , null as user_mitxpro_username - , mitxresidential_user_view.user_id as residential_openedx_user_id - , mitxresidential_user_view.user_username as user_residential_username - , null as edxorg_openedx_user_id - , null as user_edxorg_username - , null as emeritus_user_id - , null as global_alumni_user_id - , mitxresidential_user_view.user_email as email - , mitxresidential_user_view.user_full_name as full_name - , mitxresidential_user_view.user_address_country as address_country - , mitxresidential_user_view.user_highest_education as highest_education - , mitxresidential_user_view.user_gender as gender - , mitxresidential_user_view.user_birth_year as birth_year - , null as company - , null as job_title - , null as industry - , null as user_is_active_on_mitlearn - , null as user_joined_on_mitlearn - , null as user_is_active_on_mitxonline - , null as user_joined_on_mitxonline - , null as user_is_active_on_edxorg - , null as user_joined_on_edxorg - , null as user_is_active_on_mitxpro - , null as user_joined_on_mitxpro - , mitxresidential_user_view.user_is_active as user_is_active_on_residential - , mitxresidential_user_view.user_joined_on as user_joined_on_residential - from mitxresidential_user_view - -) - -, ranked_users as ( - select - * - , row_number() over ( - partition by user_pk - order by - greatest( - user_joined_on_mitlearn - , user_joined_on_mitxonline - , user_joined_on_edxorg - , user_joined_on_mitxpro - , user_joined_on_residential - ) desc - ) as row_num - from combined_users -) - -, base_info as ( - select * - from ranked_users - where row_num = 1 -) - -, agg_view as ( - select - user_pk - , max(user_global_id) as user_global_id - , max(mitlearn_user_id) as mitlearn_user_id - , max(mitxonline_openedx_user_id) as mitxonline_openedx_user_id - , max(mitxonline_application_user_id) as mitxonline_application_user_id - , max(user_mitxonline_username) as user_mitxonline_username - , max(mitxpro_openedx_user_id) as mitxpro_openedx_user_id - , max(mitxpro_application_user_id) as mitxpro_application_user_id - , max(user_mitxpro_username) as user_mitxpro_username - , max(residential_openedx_user_id) as residential_openedx_user_id - , max(user_residential_username) as user_residential_username - , max(edxorg_openedx_user_id) as edxorg_openedx_user_id - , max(emeritus_user_id) as emeritus_user_id - , max(global_alumni_user_id) as global_alumni_user_id - , max(user_edxorg_username) as user_edxorg_username - , max(user_is_active_on_mitlearn) as user_is_active_on_mitlearn - , max(user_joined_on_mitlearn) as user_joined_on_mitlearn - , max(user_is_active_on_mitxonline) as user_is_active_on_mitxonline - , max(user_joined_on_mitxonline) as user_joined_on_mitxonline - , max(user_is_active_on_edxorg) as user_is_active_on_edxorg - , max(user_joined_on_edxorg) as user_joined_on_edxorg - , max(user_is_active_on_mitxpro) as user_is_active_on_mitxpro - , max(user_joined_on_mitxpro) as user_joined_on_mitxpro - , max(user_is_active_on_residential) as user_is_active_on_residential - , max(user_joined_on_residential) as user_joined_on_residential - from combined_users - group by user_pk -) select - base.user_pk - , agg.user_global_id - , agg.mitlearn_user_id - , agg.mitxonline_openedx_user_id - , agg.mitxonline_application_user_id - , agg.user_mitxonline_username - , agg.mitxpro_openedx_user_id - , agg.mitxpro_application_user_id - , agg.user_mitxpro_username - , agg.residential_openedx_user_id - , agg.user_residential_username - , agg.edxorg_openedx_user_id - , agg.user_edxorg_username - , agg.emeritus_user_id - , agg.global_alumni_user_id - , base.email - , base.full_name - , base.address_country - , base.highest_education - , base.gender - , base.birth_year - , base.company - , base.job_title - , base.industry - , learn_user_topic_interests.topic_interests as topic_interests - , learn_profile.user_goals as goals - , learn_profile.user_delivery_preference as delivery_preference - , learn_profile.user_completed_onboarding as completed_onboarding - , learn_profile.user_certificate_desired as certificate_desired - , agg.user_is_active_on_mitlearn - , agg.user_joined_on_mitlearn - , agg.user_is_active_on_mitxonline - , agg.user_joined_on_mitxonline - , agg.user_is_active_on_edxorg - , agg.user_joined_on_edxorg - , agg.user_is_active_on_mitxpro - , agg.user_joined_on_mitxpro - , agg.user_is_active_on_residential - , agg.user_joined_on_residential + base.user_pk, + agg.user_global_id, + agg.mitlearn_user_id, + agg.mitxonline_openedx_user_id, + agg.mitxonline_application_user_id, + agg.user_mitxonline_username, + agg.mitxpro_openedx_user_id, + agg.mitxpro_application_user_id, + agg.user_mitxpro_username, + agg.residential_openedx_user_id, + agg.user_residential_username, + agg.edxorg_openedx_user_id, + agg.user_edxorg_username, + agg.emeritus_user_id, + agg.global_alumni_user_id, + base.email, + base.full_name, + base.address_country, + base.highest_education, + base.gender, + base.birth_year, + base.company, + base.job_title, + base.industry, + learn_user_topic_interests.topic_interests as topic_interests, + learn_profile.user_goals as goals, + learn_profile.user_delivery_preference as delivery_preference, + learn_profile.user_completed_onboarding as completed_onboarding, + learn_profile.user_certificate_desired as certificate_desired, + agg.user_is_active_on_mitlearn, + agg.user_joined_on_mitlearn, + agg.user_is_active_on_mitxonline, + agg.user_joined_on_mitxonline, + agg.user_is_active_on_edxorg, + agg.user_joined_on_edxorg, + agg.user_is_active_on_mitxpro, + agg.user_joined_on_mitxpro, + agg.user_is_active_on_residential, + agg.user_joined_on_residential from base_info as base inner join agg_view as agg on base.user_pk = agg.user_pk left join learn_profile on base.mitlearn_user_id = learn_profile.user_id diff --git a/src/ol_dbt/models/dimensional/dim_video.sql b/src/ol_dbt/models/dimensional/dim_video.sql index 8edb352aa..447401f71 100644 --- a/src/ol_dbt/models/dimensional/dim_video.sql +++ b/src/ol_dbt/models/dimensional/dim_video.sql @@ -1,102 +1,82 @@ -with video_content as ( - select - content_block_pk - , block_id - , block_title - , courserun_readable_id - , element_at(split(block_id, '@'), -1) as video_id - , nullif(json_query(block_metadata, 'lax $.display_name' omit quotes), 'null') as video_name - , nullif(json_query(block_metadata, 'lax $.start' omit quotes), 'null') as start_date - , nullif(json_query(block_metadata, 'lax $.due' omit quotes), 'null') as due_date - , nullif(json_query(block_metadata, 'lax $.edx_video_id' omit quotes), 'null') as edx_video_id - , nullif(json_query(block_metadata, 'lax $.html5_sources' omit quotes), 'null') as html5_sources - , nullif(json_query(block_metadata, 'lax $.transcripts' omit quotes), 'null') as transcripts - , nullif(json_query(block_metadata, 'lax $.edx_video_id' omit quotes), 'null') as video_edx_uuid - , row_number() over ( - partition by block_id - order by is_latest desc, retrieved_at desc - ) as row_num - from {{ ref('dim_course_content') }} - where block_category = 'video' -) +with + video_content as ( + select + content_block_pk, + block_id, + block_title, + courserun_readable_id, + element_at(split(block_id, '@'), -1) as video_id, + nullif(json_query(block_metadata, 'lax $.display_name' omit quotes), 'null') as video_name, + nullif(json_query(block_metadata, 'lax $.start' omit quotes), 'null') as start_date, + nullif(json_query(block_metadata, 'lax $.due' omit quotes), 'null') as due_date, + nullif(json_query(block_metadata, 'lax $.edx_video_id' omit quotes), 'null') as edx_video_id, + nullif(json_query(block_metadata, 'lax $.html5_sources' omit quotes), 'null') as html5_sources, + nullif(json_query(block_metadata, 'lax $.transcripts' omit quotes), 'null') as transcripts, + nullif(json_query(block_metadata, 'lax $.edx_video_id' omit quotes), 'null') as video_edx_uuid, + row_number() over (partition by block_id order by is_latest desc, retrieved_at desc) as row_num + from {{ ref("dim_course_content") }} + where block_category = 'video' + ), + video as ( -, video as ( + select video_edx_uuid, video_duration, video_created_on + from {{ ref("stg__mitxonline__openedx__mysql__edxval_video") }} - select - video_edx_uuid - , video_duration - , video_created_on - from {{ ref('stg__mitxonline__openedx__mysql__edxval_video') }} + union all - union all + select video_edx_uuid, video_duration, video_created_on + from {{ ref("stg__mitxpro__openedx__mysql__edxval_video") }} - select - video_edx_uuid - , video_duration - , video_created_on - from {{ ref('stg__mitxpro__openedx__mysql__edxval_video') }} + union all - union all + select video_edx_uuid, video_duration, video_created_on + from {{ ref("stg__mitxresidential__openedx__edxval_video") }} - select - video_edx_uuid - , video_duration - , video_created_on - from {{ ref('stg__mitxresidential__openedx__edxval_video') }} + union all - union all + select video_block_id as video_edx_uuid, video_duration, null as video_created_on + from {{ ref("stg__edxorg__s3__course_video") }} - select - video_block_id as video_edx_uuid - , video_duration - , null as video_created_on - from {{ ref('stg__edxorg__s3__course_video') }} - -) - -, video_duration as ( - select - video_edx_uuid - , video_duration - , row_number() over ( - partition by video_edx_uuid - order by video_created_on desc - ) as row_num - from video -) - -, combined as ( - select - video_content.block_id as video_block_pk - , video_content.video_id - , video_content.content_block_pk as content_block_fk - , video_content.courserun_readable_id - , video_content.video_edx_uuid - , video_duration.video_duration - , video_content.video_name - , video_content.start_date - , video_content.due_date - , video_content.edx_video_id - , video_content.html5_sources - , video_content.transcripts - from video_content - left join video_duration - on - video_content.video_edx_uuid = video_duration.video_edx_uuid + ), + video_duration as ( + select + video_edx_uuid, + video_duration, + row_number() over (partition by video_edx_uuid order by video_created_on desc) as row_num + from video + ), + combined as ( + select + video_content.block_id as video_block_pk, + video_content.video_id, + video_content.content_block_pk as content_block_fk, + video_content.courserun_readable_id, + video_content.video_edx_uuid, + video_duration.video_duration, + video_content.video_name, + video_content.start_date, + video_content.due_date, + video_content.edx_video_id, + video_content.html5_sources, + video_content.transcripts + from video_content + left join + video_duration + on video_content.video_edx_uuid = video_duration.video_edx_uuid and video_duration.row_num = 1 - where video_content.row_num = 1 -) + where video_content.row_num = 1 + ) select - video_block_pk - , content_block_fk - , courserun_readable_id - , video_edx_uuid - , video_name - , start_date - , due_date - , edx_video_id - , html5_sources - , transcripts - , video_duration + video_block_pk, + content_block_fk, + courserun_readable_id, + video_edx_uuid, + video_name, + start_date, + due_date, + edx_video_id, + html5_sources, + transcripts, + video_duration from combined diff --git a/src/ol_dbt/models/dimensional/tfact_course_navigation_events.sql b/src/ol_dbt/models/dimensional/tfact_course_navigation_events.sql index 9f49a09da..26d8d3eb1 100644 --- a/src/ol_dbt/models/dimensional/tfact_course_navigation_events.sql +++ b/src/ol_dbt/models/dimensional/tfact_course_navigation_events.sql @@ -1,247 +1,224 @@ -{% set navigation_events = - ( - 'edx.ui.lms.jump_nav.selected' - , 'edx.ui.lms.link_clicked' - , 'edx.ui.lms.outline.selected' - , 'edx.ui.lms.sequence.next_selected' - , 'edx.ui.lms.sequence.previous_selected' - , 'edx.ui.lms.sequence.tab_selected' - ) -%} +{% set navigation_events = ( + "edx.ui.lms.jump_nav.selected", + "edx.ui.lms.link_clicked", + "edx.ui.lms.outline.selected", + "edx.ui.lms.sequence.next_selected", + "edx.ui.lms.sequence.previous_selected", + "edx.ui.lms.sequence.tab_selected", +) %} -with mitxonline_navigation_events as ( - select - user_username - , openedx_user_id - , courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_event_object, 'lax $.old' omit quotes) as starting_tab - , json_query(useractivity_event_object, 'lax $.new' omit quotes) as ending_tab - , json_query(useractivity_event_object, 'lax $.current_url' omit quotes) as starting_url - , json_query(useractivity_event_object, 'lax $.target_url' omit quotes) as ending_url - , case - when json_query(useractivity_event_object, 'lax $.id' omit quotes) not like 'block-v1%' +with + mitxonline_navigation_events as ( + select + user_username, + openedx_user_id, + courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_event_object, 'lax $.old' omit quotes) as starting_tab, + json_query(useractivity_event_object, 'lax $.new' omit quotes) as ending_tab, + json_query(useractivity_event_object, 'lax $.current_url' omit quotes) as starting_url, + json_query(useractivity_event_object, 'lax $.target_url' omit quotes) as ending_url, + case + when json_query(useractivity_event_object, 'lax $.id' omit quotes) not like 'block-v1%' then regexp_extract(json_query(useractivity_event_object, 'lax $.id' omit quotes), 'block-v1.*') - else json_query(useractivity_event_object, 'lax $.id' omit quotes) - end as block_id - , json_query(useractivity_event_object, 'lax $.current_tab' omit quotes) as current_tab - , json_query(useractivity_event_object, 'lax $.tab_count' omit quotes) as tab_count - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type in {{ navigation_events }} -) - -, xpro_navigation_events as ( - select - user_username - , openedx_user_id - , courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_event_object, 'lax $.old' omit quotes) as starting_tab - , json_query(useractivity_event_object, 'lax $.new' omit quotes) as ending_tab - , json_query(useractivity_event_object, 'lax $.current_url' omit quotes) as starting_url - , json_query(useractivity_event_object, 'lax $.target_url' omit quotes) as ending_url - , case - when json_query(useractivity_event_object, 'lax $.id' omit quotes) not like 'block-v1%' + else json_query(useractivity_event_object, 'lax $.id' omit quotes) + end as block_id, + json_query(useractivity_event_object, 'lax $.current_tab' omit quotes) as current_tab, + json_query(useractivity_event_object, 'lax $.tab_count' omit quotes) as tab_count, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__mitxonline__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null and useractivity_event_type in {{ navigation_events }} + ), + xpro_navigation_events as ( + select + user_username, + openedx_user_id, + courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_event_object, 'lax $.old' omit quotes) as starting_tab, + json_query(useractivity_event_object, 'lax $.new' omit quotes) as ending_tab, + json_query(useractivity_event_object, 'lax $.current_url' omit quotes) as starting_url, + json_query(useractivity_event_object, 'lax $.target_url' omit quotes) as ending_url, + case + when json_query(useractivity_event_object, 'lax $.id' omit quotes) not like 'block-v1%' then regexp_extract(json_query(useractivity_event_object, 'lax $.id' omit quotes), 'block-v1.*') - else json_query(useractivity_event_object, 'lax $.id' omit quotes) - end as block_id - , json_query(useractivity_event_object, 'lax $.current_tab' omit quotes) as current_tab - , json_query(useractivity_event_object, 'lax $.tab_count' omit quotes) as tab_count - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__mitxpro__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type in {{ navigation_events }} -) - -, mitxresidential_navigation_events as ( - select - user_username - , user_id - , courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_event_object, 'lax $.old' omit quotes) as starting_tab - , json_query(useractivity_event_object, 'lax $.new' omit quotes) as ending_tab - , json_query(useractivity_event_object, 'lax $.current_url' omit quotes) as starting_url - , json_query(useractivity_event_object, 'lax $.target_url' omit quotes) as ending_url - , case - when json_query(useractivity_event_object, 'lax $.id' omit quotes) not like 'block-v1%' + else json_query(useractivity_event_object, 'lax $.id' omit quotes) + end as block_id, + json_query(useractivity_event_object, 'lax $.current_tab' omit quotes) as current_tab, + json_query(useractivity_event_object, 'lax $.tab_count' omit quotes) as tab_count, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__mitxpro__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null and useractivity_event_type in {{ navigation_events }} + ), + mitxresidential_navigation_events as ( + select + user_username, + user_id, + courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_event_object, 'lax $.old' omit quotes) as starting_tab, + json_query(useractivity_event_object, 'lax $.new' omit quotes) as ending_tab, + json_query(useractivity_event_object, 'lax $.current_url' omit quotes) as starting_url, + json_query(useractivity_event_object, 'lax $.target_url' omit quotes) as ending_url, + case + when json_query(useractivity_event_object, 'lax $.id' omit quotes) not like 'block-v1%' then regexp_extract(json_query(useractivity_event_object, 'lax $.id' omit quotes), 'block-v1.*') - else json_query(useractivity_event_object, 'lax $.id' omit quotes) - end as block_id - , json_query(useractivity_event_object, 'lax $.current_tab' omit quotes) as current_tab - , json_query(useractivity_event_object, 'lax $.tab_count' omit quotes) as tab_count - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__mitxresidential__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type in {{ navigation_events }} -) - -, edxorg_navigation_events as ( - select - user_username - , user_id - , {{ format_course_id('courserun_readable_id') }} as courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_event_object, 'lax $.old' omit quotes) as starting_tab - , json_query(useractivity_event_object, 'lax $.new' omit quotes) as ending_tab - , json_query(useractivity_event_object, 'lax $.current_url' omit quotes) as starting_url - , json_query(useractivity_event_object, 'lax $.target_url' omit quotes) as ending_url - , case - when json_query(useractivity_event_object, 'lax $.id' omit quotes) not like 'block-v1%' + else json_query(useractivity_event_object, 'lax $.id' omit quotes) + end as block_id, + json_query(useractivity_event_object, 'lax $.current_tab' omit quotes) as current_tab, + json_query(useractivity_event_object, 'lax $.tab_count' omit quotes) as tab_count, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__mitxresidential__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null and useractivity_event_type in {{ navigation_events }} + ), + edxorg_navigation_events as ( + select + user_username, + user_id, + {{ format_course_id("courserun_readable_id") }} as courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_event_object, 'lax $.old' omit quotes) as starting_tab, + json_query(useractivity_event_object, 'lax $.new' omit quotes) as ending_tab, + json_query(useractivity_event_object, 'lax $.current_url' omit quotes) as starting_url, + json_query(useractivity_event_object, 'lax $.target_url' omit quotes) as ending_url, + case + when json_query(useractivity_event_object, 'lax $.id' omit quotes) not like 'block-v1%' then regexp_extract(json_query(useractivity_event_object, 'lax $.id' omit quotes), 'block-v1.*') - else json_query(useractivity_event_object, 'lax $.id' omit quotes) - end as block_id - , json_query(useractivity_event_object, 'lax $.current_tab' omit quotes) as current_tab - , json_query(useractivity_event_object, 'lax $.tab_count' omit quotes) as tab_count - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__edxorg__s3__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type in {{ navigation_events }} -) - -, users as ( - select * from {{ ref('dim_user') }} -) - -, platform as ( - select * from {{ ref('dim_platform') }} -) - -, combined as ( - select - 'mitxonline' as platform - , users.user_pk as user_fk - , mitxonline_navigation_events.openedx_user_id - , mitxonline_navigation_events.user_username - , mitxonline_navigation_events.courserun_readable_id - , mitxonline_navigation_events.event_type - , mitxonline_navigation_events.event_json - , mitxonline_navigation_events.block_id - , coalesce( - mitxonline_navigation_events.current_tab - , mitxonline_navigation_events.starting_tab - , mitxonline_navigation_events.starting_url - ) as starting_position - , coalesce(mitxonline_navigation_events.ending_tab, mitxonline_navigation_events.ending_url) - as ending_position - , mitxonline_navigation_events.current_tab - , mitxonline_navigation_events.tab_count - , mitxonline_navigation_events.event_timestamp - from mitxonline_navigation_events - left join users - on - mitxonline_navigation_events.openedx_user_id = users.mitxonline_openedx_user_id + else json_query(useractivity_event_object, 'lax $.id' omit quotes) + end as block_id, + json_query(useractivity_event_object, 'lax $.current_tab' omit quotes) as current_tab, + json_query(useractivity_event_object, 'lax $.tab_count' omit quotes) as tab_count, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__edxorg__s3__tracking_logs__user_activity") }} + where courserun_readable_id is not null and useractivity_event_type in {{ navigation_events }} + ), + users as (select * from {{ ref("dim_user") }}), + platform as (select * from {{ ref("dim_platform") }}), + combined as ( + select + 'mitxonline' as platform, + users.user_pk as user_fk, + mitxonline_navigation_events.openedx_user_id, + mitxonline_navigation_events.user_username, + mitxonline_navigation_events.courserun_readable_id, + mitxonline_navigation_events.event_type, + mitxonline_navigation_events.event_json, + mitxonline_navigation_events.block_id, + coalesce( + mitxonline_navigation_events.current_tab, + mitxonline_navigation_events.starting_tab, + mitxonline_navigation_events.starting_url + ) as starting_position, + coalesce( + mitxonline_navigation_events.ending_tab, mitxonline_navigation_events.ending_url + ) as ending_position, + mitxonline_navigation_events.current_tab, + mitxonline_navigation_events.tab_count, + mitxonline_navigation_events.event_timestamp + from mitxonline_navigation_events + left join + users + on mitxonline_navigation_events.openedx_user_id = users.mitxonline_openedx_user_id and mitxonline_navigation_events.user_username = users.user_mitxonline_username - union all + union all - select - 'mitxpro' as platform - , users.user_pk as user_fk - , xpro_navigation_events.openedx_user_id - , xpro_navigation_events.user_username - , xpro_navigation_events.courserun_readable_id - , xpro_navigation_events.event_type - , xpro_navigation_events.event_json - , xpro_navigation_events.block_id - , coalesce( - xpro_navigation_events.current_tab - , xpro_navigation_events.starting_tab - , xpro_navigation_events.starting_url - ) as starting_position - , coalesce(xpro_navigation_events.ending_tab, xpro_navigation_events.ending_url) as ending_position - , xpro_navigation_events.current_tab - , xpro_navigation_events.tab_count - , xpro_navigation_events.event_timestamp - from xpro_navigation_events - left join users - on - xpro_navigation_events.openedx_user_id = users.mitxpro_openedx_user_id + select + 'mitxpro' as platform, + users.user_pk as user_fk, + xpro_navigation_events.openedx_user_id, + xpro_navigation_events.user_username, + xpro_navigation_events.courserun_readable_id, + xpro_navigation_events.event_type, + xpro_navigation_events.event_json, + xpro_navigation_events.block_id, + coalesce( + xpro_navigation_events.current_tab, + xpro_navigation_events.starting_tab, + xpro_navigation_events.starting_url + ) as starting_position, + coalesce(xpro_navigation_events.ending_tab, xpro_navigation_events.ending_url) as ending_position, + xpro_navigation_events.current_tab, + xpro_navigation_events.tab_count, + xpro_navigation_events.event_timestamp + from xpro_navigation_events + left join + users + on xpro_navigation_events.openedx_user_id = users.mitxpro_openedx_user_id and xpro_navigation_events.user_username = users.user_mitxpro_username - union all + union all - select - 'residential' as platform - , users.user_pk as user_fk - , mitxresidential_navigation_events.user_id as openedx_user_id - , mitxresidential_navigation_events.user_username - , mitxresidential_navigation_events.courserun_readable_id - , mitxresidential_navigation_events.event_type - , mitxresidential_navigation_events.event_json - , mitxresidential_navigation_events.block_id - , coalesce( - mitxresidential_navigation_events.current_tab - , mitxresidential_navigation_events.starting_tab - , mitxresidential_navigation_events.starting_url - ) as starting_position - , coalesce( - mitxresidential_navigation_events.ending_tab - , mitxresidential_navigation_events.ending_url - ) as ending_position - , mitxresidential_navigation_events.current_tab - , mitxresidential_navigation_events.tab_count - , mitxresidential_navigation_events.event_timestamp - from mitxresidential_navigation_events - left join users - on - mitxresidential_navigation_events.user_id = users.residential_openedx_user_id + select + 'residential' as platform, + users.user_pk as user_fk, + mitxresidential_navigation_events.user_id as openedx_user_id, + mitxresidential_navigation_events.user_username, + mitxresidential_navigation_events.courserun_readable_id, + mitxresidential_navigation_events.event_type, + mitxresidential_navigation_events.event_json, + mitxresidential_navigation_events.block_id, + coalesce( + mitxresidential_navigation_events.current_tab, + mitxresidential_navigation_events.starting_tab, + mitxresidential_navigation_events.starting_url + ) as starting_position, + coalesce( + mitxresidential_navigation_events.ending_tab, mitxresidential_navigation_events.ending_url + ) as ending_position, + mitxresidential_navigation_events.current_tab, + mitxresidential_navigation_events.tab_count, + mitxresidential_navigation_events.event_timestamp + from mitxresidential_navigation_events + left join + users + on mitxresidential_navigation_events.user_id = users.residential_openedx_user_id and mitxresidential_navigation_events.user_username = users.user_residential_username - union all + union all - select - 'edxorg' as platform - , users.user_pk as user_fk - , edxorg_navigation_events.user_id as openedx_user_id - , edxorg_navigation_events.user_username - , edxorg_navigation_events.courserun_readable_id - , edxorg_navigation_events.event_type - , edxorg_navigation_events.event_json - , edxorg_navigation_events.block_id - , coalesce( - edxorg_navigation_events.current_tab - , edxorg_navigation_events.starting_tab - , edxorg_navigation_events.starting_url - ) as starting_position - , coalesce( - edxorg_navigation_events.ending_tab - , edxorg_navigation_events.ending_url - ) as ending_position - , edxorg_navigation_events.current_tab - , edxorg_navigation_events.tab_count - , edxorg_navigation_events.event_timestamp - from edxorg_navigation_events - left join users - on - edxorg_navigation_events.user_id = users.edxorg_openedx_user_id + select + 'edxorg' as platform, + users.user_pk as user_fk, + edxorg_navigation_events.user_id as openedx_user_id, + edxorg_navigation_events.user_username, + edxorg_navigation_events.courserun_readable_id, + edxorg_navigation_events.event_type, + edxorg_navigation_events.event_json, + edxorg_navigation_events.block_id, + coalesce( + edxorg_navigation_events.current_tab, + edxorg_navigation_events.starting_tab, + edxorg_navigation_events.starting_url + ) as starting_position, + coalesce(edxorg_navigation_events.ending_tab, edxorg_navigation_events.ending_url) as ending_position, + edxorg_navigation_events.current_tab, + edxorg_navigation_events.tab_count, + edxorg_navigation_events.event_timestamp + from edxorg_navigation_events + left join + users + on edxorg_navigation_events.user_id = users.edxorg_openedx_user_id and edxorg_navigation_events.user_username = users.user_edxorg_username -) + ) select - platform.platform_pk as platform_fk - , combined.user_fk - , combined.platform - , combined.openedx_user_id - , combined.user_username - , combined.courserun_readable_id - , combined.event_type - , combined.block_id as block_fk - , combined.starting_position - , combined.ending_position - , combined.event_timestamp - , combined.event_json + platform.platform_pk as platform_fk, + combined.user_fk, + combined.platform, + combined.openedx_user_id, + combined.user_username, + combined.courserun_readable_id, + combined.event_type, + combined.block_id as block_fk, + combined.starting_position, + combined.ending_position, + combined.event_timestamp, + combined.event_json from combined -left join platform - on combined.platform = platform.platform_readable_id +left join platform on combined.platform = platform.platform_readable_id diff --git a/src/ol_dbt/models/dimensional/tfact_discussion_events.sql b/src/ol_dbt/models/dimensional/tfact_discussion_events.sql index be516973a..55effebeb 100644 --- a/src/ol_dbt/models/dimensional/tfact_discussion_events.sql +++ b/src/ol_dbt/models/dimensional/tfact_discussion_events.sql @@ -1,215 +1,201 @@ -{% set discussion_events = 'edx.forum.%' %} +{% set discussion_events = "edx.forum.%" %} -with mitxonline_discussion_events as ( - select - user_username - , openedx_user_id - , courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as post_id - , json_query(useractivity_event_object, 'lax $.title' omit quotes) as post_title - , json_query(useractivity_event_object, 'lax $.body' omit quotes) as post_content - , json_query(useractivity_event_object, 'lax $.commentable_id' omit quotes) as commentable_id - , json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as discussion_component_id - , json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as discussion_component_name - , json_query(useractivity_event_object, 'lax $.url' omit quotes) as page_url - , json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as user_forums_roles - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type like '{{ discussion_events }}' -) +with + mitxonline_discussion_events as ( + select + user_username, + openedx_user_id, + courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as post_id, + json_query(useractivity_event_object, 'lax $.title' omit quotes) as post_title, + json_query(useractivity_event_object, 'lax $.body' omit quotes) as post_content, + json_query(useractivity_event_object, 'lax $.commentable_id' omit quotes) as commentable_id, + json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as discussion_component_id, + json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as discussion_component_name, + json_query(useractivity_event_object, 'lax $.url' omit quotes) as page_url, + json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as user_forums_roles, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__mitxonline__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null and useractivity_event_type like '{{ discussion_events }}' + ), + xpro_discussion_events as ( -, xpro_discussion_events as ( + select + user_username, + openedx_user_id, + courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as post_id, + json_query(useractivity_event_object, 'lax $.title' omit quotes) as post_title, + json_query(useractivity_event_object, 'lax $.body' omit quotes) as post_content, + json_query(useractivity_event_object, 'lax $.commentable_id' omit quotes) as commentable_id, + json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as discussion_component_id, + json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as discussion_component_name, + json_query(useractivity_event_object, 'lax $.url' omit quotes) as page_url, + json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as user_forums_roles, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__mitxpro__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null and useractivity_event_type like '{{ discussion_events }}' + ), + mitxresidential_discussion_events as ( - select - user_username - , openedx_user_id - , courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as post_id - , json_query(useractivity_event_object, 'lax $.title' omit quotes) as post_title - , json_query(useractivity_event_object, 'lax $.body' omit quotes) as post_content - , json_query(useractivity_event_object, 'lax $.commentable_id' omit quotes) as commentable_id - , json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as discussion_component_id - , json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as discussion_component_name - , json_query(useractivity_event_object, 'lax $.url' omit quotes) as page_url - , json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as user_forums_roles - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__mitxpro__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type like '{{ discussion_events }}' -) - -, mitxresidential_discussion_events as ( - - select - user_username - , user_id as openedx_user_id - , courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as post_id - , json_query(useractivity_event_object, 'lax $.title' omit quotes) as post_title - , json_query(useractivity_event_object, 'lax $.body' omit quotes) as post_content - , json_query(useractivity_event_object, 'lax $.commentable_id' omit quotes) as commentable_id - , json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as discussion_component_id - , json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as discussion_component_name - , json_query(useractivity_event_object, 'lax $.url' omit quotes) as page_url - , json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as user_forums_roles - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__mitxresidential__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type like '{{ discussion_events }}' -) - -, edxorg_discussion_events as ( - select - user_username - , user_id as openedx_user_id - , {{ format_course_id('courserun_readable_id') }} as courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as post_id - , json_query(useractivity_event_object, 'lax $.title' omit quotes) as post_title - , json_query(useractivity_event_object, 'lax $.body' omit quotes) as post_content - , json_query(useractivity_event_object, 'lax $.commentable_id' omit quotes) as commentable_id - , json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as discussion_component_id - , json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as discussion_component_name - , json_query(useractivity_event_object, 'lax $.url' omit quotes) as page_url - , json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as user_forums_roles - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__edxorg__s3__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type like '{{ discussion_events }}' -) - -, users as ( - select * from {{ ref('dim_user') }} -) - -, combined as ( - select - 'mitxonline' as platform - , mitxonline_discussion_events.openedx_user_id - , mitxonline_discussion_events.user_username - , users.user_pk as user_fk - , mitxonline_discussion_events.courserun_readable_id - , mitxonline_discussion_events.event_type - , mitxonline_discussion_events.event_json - , mitxonline_discussion_events.post_id - , mitxonline_discussion_events.post_title - , mitxonline_discussion_events.post_content - , mitxonline_discussion_events.commentable_id - , mitxonline_discussion_events.discussion_component_id - , mitxonline_discussion_events.discussion_component_name - , mitxonline_discussion_events.page_url - , mitxonline_discussion_events.user_forums_roles - , mitxonline_discussion_events.event_timestamp - from mitxonline_discussion_events - left join users - on - mitxonline_discussion_events.openedx_user_id = users.mitxonline_openedx_user_id + select + user_username, + user_id as openedx_user_id, + courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as post_id, + json_query(useractivity_event_object, 'lax $.title' omit quotes) as post_title, + json_query(useractivity_event_object, 'lax $.body' omit quotes) as post_content, + json_query(useractivity_event_object, 'lax $.commentable_id' omit quotes) as commentable_id, + json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as discussion_component_id, + json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as discussion_component_name, + json_query(useractivity_event_object, 'lax $.url' omit quotes) as page_url, + json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as user_forums_roles, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__mitxresidential__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null and useractivity_event_type like '{{ discussion_events }}' + ), + edxorg_discussion_events as ( + select + user_username, + user_id as openedx_user_id, + {{ format_course_id("courserun_readable_id") }} as courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as post_id, + json_query(useractivity_event_object, 'lax $.title' omit quotes) as post_title, + json_query(useractivity_event_object, 'lax $.body' omit quotes) as post_content, + json_query(useractivity_event_object, 'lax $.commentable_id' omit quotes) as commentable_id, + json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as discussion_component_id, + json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as discussion_component_name, + json_query(useractivity_event_object, 'lax $.url' omit quotes) as page_url, + json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as user_forums_roles, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__edxorg__s3__tracking_logs__user_activity") }} + where courserun_readable_id is not null and useractivity_event_type like '{{ discussion_events }}' + ), + users as (select * from {{ ref("dim_user") }}), + combined as ( + select + 'mitxonline' as platform, + mitxonline_discussion_events.openedx_user_id, + mitxonline_discussion_events.user_username, + users.user_pk as user_fk, + mitxonline_discussion_events.courserun_readable_id, + mitxonline_discussion_events.event_type, + mitxonline_discussion_events.event_json, + mitxonline_discussion_events.post_id, + mitxonline_discussion_events.post_title, + mitxonline_discussion_events.post_content, + mitxonline_discussion_events.commentable_id, + mitxonline_discussion_events.discussion_component_id, + mitxonline_discussion_events.discussion_component_name, + mitxonline_discussion_events.page_url, + mitxonline_discussion_events.user_forums_roles, + mitxonline_discussion_events.event_timestamp + from mitxonline_discussion_events + left join + users + on mitxonline_discussion_events.openedx_user_id = users.mitxonline_openedx_user_id and mitxonline_discussion_events.user_username = users.user_mitxonline_username - union all + union all - select - 'mitxpro' as platform - , xpro_discussion_events.openedx_user_id - , xpro_discussion_events.user_username - , users.user_pk as user_fk - , xpro_discussion_events.courserun_readable_id - , xpro_discussion_events.event_type - , xpro_discussion_events.event_json - , xpro_discussion_events.post_id - , xpro_discussion_events.post_title - , xpro_discussion_events.post_content - , xpro_discussion_events.commentable_id - , xpro_discussion_events.discussion_component_id - , xpro_discussion_events.discussion_component_name - , xpro_discussion_events.page_url - , xpro_discussion_events.user_forums_roles - , xpro_discussion_events.event_timestamp - from xpro_discussion_events - left join users - on - xpro_discussion_events.openedx_user_id = users.mitxonline_openedx_user_id + select + 'mitxpro' as platform, + xpro_discussion_events.openedx_user_id, + xpro_discussion_events.user_username, + users.user_pk as user_fk, + xpro_discussion_events.courserun_readable_id, + xpro_discussion_events.event_type, + xpro_discussion_events.event_json, + xpro_discussion_events.post_id, + xpro_discussion_events.post_title, + xpro_discussion_events.post_content, + xpro_discussion_events.commentable_id, + xpro_discussion_events.discussion_component_id, + xpro_discussion_events.discussion_component_name, + xpro_discussion_events.page_url, + xpro_discussion_events.user_forums_roles, + xpro_discussion_events.event_timestamp + from xpro_discussion_events + left join + users + on xpro_discussion_events.openedx_user_id = users.mitxonline_openedx_user_id and xpro_discussion_events.user_username = users.user_mitxonline_username - union all + union all - select - 'residential' as platform - , mitxresidential_discussion_events.openedx_user_id - , mitxresidential_discussion_events.user_username - , users.user_pk as user_fk - , mitxresidential_discussion_events.courserun_readable_id - , mitxresidential_discussion_events.event_type - , mitxresidential_discussion_events.event_json - , mitxresidential_discussion_events.post_id - , mitxresidential_discussion_events.post_title - , mitxresidential_discussion_events.post_content - , mitxresidential_discussion_events.commentable_id - , mitxresidential_discussion_events.discussion_component_id - , mitxresidential_discussion_events.discussion_component_name - , mitxresidential_discussion_events.page_url - , mitxresidential_discussion_events.user_forums_roles - , mitxresidential_discussion_events.event_timestamp - from mitxresidential_discussion_events - left join users - on - mitxresidential_discussion_events.openedx_user_id = users.residential_openedx_user_id + select + 'residential' as platform, + mitxresidential_discussion_events.openedx_user_id, + mitxresidential_discussion_events.user_username, + users.user_pk as user_fk, + mitxresidential_discussion_events.courserun_readable_id, + mitxresidential_discussion_events.event_type, + mitxresidential_discussion_events.event_json, + mitxresidential_discussion_events.post_id, + mitxresidential_discussion_events.post_title, + mitxresidential_discussion_events.post_content, + mitxresidential_discussion_events.commentable_id, + mitxresidential_discussion_events.discussion_component_id, + mitxresidential_discussion_events.discussion_component_name, + mitxresidential_discussion_events.page_url, + mitxresidential_discussion_events.user_forums_roles, + mitxresidential_discussion_events.event_timestamp + from mitxresidential_discussion_events + left join + users + on mitxresidential_discussion_events.openedx_user_id = users.residential_openedx_user_id and mitxresidential_discussion_events.user_username = users.user_residential_username - union all + union all - select - 'edxorg' as platform - , edxorg_discussion_events.openedx_user_id - , edxorg_discussion_events.user_username - , users.user_pk as user_fk - , edxorg_discussion_events.courserun_readable_id - , edxorg_discussion_events.event_type - , edxorg_discussion_events.event_json - , edxorg_discussion_events.post_id - , edxorg_discussion_events.post_title - , edxorg_discussion_events.post_content - , edxorg_discussion_events.commentable_id - , edxorg_discussion_events.discussion_component_id - , edxorg_discussion_events.discussion_component_name - , edxorg_discussion_events.page_url - , edxorg_discussion_events.user_forums_roles - , edxorg_discussion_events.event_timestamp - from edxorg_discussion_events - left join users - on - edxorg_discussion_events.openedx_user_id = users.edxorg_openedx_user_id + select + 'edxorg' as platform, + edxorg_discussion_events.openedx_user_id, + edxorg_discussion_events.user_username, + users.user_pk as user_fk, + edxorg_discussion_events.courserun_readable_id, + edxorg_discussion_events.event_type, + edxorg_discussion_events.event_json, + edxorg_discussion_events.post_id, + edxorg_discussion_events.post_title, + edxorg_discussion_events.post_content, + edxorg_discussion_events.commentable_id, + edxorg_discussion_events.discussion_component_id, + edxorg_discussion_events.discussion_component_name, + edxorg_discussion_events.page_url, + edxorg_discussion_events.user_forums_roles, + edxorg_discussion_events.event_timestamp + from edxorg_discussion_events + left join + users + on edxorg_discussion_events.openedx_user_id = users.edxorg_openedx_user_id and edxorg_discussion_events.user_username = users.user_edxorg_username -) + ) select - {{ dbt_utils.generate_surrogate_key(['platform']) }} as platform_fk - , platform - , user_fk - , openedx_user_id - , user_username - , courserun_readable_id - , event_type - , event_json - , post_id - , post_title - , post_content - , commentable_id - , discussion_component_id - , discussion_component_name - , page_url - , user_forums_roles - , event_timestamp + {{ dbt_utils.generate_surrogate_key(["platform"]) }} as platform_fk, + platform, + user_fk, + openedx_user_id, + user_username, + courserun_readable_id, + event_type, + event_json, + post_id, + post_title, + post_content, + commentable_id, + discussion_component_id, + discussion_component_name, + page_url, + user_forums_roles, + event_timestamp from combined diff --git a/src/ol_dbt/models/dimensional/tfact_problem_events.sql b/src/ol_dbt/models/dimensional/tfact_problem_events.sql index cf14f2a48..c4d5b3628 100644 --- a/src/ol_dbt/models/dimensional/tfact_problem_events.sql +++ b/src/ol_dbt/models/dimensional/tfact_problem_events.sql @@ -1,253 +1,236 @@ -{% set problem_events = - ( - 'problem_check' - , 'showanswer' - ) -%} +{% set problem_events = ("problem_check", "showanswer") %} -- data from tracking logs -with mitxonline_problem_events as ( - select - user_username - , openedx_user_id - , courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id - , json_query(useractivity_event_object, 'lax $.answers' omit quotes) as answers - , json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as attempt - , json_query(useractivity_event_object, 'lax $.success' omit quotes) as success - , json_query(useractivity_event_object, 'lax $.grade' omit quotes) as grade - , json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as max_grade - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type in {{ problem_events }} - and useractivity_event_source = 'server' -) - -, xpro_problem_events as ( - select - user_username - , openedx_user_id - , courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id - , json_query(useractivity_event_object, 'lax $.answers' omit quotes) as answers - , json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as attempt - , json_query(useractivity_event_object, 'lax $.success' omit quotes) as success - , json_query(useractivity_event_object, 'lax $.grade' omit quotes) as grade - , json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as max_grade - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__mitxpro__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type in {{ problem_events }} - and useractivity_event_source = 'server' -) - -, mitxresidential_problem_events as ( - select - user_username - , user_id - , courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id - , json_query(useractivity_event_object, 'lax $.answers' omit quotes) as answers - , json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as attempt - , json_query(useractivity_event_object, 'lax $.success' omit quotes) as success - , json_query(useractivity_event_object, 'lax $.grade' omit quotes) as grade - , json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as max_grade - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__mitxresidential__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type in {{ problem_events }} - and useractivity_event_source = 'server' -) - -, edxorg_problem_events as ( - select - user_username - , user_id - , {{ format_course_id('courserun_readable_id') }} as courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id - , json_query(useractivity_event_object, 'lax $.answers' omit quotes) as answers - , json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as attempt - , json_query(useractivity_event_object, 'lax $.success' omit quotes) as success - , json_query(useractivity_event_object, 'lax $.grade' omit quotes) as grade - , json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as max_grade - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__edxorg__s3__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type in {{ problem_events }} - and useractivity_event_source = 'server' -) - -, users as ( - select * from {{ ref('dim_user') }} -) - -, platform as ( - select * from {{ ref('dim_platform') }} -) - --- data from studentmodule and studentmodulehistoryextended -, combined_studentmodule as ( - select * from {{ ref('tfact_studentmodule_problems') }} -) - -, combined as ( - select - 'mitxonline' as platform - , users.user_pk as user_fk - , mitxonline_problem_events.openedx_user_id - , mitxonline_problem_events.user_username - , mitxonline_problem_events.courserun_readable_id - , mitxonline_problem_events.event_type - , mitxonline_problem_events.event_json - , mitxonline_problem_events.problem_block_id - , mitxonline_problem_events.answers - , mitxonline_problem_events.attempt - , mitxonline_problem_events.success - , mitxonline_problem_events.grade - , mitxonline_problem_events.max_grade - , mitxonline_problem_events.event_timestamp - from mitxonline_problem_events - left join users - on - mitxonline_problem_events.openedx_user_id = users.mitxonline_openedx_user_id +with + mitxonline_problem_events as ( + select + user_username, + openedx_user_id, + courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id, + json_query(useractivity_event_object, 'lax $.answers' omit quotes) as answers, + json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as attempt, + json_query(useractivity_event_object, 'lax $.success' omit quotes) as success, + json_query(useractivity_event_object, 'lax $.grade' omit quotes) as grade, + json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as max_grade, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__mitxonline__openedx__tracking_logs__user_activity") }} + where + courserun_readable_id is not null + and useractivity_event_type in {{ problem_events }} + and useractivity_event_source = 'server' + ), + xpro_problem_events as ( + select + user_username, + openedx_user_id, + courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id, + json_query(useractivity_event_object, 'lax $.answers' omit quotes) as answers, + json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as attempt, + json_query(useractivity_event_object, 'lax $.success' omit quotes) as success, + json_query(useractivity_event_object, 'lax $.grade' omit quotes) as grade, + json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as max_grade, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__mitxpro__openedx__tracking_logs__user_activity") }} + where + courserun_readable_id is not null + and useractivity_event_type in {{ problem_events }} + and useractivity_event_source = 'server' + ), + mitxresidential_problem_events as ( + select + user_username, + user_id, + courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id, + json_query(useractivity_event_object, 'lax $.answers' omit quotes) as answers, + json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as attempt, + json_query(useractivity_event_object, 'lax $.success' omit quotes) as success, + json_query(useractivity_event_object, 'lax $.grade' omit quotes) as grade, + json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as max_grade, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__mitxresidential__openedx__tracking_logs__user_activity") }} + where + courserun_readable_id is not null + and useractivity_event_type in {{ problem_events }} + and useractivity_event_source = 'server' + ), + edxorg_problem_events as ( + select + user_username, + user_id, + {{ format_course_id("courserun_readable_id") }} as courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as problem_name, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as problem_block_id, + json_query(useractivity_event_object, 'lax $.answers' omit quotes) as answers, + json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as attempt, + json_query(useractivity_event_object, 'lax $.success' omit quotes) as success, + json_query(useractivity_event_object, 'lax $.grade' omit quotes) as grade, + json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as max_grade, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__edxorg__s3__tracking_logs__user_activity") }} + where + courserun_readable_id is not null + and useractivity_event_type in {{ problem_events }} + and useractivity_event_source = 'server' + ), + users as (select * from {{ ref("dim_user") }}), + platform as (select * from {{ ref("dim_platform") }}), + -- data from studentmodule and studentmodulehistoryextended + combined_studentmodule as (select * from {{ ref("tfact_studentmodule_problems") }}), + combined as ( + select + 'mitxonline' as platform, + users.user_pk as user_fk, + mitxonline_problem_events.openedx_user_id, + mitxonline_problem_events.user_username, + mitxonline_problem_events.courserun_readable_id, + mitxonline_problem_events.event_type, + mitxonline_problem_events.event_json, + mitxonline_problem_events.problem_block_id, + mitxonline_problem_events.answers, + mitxonline_problem_events.attempt, + mitxonline_problem_events.success, + mitxonline_problem_events.grade, + mitxonline_problem_events.max_grade, + mitxonline_problem_events.event_timestamp + from mitxonline_problem_events + left join + users + on mitxonline_problem_events.openedx_user_id = users.mitxonline_openedx_user_id and mitxonline_problem_events.user_username = users.user_mitxonline_username - union all + union all - select - 'mitxpro' as platform - , users.user_pk as user_fk - , xpro_problem_events.openedx_user_id - , xpro_problem_events.user_username - , xpro_problem_events.courserun_readable_id - , xpro_problem_events.event_type - , xpro_problem_events.event_json - , xpro_problem_events.problem_block_id - , xpro_problem_events.answers - , xpro_problem_events.attempt - , xpro_problem_events.success - , xpro_problem_events.grade - , xpro_problem_events.max_grade - , xpro_problem_events.event_timestamp - from xpro_problem_events - left join users - on - xpro_problem_events.openedx_user_id = users.mitxpro_openedx_user_id + select + 'mitxpro' as platform, + users.user_pk as user_fk, + xpro_problem_events.openedx_user_id, + xpro_problem_events.user_username, + xpro_problem_events.courserun_readable_id, + xpro_problem_events.event_type, + xpro_problem_events.event_json, + xpro_problem_events.problem_block_id, + xpro_problem_events.answers, + xpro_problem_events.attempt, + xpro_problem_events.success, + xpro_problem_events.grade, + xpro_problem_events.max_grade, + xpro_problem_events.event_timestamp + from xpro_problem_events + left join + users + on xpro_problem_events.openedx_user_id = users.mitxpro_openedx_user_id and xpro_problem_events.user_username = users.user_mitxpro_username - union all + union all - select - 'residential' as platform - , users.user_pk as user_fk - , mitxresidential_problem_events.user_id - , mitxresidential_problem_events.user_username - , mitxresidential_problem_events.courserun_readable_id - , mitxresidential_problem_events.event_type - , mitxresidential_problem_events.event_json - , mitxresidential_problem_events.problem_block_id - , mitxresidential_problem_events.answers - , mitxresidential_problem_events.attempt - , mitxresidential_problem_events.success - , mitxresidential_problem_events.grade - , mitxresidential_problem_events.max_grade - , mitxresidential_problem_events.event_timestamp - from mitxresidential_problem_events - left join users - on - mitxresidential_problem_events.user_id = users.residential_openedx_user_id + select + 'residential' as platform, + users.user_pk as user_fk, + mitxresidential_problem_events.user_id, + mitxresidential_problem_events.user_username, + mitxresidential_problem_events.courserun_readable_id, + mitxresidential_problem_events.event_type, + mitxresidential_problem_events.event_json, + mitxresidential_problem_events.problem_block_id, + mitxresidential_problem_events.answers, + mitxresidential_problem_events.attempt, + mitxresidential_problem_events.success, + mitxresidential_problem_events.grade, + mitxresidential_problem_events.max_grade, + mitxresidential_problem_events.event_timestamp + from mitxresidential_problem_events + left join + users + on mitxresidential_problem_events.user_id = users.residential_openedx_user_id and mitxresidential_problem_events.user_username = users.user_residential_username - union all + union all - select - 'edxorg' as platform - , users.user_pk as user_fk - , edxorg_problem_events.user_id - , edxorg_problem_events.user_username - , edxorg_problem_events.courserun_readable_id - , edxorg_problem_events.event_type - , edxorg_problem_events.event_json - , edxorg_problem_events.problem_block_id - , edxorg_problem_events.answers - , edxorg_problem_events.attempt - , edxorg_problem_events.success - , edxorg_problem_events.grade - , edxorg_problem_events.max_grade - , edxorg_problem_events.event_timestamp - from edxorg_problem_events - left join users - on - edxorg_problem_events.user_id = users.edxorg_openedx_user_id + select + 'edxorg' as platform, + users.user_pk as user_fk, + edxorg_problem_events.user_id, + edxorg_problem_events.user_username, + edxorg_problem_events.courserun_readable_id, + edxorg_problem_events.event_type, + edxorg_problem_events.event_json, + edxorg_problem_events.problem_block_id, + edxorg_problem_events.answers, + edxorg_problem_events.attempt, + edxorg_problem_events.success, + edxorg_problem_events.grade, + edxorg_problem_events.max_grade, + edxorg_problem_events.event_timestamp + from edxorg_problem_events + left join + users + on edxorg_problem_events.user_id = users.edxorg_openedx_user_id and edxorg_problem_events.user_username = users.user_edxorg_username - union all - - select - platform - , user_fk - , openedx_user_id - , user_username - , courserun_readable_id - , event_type - , event_json - , problem_block_id - , answers - , attempt - , success - , grade - , max_grade - , event_timestamp - from combined_studentmodule -) + union all --- dedupe the tracking log and student module data based on user_id, course_run, problem, and time --- The dbt model definition has a test against the same composite unique key. -, deduped_combined as ( - select * - from ( select - * - , row_number() over ( - partition by platform, openedx_user_id, courserun_readable_id, problem_block_id, attempt - order by event_timestamp - ) as rn - from combined + platform, + user_fk, + openedx_user_id, + user_username, + courserun_readable_id, + event_type, + event_json, + problem_block_id, + answers, + attempt, + success, + grade, + max_grade, + event_timestamp + from combined_studentmodule + ), + -- dedupe the tracking log and student module data based on user_id, course_run, problem, and time + -- The dbt model definition has a test against the same composite unique key. + deduped_combined as ( + select * + from + ( + select + *, + row_number() over ( + partition by platform, openedx_user_id, courserun_readable_id, problem_block_id, attempt + order by event_timestamp + ) as rn + from combined + ) + where rn = 1 or event_type != 'problem_check' ) - where rn = 1 or event_type != 'problem_check' -) select - platform.platform_pk as platform_fk - , deduped_combined.user_fk - , deduped_combined.platform - , deduped_combined.openedx_user_id - , deduped_combined.user_username - , deduped_combined.courserun_readable_id - , deduped_combined.event_type - , deduped_combined.problem_block_id as problem_block_fk - , deduped_combined.answers - , deduped_combined.attempt - , deduped_combined.success - , deduped_combined.grade - , deduped_combined.max_grade - , deduped_combined.event_timestamp - , deduped_combined.event_json + platform.platform_pk as platform_fk, + deduped_combined.user_fk, + deduped_combined.platform, + deduped_combined.openedx_user_id, + deduped_combined.user_username, + deduped_combined.courserun_readable_id, + deduped_combined.event_type, + deduped_combined.problem_block_id as problem_block_fk, + deduped_combined.answers, + deduped_combined.attempt, + deduped_combined.success, + deduped_combined.grade, + deduped_combined.max_grade, + deduped_combined.event_timestamp, + deduped_combined.event_json from deduped_combined left join platform on deduped_combined.platform = platform.platform_readable_id diff --git a/src/ol_dbt/models/dimensional/tfact_studentmodule_problems.sql b/src/ol_dbt/models/dimensional/tfact_studentmodule_problems.sql index 7e8996921..10496bb8e 100644 --- a/src/ol_dbt/models/dimensional/tfact_studentmodule_problems.sql +++ b/src/ol_dbt/models/dimensional/tfact_studentmodule_problems.sql @@ -1,139 +1,137 @@ -with mitxonline_studentmodule_problems as ( - {{ generate_studentmodule_problem_events( - ref('stg__mitxonline__openedx__mysql__courseware_studentmodule'), - ref('stg__mitxonline__openedx__courseware_studentmodulehistoryextended'), - 'openedx_user_id' - ) }} -) - -, mitxpro_studentmodule_problems as ( - {{ generate_studentmodule_problem_events( - ref('stg__mitxpro__openedx__mysql__courseware_studentmodule'), - ref('stg__mitxpro__openedx__courseware_studentmodulehistoryextended'), - 'openedx_user_id' - ) }} -) - -, mitxresidential_studentmodule_problems as ( - {{ generate_studentmodule_problem_events( - ref('stg__mitxresidential__openedx__courseware_studentmodule'), - ref('stg__mitxresidential__openedx__courseware_studentmodulehistoryextended'), - 'user_id' - ) }} -) - -, users as ( - select * from {{ ref('dim_user') }} -) - -, mitxonline_studentmodule_combined as ( - select - users.user_pk as user_fk - , 'mitxonline' as platform - , sm.user_id as openedx_user_id - , users.user_mitxonline_username as user_username - , sm.courserun_readable_id - , sm.studentmodule_id - , 'problem_check' as event_type - , sm.studentmodule_state_data as event_json - , sm.coursestructure_block_id as problem_block_id - , sm.answers - , sm.attempt - , sm.event_timestamp - , sm.grade - , sm.max_grade - , sm.success - from mitxonline_studentmodule_problems as sm - left join users on sm.user_id = users.mitxonline_openedx_user_id -) - -, mitxpro_studentmodule_combined as ( - select - users.user_pk as user_fk - , 'mitxpro' as platform - , sm.user_id as openedx_user_id - , users.user_mitxpro_username as user_username - , sm.courserun_readable_id - , sm.studentmodule_id - , 'problem_check' as event_type - , sm.studentmodule_state_data as event_json - , sm.coursestructure_block_id as problem_block_id - , sm.answers - , sm.attempt - , sm.event_timestamp - , sm.grade - , sm.max_grade - , sm.success - from mitxpro_studentmodule_problems as sm - left join users on sm.user_id = users.mitxpro_openedx_user_id -) - -, residential_studentmodule_combined as ( - select - users.user_pk as user_fk - , 'residential' as platform - , sm.user_id as openedx_user_id - , users.user_residential_username as user_username - , sm.courserun_readable_id - , sm.studentmodule_id - , 'problem_check' as event_type - , sm.studentmodule_state_data as event_json - , sm.coursestructure_block_id as problem_block_id - , sm.answers - , sm.attempt - , sm.event_timestamp - , sm.grade - , sm.max_grade - , sm.success - from mitxresidential_studentmodule_problems as sm - left join users on sm.user_id = users.residential_openedx_user_id -) - -, combined as ( - select * - from mitxonline_studentmodule_combined - - union all +with + mitxonline_studentmodule_problems as ( + {{ + generate_studentmodule_problem_events( + ref("stg__mitxonline__openedx__mysql__courseware_studentmodule"), + ref("stg__mitxonline__openedx__courseware_studentmodulehistoryextended"), + "openedx_user_id", + ) + }} + ), + mitxpro_studentmodule_problems as ( + {{ + generate_studentmodule_problem_events( + ref("stg__mitxpro__openedx__mysql__courseware_studentmodule"), + ref("stg__mitxpro__openedx__courseware_studentmodulehistoryextended"), + "openedx_user_id", + ) + }} + ), + mitxresidential_studentmodule_problems as ( + {{ + generate_studentmodule_problem_events( + ref("stg__mitxresidential__openedx__courseware_studentmodule"), + ref("stg__mitxresidential__openedx__courseware_studentmodulehistoryextended"), + "user_id", + ) + }} + ), + users as (select * from {{ ref("dim_user") }}), + mitxonline_studentmodule_combined as ( + select + users.user_pk as user_fk, + 'mitxonline' as platform, + sm.user_id as openedx_user_id, + users.user_mitxonline_username as user_username, + sm.courserun_readable_id, + sm.studentmodule_id, + 'problem_check' as event_type, + sm.studentmodule_state_data as event_json, + sm.coursestructure_block_id as problem_block_id, + sm.answers, + sm.attempt, + sm.event_timestamp, + sm.grade, + sm.max_grade, + sm.success + from mitxonline_studentmodule_problems as sm + left join users on sm.user_id = users.mitxonline_openedx_user_id + ), + mitxpro_studentmodule_combined as ( + select + users.user_pk as user_fk, + 'mitxpro' as platform, + sm.user_id as openedx_user_id, + users.user_mitxpro_username as user_username, + sm.courserun_readable_id, + sm.studentmodule_id, + 'problem_check' as event_type, + sm.studentmodule_state_data as event_json, + sm.coursestructure_block_id as problem_block_id, + sm.answers, + sm.attempt, + sm.event_timestamp, + sm.grade, + sm.max_grade, + sm.success + from mitxpro_studentmodule_problems as sm + left join users on sm.user_id = users.mitxpro_openedx_user_id + ), + residential_studentmodule_combined as ( + select + users.user_pk as user_fk, + 'residential' as platform, + sm.user_id as openedx_user_id, + users.user_residential_username as user_username, + sm.courserun_readable_id, + sm.studentmodule_id, + 'problem_check' as event_type, + sm.studentmodule_state_data as event_json, + sm.coursestructure_block_id as problem_block_id, + sm.answers, + sm.attempt, + sm.event_timestamp, + sm.grade, + sm.max_grade, + sm.success + from mitxresidential_studentmodule_problems as sm + left join users on sm.user_id = users.residential_openedx_user_id + ), + combined as ( + select * + from mitxonline_studentmodule_combined - select * - from mitxpro_studentmodule_combined + union all - union all + select * + from mitxpro_studentmodule_combined - select * - from residential_studentmodule_combined -) + union all --- dedupe the tracking log and student module data based on user_id, course_run, problem, and time --- The dbt model definition has a test against the same composite unique key. -, deduped_combined as ( - select * - from ( - select - * - , row_number() over ( - partition by platform, openedx_user_id, courserun_readable_id, problem_block_id, attempt - order by event_timestamp - ) as rn - from combined + select * + from residential_studentmodule_combined + ), + -- dedupe the tracking log and student module data based on user_id, course_run, problem, and time + -- The dbt model definition has a test against the same composite unique key. + deduped_combined as ( + select * + from + ( + select + *, + row_number() over ( + partition by platform, openedx_user_id, courserun_readable_id, problem_block_id, attempt + order by event_timestamp + ) as rn + from combined + ) + where rn = 1 ) - where rn = 1 -) select - user_fk - , platform - , openedx_user_id - , user_username - , courserun_readable_id - , studentmodule_id - , event_type - , event_json - , problem_block_id - , answers - , attempt - , event_timestamp - , grade - , max_grade - , success + user_fk, + platform, + openedx_user_id, + user_username, + courserun_readable_id, + studentmodule_id, + event_type, + event_json, + problem_block_id, + answers, + attempt, + event_timestamp, + grade, + max_grade, + success from deduped_combined diff --git a/src/ol_dbt/models/dimensional/tfact_video_events.sql b/src/ol_dbt/models/dimensional/tfact_video_events.sql index f0aafcb5e..9e9b5ab53 100644 --- a/src/ol_dbt/models/dimensional/tfact_video_events.sql +++ b/src/ol_dbt/models/dimensional/tfact_video_events.sql @@ -1,219 +1,203 @@ -{% set video_events = - ( - 'load_video' - , 'play_video' - , 'seek_video' - , 'pause_video' - , 'stop_video' - , 'complete_video' - , 'speed_change_video' - , 'show_transcript' - , 'hide_transcript' - , 'edx.video.closed_captions.hidden' - , 'edx.video.closed_captions.shown' - ) -%} - -with mitxonline_video_events as ( - select - user_username - , openedx_user_id - , courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as video_id - , case - when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' then null - else cast(json_query(useractivity_event_object, 'lax $.duration' omit quotes) as decimal(38, 4)) - end as video_duration - , json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as video_position - , json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as starting_position - , json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as ending_position - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type in {{ video_events }} -) - -, xpro_video_events as ( - select - user_username - , openedx_user_id - , courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as video_id - , case - when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' then null - else cast(json_query(useractivity_event_object, 'lax $.duration' omit quotes) as decimal(38, 4)) - end as video_duration - , json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as video_position - , json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as starting_position - , json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as ending_position - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__mitxpro__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type in {{ video_events }} -) - -, mitxresidential_video_events as ( - select - user_username - , user_id - , courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as video_id - , case - when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' then null - else cast(json_query(useractivity_event_object, 'lax $.duration' omit quotes) as decimal(38, 4)) - end as video_duration - , json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as video_position - , json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as starting_position - , json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as ending_position - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__mitxresidential__openedx__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type in {{ video_events }} -) +{% set video_events = ( + "load_video", + "play_video", + "seek_video", + "pause_video", + "stop_video", + "complete_video", + "speed_change_video", + "show_transcript", + "hide_transcript", + "edx.video.closed_captions.hidden", + "edx.video.closed_captions.shown", +) %} -, edxorg_video_events as ( - select - user_username - , user_id - , {{ format_course_id('courserun_readable_id') }} as courserun_readable_id - , useractivity_event_type as event_type - , useractivity_event_object as event_json - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as video_id - , case - when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' then null - else cast(json_query(useractivity_event_object, 'lax $.duration' omit quotes) as decimal(38, 4)) - end as video_duration - , json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as video_position - , json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as starting_position - , json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as ending_position - , from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp - from {{ ref('stg__edxorg__s3__tracking_logs__user_activity') }} - where - courserun_readable_id is not null - and useractivity_event_type in {{ video_events }} -) - -, users as ( - select * from {{ ref('dim_user') }} -) - -, platform as ( - select * from {{ ref('dim_platform') }} -) - -, combined as ( - select - 'mitxonline' as platform - , users.user_pk as user_fk - , mitxonline_video_events.openedx_user_id - , mitxonline_video_events.user_username - , mitxonline_video_events.courserun_readable_id - , mitxonline_video_events.event_type - , mitxonline_video_events.event_json - , mitxonline_video_events.video_id - , mitxonline_video_events.video_duration - , mitxonline_video_events.video_position - , mitxonline_video_events.starting_position - , mitxonline_video_events.ending_position - , mitxonline_video_events.event_timestamp - from mitxonline_video_events - left join users - on - mitxonline_video_events.openedx_user_id = users.mitxonline_openedx_user_id +with + mitxonline_video_events as ( + select + user_username, + openedx_user_id, + courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as video_id, + case + when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' + then null + else cast(json_query(useractivity_event_object, 'lax $.duration' omit quotes) as decimal(38, 4)) + end as video_duration, + json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as video_position, + json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as starting_position, + json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as ending_position, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__mitxonline__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null and useractivity_event_type in {{ video_events }} + ), + xpro_video_events as ( + select + user_username, + openedx_user_id, + courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as video_id, + case + when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' + then null + else cast(json_query(useractivity_event_object, 'lax $.duration' omit quotes) as decimal(38, 4)) + end as video_duration, + json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as video_position, + json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as starting_position, + json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as ending_position, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__mitxpro__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null and useractivity_event_type in {{ video_events }} + ), + mitxresidential_video_events as ( + select + user_username, + user_id, + courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as video_id, + case + when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' + then null + else cast(json_query(useractivity_event_object, 'lax $.duration' omit quotes) as decimal(38, 4)) + end as video_duration, + json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as video_position, + json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as starting_position, + json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as ending_position, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__mitxresidential__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null and useractivity_event_type in {{ video_events }} + ), + edxorg_video_events as ( + select + user_username, + user_id, + {{ format_course_id("courserun_readable_id") }} as courserun_readable_id, + useractivity_event_type as event_type, + useractivity_event_object as event_json, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as video_id, + case + when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' + then null + else cast(json_query(useractivity_event_object, 'lax $.duration' omit quotes) as decimal(38, 4)) + end as video_duration, + json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as video_position, + json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as starting_position, + json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as ending_position, + from_iso8601_timestamp_nanos(useractivity_timestamp) as event_timestamp + from {{ ref("stg__edxorg__s3__tracking_logs__user_activity") }} + where courserun_readable_id is not null and useractivity_event_type in {{ video_events }} + ), + users as (select * from {{ ref("dim_user") }}), + platform as (select * from {{ ref("dim_platform") }}), + combined as ( + select + 'mitxonline' as platform, + users.user_pk as user_fk, + mitxonline_video_events.openedx_user_id, + mitxonline_video_events.user_username, + mitxonline_video_events.courserun_readable_id, + mitxonline_video_events.event_type, + mitxonline_video_events.event_json, + mitxonline_video_events.video_id, + mitxonline_video_events.video_duration, + mitxonline_video_events.video_position, + mitxonline_video_events.starting_position, + mitxonline_video_events.ending_position, + mitxonline_video_events.event_timestamp + from mitxonline_video_events + left join + users + on mitxonline_video_events.openedx_user_id = users.mitxonline_openedx_user_id and mitxonline_video_events.user_username = users.user_mitxonline_username - union all + union all - select - 'mitxpro' as platform - , users.user_pk as user_fk - , xpro_video_events.openedx_user_id - , xpro_video_events.user_username - , xpro_video_events.courserun_readable_id - , xpro_video_events.event_type - , xpro_video_events.event_json - , xpro_video_events.video_id - , xpro_video_events.video_duration - , xpro_video_events.video_position - , xpro_video_events.starting_position - , xpro_video_events.ending_position - , xpro_video_events.event_timestamp - from xpro_video_events - left join users - on - xpro_video_events.openedx_user_id = users.mitxpro_openedx_user_id + select + 'mitxpro' as platform, + users.user_pk as user_fk, + xpro_video_events.openedx_user_id, + xpro_video_events.user_username, + xpro_video_events.courserun_readable_id, + xpro_video_events.event_type, + xpro_video_events.event_json, + xpro_video_events.video_id, + xpro_video_events.video_duration, + xpro_video_events.video_position, + xpro_video_events.starting_position, + xpro_video_events.ending_position, + xpro_video_events.event_timestamp + from xpro_video_events + left join + users + on xpro_video_events.openedx_user_id = users.mitxpro_openedx_user_id and xpro_video_events.user_username = users.user_mitxpro_username - union all + union all - select - 'residential' as platform - , users.user_pk as user_fk - , mitxresidential_video_events.user_id as openedx_user_id - , mitxresidential_video_events.user_username - , mitxresidential_video_events.courserun_readable_id - , mitxresidential_video_events.event_type - , mitxresidential_video_events.event_json - , mitxresidential_video_events.video_id - , mitxresidential_video_events.video_duration - , mitxresidential_video_events.video_position - , mitxresidential_video_events.starting_position - , mitxresidential_video_events.ending_position - , mitxresidential_video_events.event_timestamp - from mitxresidential_video_events - left join users - on - mitxresidential_video_events.user_id = users.residential_openedx_user_id + select + 'residential' as platform, + users.user_pk as user_fk, + mitxresidential_video_events.user_id as openedx_user_id, + mitxresidential_video_events.user_username, + mitxresidential_video_events.courserun_readable_id, + mitxresidential_video_events.event_type, + mitxresidential_video_events.event_json, + mitxresidential_video_events.video_id, + mitxresidential_video_events.video_duration, + mitxresidential_video_events.video_position, + mitxresidential_video_events.starting_position, + mitxresidential_video_events.ending_position, + mitxresidential_video_events.event_timestamp + from mitxresidential_video_events + left join + users + on mitxresidential_video_events.user_id = users.residential_openedx_user_id and mitxresidential_video_events.user_username = users.user_residential_username - union all + union all - select - 'edxorg' as platform - , users.user_pk as user_fk - , edxorg_video_events.user_id as openedx_user_id - , edxorg_video_events.user_username - , edxorg_video_events.courserun_readable_id - , edxorg_video_events.event_type - , edxorg_video_events.event_json - , edxorg_video_events.video_id - , edxorg_video_events.video_duration - , edxorg_video_events.video_position - , edxorg_video_events.starting_position - , edxorg_video_events.ending_position - , edxorg_video_events.event_timestamp - from edxorg_video_events - left join users - on - edxorg_video_events.user_id = users.edxorg_openedx_user_id + select + 'edxorg' as platform, + users.user_pk as user_fk, + edxorg_video_events.user_id as openedx_user_id, + edxorg_video_events.user_username, + edxorg_video_events.courserun_readable_id, + edxorg_video_events.event_type, + edxorg_video_events.event_json, + edxorg_video_events.video_id, + edxorg_video_events.video_duration, + edxorg_video_events.video_position, + edxorg_video_events.starting_position, + edxorg_video_events.ending_position, + edxorg_video_events.event_timestamp + from edxorg_video_events + left join + users + on edxorg_video_events.user_id = users.edxorg_openedx_user_id and edxorg_video_events.user_username = users.user_edxorg_username -) + ) select distinct - platform.platform_pk as platform_fk - , combined.user_fk - , combined.platform - , combined.openedx_user_id - , combined.user_username - , combined.courserun_readable_id - , combined.event_type - , combined.video_id as video_block_fk - , combined.video_duration - , combined.video_position - , combined.starting_position - , combined.ending_position - , combined.event_timestamp - , combined.event_json + platform.platform_pk as platform_fk, + combined.user_fk, + combined.platform, + combined.openedx_user_id, + combined.user_username, + combined.courserun_readable_id, + combined.event_type, + combined.video_id as video_block_fk, + combined.video_duration, + combined.video_position, + combined.starting_position, + combined.ending_position, + combined.event_timestamp, + combined.event_json from combined -left join platform - on combined.platform = platform.platform_readable_id +left join platform on combined.platform = platform.platform_readable_id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__bigquery__email_opt_in.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__bigquery__email_opt_in.sql index 451a92cae..4ad529998 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__bigquery__email_opt_in.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__bigquery__email_opt_in.sql @@ -1,11 +1,4 @@ -with email_opt_in as ( - select * from {{ source('ol_warehouse_raw_data','raw__irx__edxorg__bigquery__email_opt_in') }} -) +with email_opt_in as (select * from {{ source("ol_warehouse_raw_data", "raw__irx__edxorg__bigquery__email_opt_in") }}) -select - is_opted_in_for_email - , course_id - , full_name - , preference_set_datetime - , user_id +select is_opted_in_for_email, course_id, full_name, preference_set_datetime, user_id from email_opt_in diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aiclassifier.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aiclassifier.sql index b487331a3..d1936ab42 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aiclassifier.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aiclassifier.sql @@ -1,5 +1,2 @@ -select - id - , classifier_set_id - , classifier_data -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aiclassifier') }} +select id, classifier_set_id, classifier_data +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aiclassifier") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aiclassifierset.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aiclassifierset.sql index 54fb8dffd..96698e022 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aiclassifierset.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aiclassifierset.sql @@ -1,5 +1,2 @@ -select - course_id - , id - , item_id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aiclassifierset') }} +select course_id, id, item_id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aiclassifierset") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aigradingworkflow.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aigradingworkflow.sql index af00c31d0..4d482444a 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aigradingworkflow.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aigradingworkflow.sql @@ -1,5 +1,2 @@ -select - scheduled_at - , submission_uuid - , algorithm_id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aigradingworkflow') }} +select scheduled_at, submission_uuid, algorithm_id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aigradingworkflow") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aitrainingworkflow.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aitrainingworkflow.sql index bbcb267a9..6c8935152 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aitrainingworkflow.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aitrainingworkflow.sql @@ -1,6 +1,2 @@ -select - course_id - , id - , item_id - as uuid -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aitrainingworkflow') }} +select course_id, id, item_id as uuid +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aitrainingworkflow") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples.sql index af423ae17..d72807c32 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples.sql @@ -1,8 +1,2 @@ -select - trainingexample_id - , id - , aitrainingworkflow_id -from - {{ source( - 'ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples' - ) }} +select trainingexample_id, id, aitrainingworkflow_id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessment.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessment.sql index 5c84c6939..24798df95 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessment.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessment.sql @@ -1,19 +1,9 @@ -with a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessment') }} -) +with + a as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }}) -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} -) - -select - a.scorer_id - , a.id - , a.score_type +select a.scorer_id, a.id, a.score_type from a inner join s on a.submission_uuid = s.uuid inner join si on s.student_item_id = si.id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedback.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedback.sql index 068197606..1358aeeb2 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedback.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedback.sql @@ -1,31 +1,21 @@ -with af as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessmentfeedback') }} -) +with + af as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessmentfeedback") }} + ), + afa as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessmentfeedback_assessments" + ) + }} + ), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }}) -, afa as ( - select * - from - {{ - source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessmentfeedback_assessments') - }} -) - -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} -) - -select distinct - af.id - , af.submission_uuid - , af.feedback_text +select distinct af.id, af.submission_uuid, af.feedback_text from af inner join afa on af.id = afa.assessmentfeedback_id inner join a on afa.assessment_id = a.id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedback_assessments.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedback_assessments.sql index a8a5e2e9f..fbb5ea984 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedback_assessments.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedback_assessments.sql @@ -1,27 +1,18 @@ -with afa as ( - select * - from - {{ - source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessmentfeedback_assessments') - }} -) +with + afa as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessmentfeedback_assessments" + ) + }} + ), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }}) -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} -) - -select - afa.assessment_id - , afa.assessmentfeedback_id - , afa.id +select afa.assessment_id, afa.assessmentfeedback_id, afa.id from afa inner join a on afa.assessment_id = a.id inner join s on a.submission_uuid = s.uuid diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedback_options.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedback_options.sql index 2059660f2..b942ebe04 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedback_options.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedback_options.sql @@ -1,36 +1,25 @@ -with afo as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessmentfeedback_options') }} -) +with + afo as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessmentfeedback_options") }} + ), + af as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessmentfeedback") }} + ), + afa as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessmentfeedback_assessments" + ) + }} + ), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }}) -, af as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessmentfeedback') }} -) - -, afa as ( - select * - from - {{ - source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessmentfeedback_assessments') - }} -) - -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} -) - -select distinct - afo.assessmentfeedbackoption_id - , afo.assessmentfeedback_id - , afo.id +select distinct afo.assessmentfeedbackoption_id, afo.assessmentfeedback_id, afo.id from afo inner join af on afo.assessmentfeedback_id = af.id inner join afa on af.id = afa.assessmentfeedback_id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedbackoption.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedbackoption.sql index 05622bc42..3e119e483 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedbackoption.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentfeedbackoption.sql @@ -1,40 +1,29 @@ -with aafo as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessmentfeedbackoption') }} -) +with + aafo as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessmentfeedbackoption") }} + ), + afo as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessmentfeedback_options") }} + ), + af as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessmentfeedback") }} + ), + afa as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessmentfeedback_assessments" + ) + }} + ), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }}) -, afo as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessmentfeedback_options') }} -) - -, af as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessmentfeedback') }} -) - -, afa as ( - select * - from - {{ - source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessmentfeedback_assessments') - }} -) - -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} -) - -select distinct - aafo.text - , aafo.id +select distinct aafo.text, aafo.id from aafo inner join afo on aafo.id = afo.assessmentfeedbackoption_id inner join af on afo.assessmentfeedback_id = af.id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentpart.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentpart.sql index ed2f9451a..e17ba756c 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentpart.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_assessmentpart.sql @@ -1,24 +1,10 @@ -with ap as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessmentpart') }} -) +with + ap as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessmentpart") }}), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }}) -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} -) - -select - ap.assessment_id - , ap.option_id - , ap.id - , ap.feedback +select ap.assessment_id, ap.option_id, ap.id, ap.feedback from ap inner join a on ap.assessment_id = a.id inner join s on a.submission_uuid = s.uuid diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_criterion.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_criterion.sql index f573ecbb9..43a76ef80 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_criterion.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_criterion.sql @@ -1,66 +1,53 @@ -with c as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_criterion') }} -) - -, rub as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_rubric') }} -) - -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} -) - -, te as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_trainingexample') }} -) - -, ate as ( - select * - from - {{ source( - 'ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples' - ) }} -) - -, tw as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aitrainingworkflow') }} -) - -, aigw as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aigradingworkflow') }} -) - -, acs as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aiclassifierset') }} -) - -select distinct - c.order_num - , c.id - , c.rubric_id +with + c as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_criterion") }}), + rub as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_rubric") }}), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }}), + te as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_trainingexample") }} + ), + ate as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples", + ) + }} + ), + tw as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aitrainingworkflow") }} + ), + aigw as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aigradingworkflow") }} + ), + acs as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aiclassifierset") }} + ) + +select distinct c.order_num, c.id, c.rubric_id from c -where c.rubric_id in ( - select distinct rub.id from rub - inner join a on rub.id = a.rubric_id - inner join s on a.submission_uuid = s.uuid - inner join si on s.student_item_id = si.id - union distinct - select distinct rub.id from rub - inner join te on rub.id = te.rubric_id - inner join ate on te.id = ate.trainingexample_id - inner join tw on ate.aitrainingworkflow_id = tw.id - union distinct - select distinct rub.id from rub - inner join aigw on rub.id = aigw.rubric_id - union distinct - select distinct rub.id from rub - inner join acs on rub.id = acs.rubric_id -) +where + c.rubric_id in ( + select distinct rub.id + from rub + inner join a on rub.id = a.rubric_id + inner join s on a.submission_uuid = s.uuid + inner join si on s.student_item_id = si.id + union distinct + select distinct rub.id + from rub + inner join te on rub.id = te.rubric_id + inner join ate on te.id = ate.trainingexample_id + inner join tw on ate.aitrainingworkflow_id = tw.id + union distinct + select distinct rub.id + from rub + inner join aigw on rub.id = aigw.rubric_id + union distinct + select distinct rub.id + from rub + inner join acs on rub.id = acs.rubric_id + ) diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_criterionoption.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_criterionoption.sql index 6a5928a2e..7c8caf0d1 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_criterionoption.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_criterionoption.sql @@ -1,73 +1,61 @@ -with c as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_criterion') }} -) - -, co as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_criterionoption') }} -) - -, rub as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_rubric') }} -) - -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} -) - -, te as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_trainingexample') }} -) - -, ate as ( - select * - from - {{ source( - 'ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples' - ) }} -) - -, tw as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aitrainingworkflow') }} -) - -, aigw as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aigradingworkflow') }} -) - -, acs as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aiclassifierset') }} -) +with + c as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_criterion") }}), + co as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_criterionoption") }} + ), + rub as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_rubric") }}), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }}), + te as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_trainingexample") }} + ), + ate as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples", + ) + }} + ), + tw as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aitrainingworkflow") }} + ), + aigw as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aigradingworkflow") }} + ), + acs as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aiclassifierset") }} + ) -select distinct - co.order_num - , co.id - , co.points +select distinct co.order_num, co.id, co.points from co -where co.criterion_id in ( - select c.id from c - where c.rubric_id in ( - select distinct rub.id from rub - inner join a on rub.id = a.rubric_id - inner join s on a.submission_uuid = s.uuid - inner join si on s.student_item_id = si.id - union distinct - select distinct rub.id from rub - inner join te on rub.id = te.rubric_id - inner join ate on te.id = ate.trainingexample_id - inner join tw on ate.aitrainingworkflow_id = tw.id - union distinct - select distinct rub.id from rub - inner join aigw on rub.id = aigw.rubric_id - union distinct - select distinct rub.id from rub - inner join acs on rub.id = acs.rubric_id +where + co.criterion_id in ( + select c.id + from c + where + c.rubric_id in ( + select distinct rub.id + from rub + inner join a on rub.id = a.rubric_id + inner join s on a.submission_uuid = s.uuid + inner join si on s.student_item_id = si.id + union distinct + select distinct rub.id + from rub + inner join te on rub.id = te.rubric_id + inner join ate on te.id = ate.trainingexample_id + inner join tw on ate.aitrainingworkflow_id = tw.id + union distinct + select distinct rub.id + from rub + inner join aigw on rub.id = aigw.rubric_id + union distinct + select distinct rub.id + from rub + inner join acs on rub.id = acs.rubric_id + ) ) -) diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_peerworkflow.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_peerworkflow.sql index ab8f65923..4756709ff 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_peerworkflow.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_peerworkflow.sql @@ -1,6 +1,2 @@ -select - cancelled_at - , grading_completed_at - , student_id - , course_id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_peerworkflow') }} +select cancelled_at, grading_completed_at, student_id, course_id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_peerworkflow") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_peerworkflowitem.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_peerworkflowitem.sql index 7d4fa2dc4..72d171e9a 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_peerworkflowitem.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_peerworkflowitem.sql @@ -1,6 +1,2 @@ -select - started_at - , assessment_id - , author_id - , id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_peerworkflowitem') }} +select started_at, assessment_id, author_id, id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_peerworkflowitem") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_rubric.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_rubric.sql index cfb2e2a45..caca809b8 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_rubric.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_rubric.sql @@ -1,55 +1,47 @@ -with rub as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_rubric') }} -) - -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} -) - -, te as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_trainingexample') }} -) - -, ate as ( - select * - from - {{ source( - 'ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples' - ) }} -) - -, tw as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aitrainingworkflow') }} -) - -, aigw as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aigradingworkflow') }} -) - -, acs as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aiclassifierset') }} -) - -select distinct rub.* from rub +with + rub as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_rubric") }}), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }}), + te as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_trainingexample") }} + ), + ate as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples", + ) + }} + ), + tw as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aitrainingworkflow") }} + ), + aigw as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aigradingworkflow") }} + ), + acs as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aiclassifierset") }} + ) + +select distinct rub.* +from rub inner join a on rub.id = a.rubric_id inner join s on a.submission_uuid = s.uuid inner join si on s.student_item_id = si.id union distinct -select distinct rub.* from rub +select distinct rub.* +from rub inner join te on rub.id = te.rubric_id inner join ate on te.id = ate.trainingexample_id inner join tw on ate.aitrainingworkflow_id = tw.id union distinct -select distinct rub.* from rub +select distinct rub.* +from rub inner join aigw on rub.id = aigw.rubric_id union distinct -select distinct rub.* from rub +select distinct rub.* +from rub inner join acs on rub.id = acs.rubric_id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_studenttrainingworkflow.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_studenttrainingworkflow.sql index f885e6d68..7c8e88775 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_studenttrainingworkflow.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_studenttrainingworkflow.sql @@ -1,6 +1,2 @@ -select - student_id - , course_id - , id - , item_id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_studenttrainingworkflow') }} +select student_id, course_id, id, item_id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_studenttrainingworkflow") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_studenttrainingworkflowitem.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_studenttrainingworkflowitem.sql index e8f9eb7dc..ce4dca5a1 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_studenttrainingworkflowitem.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_studenttrainingworkflowitem.sql @@ -1,6 +1,2 @@ -select - workflow_id - , order_num - , started_at - , id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_studenttrainingworkflowitem') }} +select workflow_id, order_num, started_at, id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_studenttrainingworkflowitem") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_trainingexample.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_trainingexample.sql index 6d50c932e..71c12a1eb 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_trainingexample.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_trainingexample.sql @@ -1,33 +1,35 @@ -with te as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_trainingexample') }} -) +with + te as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_trainingexample") }} + ), + ate as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples", + ) + }} + ), + tw as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aitrainingworkflow") }} + ), + stwi as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_studenttrainingworkflowitem") }} + ), + stw as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_studenttrainingworkflow") }} + ) -, ate as ( - select * - from - {{ source( - 'ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples' - ) }} -) - -, tw as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aitrainingworkflow') }} -) - -, stwi as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_studenttrainingworkflowitem') }} -) - -, stw as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_studenttrainingworkflow') }} -) - -select distinct te.* from te +select distinct te.* +from te inner join ate on te.id = ate.trainingexample_id inner join tw on ate.aitrainingworkflow_id = tw.id union distinct -select distinct te.* from te +select distinct te.* +from te inner join stwi on te.id = stwi.training_example_id inner join stw on stwi.workflow_id = stw.id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_trainingexample_options_selected.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_trainingexample_options_selected.sql index 7dcbb15f8..d7474fc5d 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_trainingexample_options_selected.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__assessment_trainingexample_options_selected.sql @@ -1,48 +1,49 @@ -with tos as ( - select * - from - {{ source ( - 'ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_trainingexample_options_selected' - ) }} -) +with + tos as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_trainingexample_options_selected" + ) + }} + ), + te as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_trainingexample") }} + ), + ate as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples", + ) + }} + ), + tw as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_aitrainingworkflow") }} + ), + stwi as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_studenttrainingworkflowitem") }} + ), + stw as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__assessment_studenttrainingworkflow") }} + ) -, te as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_trainingexample') }} -) - -, ate as ( - select * - from - {{ source( - 'ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aitrainingworkflow_training_examples' - ) }} -) - -, tw as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_aitrainingworkflow') }} -) - -, stwi as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_studenttrainingworkflowitem') }} -) - -, stw as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__assessment_studenttrainingworkflow') }} -) - -select - tos.trainingexample_id - , tos.id - , tos.criterionoption_id +select tos.trainingexample_id, tos.id, tos.criterionoption_id from tos -where tos.trainingexample_id in ( - select distinct te.id from te - inner join ate on te.id = ate.trainingexample_id - inner join tw on ate.aitrainingworkflow_id = tw.id - union distinct - select distinct te.id from te - inner join stwi on te.id = stwi.training_example_id - inner join stw on stwi.workflow_id = stw.id -) +where + tos.trainingexample_id in ( + select distinct te.id + from te + inner join ate on te.id = ate.trainingexample_id + inner join tw on ate.aitrainingworkflow_id = tw.id + union distinct + select distinct te.id + from te + inner join stwi on te.id = stwi.training_example_id + inner join stw on stwi.workflow_id = stw.id + ) diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__auth_user.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__auth_user.sql index c804b9de5..9d09e2f70 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__auth_user.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__auth_user.sql @@ -1,33 +1,31 @@ -with auth_user as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__auth_user') }} -) - -, student_courseenrollment as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__student_courseenrollment') }} -) +with + auth_user as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__auth_user") }}), + student_courseenrollment as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__student_courseenrollment") }} + ) select - auth_user.id - , auth_user.username - , auth_user.first_name - , auth_user.last_name - , auth_user.email - , '' as pass_word - , auth_user.is_staff - , auth_user.is_active - , auth_user.is_superuser - , auth_user.last_login - , auth_user.date_joined - , '' as status - , '' as email_key - , '' as avatar_type - , '' as country - , 0 as show_country - , '' as date_of_birth - , '' as interesting_tags - , '' as ignored_tags - , 0 as email_tag_filter_strategy - , 0 as display_tag_filter_strategy - , 0 as consecutive_days_visit_count + auth_user.id, + auth_user.username, + auth_user.first_name, + auth_user.last_name, + auth_user.email, + '' as pass_word, + auth_user.is_staff, + auth_user.is_active, + auth_user.is_superuser, + auth_user.last_login, + auth_user.date_joined, + '' as status, + '' as email_key, + '' as avatar_type, + '' as country, + 0 as show_country, + '' as date_of_birth, + '' as interesting_tags, + '' as ignored_tags, + 0 as email_tag_filter_strategy, + 0 as display_tag_filter_strategy, + 0 as consecutive_days_visit_count from auth_user inner join student_courseenrollment on auth_user.id = student_courseenrollment.user_id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__auth_userprofile.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__auth_userprofile.sql index 15a677ad1..374d627c3 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__auth_userprofile.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__auth_userprofile.sql @@ -1,28 +1,28 @@ -with auth_userprofile as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__auth_userprofile') }} -) - -, student_courseenrollment as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__student_courseenrollment') }} -) +with + auth_userprofile as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__auth_userprofile") }} + ), + student_courseenrollment as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__student_courseenrollment") }} + ) select - auth_userprofile.id - , auth_userprofile.user_id - , auth_userprofile.name - , auth_userprofile.language - , auth_userprofile.location - , auth_userprofile.meta - , auth_userprofile.courseware - , auth_userprofile.gender - , auth_userprofile.mailing_address - , auth_userprofile.year_of_birth - , auth_userprofile.level_of_education - , auth_userprofile.goals - , auth_userprofile.country - , auth_userprofile.city - , auth_userprofile.bio - , auth_userprofile.profile_image_uploaded_at - , auth_userprofile.state + auth_userprofile.id, + auth_userprofile.user_id, + auth_userprofile.name, + auth_userprofile.language, + auth_userprofile.location, + auth_userprofile.meta, + auth_userprofile.courseware, + auth_userprofile.gender, + auth_userprofile.mailing_address, + auth_userprofile.year_of_birth, + auth_userprofile.level_of_education, + auth_userprofile.goals, + auth_userprofile.country, + auth_userprofile.city, + auth_userprofile.bio, + auth_userprofile.profile_image_uploaded_at, + auth_userprofile.state from auth_userprofile inner join student_courseenrollment on auth_userprofile.user_id = student_courseenrollment.user_id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__certificates_generatedcertificate.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__certificates_generatedcertificate.sql index ba8651ca3..f89a8bfff 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__certificates_generatedcertificate.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__certificates_generatedcertificate.sql @@ -1,23 +1,24 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__certificates_generatedcertificate') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__certificates_generatedcertificate") }} + ) -{{ deduplicate_raw_table(order_by='modified_date' , partition_columns = 'id') }} + {{ deduplicate_raw_table(order_by="modified_date", partition_columns="id") }} select - id - , user_id - , course_id - , key - , name - , distinction - , verify_uuid - , download_uuid - , download_url - , grade - , created_date - , modified_date - , mode - , status - , error_reason + id, + user_id, + course_id, + key, + name, + distinction, + verify_uuid, + download_uuid, + download_url, + grade, + created_date, + modified_date, + mode, + status, + error_reason from most_recent_source diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__course_groups_cohortmembership.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__course_groups_cohortmembership.sql index e819614b5..1300c9be2 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__course_groups_cohortmembership.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__course_groups_cohortmembership.sql @@ -1,16 +1,11 @@ -with cm as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__course_groups_cohortmembership') }} -) +with + cm as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__course_groups_cohortmembership") }} + ), + cug as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__course_groups_courseusergroup") }} + ) -, cug as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__course_groups_courseusergroup') }} -) - -select - cm.user_id - , cm.course_id - , cug.group_type - , cug.name +select cm.user_id, cm.course_id, cug.group_type, cug.name from cm inner join cug on cm.course_user_group_id = cug.id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__courseware_studentmodulehistoryextended.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__courseware_studentmodulehistoryextended.sql index 61b47cd3c..e5246fee2 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__courseware_studentmodulehistoryextended.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__courseware_studentmodulehistoryextended.sql @@ -1,19 +1,17 @@ -with studentmodulehistoryextended as ( - select * from {{ ref('stg__mitxresidential__openedx__courseware_studentmodulehistoryextended') }} -) - -, studentmodule as ( - select * from {{ ref('stg__mitxresidential__openedx__courseware_studentmodule') }} -) +with + studentmodulehistoryextended as ( + select * from {{ ref("stg__mitxresidential__openedx__courseware_studentmodulehistoryextended") }} + ), + studentmodule as (select * from {{ ref("stg__mitxresidential__openedx__courseware_studentmodule") }}) select - studentmodule.courserun_readable_id as course_id - , studentmodule.user_id - , studentmodule.coursestructure_block_id as module_id - , studentmodule.coursestructure_block_category as module_type - , studentmodulehistoryextended.studentmodule_state_data as state_data - , studentmodulehistoryextended.studentmodule_problem_grade as grade - , studentmodulehistoryextended.studentmodule_problem_max_grade as max_grade - , studentmodulehistoryextended.studentmodule_created_on as created + studentmodule.courserun_readable_id as course_id, + studentmodule.user_id, + studentmodule.coursestructure_block_id as module_id, + studentmodule.coursestructure_block_category as module_type, + studentmodulehistoryextended.studentmodule_state_data as state_data, + studentmodulehistoryextended.studentmodule_problem_grade as grade, + studentmodulehistoryextended.studentmodule_problem_max_grade as max_grade, + studentmodulehistoryextended.studentmodule_created_on as created from studentmodulehistoryextended inner join studentmodule on studentmodulehistoryextended.studentmodule_id = studentmodule.studentmodule_id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__credit_crediteligibility.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__credit_crediteligibility.sql index 679c5f988..be44b1e09 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__credit_crediteligibility.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__credit_crediteligibility.sql @@ -1,18 +1,8 @@ -with ce as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__credit_crediteligibility') }} -) +with + ce as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__credit_crediteligibility") }}), + cc as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__credit_creditcourse") }}) -, cc as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__credit_creditcourse') }} -) - -select - ce.id - , ce.created - , ce.modified - , ce.username - , ce.deadline - , cc.course_key as course_id +select ce.id, ce.created, ce.modified, ce.username, ce.deadline, cc.course_key as course_id from ce inner join cc on ce.course_id = cc.id order by ce.username diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__django_comment_client_role_users.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__django_comment_client_role_users.sql index 37d11b9a6..08453cfe5 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__django_comment_client_role_users.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__django_comment_client_role_users.sql @@ -1,16 +1,12 @@ -with dccr as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__django_comment_client_role') }} -) +with + dccr as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__django_comment_client_role") }} + ), + dccru as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__django_comment_client_role_users") }} + ) -, dccru as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__django_comment_client_role_users') }} -) - -select - dccr.course_id - , dccru.user_id - , dccr.name +select dccr.course_id, dccru.user_id, dccr.name from dccr inner join dccru on dccr.id = dccru.role_id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__grades_persistentcoursegrade.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__grades_persistentcoursegrade.sql index 616deb688..5564f0a5c 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__grades_persistentcoursegrade.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__grades_persistentcoursegrade.sql @@ -1,16 +1,9 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__grades_persistentcoursegrade') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__grades_persistentcoursegrade") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }} -select - course_id - , user_id - , grading_policy_hash - , percent_grade - , letter_grade - , passed_timestamp - , created - , modified + {{ deduplicate_raw_table(order_by="modified", partition_columns="id") }} +select course_id, user_id, grading_policy_hash, percent_grade, letter_grade, passed_timestamp, created, modified from most_recent_source order by user_id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__grades_persistentsubsectiongrade.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__grades_persistentsubsectiongrade.sql index b7c8ffe30..1d21ffa8c 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__grades_persistentsubsectiongrade.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__grades_persistentsubsectiongrade.sql @@ -1,19 +1,20 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__grades_persistentsubsectiongrade') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__grades_persistentsubsectiongrade") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }} + {{ deduplicate_raw_table(order_by="modified", partition_columns="id") }} select - course_id - , user_id - , usage_key - , earned_all - , possible_all - , earned_graded - , possible_graded - , first_attempted - , created - , modified + course_id, + user_id, + usage_key, + earned_all, + possible_all, + earned_graded, + possible_graded, + first_attempted, + created, + modified from most_recent_source order by user_id, first_attempted diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_anonymoususerid.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_anonymoususerid.sql index de16afca9..2fe8f532b 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_anonymoususerid.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_anonymoususerid.sql @@ -1,5 +1,2 @@ -select - anonymous_user_id - , course_id - , id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__student_anonymoususerid') }} +select anonymous_user_id, course_id, id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__student_anonymoususerid") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_courseaccessrole.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_courseaccessrole.sql index b80355f75..0c54ad6d4 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_courseaccessrole.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_courseaccessrole.sql @@ -1,6 +1,2 @@ -select - org - , course_id - , user_id - , role -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__student_courseaccessrole') }} +select org, course_id, user_id, role +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__student_courseaccessrole") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_courseenrollment.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_courseenrollment.sql index 7f2d4a292..72cf09300 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_courseenrollment.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_courseenrollment.sql @@ -1,6 +1,2 @@ -select - course_id - , mode - , id - , is_active -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__student_courseenrollment') }} +select course_id, mode, id, is_active +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__student_courseenrollment") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_languageproficiency.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_languageproficiency.sql index dba8edca0..4115502a4 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_languageproficiency.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__student_languageproficiency.sql @@ -1,19 +1,15 @@ -with student_languageproficiency as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__student_languageproficiency') }} -) +with + student_languageproficiency as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__student_languageproficiency") }} + ), + auth_userprofile as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__auth_userprofile") }} + ), + student_courseenrollment as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__student_courseenrollment") }} + ) -, auth_userprofile as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__auth_userprofile') }} -) - -, student_courseenrollment as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__student_courseenrollment') }} -) - -select - student_languageproficiency.user_profile_id - , student_languageproficiency.id - , student_languageproficiency.code +select student_languageproficiency.user_profile_id, student_languageproficiency.id, student_languageproficiency.code from student_languageproficiency inner join auth_userprofile on student_languageproficiency.user_profile_id = auth_userprofile.user_id inner join student_courseenrollment on auth_userprofile.user_id = student_courseenrollment.user_id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_score.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_score.sql index 9c089a3f5..eb9c75e28 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_score.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_score.sql @@ -1,11 +1,7 @@ -select - points_earned - , reset - , submission_id - , id - , student_item_id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_score') }} -where student_item_id in ( - select studentitem.id - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} as studentitem -) +select points_earned, reset, submission_id, id, student_item_id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_score") }} +where + student_item_id in ( + select studentitem.id + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }} as studentitem + ) diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_scoresummary.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_scoresummary.sql index 355ee5c3a..64b0529a7 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_scoresummary.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_scoresummary.sql @@ -1,10 +1,7 @@ -select - highest_id - , id - , latest_id - , student_item_id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_scoresummary') }} -where student_item_id in ( - select studentitem.id - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} as studentitem -) +select highest_id, id, latest_id, student_item_id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_scoresummary") }} +where + student_item_id in ( + select studentitem.id + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }} as studentitem + ) diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_studentitem.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_studentitem.sql index cb92ce5b1..78a30552f 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_studentitem.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_studentitem.sql @@ -1,6 +1,2 @@ -select - student_id - , course_id - , id - , item_id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} +select student_id, course_id, id, item_id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_submission.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_submission.sql index 35ee881fa..f3603104a 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_submission.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__submissions_submission.sql @@ -1,10 +1,7 @@ -select - team_submission_id - , submitted_at - , status - , id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_submission') }} -where student_item_id in ( - select studentitem.id - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__submissions_studentitem') }} as studentitem -) +select team_submission_id, submitted_at, status, id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_submission") }} +where + student_item_id in ( + select studentitem.id + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__submissions_studentitem") }} as studentitem + ) diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__teams.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__teams.sql index 36cbe3a1d..2bccc53a2 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__teams.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__teams.sql @@ -1,5 +1,2 @@ -select - team_size - , description - , team_id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__teams_courseteam') }} +select team_size, description, team_id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__teams_courseteam") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__teams_membership.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__teams_membership.sql index d500ab30a..795bd80c5 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__teams_membership.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__teams_membership.sql @@ -1,14 +1,11 @@ -with teams_courseteam as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__teams_courseteam') }} -) +with + teams_courseteam as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__teams_courseteam") }} + ), + teams_courseteammembership as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__teams_courseteammembership") }} + ) -, teams_courseteammembership as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__teams_courseteammembership') }} -) - -select - teams_courseteammembership.id - , teams_courseteammembership.user_id - , teams_courseteammembership.team_id +select teams_courseteammembership.id, teams_courseteammembership.user_id, teams_courseteammembership.team_id from teams_courseteam inner join teams_courseteammembership on teams_courseteam.id = teams_courseteammembership.team_id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__user_api_usercoursetag.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__user_api_usercoursetag.sql index 9fa259e87..75a70f5aa 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__user_api_usercoursetag.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__user_api_usercoursetag.sql @@ -1,5 +1,2 @@ -select - course_id - , id - , user_id -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__user_api_usercoursetag') }} +select course_id, id, user_id +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__user_api_usercoursetag") }} diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__user_id_map.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__user_id_map.sql index f98a22894..85d27f6e4 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__user_id_map.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__user_id_map.sql @@ -1,19 +1,13 @@ -with student_anonymoususerid as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__student_anonymoususerid') }} -) +with + student_anonymoususerid as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__student_anonymoususerid") }} + ), + auth_user as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__auth_user") }}), + student_courseenrollment as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__student_courseenrollment") }} + ) -, auth_user as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__auth_user') }} -) - -, student_courseenrollment as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__student_courseenrollment') }} -) - -select - student_anonymoususerid.anonymous_user_id as hash_id - , student_anonymoususerid.user_id - , auth_user.username +select student_anonymoususerid.anonymous_user_id as hash_id, student_anonymoususerid.user_id, auth_user.username from student_anonymoususerid inner join auth_user on student_anonymoususerid.user_id = auth_user.id inner join student_courseenrollment on auth_user.id = student_courseenrollment.user_id diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__workflow_assessmentworkflow.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__workflow_assessmentworkflow.sql index add748467..a75570b88 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__workflow_assessmentworkflow.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__workflow_assessmentworkflow.sql @@ -1,12 +1,8 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__workflow_assessmentworkflow') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__workflow_assessmentworkflow") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }} -select - course_id - , status - , modified - , id + {{ deduplicate_raw_table(order_by="modified", partition_columns="id") }} +select course_id, status, modified, id from most_recent_source diff --git a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__workflow_assessmentworkflowstep.sql b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__workflow_assessmentworkflowstep.sql index 2d2afc501..8a494b9ff 100644 --- a/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__workflow_assessmentworkflowstep.sql +++ b/src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__workflow_assessmentworkflowstep.sql @@ -1,12 +1,9 @@ -select - workflow_id - , skipped - , order_num - , assessment_completed_at -from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__workflow_assessmentworkflowstep') }} -where workflow_id in ( - select assessmentworkflow.id - from - {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__workflow_assessmentworkflow') }} +select workflow_id, skipped, order_num, assessment_completed_at +from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__workflow_assessmentworkflowstep") }} +where + workflow_id in ( + select assessmentworkflow.id + from + {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__workflow_assessmentworkflow") }} as assessmentworkflow -) + ) diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__bigquery__email_opt_in.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__bigquery__email_opt_in.sql index f3267fba4..5eaf1dc1b 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__bigquery__email_opt_in.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__bigquery__email_opt_in.sql @@ -1,13 +1,5 @@ -with email_opt_in as ( - select * from {{ ref('int__mitxonline__bulk_email_optin') }} -) +with email_opt_in as (select * from {{ ref("int__mitxonline__bulk_email_optin") }}) select - openedx_user_id - , user_full_name - , user_username - , user_email - , courserun_readable_id - , courserun_title - , email_opted_in + openedx_user_id, user_full_name, user_username, user_email, courserun_readable_id, courserun_title, email_opted_in from email_opt_in diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessment.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessment.sql index 2cf99b34d..d15ae7780 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessment.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessment.sql @@ -1,19 +1,15 @@ -with a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessment') }} -) +with + a as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessment") }} + ), + s as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_submission") }} + ), + si as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_studentitem") }} + ) -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_studentitem') }} -) - -select - a.scorer_id - , a.id - , a.score_type +select a.scorer_id, a.id, a.score_type from a inner join s on a.submission_uuid = s.uuid inner join si on s.student_item_id = si.id diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedback.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedback.sql index 423814f22..f5cb42d07 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedback.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedback.sql @@ -1,31 +1,29 @@ -with af as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessmentfeedback') }} -) +with + af as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessmentfeedback") }} + ), + afa as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitxonline__openedx__mysql__assessment_assessmentfeedback_assessments", + ) + }} + ), + a as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessment") }} + ), + s as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_submission") }} + ), + si as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_studentitem") }} + ) -, afa as ( - select * - from - {{ - source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessmentfeedback_assessments') - }} -) - -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_studentitem') }} -) - -select distinct - af.id - , af.submission_uuid - , af.feedback_text +select distinct af.id, af.submission_uuid, af.feedback_text from af inner join afa on af.id = afa.assessmentfeedback_id inner join a on afa.assessment_id = a.id diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedback_assessments.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedback_assessments.sql index dcbfd97fb..3ccc3d381 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedback_assessments.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedback_assessments.sql @@ -1,27 +1,25 @@ -with afa as ( - select * - from - {{ - source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessmentfeedback_assessments') - }} -) +with + afa as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitxonline__openedx__mysql__assessment_assessmentfeedback_assessments", + ) + }} + ), + a as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessment") }} + ), + s as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_submission") }} + ), + si as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_studentitem") }} + ) -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_studentitem') }} -) - -select - afa.assessment_id - , afa.assessmentfeedback_id - , afa.id +select afa.assessment_id, afa.assessmentfeedback_id, afa.id from afa inner join a on afa.assessment_id = a.id inner join s on a.submission_uuid = s.uuid diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedback_options.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedback_options.sql index f8d6bf9d8..55456e0e6 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedback_options.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedback_options.sql @@ -1,36 +1,38 @@ -with afo as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessmentfeedback_options') }} -) +with + afo as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessmentfeedback_options" + ) + }} + ), + af as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessmentfeedback") }} + ), + afa as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitxonline__openedx__mysql__assessment_assessmentfeedback_assessments", + ) + }} + ), + a as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessment") }} + ), + s as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_submission") }} + ), + si as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_studentitem") }} + ) -, af as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessmentfeedback') }} -) - -, afa as ( - select * - from - {{ - source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessmentfeedback_assessments') - }} -) - -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_studentitem') }} -) - -select distinct - afo.assessmentfeedbackoption_id - , afo.assessmentfeedback_id - , afo.id +select distinct afo.assessmentfeedbackoption_id, afo.assessmentfeedback_id, afo.id from afo inner join af on afo.assessmentfeedback_id = af.id inner join afa on af.id = afa.assessmentfeedback_id diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedbackoption.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedbackoption.sql index a0a8dc20b..d5f9e2de5 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedbackoption.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentfeedbackoption.sql @@ -1,40 +1,47 @@ -with aafo as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessmentfeedbackoption') }} -) +with + aafo as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessmentfeedbackoption" + ) + }} + ), + afo as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessmentfeedback_options" + ) + }} + ), + af as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessmentfeedback") }} + ), + afa as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitxonline__openedx__mysql__assessment_assessmentfeedback_assessments", + ) + }} + ), + a as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessment") }} + ), + s as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_submission") }} + ), + si as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_studentitem") }} + ) -, afo as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessmentfeedback_options') }} -) - -, af as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessmentfeedback') }} -) - -, afa as ( - select * - from - {{ - source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessmentfeedback_assessments') - }} -) - -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_studentitem') }} -) - -select distinct - aafo.text - , aafo.id +select distinct aafo.text, aafo.id from aafo inner join afo on aafo.id = afo.assessmentfeedbackoption_id inner join af on afo.assessmentfeedback_id = af.id diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentpart.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentpart.sql index 692110073..116ad1b38 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentpart.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_assessmentpart.sql @@ -1,24 +1,19 @@ -with ap as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessmentpart') }} -) +with + ap as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessmentpart") }} + ), + a as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_assessment") }} + ), + s as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_submission") }} + ), + si as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_studentitem") }} + ) -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_studentitem') }} -) - -select - ap.assessment_id - , ap.option_id - , ap.id - , ap.feedback +select ap.assessment_id, ap.option_id, ap.id, ap.feedback from ap inner join a on ap.assessment_id = a.id inner join s on a.submission_uuid = s.uuid diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_peerworkflow.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_peerworkflow.sql index 9afe4bd31..918f18b95 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_peerworkflow.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_peerworkflow.sql @@ -1,6 +1,2 @@ -select - cancelled_at - , grading_completed_at - , student_id - , course_id -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_peerworkflow') }} +select cancelled_at, grading_completed_at, student_id, course_id +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_peerworkflow") }} diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_peerworkflowitem.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_peerworkflowitem.sql index e4f7d6535..2d2bc7735 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_peerworkflowitem.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_peerworkflowitem.sql @@ -1,6 +1,2 @@ -select - started_at - , assessment_id - , author_id - , id -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_peerworkflowitem') }} +select started_at, assessment_id, author_id, id +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_peerworkflowitem") }} diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_studenttrainingworkflow.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_studenttrainingworkflow.sql index 0d03912fb..dd182764c 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_studenttrainingworkflow.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_studenttrainingworkflow.sql @@ -1,6 +1,2 @@ -select - student_id - , course_id - , id - , item_id -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_studenttrainingworkflow') }} +select student_id, course_id, id, item_id +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_studenttrainingworkflow") }} diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_studenttrainingworkflowitem.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_studenttrainingworkflowitem.sql index 9f871c81e..03a928fac 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_studenttrainingworkflowitem.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__assessment_studenttrainingworkflowitem.sql @@ -1,6 +1,2 @@ -select - workflow_id - , order_num - , started_at - , id -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__assessment_studenttrainingworkflowitem') }} +select workflow_id, order_num, started_at, id +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__assessment_studenttrainingworkflowitem") }} diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__auth_user.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__auth_user.sql index bbfc6a0bb..b8aa634ba 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__auth_user.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__auth_user.sql @@ -1,33 +1,31 @@ -with auth_user as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__auth_user') }} -) - -, student_courseenrollment as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__student_courseenrollment') }} -) +with + auth_user as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__auth_user") }}), + student_courseenrollment as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__student_courseenrollment") }} + ) select - auth_user.id - , auth_user.username - , auth_user.first_name - , auth_user.last_name - , auth_user.email - , '' as pass_word - , auth_user.is_staff - , auth_user.is_active - , auth_user.is_superuser - , auth_user.last_login - , auth_user.date_joined - , '' as status - , '' as email_key - , '' as avatar_type - , '' as country - , 0 as show_country - , '' as date_of_birth - , '' as interesting_tags - , '' as ignored_tags - , 0 as email_tag_filter_strategy - , 0 as display_tag_filter_strategy - , 0 as consecutive_days_visit_count + auth_user.id, + auth_user.username, + auth_user.first_name, + auth_user.last_name, + auth_user.email, + '' as pass_word, + auth_user.is_staff, + auth_user.is_active, + auth_user.is_superuser, + auth_user.last_login, + auth_user.date_joined, + '' as status, + '' as email_key, + '' as avatar_type, + '' as country, + 0 as show_country, + '' as date_of_birth, + '' as interesting_tags, + '' as ignored_tags, + 0 as email_tag_filter_strategy, + 0 as display_tag_filter_strategy, + 0 as consecutive_days_visit_count from auth_user inner join student_courseenrollment on auth_user.id = student_courseenrollment.user_id diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__auth_userprofile.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__auth_userprofile.sql index ad0178c3f..e36df5f3a 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__auth_userprofile.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__auth_userprofile.sql @@ -1,28 +1,28 @@ -with auth_userprofile as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__auth_userprofile') }} -) - -, student_courseenrollment as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__student_courseenrollment') }} -) +with + auth_userprofile as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__auth_userprofile") }} + ), + student_courseenrollment as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__student_courseenrollment") }} + ) select - auth_userprofile.id - , auth_userprofile.user_id - , auth_userprofile.name - , auth_userprofile.language - , auth_userprofile.location - , auth_userprofile.meta - , auth_userprofile.courseware - , auth_userprofile.gender - , auth_userprofile.mailing_address - , auth_userprofile.year_of_birth - , auth_userprofile.level_of_education - , auth_userprofile.goals - , auth_userprofile.country - , auth_userprofile.city - , auth_userprofile.bio - , auth_userprofile.profile_image_uploaded_at - , auth_userprofile.state + auth_userprofile.id, + auth_userprofile.user_id, + auth_userprofile.name, + auth_userprofile.language, + auth_userprofile.location, + auth_userprofile.meta, + auth_userprofile.courseware, + auth_userprofile.gender, + auth_userprofile.mailing_address, + auth_userprofile.year_of_birth, + auth_userprofile.level_of_education, + auth_userprofile.goals, + auth_userprofile.country, + auth_userprofile.city, + auth_userprofile.bio, + auth_userprofile.profile_image_uploaded_at, + auth_userprofile.state from auth_userprofile inner join student_courseenrollment on auth_userprofile.user_id = student_courseenrollment.user_id diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__certificates_generatedcertificate.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__certificates_generatedcertificate.sql index f0bf1a445..133728165 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__certificates_generatedcertificate.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__certificates_generatedcertificate.sql @@ -1,23 +1,24 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__certificates_generatedcertificate') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__certificates_generatedcertificate") }} + ) -{{ deduplicate_raw_table(order_by='modified_date' , partition_columns = 'id') }} + {{ deduplicate_raw_table(order_by="modified_date", partition_columns="id") }} select - id - , user_id - , course_id - , key - , name - , distinction - , verify_uuid - , download_uuid - , download_url - , grade - , created_date - , modified_date - , mode - , status - , error_reason + id, + user_id, + course_id, + key, + name, + distinction, + verify_uuid, + download_uuid, + download_url, + grade, + created_date, + modified_date, + mode, + status, + error_reason from most_recent_source diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__course_groups_cohortmembership.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__course_groups_cohortmembership.sql index aa90a0aae..2fd3cc808 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__course_groups_cohortmembership.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__course_groups_cohortmembership.sql @@ -1,16 +1,13 @@ -with cm as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__course_groups_cohortmembership') }} -) +with + cm as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__course_groups_cohortmembership") }} + ), + cug as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__course_groups_courseusergroup") }} + ) -, cug as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__course_groups_courseusergroup') }} -) - -select - cm.user_id - , cm.course_id - , cug.group_type - , cug.name +select cm.user_id, cm.course_id, cug.group_type, cug.name from cm inner join cug on cm.course_user_group_id = cug.id diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__credit_crediteligibility.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__credit_crediteligibility.sql index 0da7d3859..aa7b0a487 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__credit_crediteligibility.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__credit_crediteligibility.sql @@ -1,18 +1,10 @@ -with ce as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__credit_crediteligibility') }} -) +with + ce as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__credit_crediteligibility") }} + ), + cc as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__credit_creditcourse") }}) -, cc as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__credit_creditcourse') }} -) - -select - ce.id - , ce.created - , ce.modified - , ce.username - , ce.deadline - , cc.course_key as course_id +select ce.id, ce.created, ce.modified, ce.username, ce.deadline, cc.course_key as course_id from ce inner join cc on ce.course_id = cc.id order by ce.username diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__django_comment_client_role_users.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__django_comment_client_role_users.sql index d3109b292..c99c6c1ae 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__django_comment_client_role_users.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__django_comment_client_role_users.sql @@ -1,16 +1,13 @@ -with dccr as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__django_comment_client_role') }} -) +with + dccr as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__django_comment_client_role") }} + ), + dccru as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__django_comment_client_role_users") }} + ) -, dccru as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__django_comment_client_role_users') }} -) - -select - dccr.course_id - , dccru.user_id - , dccr.name +select dccr.course_id, dccru.user_id, dccr.name from dccr inner join dccru on dccr.id = dccru.role_id diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__grades_persistentcoursegrade.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__grades_persistentcoursegrade.sql index 852100a7a..0dc348426 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__grades_persistentcoursegrade.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__grades_persistentcoursegrade.sql @@ -1,16 +1,10 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__grades_persistentcoursegrade') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__grades_persistentcoursegrade") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }} -select - course_id - , user_id - , grading_policy_hash - , percent_grade - , letter_grade - , passed_timestamp - , created - , modified + {{ deduplicate_raw_table(order_by="modified", partition_columns="id") }} +select course_id, user_id, grading_policy_hash, percent_grade, letter_grade, passed_timestamp, created, modified from most_recent_source order by user_id diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__grades_persistentsubsectiongrade.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__grades_persistentsubsectiongrade.sql index fcbfd2584..da2257476 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__grades_persistentsubsectiongrade.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__grades_persistentsubsectiongrade.sql @@ -1,19 +1,20 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__grades_persistentsubsectiongrade') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__grades_persistentsubsectiongrade") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }} + {{ deduplicate_raw_table(order_by="modified", partition_columns="id") }} select - course_id - , user_id - , usage_key - , earned_all - , possible_all - , earned_graded - , possible_graded - , first_attempted - , created - , modified + course_id, + user_id, + usage_key, + earned_all, + possible_all, + earned_graded, + possible_graded, + first_attempted, + created, + modified from most_recent_source order by user_id, first_attempted diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_anonymoususerid.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_anonymoususerid.sql index ee2891f10..2c2b730c3 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_anonymoususerid.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_anonymoususerid.sql @@ -1,5 +1,2 @@ -select - anonymous_user_id - , course_id - , id -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__student_anonymoususerid') }} +select anonymous_user_id, course_id, id +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__student_anonymoususerid") }} diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_courseaccessrole.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_courseaccessrole.sql index c1aca5157..2d1e64dad 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_courseaccessrole.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_courseaccessrole.sql @@ -1,6 +1,2 @@ -select - org - , course_id - , user_id - , role -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__student_courseaccessrole') }} +select org, course_id, user_id, role +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__student_courseaccessrole") }} diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_courseenrollment.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_courseenrollment.sql index a835e51d2..bb103438f 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_courseenrollment.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_courseenrollment.sql @@ -1,6 +1,2 @@ -select - course_id - , mode - , id - , is_active -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__student_courseenrollment') }} +select course_id, mode, id, is_active +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__student_courseenrollment") }} diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_languageproficiency.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_languageproficiency.sql index 99b05d463..77f39416d 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_languageproficiency.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__student_languageproficiency.sql @@ -1,19 +1,16 @@ -with student_languageproficiency as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__student_languageproficiency') }} -) +with + student_languageproficiency as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__student_languageproficiency") }} + ), + auth_userprofile as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__auth_userprofile") }} + ), + student_courseenrollment as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__student_courseenrollment") }} + ) -, auth_userprofile as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__auth_userprofile') }} -) - -, student_courseenrollment as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__student_courseenrollment') }} -) - -select - student_languageproficiency.user_profile_id - , student_languageproficiency.id - , student_languageproficiency.code +select student_languageproficiency.user_profile_id, student_languageproficiency.id, student_languageproficiency.code from student_languageproficiency inner join auth_userprofile on student_languageproficiency.user_profile_id = auth_userprofile.user_id inner join student_courseenrollment on auth_userprofile.user_id = student_courseenrollment.user_id diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_score.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_score.sql index 14a948f5d..35ea5e4da 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_score.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_score.sql @@ -1,11 +1,9 @@ -select - points_earned - , reset - , submission_id - , id - , student_item_id -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_score') }} -where student_item_id in ( - select studentitem.id - from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_studentitem') }} as studentitem -) +select points_earned, reset, submission_id, id, student_item_id +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_score") }} +where + student_item_id in ( + select studentitem.id + from + {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_studentitem") }} + as studentitem + ) diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_scoresummary.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_scoresummary.sql index ee915f1f1..4fb828709 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_scoresummary.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_scoresummary.sql @@ -1,10 +1,9 @@ -select - highest_id - , id - , latest_id - , student_item_id -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_scoresummary') }} -where student_item_id in ( - select studentitem.id - from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_studentitem') }} as studentitem -) +select highest_id, id, latest_id, student_item_id +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_scoresummary") }} +where + student_item_id in ( + select studentitem.id + from + {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_studentitem") }} + as studentitem + ) diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_studentitem.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_studentitem.sql index b2b168c37..9b83325ef 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_studentitem.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_studentitem.sql @@ -1,6 +1,2 @@ -select - student_id - , course_id - , id - , item_id -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_studentitem') }} +select student_id, course_id, id, item_id +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_studentitem") }} diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_submission.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_submission.sql index c94af9759..4794f5fad 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_submission.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__submissions_submission.sql @@ -1,10 +1,9 @@ -select - team_submission_id - , submitted_at - , status - , id -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_submission') }} -where student_item_id in ( - select studentitem.id - from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__submissions_studentitem') }} as studentitem -) +select team_submission_id, submitted_at, status, id +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_submission") }} +where + student_item_id in ( + select studentitem.id + from + {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__submissions_studentitem") }} + as studentitem + ) diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__teams.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__teams.sql index 01361ddd5..1c63c456a 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__teams.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__teams.sql @@ -1,5 +1,2 @@ -select - team_size - , description - , team_id -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__teams_courseteam') }} +select team_size, description, team_id +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__teams_courseteam") }} diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__teams_membership.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__teams_membership.sql index 738c9c674..3b8265308 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__teams_membership.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__teams_membership.sql @@ -1,14 +1,12 @@ -with teams_courseteam as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__teams_courseteam') }} -) +with + teams_courseteam as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__teams_courseteam") }} + ), + teams_courseteammembership as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__teams_courseteammembership") }} + ) -, teams_courseteammembership as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__teams_courseteammembership') }} -) - -select - teams_courseteammembership.id - , teams_courseteammembership.user_id - , teams_courseteammembership.team_id +select teams_courseteammembership.id, teams_courseteammembership.user_id, teams_courseteammembership.team_id from teams_courseteam inner join teams_courseteammembership on teams_courseteam.id = teams_courseteammembership.team_id diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__user_api_usercoursetag.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__user_api_usercoursetag.sql index 716af8b56..997461f8d 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__user_api_usercoursetag.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__user_api_usercoursetag.sql @@ -1,5 +1,2 @@ -select - course_id - , id - , user_id -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__user_api_usercoursetag') }} +select course_id, id, user_id +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__user_api_usercoursetag") }} diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__user_id_map.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__user_id_map.sql index 1d56751f2..d3c5d4a44 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__user_id_map.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__user_id_map.sql @@ -1,19 +1,13 @@ -with student_anonymoususerid as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__student_anonymoususerid') }} -) +with + student_anonymoususerid as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__student_anonymoususerid") }} + ), + auth_user as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__auth_user") }}), + student_courseenrollment as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__student_courseenrollment") }} + ) -, auth_user as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__auth_user') }} -) - -, student_courseenrollment as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__student_courseenrollment') }} -) - -select - student_anonymoususerid.anonymous_user_id as hash_id - , student_anonymoususerid.user_id - , auth_user.username +select student_anonymoususerid.anonymous_user_id as hash_id, student_anonymoususerid.user_id, auth_user.username from student_anonymoususerid inner join auth_user on student_anonymoususerid.user_id = auth_user.id inner join student_courseenrollment on auth_user.id = student_courseenrollment.user_id diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__workflow_assessmentworkflow.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__workflow_assessmentworkflow.sql index 5892d21b5..6205d8d48 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__workflow_assessmentworkflow.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__workflow_assessmentworkflow.sql @@ -1,12 +1,9 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__workflow_assessmentworkflow') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__workflow_assessmentworkflow") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }} -select - course_id - , status - , modified - , id + {{ deduplicate_raw_table(order_by="modified", partition_columns="id") }} +select course_id, status, modified, id from most_recent_source diff --git a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__workflow_assessmentworkflowstep.sql b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__workflow_assessmentworkflowstep.sql index de1b221b6..2ad041eb8 100644 --- a/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__workflow_assessmentworkflowstep.sql +++ b/src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__workflow_assessmentworkflowstep.sql @@ -1,12 +1,9 @@ -select - workflow_id - , skipped - , order_num - , assessment_completed_at -from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__workflow_assessmentworkflowstep') }} -where workflow_id in ( - select assessmentworkflow.id - from - {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__workflow_assessmentworkflow') }} +select workflow_id, skipped, order_num, assessment_completed_at +from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__workflow_assessmentworkflowstep") }} +where + workflow_id in ( + select assessmentworkflow.id + from + {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__workflow_assessmentworkflow") }} as assessmentworkflow -) + ) diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__bigquery__email_opt_in.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__bigquery__email_opt_in.sql index 451a92cae..4ad529998 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__bigquery__email_opt_in.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__bigquery__email_opt_in.sql @@ -1,11 +1,4 @@ -with email_opt_in as ( - select * from {{ source('ol_warehouse_raw_data','raw__irx__edxorg__bigquery__email_opt_in') }} -) +with email_opt_in as (select * from {{ source("ol_warehouse_raw_data", "raw__irx__edxorg__bigquery__email_opt_in") }}) -select - is_opted_in_for_email - , course_id - , full_name - , preference_set_datetime - , user_id +select is_opted_in_for_email, course_id, full_name, preference_set_datetime, user_id from email_opt_in diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessment.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessment.sql index 95119797a..4c9fbf2f7 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessment.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessment.sql @@ -1,19 +1,9 @@ -with a as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessment') }} -) +with + a as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_studentitem") }}) -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_studentitem') }} -) - -select - a.scorer_id - , a.id - , a.score_type +select a.scorer_id, a.id, a.score_type from a inner join s on a.submission_uuid = s.uuid inner join si on s.student_item_id = si.id diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedback.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedback.sql index 5e04a82f6..8d3f73855 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedback.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedback.sql @@ -1,31 +1,21 @@ -with af as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessmentfeedback') }} -) +with + af as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessmentfeedback") }} + ), + afa as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessmentfeedback_assessments" + ) + }} + ), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_studentitem") }}) -, afa as ( - select * - from - {{ - source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessmentfeedback_assessments') - }} -) - -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_studentitem') }} -) - -select distinct - af.id - , af.submission_uuid - , af.feedback_text +select distinct af.id, af.submission_uuid, af.feedback_text from af inner join afa on af.id = afa.assessmentfeedback_id inner join a on afa.assessment_id = a.id diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedback_assessments.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedback_assessments.sql index 7f3288f31..b7b486eba 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedback_assessments.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedback_assessments.sql @@ -1,27 +1,18 @@ -with afa as ( - select * - from - {{ - source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessmentfeedback_assessments') - }} -) +with + afa as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessmentfeedback_assessments" + ) + }} + ), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_studentitem") }}) -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_studentitem') }} -) - -select - afa.assessment_id - , afa.assessmentfeedback_id - , afa.id +select afa.assessment_id, afa.assessmentfeedback_id, afa.id from afa inner join a on afa.assessment_id = a.id inner join s on a.submission_uuid = s.uuid diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedback_options.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedback_options.sql index 82b996763..7e9d8abae 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedback_options.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedback_options.sql @@ -1,36 +1,25 @@ -with afo as ( - select * - from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessmentfeedback_options') }} -) +with + afo as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessmentfeedback_options") }} + ), + af as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessmentfeedback") }} + ), + afa as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessmentfeedback_assessments" + ) + }} + ), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_studentitem") }}) -, af as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessmentfeedback') }} -) - -, afa as ( - select * - from - {{ - source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessmentfeedback_assessments') - }} -) - -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_studentitem') }} -) - -select distinct - afo.assessmentfeedbackoption_id - , afo.assessmentfeedback_id - , afo.id +select distinct afo.assessmentfeedbackoption_id, afo.assessmentfeedback_id, afo.id from afo inner join af on afo.assessmentfeedback_id = af.id inner join afa on af.id = afa.assessmentfeedback_id diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedbackoption.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedbackoption.sql index f5cd19fab..7b83f55f0 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedbackoption.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentfeedbackoption.sql @@ -1,40 +1,29 @@ -with aafo as ( - select * - from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessmentfeedbackoption') }} -) +with + aafo as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessmentfeedbackoption") }} + ), + afo as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessmentfeedback_options") }} + ), + af as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessmentfeedback") }} + ), + afa as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessmentfeedback_assessments" + ) + }} + ), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_studentitem") }}) -, afo as ( - select * - from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessmentfeedback_options') }} -) - -, af as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessmentfeedback') }} -) - -, afa as ( - select * - from - {{ - source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessmentfeedback_assessments') - }} -) - -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_studentitem') }} -) - -select distinct - aafo.text - , aafo.id +select distinct aafo.text, aafo.id from aafo inner join afo on aafo.id = afo.assessmentfeedbackoption_id inner join af on afo.assessmentfeedback_id = af.id diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentpart.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentpart.sql index a8a9c3134..0e78a051e 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentpart.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_assessmentpart.sql @@ -1,24 +1,10 @@ -with ap as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessmentpart') }} -) +with + ap as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessmentpart") }}), + a as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_assessment") }}), + s as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_submission") }}), + si as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_studentitem") }}) -, a as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_assessment') }} -) - -, s as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_submission') }} -) - -, si as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_studentitem') }} -) - -select - ap.assessment_id - , ap.option_id - , ap.id - , ap.feedback +select ap.assessment_id, ap.option_id, ap.id, ap.feedback from ap inner join a on ap.assessment_id = a.id inner join s on a.submission_uuid = s.uuid diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_peerworkflow.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_peerworkflow.sql index 1f32a6aaf..368cb4a4b 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_peerworkflow.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_peerworkflow.sql @@ -1,6 +1,2 @@ -select - cancelled_at - , grading_completed_at - , student_id - , course_id -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_peerworkflow') }} +select cancelled_at, grading_completed_at, student_id, course_id +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_peerworkflow") }} diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_peerworkflowitem.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_peerworkflowitem.sql index f87010a7d..4854a651d 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_peerworkflowitem.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_peerworkflowitem.sql @@ -1,6 +1,2 @@ -select - started_at - , assessment_id - , author_id - , id -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_peerworkflowitem') }} +select started_at, assessment_id, author_id, id +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_peerworkflowitem") }} diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_studenttrainingworkflow.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_studenttrainingworkflow.sql index 8444260bf..b4b9214b9 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_studenttrainingworkflow.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_studenttrainingworkflow.sql @@ -1,6 +1,2 @@ -select - student_id - , course_id - , id - , item_id -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_studenttrainingworkflow') }} +select student_id, course_id, id, item_id +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_studenttrainingworkflow") }} diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_studenttrainingworkflowitem.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_studenttrainingworkflowitem.sql index 5763e2cf2..2e12ec909 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_studenttrainingworkflowitem.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__assessment_studenttrainingworkflowitem.sql @@ -1,6 +1,2 @@ -select - workflow_id - , order_num - , started_at - , id -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__assessment_studenttrainingworkflowitem') }} +select workflow_id, order_num, started_at, id +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__assessment_studenttrainingworkflowitem") }} diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__auth_user.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__auth_user.sql index 55f1ec912..06932cbb8 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__auth_user.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__auth_user.sql @@ -1,33 +1,31 @@ -with auth_user as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__auth_user') }} -) - -, student_courseenrollment as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__student_courseenrollment') }} -) +with + auth_user as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__auth_user") }}), + student_courseenrollment as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__student_courseenrollment") }} + ) select - auth_user.id - , auth_user.username - , auth_user.first_name - , auth_user.last_name - , auth_user.email - , '' as pass_word - , auth_user.is_staff - , auth_user.is_active - , auth_user.is_superuser - , auth_user.last_login - , auth_user.date_joined - , '' as status - , '' as email_key - , '' as avatar_type - , '' as country - , 0 as show_country - , '' as date_of_birth - , '' as interesting_tags - , '' as ignored_tags - , 0 as email_tag_filter_strategy - , 0 as display_tag_filter_strategy - , 0 as consecutive_days_visit_count + auth_user.id, + auth_user.username, + auth_user.first_name, + auth_user.last_name, + auth_user.email, + '' as pass_word, + auth_user.is_staff, + auth_user.is_active, + auth_user.is_superuser, + auth_user.last_login, + auth_user.date_joined, + '' as status, + '' as email_key, + '' as avatar_type, + '' as country, + 0 as show_country, + '' as date_of_birth, + '' as interesting_tags, + '' as ignored_tags, + 0 as email_tag_filter_strategy, + 0 as display_tag_filter_strategy, + 0 as consecutive_days_visit_count from auth_user inner join student_courseenrollment on auth_user.id = student_courseenrollment.user_id diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__auth_userprofile.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__auth_userprofile.sql index 5c4e22903..3a158d40f 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__auth_userprofile.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__auth_userprofile.sql @@ -1,28 +1,28 @@ -with auth_userprofile as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__auth_userprofile') }} -) - -, student_courseenrollment as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__student_courseenrollment') }} -) +with + auth_userprofile as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__auth_userprofile") }} + ), + student_courseenrollment as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__student_courseenrollment") }} + ) select - auth_userprofile.id - , auth_userprofile.user_id - , auth_userprofile.name - , auth_userprofile.language - , auth_userprofile.location - , auth_userprofile.meta - , auth_userprofile.courseware - , auth_userprofile.gender - , auth_userprofile.mailing_address - , auth_userprofile.year_of_birth - , auth_userprofile.level_of_education - , auth_userprofile.goals - , auth_userprofile.country - , auth_userprofile.city - , auth_userprofile.bio - , auth_userprofile.profile_image_uploaded_at - , auth_userprofile.state + auth_userprofile.id, + auth_userprofile.user_id, + auth_userprofile.name, + auth_userprofile.language, + auth_userprofile.location, + auth_userprofile.meta, + auth_userprofile.courseware, + auth_userprofile.gender, + auth_userprofile.mailing_address, + auth_userprofile.year_of_birth, + auth_userprofile.level_of_education, + auth_userprofile.goals, + auth_userprofile.country, + auth_userprofile.city, + auth_userprofile.bio, + auth_userprofile.profile_image_uploaded_at, + auth_userprofile.state from auth_userprofile inner join student_courseenrollment on auth_userprofile.user_id = student_courseenrollment.user_id diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__certificates_generatedcertificate.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__certificates_generatedcertificate.sql index 14c821f43..805903141 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__certificates_generatedcertificate.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__certificates_generatedcertificate.sql @@ -1,23 +1,24 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__certificates_generatedcertificate') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__certificates_generatedcertificate") }} + ) -{{ deduplicate_raw_table(order_by='modified_date' , partition_columns = 'id') }} + {{ deduplicate_raw_table(order_by="modified_date", partition_columns="id") }} select - id - , user_id - , course_id - , key - , name - , distinction - , verify_uuid - , download_uuid - , download_url - , grade - , created_date - , modified_date - , mode - , status - , error_reason + id, + user_id, + course_id, + key, + name, + distinction, + verify_uuid, + download_uuid, + download_url, + grade, + created_date, + modified_date, + mode, + status, + error_reason from most_recent_source diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__course_groups_cohortmembership.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__course_groups_cohortmembership.sql index 0d1f8a221..8f29605a7 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__course_groups_cohortmembership.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__course_groups_cohortmembership.sql @@ -1,16 +1,11 @@ -with cm as ( - select * - from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__course_groups_cohortmembership') }} -) +with + cm as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__course_groups_cohortmembership") }} + ), + cug as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__course_groups_courseusergroup") }} + ) -, cug as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__course_groups_courseusergroup') }} -) - -select - cm.user_id - , cm.course_id - , cug.group_type - , cug.name +select cm.user_id, cm.course_id, cug.group_type, cug.name from cm inner join cug on cm.course_user_group_id = cug.id diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__credit_crediteligibility.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__credit_crediteligibility.sql index 6c329f63c..3b97f5367 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__credit_crediteligibility.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__credit_crediteligibility.sql @@ -1,18 +1,8 @@ -with ce as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__credit_crediteligibility') }} -) +with + ce as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__credit_crediteligibility") }}), + cc as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__credit_creditcourse") }}) -, cc as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__credit_creditcourse') }} -) - -select - ce.id - , ce.created - , ce.modified - , ce.username - , ce.deadline - , cc.course_key as course_id +select ce.id, ce.created, ce.modified, ce.username, ce.deadline, cc.course_key as course_id from ce inner join cc on ce.course_id = cc.id order by ce.username diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__django_comment_client_role_users.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__django_comment_client_role_users.sql index b35dde95e..76b45560c 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__django_comment_client_role_users.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__django_comment_client_role_users.sql @@ -1,16 +1,12 @@ -with dccr as ( - select * - from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__django_comment_client_role') }} -) +with + dccr as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__django_comment_client_role") }} + ), + dccru as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__django_comment_client_role_users") }} + ) -, dccru as ( - select * - from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__django_comment_client_role_users') }} -) - -select - dccr.course_id - , dccru.user_id - , dccr.name +select dccr.course_id, dccru.user_id, dccr.name from dccr inner join dccru on dccr.id = dccru.role_id diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__grades_persistentcoursegrade.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__grades_persistentcoursegrade.sql index 4e1659b79..2d79a7e75 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__grades_persistentcoursegrade.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__grades_persistentcoursegrade.sql @@ -1,16 +1,9 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__grades_persistentcoursegrade') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__grades_persistentcoursegrade") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }} -select - course_id - , user_id - , grading_policy_hash - , percent_grade - , letter_grade - , passed_timestamp - , created - , modified + {{ deduplicate_raw_table(order_by="modified", partition_columns="id") }} +select course_id, user_id, grading_policy_hash, percent_grade, letter_grade, passed_timestamp, created, modified from most_recent_source order by user_id diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__grades_persistentsubsectiongrade.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__grades_persistentsubsectiongrade.sql index cfd5908fb..d2878ccc8 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__grades_persistentsubsectiongrade.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__grades_persistentsubsectiongrade.sql @@ -1,19 +1,20 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__grades_persistentsubsectiongrade') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__grades_persistentsubsectiongrade") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }} + {{ deduplicate_raw_table(order_by="modified", partition_columns="id") }} select - course_id - , user_id - , usage_key - , earned_all - , possible_all - , earned_graded - , possible_graded - , first_attempted - , created - , modified + course_id, + user_id, + usage_key, + earned_all, + possible_all, + earned_graded, + possible_graded, + first_attempted, + created, + modified from most_recent_source order by user_id, first_attempted diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_anonymoususerid.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_anonymoususerid.sql index cdb512b63..691ca1a69 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_anonymoususerid.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_anonymoususerid.sql @@ -1,5 +1,2 @@ -select - anonymous_user_id - , course_id - , id -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__student_anonymoususerid') }} +select anonymous_user_id, course_id, id +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__student_anonymoususerid") }} diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_courseaccessrole.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_courseaccessrole.sql index ebe5a1fba..9fa172f03 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_courseaccessrole.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_courseaccessrole.sql @@ -1,6 +1,2 @@ -select - org - , course_id - , user_id - , role -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__student_courseaccessrole') }} +select org, course_id, user_id, role +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__student_courseaccessrole") }} diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_courseenrollment.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_courseenrollment.sql index 8b074b3a6..d1c85a07d 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_courseenrollment.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_courseenrollment.sql @@ -1,6 +1,2 @@ -select - course_id - , mode - , id - , is_active -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__student_courseenrollment') }} +select course_id, mode, id, is_active +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__student_courseenrollment") }} diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_languageproficiency.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_languageproficiency.sql index 5fffdb143..cc7df1ebe 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_languageproficiency.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__student_languageproficiency.sql @@ -1,19 +1,15 @@ -with student_languageproficiency as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__student_languageproficiency') }} -) +with + student_languageproficiency as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__student_languageproficiency") }} + ), + auth_userprofile as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__auth_userprofile") }} + ), + student_courseenrollment as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__student_courseenrollment") }} + ) -, auth_userprofile as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__auth_userprofile') }} -) - -, student_courseenrollment as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__student_courseenrollment') }} -) - -select - student_languageproficiency.user_profile_id - , student_languageproficiency.id - , student_languageproficiency.code +select student_languageproficiency.user_profile_id, student_languageproficiency.id, student_languageproficiency.code from student_languageproficiency inner join auth_userprofile on student_languageproficiency.user_profile_id = auth_userprofile.user_id inner join student_courseenrollment on auth_userprofile.user_id = student_courseenrollment.user_id diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_score.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_score.sql index c90fead76..e21a95510 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_score.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_score.sql @@ -1,11 +1,7 @@ -select - points_earned - , reset - , submission_id - , id - , student_item_id -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_score') }} -where student_item_id in ( - select studentitem.id - from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_studentitem') }} as studentitem -) +select points_earned, reset, submission_id, id, student_item_id +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_score") }} +where + student_item_id in ( + select studentitem.id + from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_studentitem") }} as studentitem + ) diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_scoresummary.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_scoresummary.sql index 949c0a673..8ba4491cd 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_scoresummary.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_scoresummary.sql @@ -1,10 +1,7 @@ -select - highest_id - , id - , latest_id - , student_item_id -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_scoresummary') }} -where student_item_id in ( - select studentitem.id - from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_studentitem') }} as studentitem -) +select highest_id, id, latest_id, student_item_id +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_scoresummary") }} +where + student_item_id in ( + select studentitem.id + from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_studentitem") }} as studentitem + ) diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_studentitem.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_studentitem.sql index 51a391d81..cf05bea9c 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_studentitem.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_studentitem.sql @@ -1,6 +1,2 @@ -select - student_id - , course_id - , id - , item_id -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_studentitem') }} +select student_id, course_id, id, item_id +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_studentitem") }} diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_submission.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_submission.sql index 8bb9fd8ee..633d60396 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_submission.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__submissions_submission.sql @@ -1,10 +1,7 @@ -select - team_submission_id - , submitted_at - , status - , id -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_submission') }} -where student_item_id in ( - select studentitem.id - from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__submissions_studentitem') }} as studentitem -) +select team_submission_id, submitted_at, status, id +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_submission") }} +where + student_item_id in ( + select studentitem.id + from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__submissions_studentitem") }} as studentitem + ) diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__teams.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__teams.sql index 04f2a1e3a..49d353f3d 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__teams.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__teams.sql @@ -1,5 +1,2 @@ -select - team_size - , description - , team_id -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__teams_courseteam') }} +select team_size, description, team_id +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__teams_courseteam") }} diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__teams_membership.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__teams_membership.sql index c19db1fe6..8288d91c5 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__teams_membership.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__teams_membership.sql @@ -1,14 +1,11 @@ -with teams_courseteam as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__teams_courseteam') }} -) +with + teams_courseteam as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__teams_courseteam") }} + ), + teams_courseteammembership as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__teams_courseteammembership") }} + ) -, teams_courseteammembership as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__teams_courseteammembership') }} -) - -select - teams_courseteammembership.id - , teams_courseteammembership.user_id - , teams_courseteammembership.team_id +select teams_courseteammembership.id, teams_courseteammembership.user_id, teams_courseteammembership.team_id from teams_courseteam inner join teams_courseteammembership on teams_courseteam.id = teams_courseteammembership.team_id diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__user_api_usercoursetag.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__user_api_usercoursetag.sql index c44240ec6..0f61cdb04 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__user_api_usercoursetag.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__user_api_usercoursetag.sql @@ -1,5 +1,2 @@ -select - course_id - , id - , user_id -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__user_api_usercoursetag') }} +select course_id, id, user_id +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__user_api_usercoursetag") }} diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__user_id_map.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__user_id_map.sql index 0e564a26b..23a8e19c1 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__user_id_map.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__user_id_map.sql @@ -1,19 +1,13 @@ -with student_anonymoususerid as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__student_anonymoususerid') }} -) +with + student_anonymoususerid as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__student_anonymoususerid") }} + ), + auth_user as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__auth_user") }}), + student_courseenrollment as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__student_courseenrollment") }} + ) -, auth_user as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__auth_user') }} -) - -, student_courseenrollment as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__student_courseenrollment') }} -) - -select - student_anonymoususerid.anonymous_user_id as hash_id - , student_anonymoususerid.user_id - , auth_user.username +select student_anonymoususerid.anonymous_user_id as hash_id, student_anonymoususerid.user_id, auth_user.username from student_anonymoususerid inner join auth_user on student_anonymoususerid.user_id = auth_user.id inner join student_courseenrollment on auth_user.id = student_courseenrollment.user_id diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__workflow_assessmentworkflow.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__workflow_assessmentworkflow.sql index 612cf0ba3..6b9d088df 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__workflow_assessmentworkflow.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__workflow_assessmentworkflow.sql @@ -1,12 +1,8 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__workflow_assessmentworkflow') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__workflow_assessmentworkflow") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }} -select - course_id - , status - , modified - , id + {{ deduplicate_raw_table(order_by="modified", partition_columns="id") }} +select course_id, status, modified, id from most_recent_source diff --git a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__workflow_assessmentworkflowstep.sql b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__workflow_assessmentworkflowstep.sql index 4de2d22a2..6a6430d61 100644 --- a/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__workflow_assessmentworkflowstep.sql +++ b/src/ol_dbt/models/external/irx/xpro/irx__xpro__openedx__mysql__workflow_assessmentworkflowstep.sql @@ -1,12 +1,9 @@ -select - workflow_id - , skipped - , order_num - , assessment_completed_at -from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__workflow_assessmentworkflowstep') }} -where workflow_id in ( - select assessmentworkflow.id - from - {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__workflow_assessmentworkflow') }} +select workflow_id, skipped, order_num, assessment_completed_at +from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__workflow_assessmentworkflowstep") }} +where + workflow_id in ( + select assessmentworkflow.id + from + {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__workflow_assessmentworkflow") }} as assessmentworkflow -) + ) diff --git a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__applications.sql b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__applications.sql index bc1cc7c0a..f2eca2429 100644 --- a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__applications.sql +++ b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__applications.sql @@ -1,100 +1,73 @@ -with applications as ( - select * from {{ ref('stg__bootcamps__app__postgres__applications_courserun_application') }} -) - -, runs as ( - select * from {{ ref('stg__bootcamps__app__postgres__courses_courserun') }} -) - -, application_steps as ( - select * from {{ ref('stg__bootcamps__app__postgres__applications_applicationstep') }} -) - -, application_last_steps as ( - select - course_id - , max(applicationstep_step_order) as last_step - from application_steps - group by course_id -) - -, courserun_applicationstep as ( - select * from {{ ref('stg__bootcamps__app__postgres__applications_courserun_applicationstep') }} -) - -, applicationstep_submissions as ( - select * from {{ ref('stg__bootcamps__app__postgres__applications_applicationstep_submission') }} -) - -, personal_prices as ( - select * from {{ ref('stg__bootcamps__app__postgres__courses_personalprice') }} -) - -, installments as ( - select - courserun_id - , sum(installment_amount) as list_price - from {{ ref('stg__bootcamps__app__postgres__courses_installment') }} - group by courserun_id -) - -, fulfilled_orders as ( - select - application_id - , sum(line_price) as total_paid - from {{ ref('int__bootcamps__ecommerce_order') }} - where order_state = 'fulfilled' - group by application_id -) - -, courserun_last_steps as ( - select - courserun_applicationstep.courserun_applicationstep_id - , courserun_applicationstep.courserun_id - , application_steps.applicationstep_step_order - from - courserun_applicationstep - inner join application_steps on courserun_applicationstep.applicationstep_id = application_steps.applicationstep_id - inner join application_last_steps - on - application_steps.course_id = application_last_steps.course_id +with + applications as (select * from {{ ref("stg__bootcamps__app__postgres__applications_courserun_application") }}), + runs as (select * from {{ ref("stg__bootcamps__app__postgres__courses_courserun") }}), + application_steps as (select * from {{ ref("stg__bootcamps__app__postgres__applications_applicationstep") }}), + application_last_steps as ( + select course_id, max(applicationstep_step_order) as last_step from application_steps group by course_id + ), + courserun_applicationstep as ( + select * from {{ ref("stg__bootcamps__app__postgres__applications_courserun_applicationstep") }} + ), + applicationstep_submissions as ( + select * from {{ ref("stg__bootcamps__app__postgres__applications_applicationstep_submission") }} + ), + personal_prices as (select * from {{ ref("stg__bootcamps__app__postgres__courses_personalprice") }}), + installments as ( + select courserun_id, sum(installment_amount) as list_price + from {{ ref("stg__bootcamps__app__postgres__courses_installment") }} + group by courserun_id + ), + fulfilled_orders as ( + select application_id, sum(line_price) as total_paid + from {{ ref("int__bootcamps__ecommerce_order") }} + where order_state = 'fulfilled' + group by application_id + ), + courserun_last_steps as ( + select + courserun_applicationstep.courserun_applicationstep_id, + courserun_applicationstep.courserun_id, + application_steps.applicationstep_step_order + from courserun_applicationstep + inner join + application_steps on courserun_applicationstep.applicationstep_id = application_steps.applicationstep_id + inner join + application_last_steps + on application_steps.course_id = application_last_steps.course_id and application_steps.applicationstep_step_order = application_last_steps.last_step -) + ) select - applications.application_id - , applications.user_id - , applications.courserun_id - , runs.courserun_readable_id - , runs.courserun_title - , runs.courserun_start_on - , runs.courserun_end_on - , applications.application_linkedin_url - , applications.application_resume_file - , applications.application_resume_uploaded_on - , applications.application_state - , applications.application_created_on - , applications.application_updated_on - , personal_prices.personalprice as personal_price - , installments.list_price - , fulfilled_orders.total_paid - , case + applications.application_id, + applications.user_id, + applications.courserun_id, + runs.courserun_readable_id, + runs.courserun_title, + runs.courserun_start_on, + runs.courserun_end_on, + applications.application_linkedin_url, + applications.application_resume_file, + applications.application_resume_uploaded_on, + applications.application_state, + applications.application_created_on, + applications.application_updated_on, + personal_prices.personalprice as personal_price, + installments.list_price, + fulfilled_orders.total_paid, + case when applicationstep_submissions.submission_review_status = 'approved' - then applicationstep_submissions.submission_reviewed_on + then applicationstep_submissions.submission_reviewed_on end as application_admitted_date from applications inner join runs on applications.courserun_id = runs.courserun_id -left join courserun_last_steps - on applications.courserun_id = courserun_last_steps.courserun_id -left join applicationstep_submissions - on - courserun_last_steps.courserun_applicationstep_id = applicationstep_submissions.courserun_applicationstep_id - and applications.application_id = applicationstep_submissions.application_id -left join installments - on applications.courserun_id = installments.courserun_id -left join personal_prices - on - applications.user_id = personal_prices.user_id - and applications.courserun_id = personal_prices.courserun_id -left join fulfilled_orders - on applications.application_id = fulfilled_orders.application_id +left join courserun_last_steps on applications.courserun_id = courserun_last_steps.courserun_id +left join + applicationstep_submissions + on courserun_last_steps.courserun_applicationstep_id = applicationstep_submissions.courserun_applicationstep_id + and applications.application_id = applicationstep_submissions.application_id +left join installments on applications.courserun_id = installments.courserun_id +left join + personal_prices + on applications.user_id = personal_prices.user_id + and applications.courserun_id = personal_prices.courserun_id +left join fulfilled_orders on applications.application_id = fulfilled_orders.application_id diff --git a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__course_runs.sql b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__course_runs.sql index 57cb96aa7..d1688ace3 100644 --- a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__course_runs.sql +++ b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__course_runs.sql @@ -1,15 +1,5 @@ -- Course Runs information for Bootcamps +with runs as (select * from {{ ref("stg__bootcamps__app__postgres__courses_courserun") }}) -with runs as ( - select * - from {{ ref('stg__bootcamps__app__postgres__courses_courserun') }} -) - -select - courserun_id - , course_id - , courserun_title - , courserun_readable_id - , courserun_start_on - , courserun_end_on +select courserun_id, course_id, courserun_title, courserun_readable_id, courserun_start_on, courserun_end_on from runs diff --git a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courserun_certificates.sql b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courserun_certificates.sql index dc40cffd0..1088af57c 100644 --- a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courserun_certificates.sql +++ b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courserun_certificates.sql @@ -1,36 +1,28 @@ -- Course Certificate information for Bootcamps +with + certificates as (select * from {{ ref("stg__bootcamps__app__postgres__courses_courseruncertificate") }}), + runs as (select * from {{ ref("stg__bootcamps__app__postgres__courses_courserun") }}), + users as (select * from {{ ref("int__bootcamps__users") }}), + bootcamps_certificates as ( + select + certificates.courseruncertificate_id, + certificates.courseruncertificate_uuid, + certificates.courserun_id, + runs.courserun_title, + runs.courserun_readable_id, + runs.course_id, + certificates.courseruncertificate_is_revoked, + certificates.courseruncertificate_url, + certificates.courseruncertificate_created_on, + certificates.courseruncertificate_updated_on, + certificates.user_id, + users.user_username, + users.user_email, + users.user_full_name + from certificates + inner join runs on certificates.courserun_id = runs.courserun_id + inner join users on certificates.user_id = users.user_id + ) -with certificates as ( - select * from {{ ref('stg__bootcamps__app__postgres__courses_courseruncertificate') }} -) - -, runs as ( - select * from {{ ref('stg__bootcamps__app__postgres__courses_courserun') }} -) - -, users as ( - select * from {{ ref('int__bootcamps__users') }} -) - -, bootcamps_certificates as ( - select - certificates.courseruncertificate_id - , certificates.courseruncertificate_uuid - , certificates.courserun_id - , runs.courserun_title - , runs.courserun_readable_id - , runs.course_id - , certificates.courseruncertificate_is_revoked - , certificates.courseruncertificate_url - , certificates.courseruncertificate_created_on - , certificates.courseruncertificate_updated_on - , certificates.user_id - , users.user_username - , users.user_email - , users.user_full_name - from certificates - inner join runs on certificates.courserun_id = runs.courserun_id - inner join users on certificates.user_id = users.user_id -) - -select * from bootcamps_certificates +select * +from bootcamps_certificates diff --git a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courserunenrollments.sql b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courserunenrollments.sql index 7fab64fcd..0a3531cb3 100644 --- a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courserunenrollments.sql +++ b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courserunenrollments.sql @@ -1,35 +1,27 @@ -- Enrollment information for Bootcamps +with + enrollments as (select * from {{ ref("stg__bootcamps__app__postgres__courserunenrollment") }}), + runs as (select * from {{ ref("stg__bootcamps__app__postgres__courses_courserun") }}), + users as (select * from {{ ref("int__bootcamps__users") }}), + bootcamps_enrollments as ( + select + enrollments.courserunenrollment_id, + enrollments.courserunenrollment_is_active, + enrollments.user_id, + enrollments.courserun_id, + enrollments.courserunenrollment_created_on, + enrollments.courserunenrollment_enrollment_status, + runs.courserun_readable_id, + runs.courserun_title, + runs.courserun_start_on, + users.user_username, + users.user_email, + users.user_full_name, + users.user_address_country + from enrollments + inner join runs on enrollments.courserun_id = runs.courserun_id + inner join users on enrollments.user_id = users.user_id + ) -with enrollments as ( - select * from {{ ref('stg__bootcamps__app__postgres__courserunenrollment') }} -) - -, runs as ( - select * from {{ ref('stg__bootcamps__app__postgres__courses_courserun') }} -) - -, users as ( - select * from {{ ref('int__bootcamps__users') }} -) - -, bootcamps_enrollments as ( - select - enrollments.courserunenrollment_id - , enrollments.courserunenrollment_is_active - , enrollments.user_id - , enrollments.courserun_id - , enrollments.courserunenrollment_created_on - , enrollments.courserunenrollment_enrollment_status - , runs.courserun_readable_id - , runs.courserun_title - , runs.courserun_start_on - , users.user_username - , users.user_email - , users.user_full_name - , users.user_address_country - from enrollments - inner join runs on enrollments.courserun_id = runs.courserun_id - inner join users on enrollments.user_id = users.user_id -) - -select * from bootcamps_enrollments +select * +from bootcamps_enrollments diff --git a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courses.sql b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courses.sql index 7de80f4b2..4ccc5d5f4 100644 --- a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courses.sql +++ b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courses.sql @@ -1,12 +1,5 @@ -- Course information for Bootcamps +with courses as (select * from {{ ref("stg__bootcamps__app__postgres__courses_course") }}) -with courses as ( - select * - from {{ ref('stg__bootcamps__app__postgres__courses_course') }} -) - -select - course_id - , course_title - , course_readable_id +select course_id, course_title, course_readable_id from courses diff --git a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__ecommerce_order.sql b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__ecommerce_order.sql index 0976f1ce5..3eb415ec6 100644 --- a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__ecommerce_order.sql +++ b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__ecommerce_order.sql @@ -1,49 +1,34 @@ -with orders as ( - select * - from {{ ref('stg__bootcamps__app__postgres__ecommerce_order') }} -) - -, lines as ( - select * - from {{ ref('stg__bootcamps__app__postgres__ecommerce_line') }} -) - -, receipts as ( - select * from {{ ref('int__bootcamps__ecommerce_receipt') }} -) - -, users as ( - select * from {{ ref('int__bootcamps__users') }} -) - -, runs as ( - select * from {{ ref('stg__bootcamps__app__postgres__courses_courserun') }} -) +with + orders as (select * from {{ ref("stg__bootcamps__app__postgres__ecommerce_order") }}), + lines as (select * from {{ ref("stg__bootcamps__app__postgres__ecommerce_line") }}), + receipts as (select * from {{ ref("int__bootcamps__ecommerce_receipt") }}), + users as (select * from {{ ref("int__bootcamps__users") }}), + runs as (select * from {{ ref("stg__bootcamps__app__postgres__courses_courserun") }}) select - orders.order_id - , orders.order_reference_number - , orders.order_state - , orders.order_purchaser_user_id - , orders.application_id - , orders.order_payment_type - , orders.order_total_price_paid - , orders.order_created_on - , orders.order_updated_on - , lines.line_id - , lines.line_description - , lines.courserun_id - , runs.courserun_readable_id - , lines.line_price - , receipts.receipt_transaction_id - , receipts.receipt_reference_number - , receipts.receipt_payment_method - , receipts.receipt_authorization_code - , receipts.receipt_bill_to_address_state - , receipts.receipt_bill_to_address_country - , users.user_username - , users.user_email - , users.user_full_name + orders.order_id, + orders.order_reference_number, + orders.order_state, + orders.order_purchaser_user_id, + orders.application_id, + orders.order_payment_type, + orders.order_total_price_paid, + orders.order_created_on, + orders.order_updated_on, + lines.line_id, + lines.line_description, + lines.courserun_id, + runs.courserun_readable_id, + lines.line_price, + receipts.receipt_transaction_id, + receipts.receipt_reference_number, + receipts.receipt_payment_method, + receipts.receipt_authorization_code, + receipts.receipt_bill_to_address_state, + receipts.receipt_bill_to_address_country, + users.user_username, + users.user_email, + users.user_full_name from lines inner join orders on lines.order_id = orders.order_id inner join users on orders.order_purchaser_user_id = users.user_id diff --git a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__ecommerce_receipt.sql b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__ecommerce_receipt.sql index e4e1a0dc1..d4310e127 100644 --- a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__ecommerce_receipt.sql +++ b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__ecommerce_receipt.sql @@ -1,33 +1,30 @@ -with receipts as ( - select * - from {{ ref('stg__bootcamps__app__postgres__ecommerce_receipt') }} -) +with receipts as (select * from {{ ref("stg__bootcamps__app__postgres__ecommerce_receipt") }}) select - receipt_id - , receipt_created_on - , receipt_updated_on - , order_id - , receipt_data - , cast(json_query(receipt_data, 'lax $.req_amount' omit quotes) as decimal(38, 2)) as receipt_payment_amount - , json_query(receipt_data, 'lax $.req_transaction_uuid' omit quotes) as receipt_transaction_uuid - , json_query(receipt_data, 'lax $.decision' omit quotes) as receipt_transaction_status - , json_query(receipt_data, 'lax $.transaction_id' omit quotes) as receipt_transaction_id - , json_query(receipt_data, 'lax $.req_transaction_type' omit quotes) as receipt_transaction_type - , json_query(receipt_data, 'lax $.auth_code' omit quotes) as receipt_authorization_code - , json_query(receipt_data, 'lax $.req_payment_method' omit quotes) as receipt_payment_method - , json_query(receipt_data, 'lax $.req_reference_number' omit quotes) as receipt_reference_number - , json_query(receipt_data, 'lax $.req_bill_to_address_state' omit quotes) as receipt_bill_to_address_state - , json_query(receipt_data, 'lax $.req_bill_to_address_country' omit quotes) as receipt_bill_to_address_country - , json_query(receipt_data, 'lax $.req_currency' omit quotes) as receipt_payment_currency - , json_query(receipt_data, 'lax $.req_bill_to_email' omit quotes) as receipt_payer_email - , json_query(receipt_data, 'lax $.req_card_number' omit quotes) as receipt_payment_card_number - , json_query(receipt_data, 'lax $.req_customer_ip_address' omit quotes) as receipt_payer_ip_address - , json_query(receipt_data, 'lax $.card_type_name' omit quotes) as receipt_payment_card_type - , concat( - json_query(receipt_data, 'lax $.req_bill_to_forename' omit quotes) - , ' ' - , json_query(receipt_data, 'lax $.req_bill_to_surname' omit quotes) - ) as receipt_payer_name - , json_query(receipt_data, 'lax $.signed_date_time' omit quotes) as receipt_payment_timestamp + receipt_id, + receipt_created_on, + receipt_updated_on, + order_id, + receipt_data, + cast(json_query(receipt_data, 'lax $.req_amount' omit quotes) as decimal(38, 2)) as receipt_payment_amount, + json_query(receipt_data, 'lax $.req_transaction_uuid' omit quotes) as receipt_transaction_uuid, + json_query(receipt_data, 'lax $.decision' omit quotes) as receipt_transaction_status, + json_query(receipt_data, 'lax $.transaction_id' omit quotes) as receipt_transaction_id, + json_query(receipt_data, 'lax $.req_transaction_type' omit quotes) as receipt_transaction_type, + json_query(receipt_data, 'lax $.auth_code' omit quotes) as receipt_authorization_code, + json_query(receipt_data, 'lax $.req_payment_method' omit quotes) as receipt_payment_method, + json_query(receipt_data, 'lax $.req_reference_number' omit quotes) as receipt_reference_number, + json_query(receipt_data, 'lax $.req_bill_to_address_state' omit quotes) as receipt_bill_to_address_state, + json_query(receipt_data, 'lax $.req_bill_to_address_country' omit quotes) as receipt_bill_to_address_country, + json_query(receipt_data, 'lax $.req_currency' omit quotes) as receipt_payment_currency, + json_query(receipt_data, 'lax $.req_bill_to_email' omit quotes) as receipt_payer_email, + json_query(receipt_data, 'lax $.req_card_number' omit quotes) as receipt_payment_card_number, + json_query(receipt_data, 'lax $.req_customer_ip_address' omit quotes) as receipt_payer_ip_address, + json_query(receipt_data, 'lax $.card_type_name' omit quotes) as receipt_payment_card_type, + concat( + json_query(receipt_data, 'lax $.req_bill_to_forename' omit quotes), + ' ', + json_query(receipt_data, 'lax $.req_bill_to_surname' omit quotes) + ) as receipt_payer_name, + json_query(receipt_data, 'lax $.signed_date_time' omit quotes) as receipt_payment_timestamp from receipts diff --git a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__ecommerce_wiretransferreceipt.sql b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__ecommerce_wiretransferreceipt.sql index 469832dca..3883edec2 100644 --- a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__ecommerce_wiretransferreceipt.sql +++ b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__ecommerce_wiretransferreceipt.sql @@ -1,12 +1,9 @@ -with receipts as ( - select * - from {{ ref('stg__bootcamps__app__postgres__ecommerce_wiretransferreceipt') }} -) +with receipts as (select * from {{ ref("stg__bootcamps__app__postgres__ecommerce_wiretransferreceipt") }}) select - wiretransferreceipt_id - , wiretransferreceipt_created_on - , wiretransferreceipt_updated_on - , wiretransferreceipt_data - , order_id + wiretransferreceipt_id, + wiretransferreceipt_created_on, + wiretransferreceipt_updated_on, + wiretransferreceipt_data, + order_id from receipts diff --git a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__users.sql b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__users.sql index 256e06c77..61c801db9 100644 --- a/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__users.sql +++ b/src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__users.sql @@ -1,40 +1,30 @@ -with users as ( - select * - from {{ ref('stg__bootcamps__app__postgres__auth_user') }} -) - -, users_legaladdress as ( - select * - from {{ ref('stg__bootcamps__app__postgres__profiles_legaladdress') }} -) - -, users_profile as ( - select * - from {{ ref('stg__bootcamps__app__postgres__profiles_profile') }} -) +with + users as (select * from {{ ref("stg__bootcamps__app__postgres__auth_user") }}), + users_legaladdress as (select * from {{ ref("stg__bootcamps__app__postgres__profiles_legaladdress") }}), + users_profile as (select * from {{ ref("stg__bootcamps__app__postgres__profiles_profile") }}) select - users.user_id - , users.user_username - , users_profile.user_full_name - , users.user_email - , users.user_joined_on - , users.user_last_login - , users.user_is_active - , users_legaladdress.user_address_country - , users_legaladdress.user_address_state_or_territory - , users_legaladdress.user_address_city - , users_legaladdress.user_street_address - , users_legaladdress.user_address_postal_code - , users_profile.user_birth_year - , users_profile.user_company - , users_profile.user_job_title - , users_profile.user_industry - , users_profile.user_job_function - , users_profile.user_highest_education - , users_profile.user_gender - , users_profile.user_company_size - , users_profile.user_years_experience + users.user_id, + users.user_username, + users_profile.user_full_name, + users.user_email, + users.user_joined_on, + users.user_last_login, + users.user_is_active, + users_legaladdress.user_address_country, + users_legaladdress.user_address_state_or_territory, + users_legaladdress.user_address_city, + users_legaladdress.user_street_address, + users_legaladdress.user_address_postal_code, + users_profile.user_birth_year, + users_profile.user_company, + users_profile.user_job_title, + users_profile.user_industry, + users_profile.user_job_function, + users_profile.user_highest_education, + users_profile.user_gender, + users_profile.user_company_size, + users_profile.user_years_experience from users left join users_legaladdress on users.user_id = users_legaladdress.user_id left join users_profile on users.user_id = users_profile.user_id diff --git a/src/ol_dbt/models/intermediate/combined/int__combined__course_runs.sql b/src/ol_dbt/models/intermediate/combined/int__combined__course_runs.sql index cde890ded..fcf45e4b6 100644 --- a/src/ol_dbt/models/intermediate/combined/int__combined__course_runs.sql +++ b/src/ol_dbt/models/intermediate/combined/int__combined__course_runs.sql @@ -1,287 +1,262 @@ -with mitx_courses as ( - select * from {{ ref('int__mitx__courses') }} -) - -, mitxonline_runs as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, edxorg_runs as ( - select * from {{ ref('int__edxorg__mitx_courseruns') }} -) - -, mitxpro_courses as ( - select * from {{ ref('int__mitxpro__courses') }} -) - -, mitxpro_runs as ( - select * from {{ ref('int__mitxpro__course_runs') }} -) - -, bootcamps_courses as ( - select * from {{ ref('int__bootcamps__courses') }} -) - -, bootcamps_runs as ( - select * from {{ ref('int__bootcamps__course_runs') }} -) - -, micromasters_runs as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_courserun') }} -) - -, emeritus_runs as ( - select * from ( - select - courserun_external_readable_id - , courserun_id as course_number - , courserun_title - , courserun_start_on - , courserun_end_on - , row_number() over ( - partition by courserun_external_readable_id - order by courserun_start_on desc, courserun_end_on desc, enrollment_created_on desc - ) as row_num - from {{ ref('stg__emeritus__api__bigquery__user_enrollments') }} - ) - where row_num = 1 -) - -, global_alumni_runs as ( - select * from ( +with + mitx_courses as (select * from {{ ref("int__mitx__courses") }}), + mitxonline_runs as (select * from {{ ref("int__mitxonline__course_runs") }}), + edxorg_runs as (select * from {{ ref("int__edxorg__mitx_courseruns") }}), + mitxpro_courses as (select * from {{ ref("int__mitxpro__courses") }}), + mitxpro_runs as (select * from {{ ref("int__mitxpro__course_runs") }}), + bootcamps_courses as (select * from {{ ref("int__bootcamps__courses") }}), + bootcamps_runs as (select * from {{ ref("int__bootcamps__course_runs") }}), + micromasters_runs as (select * from {{ ref("stg__micromasters__app__postgres__courses_courserun") }}), + emeritus_runs as ( + select * + from + ( + select + courserun_external_readable_id, + courserun_id as course_number, + courserun_title, + courserun_start_on, + courserun_end_on, + row_number() over ( + partition by courserun_external_readable_id + order by courserun_start_on desc, courserun_end_on desc, enrollment_created_on desc + ) as row_num + from {{ ref("stg__emeritus__api__bigquery__user_enrollments") }} + ) + where row_num = 1 + ), + global_alumni_runs as ( + select * + from + ( + select + courserun_external_readable_id, + courserun_id as course_number, + courserun_title, + courserun_start_on, + courserun_end_on, + row_number() over ( + partition by courserun_external_readable_id + order by courserun_start_on desc, courserun_end_on desc, user_gdpr_consent_date desc + ) as row_num + from {{ ref("stg__global_alumni__api__bigquery__user_enrollments") }} + ) + where row_num = 1 + ), + residential_runs as ( + select *, {{ extract_course_readable_id("courserun_readable_id") }} as course_readable_id + from {{ ref("int__mitxresidential__courseruns") }} + ), + combined_runs as ( select - courserun_external_readable_id - , courserun_id as course_number - , courserun_title - , courserun_start_on - , courserun_end_on - , row_number() over ( - partition by courserun_external_readable_id - order by courserun_start_on desc, courserun_end_on desc, user_gdpr_consent_date desc - ) as row_num - from {{ ref('stg__global_alumni__api__bigquery__user_enrollments') }} - ) - where row_num = 1 -) - -, residential_runs as ( - select - * - , {{ extract_course_readable_id('courserun_readable_id') }} as course_readable_id - from {{ ref('int__mitxresidential__courseruns') }} -) - -, combined_runs as ( - select - '{{ var("mitxonline") }}' as platform - , mitx_courses.course_title - , mitx_courses.course_readable_id - , mitxonline_runs.courserun_title - , mitxonline_runs.courserun_readable_id - , mitxonline_runs.courserun_url - , mitxonline_runs.courserun_start_on - , mitxonline_runs.courserun_end_on - , mitxonline_runs.courserun_upgrade_deadline - , mitxonline_runs.courserun_is_live - , mitxonline_runs.course_number - , case - when - mitxonline_runs.courserun_end_on is null - and from_iso8601_timestamp(mitxonline_runs.courserun_start_on) <= current_date + '{{ var("mitxonline") }}' as platform, + mitx_courses.course_title, + mitx_courses.course_readable_id, + mitxonline_runs.courserun_title, + mitxonline_runs.courserun_readable_id, + mitxonline_runs.courserun_url, + mitxonline_runs.courserun_start_on, + mitxonline_runs.courserun_end_on, + mitxonline_runs.courserun_upgrade_deadline, + mitxonline_runs.courserun_is_live, + mitxonline_runs.course_number, + case + when + mitxonline_runs.courserun_end_on is null + and from_iso8601_timestamp(mitxonline_runs.courserun_start_on) <= current_date then true - when - from_iso8601_timestamp(mitxonline_runs.courserun_start_on) <= current_date - and from_iso8601_timestamp(mitxonline_runs.courserun_end_on) > current_date + when + from_iso8601_timestamp(mitxonline_runs.courserun_start_on) <= current_date + and from_iso8601_timestamp(mitxonline_runs.courserun_end_on) > current_date then true - else false - end as courserun_is_current - from mitxonline_runs - left join mitx_courses on mitxonline_runs.course_id = mitx_courses.mitxonline_course_id - where mitxonline_runs.courserun_platform = '{{ var("mitxonline") }}' + else false + end as courserun_is_current + from mitxonline_runs + left join mitx_courses on mitxonline_runs.course_id = mitx_courses.mitxonline_course_id + where mitxonline_runs.courserun_platform = '{{ var("mitxonline") }}' - union all + union all - select - '{{ var("edxorg") }}' as platform - , mitx_courses.course_title - , mitx_courses.course_readable_id - , edxorg_runs.courserun_title - , edxorg_runs.courserun_readable_id - , edxorg_runs.courserun_url - , edxorg_runs.courserun_start_date as courserun_start_on - , edxorg_runs.courserun_end_date as courserun_end_on - , micromasters_runs.courserun_upgrade_deadline - , null as courserun_is_live - , edxorg_runs.course_number - , case - when - edxorg_runs.courserun_end_date is null - and from_iso8601_timestamp(edxorg_runs.courserun_start_date) <= current_date + select + '{{ var("edxorg") }}' as platform, + mitx_courses.course_title, + mitx_courses.course_readable_id, + edxorg_runs.courserun_title, + edxorg_runs.courserun_readable_id, + edxorg_runs.courserun_url, + edxorg_runs.courserun_start_date as courserun_start_on, + edxorg_runs.courserun_end_date as courserun_end_on, + micromasters_runs.courserun_upgrade_deadline, + null as courserun_is_live, + edxorg_runs.course_number, + case + when + edxorg_runs.courserun_end_date is null + and from_iso8601_timestamp(edxorg_runs.courserun_start_date) <= current_date then true - when - from_iso8601_timestamp(edxorg_runs.courserun_start_date) <= current_date - and from_iso8601_timestamp(edxorg_runs.courserun_end_date) > current_date + when + from_iso8601_timestamp(edxorg_runs.courserun_start_date) <= current_date + and from_iso8601_timestamp(edxorg_runs.courserun_end_date) > current_date then true - else false - end as courserun_is_current - from edxorg_runs - left join mitx_courses on edxorg_runs.course_number = mitx_courses.course_number - left join micromasters_runs on edxorg_runs.courserun_readable_id = micromasters_runs.courserun_edxorg_readable_id + else false + end as courserun_is_current + from edxorg_runs + left join mitx_courses on edxorg_runs.course_number = mitx_courses.course_number + left join + micromasters_runs on edxorg_runs.courserun_readable_id = micromasters_runs.courserun_edxorg_readable_id - union all + union all - select - case - when mitxpro_runs.platform_id = 2 then '{{ var("mitxpro") }}' - else 'xPRO ' || mitxpro_runs.platform_name - end as platform - , mitxpro_courses.course_title - , mitxpro_courses.course_readable_id - , mitxpro_runs.courserun_title - , mitxpro_runs.courserun_readable_id - , mitxpro_runs.courserun_url - , mitxpro_runs.courserun_start_on - , mitxpro_runs.courserun_end_on - , null as courserun_upgrade_deadline - , mitxpro_runs.courserun_is_live - , case - when cardinality(split(mitxpro_runs.courserun_readable_id, '+')) >= 2 + select + case + when mitxpro_runs.platform_id = 2 then '{{ var("mitxpro") }}' else 'xPRO ' || mitxpro_runs.platform_name + end as platform, + mitxpro_courses.course_title, + mitxpro_courses.course_readable_id, + mitxpro_runs.courserun_title, + mitxpro_runs.courserun_readable_id, + mitxpro_runs.courserun_url, + mitxpro_runs.courserun_start_on, + mitxpro_runs.courserun_end_on, + null as courserun_upgrade_deadline, + mitxpro_runs.courserun_is_live, + case + when cardinality(split(mitxpro_runs.courserun_readable_id, '+')) >= 2 then split(mitxpro_runs.courserun_readable_id, '+')[2] - end as course_number - , case - when - mitxpro_runs.courserun_end_on is null - and from_iso8601_timestamp(mitxpro_runs.courserun_start_on) <= current_date + end as course_number, + case + when + mitxpro_runs.courserun_end_on is null + and from_iso8601_timestamp(mitxpro_runs.courserun_start_on) <= current_date then true - when - from_iso8601_timestamp(mitxpro_runs.courserun_start_on) <= current_date - and from_iso8601_timestamp(mitxpro_runs.courserun_end_on) > current_date + when + from_iso8601_timestamp(mitxpro_runs.courserun_start_on) <= current_date + and from_iso8601_timestamp(mitxpro_runs.courserun_end_on) > current_date then true - else false - end as courserun_is_current - from mitxpro_runs - left join mitxpro_courses on mitxpro_runs.course_id = mitxpro_courses.course_id + else false + end as courserun_is_current + from mitxpro_runs + left join mitxpro_courses on mitxpro_runs.course_id = mitxpro_courses.course_id - union all + union all - select - '{{ var("emeritus") }}' as platform - , emeritus_runs.courserun_title as course_title - , emeritus_runs.courserun_external_readable_id as course_readable_id - , emeritus_runs.courserun_title - , emeritus_runs.courserun_external_readable_id - , null as courserun_url - , emeritus_runs.courserun_start_on - , emeritus_runs.courserun_end_on - , null as courserun_upgrade_deadline - , null as courserun_is_live - , emeritus_runs.course_number - , case - when - emeritus_runs.courserun_end_on is null - and from_iso8601_timestamp(mitxpro_runs.courserun_start_on) <= current_date + select + '{{ var("emeritus") }}' as platform, + emeritus_runs.courserun_title as course_title, + emeritus_runs.courserun_external_readable_id as course_readable_id, + emeritus_runs.courserun_title, + emeritus_runs.courserun_external_readable_id, + null as courserun_url, + emeritus_runs.courserun_start_on, + emeritus_runs.courserun_end_on, + null as courserun_upgrade_deadline, + null as courserun_is_live, + emeritus_runs.course_number, + case + when + emeritus_runs.courserun_end_on is null + and from_iso8601_timestamp(mitxpro_runs.courserun_start_on) <= current_date then true - when - from_iso8601_timestamp(emeritus_runs.courserun_start_on) <= current_date - and from_iso8601_timestamp(emeritus_runs.courserun_end_on) > current_date + when + from_iso8601_timestamp(emeritus_runs.courserun_start_on) <= current_date + and from_iso8601_timestamp(emeritus_runs.courserun_end_on) > current_date then true - else false - end as courserun_is_current - from emeritus_runs - left join mitxpro_runs - on emeritus_runs.courserun_external_readable_id = mitxpro_runs.courserun_external_readable_id - where mitxpro_runs.courserun_external_readable_id is null + else false + end as courserun_is_current + from emeritus_runs + left join + mitxpro_runs on emeritus_runs.courserun_external_readable_id = mitxpro_runs.courserun_external_readable_id + where mitxpro_runs.courserun_external_readable_id is null - union all - -- any remaining global alumni runs that are not in mitxpro - select - '{{ var("global_alumni") }}' as platform - , global_alumni_runs.courserun_title as course_title - , global_alumni_runs.courserun_external_readable_id as course_readable_id - , global_alumni_runs.courserun_title - , global_alumni_runs.courserun_external_readable_id - , null as courserun_url - , global_alumni_runs.courserun_start_on - , global_alumni_runs.courserun_end_on - , null as courserun_upgrade_deadline - , null as courserun_is_live - , global_alumni_runs.course_number - , case - when - global_alumni_runs.courserun_end_on is null - and from_iso8601_timestamp(global_alumni_runs.courserun_start_on) <= current_date + union all + -- any remaining global alumni runs that are not in mitxpro + select + '{{ var("global_alumni") }}' as platform, + global_alumni_runs.courserun_title as course_title, + global_alumni_runs.courserun_external_readable_id as course_readable_id, + global_alumni_runs.courserun_title, + global_alumni_runs.courserun_external_readable_id, + null as courserun_url, + global_alumni_runs.courserun_start_on, + global_alumni_runs.courserun_end_on, + null as courserun_upgrade_deadline, + null as courserun_is_live, + global_alumni_runs.course_number, + case + when + global_alumni_runs.courserun_end_on is null + and from_iso8601_timestamp(global_alumni_runs.courserun_start_on) <= current_date then true - when - from_iso8601_timestamp(global_alumni_runs.courserun_start_on) <= current_date - and from_iso8601_timestamp(global_alumni_runs.courserun_end_on) > current_date + when + from_iso8601_timestamp(global_alumni_runs.courserun_start_on) <= current_date + and from_iso8601_timestamp(global_alumni_runs.courserun_end_on) > current_date then true - else false - end as courserun_is_current - from global_alumni_runs - left join mitxpro_runs - on global_alumni_runs.courserun_external_readable_id = mitxpro_runs.courserun_external_readable_id - where mitxpro_runs.courserun_external_readable_id is null + else false + end as courserun_is_current + from global_alumni_runs + left join + mitxpro_runs + on global_alumni_runs.courserun_external_readable_id = mitxpro_runs.courserun_external_readable_id + where mitxpro_runs.courserun_external_readable_id is null - union all + union all - select - '{{ var("bootcamps") }}' as platform - , bootcamps_courses.course_title - , bootcamps_courses.course_readable_id - , bootcamps_runs.courserun_title - , bootcamps_runs.courserun_readable_id - , null as courserun_url - , bootcamps_runs.courserun_start_on - , bootcamps_runs.courserun_end_on - , null as courserun_upgrade_deadline - , null as courserun_is_live - , case - when cardinality(split(bootcamps_runs.courserun_readable_id, '+')) >= 2 + select + '{{ var("bootcamps") }}' as platform, + bootcamps_courses.course_title, + bootcamps_courses.course_readable_id, + bootcamps_runs.courserun_title, + bootcamps_runs.courserun_readable_id, + null as courserun_url, + bootcamps_runs.courserun_start_on, + bootcamps_runs.courserun_end_on, + null as courserun_upgrade_deadline, + null as courserun_is_live, + case + when cardinality(split(bootcamps_runs.courserun_readable_id, '+')) >= 2 then split(bootcamps_runs.courserun_readable_id, '+')[2] - end as course_number - , case - when - bootcamps_runs.courserun_end_on is null - and from_iso8601_timestamp(bootcamps_runs.courserun_start_on) <= current_date + end as course_number, + case + when + bootcamps_runs.courserun_end_on is null + and from_iso8601_timestamp(bootcamps_runs.courserun_start_on) <= current_date then true - when - from_iso8601_timestamp(bootcamps_runs.courserun_start_on) <= current_date - and from_iso8601_timestamp(bootcamps_runs.courserun_end_on) > current_date + when + from_iso8601_timestamp(bootcamps_runs.courserun_start_on) <= current_date + and from_iso8601_timestamp(bootcamps_runs.courserun_end_on) > current_date then true - else false - end as courserun_is_current - from bootcamps_runs - left join bootcamps_courses on bootcamps_runs.course_id = bootcamps_courses.course_id + else false + end as courserun_is_current + from bootcamps_runs + left join bootcamps_courses on bootcamps_runs.course_id = bootcamps_courses.course_id - union all + union all - select - '{{ var("residential") }}' as platform - , courserun_title as course_title - , course_readable_id - , courserun_title - , courserun_readable_id - , null as courserun_url - , courserun_start_on - , courserun_end_on - , null as courserun_upgrade_deadline - , null as courserun_is_live - , case - when cardinality(split(courserun_readable_id, '+')) >= 2 - then split(courserun_readable_id, '+')[2] - end as course_number - , case - when - courserun_end_on is null - and from_iso8601_timestamp(courserun_start_on) <= current_date + select + '{{ var("residential") }}' as platform, + courserun_title as course_title, + course_readable_id, + courserun_title, + courserun_readable_id, + null as courserun_url, + courserun_start_on, + courserun_end_on, + null as courserun_upgrade_deadline, + null as courserun_is_live, + case + when cardinality(split(courserun_readable_id, '+')) >= 2 then split(courserun_readable_id, '+')[2] + end as course_number, + case + when courserun_end_on is null and from_iso8601_timestamp(courserun_start_on) <= current_date then true - when - from_iso8601_timestamp(courserun_start_on) <= current_date - and from_iso8601_timestamp(courserun_end_on) > current_date + when + from_iso8601_timestamp(courserun_start_on) <= current_date + and from_iso8601_timestamp(courserun_end_on) > current_date then true - else false - end as courserun_is_current - from residential_runs -) + else false + end as courserun_is_current + from residential_runs + ) -select * from combined_runs +select * +from combined_runs diff --git a/src/ol_dbt/models/intermediate/combined/int__combined__course_structure.sql b/src/ol_dbt/models/intermediate/combined/int__combined__course_structure.sql index 98c22da92..3cebaf03c 100644 --- a/src/ol_dbt/models/intermediate/combined/int__combined__course_structure.sql +++ b/src/ol_dbt/models/intermediate/combined/int__combined__course_structure.sql @@ -1,89 +1,79 @@ -with mitxonline_course_structure as ( - select * from {{ ref('int__mitxonline__course_structure') }} -) +with + mitxonline_course_structure as (select * from {{ ref("int__mitxonline__course_structure") }}), + edxorg_course_structure as (select * from {{ ref("int__edxorg__mitx_course_structure") }}), + xpro_course_structure as (select * from {{ ref("int__mitxpro__course_structure") }}), + residential_course_structure as (select * from {{ ref("int__mitxresidential__course_structure") }}), + combined as ( + select + '{{ var("mitxonline") }}' as platform, + courserun_readable_id, + courserun_title, + coursestructure_block_index, + coursestructure_block_id, + coursestructure_parent_block_id, + coursestructure_block_category, + coursestructure_block_title, + coursestructure_block_metadata, + coursestructure_retrieved_at, + coursestructure_chapter_id, + coursestructure_chapter_title + from mitxonline_course_structure + where coursestructure_is_latest = true -, edxorg_course_structure as ( - select * from {{ ref('int__edxorg__mitx_course_structure') }} -) + union all -, xpro_course_structure as ( - select * from {{ ref('int__mitxpro__course_structure') }} -) + select + '{{ var("edxorg") }}' as platform, + courserun_readable_id, + courserun_title, + coursestructure_block_index, + coursestructure_block_id, + coursestructure_parent_block_id, + coursestructure_block_category, + coursestructure_block_title, + coursestructure_block_metadata, + coursestructure_retrieved_at, + coursestructure_chapter_id, + coursestructure_chapter_title + from edxorg_course_structure + where coursestructure_is_latest = true -, residential_course_structure as ( - select * from {{ ref('int__mitxresidential__course_structure') }} -) + union all -, combined as ( - select - '{{ var("mitxonline") }}' as platform - , courserun_readable_id - , courserun_title - , coursestructure_block_index - , coursestructure_block_id - , coursestructure_parent_block_id - , coursestructure_block_category - , coursestructure_block_title - , coursestructure_block_metadata - , coursestructure_retrieved_at - , coursestructure_chapter_id - , coursestructure_chapter_title - from mitxonline_course_structure - where coursestructure_is_latest = true + select + '{{ var("mitxpro") }}' as platform, + courserun_readable_id, + courserun_title, + coursestructure_block_index, + coursestructure_block_id, + coursestructure_parent_block_id, + coursestructure_block_category, + coursestructure_block_title, + coursestructure_block_metadata, + coursestructure_retrieved_at, + coursestructure_chapter_id, + coursestructure_chapter_title + from xpro_course_structure + where coursestructure_is_latest = true - union all + union all - select - '{{ var("edxorg") }}' as platform - , courserun_readable_id - , courserun_title - , coursestructure_block_index - , coursestructure_block_id - , coursestructure_parent_block_id - , coursestructure_block_category - , coursestructure_block_title - , coursestructure_block_metadata - , coursestructure_retrieved_at - , coursestructure_chapter_id - , coursestructure_chapter_title - from edxorg_course_structure - where coursestructure_is_latest = true + select + '{{ var("residential") }}' as platform, + courserun_readable_id, + courserun_title, + coursestructure_block_index, + coursestructure_block_id, + coursestructure_parent_block_id, + coursestructure_block_category, + coursestructure_block_title, + coursestructure_block_metadata, + coursestructure_retrieved_at, + coursestructure_chapter_id, + coursestructure_chapter_title + from residential_course_structure + where coursestructure_is_latest = true + ) - union all - - select - '{{ var("mitxpro") }}' as platform - , courserun_readable_id - , courserun_title - , coursestructure_block_index - , coursestructure_block_id - , coursestructure_parent_block_id - , coursestructure_block_category - , coursestructure_block_title - , coursestructure_block_metadata - , coursestructure_retrieved_at - , coursestructure_chapter_id - , coursestructure_chapter_title - from xpro_course_structure - where coursestructure_is_latest = true - - union all - - select - '{{ var("residential") }}' as platform - , courserun_readable_id - , courserun_title - , coursestructure_block_index - , coursestructure_block_id - , coursestructure_parent_block_id - , coursestructure_block_category - , coursestructure_block_title - , coursestructure_block_metadata - , coursestructure_retrieved_at - , coursestructure_chapter_id - , coursestructure_chapter_title - from residential_course_structure - where coursestructure_is_latest = true -) - -select * from combined +select * +from combined diff --git a/src/ol_dbt/models/intermediate/combined/int__combined__course_videos.sql b/src/ol_dbt/models/intermediate/combined/int__combined__course_videos.sql index 8c63e763d..506961858 100644 --- a/src/ol_dbt/models/intermediate/combined/int__combined__course_videos.sql +++ b/src/ol_dbt/models/intermediate/combined/int__combined__course_videos.sql @@ -1,69 +1,57 @@ -with video_structure as ( - select - * - , json_query(coursestructure_block_metadata, 'lax $.edx_video_id' omit quotes) as video_edx_uuid - , element_at(split(coursestructure_block_id, '@'), -1) as video_id - from {{ ref('int__combined__course_structure') }} - where coursestructure_block_category = 'video' -) +with + video_structure as ( + select + *, + json_query(coursestructure_block_metadata, 'lax $.edx_video_id' omit quotes) as video_edx_uuid, + element_at(split(coursestructure_block_id, '@'), -1) as video_id + from {{ ref("int__combined__course_structure") }} + where coursestructure_block_category = 'video' + ), + combined_videos as ( -, combined_videos as ( + select '{{ var("mitxonline") }}' as platform, courserun_readable_id, video_edx_uuid, video_duration + from {{ ref("int__mitxonline__courserun_videos") }} - select - '{{ var("mitxonline") }}' as platform - , courserun_readable_id - , video_edx_uuid - , video_duration - from {{ ref('int__mitxonline__courserun_videos') }} + union all - union all + select '{{ var("mitxpro") }}' as platform, courserun_readable_id, video_edx_uuid, video_duration + from {{ ref("int__mitxpro__courserun_videos") }} - select - '{{ var("mitxpro") }}' as platform - , courserun_readable_id - , video_edx_uuid - , video_duration - from {{ ref('int__mitxpro__courserun_videos') }} + union all - union all + select '{{ var("residential") }}' as platform, courserun_readable_id, video_edx_uuid, video_duration + from {{ ref("int__mitxresidential__courserun_videos") }} - select - '{{ var("residential") }}' as platform - , courserun_readable_id - , video_edx_uuid - , video_duration - from {{ ref('int__mitxresidential__courserun_videos') }} + union all - union all + select + '{{ var("edxorg") }}' as platform, + courserun_old_readable_id as courserun_readable_id, + video_block_id as video_edx_uuid, + video_duration + from {{ ref("stg__edxorg__s3__course_video") }} - select - '{{ var("edxorg") }}' as platform - , courserun_old_readable_id as courserun_readable_id - , video_block_id as video_edx_uuid - , video_duration - from {{ ref('stg__edxorg__s3__course_video') }} - -) + ) select - video_structure.platform - , video_structure.courserun_readable_id - , video_structure.video_id - , combined_videos.video_edx_uuid as video_edx_id - , video_structure.coursestructure_block_id as video_block_id - , video_structure.coursestructure_parent_block_id as video_parent_block_id - , video_structure.coursestructure_block_metadata as video_metadata - , video_structure.coursestructure_block_title as video_title - , video_structure.coursestructure_block_index as video_index - , video_structure.coursestructure_chapter_title as chapter_title - , video_structure.coursestructure_chapter_id as chapter_id - , combined_videos.video_duration + video_structure.platform, + video_structure.courserun_readable_id, + video_structure.video_id, + combined_videos.video_edx_uuid as video_edx_id, + video_structure.coursestructure_block_id as video_block_id, + video_structure.coursestructure_parent_block_id as video_parent_block_id, + video_structure.coursestructure_block_metadata as video_metadata, + video_structure.coursestructure_block_title as video_title, + video_structure.coursestructure_block_index as video_index, + video_structure.coursestructure_chapter_title as chapter_title, + video_structure.coursestructure_chapter_id as chapter_id, + combined_videos.video_duration from video_structure -left join combined_videos - on - video_structure.courserun_readable_id = combined_videos.courserun_readable_id - and video_structure.platform = combined_videos.platform - and ( - video_structure.video_edx_uuid = combined_videos.video_edx_uuid - or video_structure.video_id = combined_videos.video_edx_uuid - ) +left join + combined_videos + on video_structure.courserun_readable_id = combined_videos.courserun_readable_id + and video_structure.platform = combined_videos.platform + and ( + video_structure.video_edx_uuid = combined_videos.video_edx_uuid + or video_structure.video_id = combined_videos.video_edx_uuid + ) diff --git a/src/ol_dbt/models/intermediate/combined/int__combined__courserun_certificates.sql b/src/ol_dbt/models/intermediate/combined/int__combined__courserun_certificates.sql index b6f195a5f..87e1f3fb6 100644 --- a/src/ol_dbt/models/intermediate/combined/int__combined__courserun_certificates.sql +++ b/src/ol_dbt/models/intermediate/combined/int__combined__courserun_certificates.sql @@ -1,71 +1,68 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with mitx_certificates as ( - --revoked certificates are already filtered - select - platform - , courseruncertificate_uuid - , courseruncertificate_url - , courseruncertificate_created_on - , courserun_title - , user_mitxonline_username - , user_edxorg_username - , user_email - , user_full_name - , if( - platform = '{{ var("edxorg") }}' - , replace(replace(courserun_readable_id, 'course-v1:', ''), '+', '/') - , courserun_readable_id - ) as courserun_readable_id - from {{ ref('int__mitx__courserun_certificates') }} -) - -, mitxpro_certificates as ( - select * from {{ ref('int__mitxpro__courserun_certificates') }} - where courseruncertificate_is_revoked = false -) - -, bootcamps_certificates as ( - select * from {{ ref('int__bootcamps__courserun_certificates') }} - where courseruncertificate_is_revoked = false -) +with + mitx_certificates as ( + -- revoked certificates are already filtered + select + platform, + courseruncertificate_uuid, + courseruncertificate_url, + courseruncertificate_created_on, + courserun_title, + user_mitxonline_username, + user_edxorg_username, + user_email, + user_full_name, + if( + platform = '{{ var("edxorg") }}', + replace(replace(courserun_readable_id, 'course-v1:', ''), '+', '/'), + courserun_readable_id + ) as courserun_readable_id + from {{ ref("int__mitx__courserun_certificates") }} + ), + mitxpro_certificates as ( + select * from {{ ref("int__mitxpro__courserun_certificates") }} where courseruncertificate_is_revoked = false + ), + bootcamps_certificates as ( + select * from {{ ref("int__bootcamps__courserun_certificates") }} where courseruncertificate_is_revoked = false + ) select - platform - , courseruncertificate_uuid - , courseruncertificate_url - , courseruncertificate_created_on - , courserun_title - , courserun_readable_id - , if(platform = '{{ var("mitxonline") }}', user_mitxonline_username, user_edxorg_username) as user_username - , user_email - , user_full_name + platform, + courseruncertificate_uuid, + courseruncertificate_url, + courseruncertificate_created_on, + courserun_title, + courserun_readable_id, + if(platform = '{{ var("mitxonline") }}', user_mitxonline_username, user_edxorg_username) as user_username, + user_email, + user_full_name from mitx_certificates union all select - '{{ var("mitxpro") }}' as platform - , courseruncertificate_uuid - , courseruncertificate_url - , courseruncertificate_created_on - , courserun_title - , courserun_readable_id - , user_username - , user_email - , user_full_name + '{{ var("mitxpro") }}' as platform, + courseruncertificate_uuid, + courseruncertificate_url, + courseruncertificate_created_on, + courserun_title, + courserun_readable_id, + user_username, + user_email, + user_full_name from mitxpro_certificates union all select - '{{ var("bootcamps") }}' as platform - , courseruncertificate_uuid - , courseruncertificate_url - , courseruncertificate_created_on - , courserun_title - , courserun_readable_id - , user_username - , user_email - , user_full_name + '{{ var("bootcamps") }}' as platform, + courseruncertificate_uuid, + courseruncertificate_url, + courseruncertificate_created_on, + courserun_title, + courserun_readable_id, + user_username, + user_email, + user_full_name from bootcamps_certificates diff --git a/src/ol_dbt/models/intermediate/combined/int__combined__courserun_enrollments.sql b/src/ol_dbt/models/intermediate/combined/int__combined__courserun_enrollments.sql index e9875a1e3..18cb2c532 100644 --- a/src/ol_dbt/models/intermediate/combined/int__combined__courserun_enrollments.sql +++ b/src/ol_dbt/models/intermediate/combined/int__combined__courserun_enrollments.sql @@ -1,249 +1,217 @@ -with mitx_enrollments as ( - select * from {{ ref('int__mitx__courserun_enrollments') }} -) - -, mitxpro_enrollments as ( - select * from {{ ref('int__mitxpro__courserunenrollments') }} -) - -, emeritus_enrollments as ( - select * from {{ ref('stg__emeritus__api__bigquery__user_enrollments') }} -) - -, global_alumni_enrollments as ( - select * from {{ ref('stg__global_alumni__api__bigquery__user_enrollments') }} -) - -, bootcamps_enrollments as ( - select * from {{ ref('int__bootcamps__courserunenrollments') }} -) - -, residential_enrollments as ( - select * from {{ ref('int__mitxresidential__courserun_enrollments') }} -) - -, residential_grades as ( - select * from {{ ref('int__mitxresidential__courserun_grades') }} -) - -, mitx_grades as ( - select * from {{ ref('int__mitx__courserun_grades') }} -) - -, mitxpro_grades as ( - select * from {{ ref('int__mitxpro__courserun_grades') }} -) - -, combined_certificates as ( - select * from {{ ref('int__combined__courserun_certificates') }} -) - -, combined_courseruns as ( - select * from {{ ref('int__combined__course_runs') }} -) - -, mitxpro_courseruns as ( - select * from {{ ref('int__mitxpro__course_runs') }} -) - -, combined_enrollments as ( - select - mitx_enrollments.platform - , mitx_enrollments.courserunenrollment_id - , mitx_enrollments.courserunenrollment_is_active - , mitx_enrollments.courserunenrollment_created_on - , mitx_enrollments.courserunenrollment_enrollment_mode - , mitx_enrollments.courserunenrollment_enrollment_status - , mitx_enrollments.courserunenrollment_is_edx_enrolled - , mitx_enrollments.courserun_upgrade_deadline - , cast(mitx_enrollments.user_id as varchar) as user_id - , mitx_enrollments.courserun_id - , mitx_enrollments.courserun_title - , mitx_enrollments.courserun_readable_id - , mitx_enrollments.user_username - , mitx_enrollments.user_email - , mitx_enrollments.user_full_name - , mitx_grades.courserungrade_grade - , mitx_grades.courserungrade_is_passing - from mitx_enrollments - left join mitx_grades - on - mitx_enrollments.courserun_readable_id = mitx_grades.courserun_readable_id +with + mitx_enrollments as (select * from {{ ref("int__mitx__courserun_enrollments") }}), + mitxpro_enrollments as (select * from {{ ref("int__mitxpro__courserunenrollments") }}), + emeritus_enrollments as (select * from {{ ref("stg__emeritus__api__bigquery__user_enrollments") }}), + global_alumni_enrollments as (select * from {{ ref("stg__global_alumni__api__bigquery__user_enrollments") }}), + bootcamps_enrollments as (select * from {{ ref("int__bootcamps__courserunenrollments") }}), + residential_enrollments as (select * from {{ ref("int__mitxresidential__courserun_enrollments") }}), + residential_grades as (select * from {{ ref("int__mitxresidential__courserun_grades") }}), + mitx_grades as (select * from {{ ref("int__mitx__courserun_grades") }}), + mitxpro_grades as (select * from {{ ref("int__mitxpro__courserun_grades") }}), + combined_certificates as (select * from {{ ref("int__combined__courserun_certificates") }}), + combined_courseruns as (select * from {{ ref("int__combined__course_runs") }}), + mitxpro_courseruns as (select * from {{ ref("int__mitxpro__course_runs") }}), + combined_enrollments as ( + select + mitx_enrollments.platform, + mitx_enrollments.courserunenrollment_id, + mitx_enrollments.courserunenrollment_is_active, + mitx_enrollments.courserunenrollment_created_on, + mitx_enrollments.courserunenrollment_enrollment_mode, + mitx_enrollments.courserunenrollment_enrollment_status, + mitx_enrollments.courserunenrollment_is_edx_enrolled, + mitx_enrollments.courserun_upgrade_deadline, + cast(mitx_enrollments.user_id as varchar) as user_id, + mitx_enrollments.courserun_id, + mitx_enrollments.courserun_title, + mitx_enrollments.courserun_readable_id, + mitx_enrollments.user_username, + mitx_enrollments.user_email, + mitx_enrollments.user_full_name, + mitx_grades.courserungrade_grade, + mitx_grades.courserungrade_is_passing + from mitx_enrollments + left join + mitx_grades + on mitx_enrollments.courserun_readable_id = mitx_grades.courserun_readable_id and mitx_enrollments.user_mitxonline_username = mitx_grades.user_mitxonline_username - where mitx_enrollments.platform = '{{ var("mitxonline") }}' - - union all - - select - mitx_enrollments.platform - , mitx_enrollments.courserunenrollment_id - , mitx_enrollments.courserunenrollment_is_active - , mitx_enrollments.courserunenrollment_created_on - , mitx_enrollments.courserunenrollment_enrollment_mode - , mitx_enrollments.courserunenrollment_enrollment_status - , mitx_enrollments.courserunenrollment_is_edx_enrolled - , mitx_enrollments.courserun_upgrade_deadline - , cast(mitx_enrollments.user_id as varchar) as user_id - , mitx_enrollments.courserun_id - , mitx_enrollments.courserun_title - , mitx_enrollments.courserun_readable_id - , mitx_enrollments.user_username - , mitx_enrollments.user_email - , mitx_enrollments.user_full_name - , mitx_grades.courserungrade_grade - , mitx_grades.courserungrade_is_passing - from mitx_enrollments - left join mitx_grades - on - mitx_enrollments.courserun_readable_id = mitx_grades.courserun_readable_id + where mitx_enrollments.platform = '{{ var("mitxonline") }}' + + union all + + select + mitx_enrollments.platform, + mitx_enrollments.courserunenrollment_id, + mitx_enrollments.courserunenrollment_is_active, + mitx_enrollments.courserunenrollment_created_on, + mitx_enrollments.courserunenrollment_enrollment_mode, + mitx_enrollments.courserunenrollment_enrollment_status, + mitx_enrollments.courserunenrollment_is_edx_enrolled, + mitx_enrollments.courserun_upgrade_deadline, + cast(mitx_enrollments.user_id as varchar) as user_id, + mitx_enrollments.courserun_id, + mitx_enrollments.courserun_title, + mitx_enrollments.courserun_readable_id, + mitx_enrollments.user_username, + mitx_enrollments.user_email, + mitx_enrollments.user_full_name, + mitx_grades.courserungrade_grade, + mitx_grades.courserungrade_is_passing + from mitx_enrollments + left join + mitx_grades + on mitx_enrollments.courserun_readable_id = mitx_grades.courserun_readable_id and mitx_enrollments.user_edxorg_username = mitx_grades.user_edxorg_username - where mitx_enrollments.platform = '{{ var("edxorg") }}' - - union all - - select - '{{ var("mitxpro") }}' as platform - , mitxpro_enrollments.courserunenrollment_id - , mitxpro_enrollments.courserunenrollment_is_active - , mitxpro_enrollments.courserunenrollment_created_on - , mitxpro_enrollments.courserunenrollment_enrollment_mode - , mitxpro_enrollments.courserunenrollment_enrollment_status - , mitxpro_enrollments.courserunenrollment_is_edx_enrolled - , null as courserun_upgrade_deadline - , cast(mitxpro_enrollments.user_id as varchar) as user_id - , mitxpro_enrollments.courserun_id - , mitxpro_enrollments.courserun_title - , mitxpro_enrollments.courserun_readable_id - , mitxpro_enrollments.user_username - , mitxpro_enrollments.user_email - , mitxpro_enrollments.user_full_name - , mitxpro_grades.courserungrade_grade - , mitxpro_grades.courserungrade_is_passing - from mitxpro_enrollments - left join mitxpro_grades - on - mitxpro_enrollments.courserun_readable_id = mitxpro_grades.courserun_readable_id + where mitx_enrollments.platform = '{{ var("edxorg") }}' + + union all + + select + '{{ var("mitxpro") }}' as platform, + mitxpro_enrollments.courserunenrollment_id, + mitxpro_enrollments.courserunenrollment_is_active, + mitxpro_enrollments.courserunenrollment_created_on, + mitxpro_enrollments.courserunenrollment_enrollment_mode, + mitxpro_enrollments.courserunenrollment_enrollment_status, + mitxpro_enrollments.courserunenrollment_is_edx_enrolled, + null as courserun_upgrade_deadline, + cast(mitxpro_enrollments.user_id as varchar) as user_id, + mitxpro_enrollments.courserun_id, + mitxpro_enrollments.courserun_title, + mitxpro_enrollments.courserun_readable_id, + mitxpro_enrollments.user_username, + mitxpro_enrollments.user_email, + mitxpro_enrollments.user_full_name, + mitxpro_grades.courserungrade_grade, + mitxpro_grades.courserungrade_is_passing + from mitxpro_enrollments + left join + mitxpro_grades + on mitxpro_enrollments.courserun_readable_id = mitxpro_grades.courserun_readable_id and mitxpro_enrollments.user_username = mitxpro_grades.user_username - union all - - select - '{{ var("emeritus") }}' as platform - , null as courserunenrollment_id - , emeritus_enrollments.is_enrolled as courserunenrollment_is_active - , emeritus_enrollments.enrollment_created_on as courserunenrollment_created_on - , null as courserunenrollment_enrollment_mode - , emeritus_enrollments.enrollment_status as courserunenrollment_enrollment_status - , null as courserunenrollment_is_edx_enrolled - , null as courserun_upgrade_deadline - , emeritus_enrollments.user_id - , mitxpro_courseruns.courserun_id - , coalesce(mitxpro_courseruns.courserun_title, emeritus_enrollments.courserun_title) as courserun_title - , coalesce(mitxpro_courseruns.courserun_readable_id, emeritus_enrollments.courserun_external_readable_id) - as courserun_readable_id - , null as user_username - , emeritus_enrollments.user_email - , emeritus_enrollments.user_full_name - , null as courserungrade_grade - , null as courserungrade_is_passing - from emeritus_enrollments - left join mitxpro_courseruns - on - emeritus_enrollments.courserun_external_readable_id = mitxpro_courseruns.courserun_external_readable_id - - union all - - select - '{{ var("global_alumni") }}' as platform - , null as courserunenrollment_id - , global_alumni_enrollments.is_enrolled as courserunenrollment_is_active - , null as courserunenrollment_created_on - , null as courserunenrollment_enrollment_mode - , global_alumni_enrollments.enrollment_status as courserunenrollment_enrollment_status - , null as courserunenrollment_is_edx_enrolled - , null as courserun_upgrade_deadline - , global_alumni_enrollments.user_id - , mitxpro_courseruns.courserun_id - , coalesce(mitxpro_courseruns.courserun_title, global_alumni_enrollments.courserun_title) as courserun_title - , coalesce(mitxpro_courseruns.courserun_readable_id, global_alumni_enrollments.courserun_external_readable_id) - as courserun_readable_id - , null as user_username - , global_alumni_enrollments.user_email - , global_alumni_enrollments.user_full_name - , null as courserungrade_grade - , null as courserungrade_is_passing - from global_alumni_enrollments - left join mitxpro_courseruns - on - global_alumni_enrollments.courserun_external_readable_id = mitxpro_courseruns.courserun_external_readable_id - - union all - - select - '{{ var("bootcamps") }}' as platform - , courserunenrollment_id - , courserunenrollment_is_active - , courserunenrollment_created_on - , null as courserunenrollment_enrollment_mode - , courserunenrollment_enrollment_status - , null as courserunenrollment_is_edx_enrolled - , null as courserun_upgrade_deadline - , cast(user_id as varchar) as user_id - , courserun_id - , courserun_title - , courserun_readable_id - , user_username - , user_email - , user_full_name - , null as courserungrade_grade - , null as courserungrade_is_passing - from bootcamps_enrollments - - union all - - select - '{{ var("residential") }}' as platform - , residential_enrollments.courserunenrollment_id - , residential_enrollments.courserunenrollment_is_active - , residential_enrollments.courserunenrollment_created_on - , residential_enrollments.courserunenrollment_enrollment_mode - , null as courserunenrollment_enrollment_status - , true as courserunenrollment_is_edx_enrolled - , null as courserun_upgrade_deadline - , cast(residential_enrollments.user_id as varchar) as user_id - , null as courserun_id - , residential_enrollments.courserun_title - , residential_enrollments.courserun_readable_id - , residential_enrollments.user_username - , residential_enrollments.user_email - , residential_enrollments.user_full_name - , residential_grades.courserungrade_grade - , if(residential_grades.courserungrade_letter_grade != '', true, false) as courserungrade_is_passing - from residential_enrollments - left join residential_grades - on - residential_enrollments.courserun_readable_id = residential_grades.courserun_readable_id + union all + + select + '{{ var("emeritus") }}' as platform, + null as courserunenrollment_id, + emeritus_enrollments.is_enrolled as courserunenrollment_is_active, + emeritus_enrollments.enrollment_created_on as courserunenrollment_created_on, + null as courserunenrollment_enrollment_mode, + emeritus_enrollments.enrollment_status as courserunenrollment_enrollment_status, + null as courserunenrollment_is_edx_enrolled, + null as courserun_upgrade_deadline, + emeritus_enrollments.user_id, + mitxpro_courseruns.courserun_id, + coalesce(mitxpro_courseruns.courserun_title, emeritus_enrollments.courserun_title) as courserun_title, + coalesce( + mitxpro_courseruns.courserun_readable_id, emeritus_enrollments.courserun_external_readable_id + ) as courserun_readable_id, + null as user_username, + emeritus_enrollments.user_email, + emeritus_enrollments.user_full_name, + null as courserungrade_grade, + null as courserungrade_is_passing + from emeritus_enrollments + left join + mitxpro_courseruns + on emeritus_enrollments.courserun_external_readable_id = mitxpro_courseruns.courserun_external_readable_id + + union all + + select + '{{ var("global_alumni") }}' as platform, + null as courserunenrollment_id, + global_alumni_enrollments.is_enrolled as courserunenrollment_is_active, + null as courserunenrollment_created_on, + null as courserunenrollment_enrollment_mode, + global_alumni_enrollments.enrollment_status as courserunenrollment_enrollment_status, + null as courserunenrollment_is_edx_enrolled, + null as courserun_upgrade_deadline, + global_alumni_enrollments.user_id, + mitxpro_courseruns.courserun_id, + coalesce(mitxpro_courseruns.courserun_title, global_alumni_enrollments.courserun_title) as courserun_title, + coalesce( + mitxpro_courseruns.courserun_readable_id, global_alumni_enrollments.courserun_external_readable_id + ) as courserun_readable_id, + null as user_username, + global_alumni_enrollments.user_email, + global_alumni_enrollments.user_full_name, + null as courserungrade_grade, + null as courserungrade_is_passing + from global_alumni_enrollments + left join + mitxpro_courseruns + on global_alumni_enrollments.courserun_external_readable_id + = mitxpro_courseruns.courserun_external_readable_id + + union all + + select + '{{ var("bootcamps") }}' as platform, + courserunenrollment_id, + courserunenrollment_is_active, + courserunenrollment_created_on, + null as courserunenrollment_enrollment_mode, + courserunenrollment_enrollment_status, + null as courserunenrollment_is_edx_enrolled, + null as courserun_upgrade_deadline, + cast(user_id as varchar) as user_id, + courserun_id, + courserun_title, + courserun_readable_id, + user_username, + user_email, + user_full_name, + null as courserungrade_grade, + null as courserungrade_is_passing + from bootcamps_enrollments + + union all + + select + '{{ var("residential") }}' as platform, + residential_enrollments.courserunenrollment_id, + residential_enrollments.courserunenrollment_is_active, + residential_enrollments.courserunenrollment_created_on, + residential_enrollments.courserunenrollment_enrollment_mode, + null as courserunenrollment_enrollment_status, + true as courserunenrollment_is_edx_enrolled, + null as courserun_upgrade_deadline, + cast(residential_enrollments.user_id as varchar) as user_id, + null as courserun_id, + residential_enrollments.courserun_title, + residential_enrollments.courserun_readable_id, + residential_enrollments.user_username, + residential_enrollments.user_email, + residential_enrollments.user_full_name, + residential_grades.courserungrade_grade, + if(residential_grades.courserungrade_letter_grade != '', true, false) as courserungrade_is_passing + from residential_enrollments + left join + residential_grades + on residential_enrollments.courserun_readable_id = residential_grades.courserun_readable_id and residential_enrollments.user_id = residential_grades.user_id -) + ) select - combined_enrollments.* - , coalesce(combined_courseruns.course_title, combined_enrollments.courserun_title) as course_title - , coalesce( - combined_courseruns.course_readable_id - , {{ extract_course_readable_id('combined_enrollments.courserun_readable_id') }} - ) as course_readable_id - , combined_certificates.courseruncertificate_created_on - , combined_certificates.courseruncertificate_url - , combined_certificates.courseruncertificate_uuid - , if(combined_certificates.courseruncertificate_url is not null, true, false) as courseruncertificate_is_earned + combined_enrollments.*, + coalesce(combined_courseruns.course_title, combined_enrollments.courserun_title) as course_title, + coalesce( + combined_courseruns.course_readable_id, + {{ extract_course_readable_id("combined_enrollments.courserun_readable_id") }} + ) as course_readable_id, + combined_certificates.courseruncertificate_created_on, + combined_certificates.courseruncertificate_url, + combined_certificates.courseruncertificate_uuid, + if(combined_certificates.courseruncertificate_url is not null, true, false) as courseruncertificate_is_earned from combined_enrollments -left join combined_courseruns - on - combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id - and combined_enrollments.platform = combined_courseruns.platform -left join combined_certificates - on - combined_enrollments.platform = combined_certificates.platform - and combined_enrollments.user_username = combined_certificates.user_username - and combined_enrollments.courserun_readable_id = combined_certificates.courserun_readable_id +left join + combined_courseruns + on combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id + and combined_enrollments.platform = combined_courseruns.platform +left join + combined_certificates + on combined_enrollments.platform = combined_certificates.platform + and combined_enrollments.user_username = combined_certificates.user_username + and combined_enrollments.courserun_readable_id = combined_certificates.courserun_readable_id diff --git a/src/ol_dbt/models/intermediate/combined/int__combined__user_course_roles.sql b/src/ol_dbt/models/intermediate/combined/int__combined__user_course_roles.sql index 7c62e5f72..a6e5f9595 100644 --- a/src/ol_dbt/models/intermediate/combined/int__combined__user_course_roles.sql +++ b/src/ol_dbt/models/intermediate/combined/int__combined__user_course_roles.sql @@ -1,172 +1,133 @@ -with mitxonline_courseroles as ( - select * from {{ ref('stg__mitxonline__openedx__mysql__user_courseaccessrole') }} -) - -, mitxonline_app_users as ( - select * from {{ ref('stg__mitxonline__app__postgres__users_user') }} -) - -, mitxonline_openedxuser_mapping as ( - select * from {{ ref('stg__mitxonline__app__postgres__openedx_openedxuser') }} -) - -, mitxonline_openedx_users as ( - select * from {{ ref('stg__mitxonline__openedx__mysql__auth_user') }} -) - -, mitxpro_courseroles as ( - select * from {{ ref('stg__mitxpro__openedx__mysql__user_courseaccessrole') }} -) - -, mitxpro_app_users as ( - select * from {{ ref('stg__mitxpro__app__postgres__users_user') }} -) - -, mitxpro_openedx_users as ( - select * from {{ ref('stg__mitxpro__openedx__mysql__auth_user') }} -) - -, residential_courseroles as ( - select * from {{ ref('stg__mitxresidential__openedx__user_courseaccessrole') }} -) - -, residential_openedx_users as ( - select * from {{ ref('stg__mitxresidential__openedx__auth_user') }} -) - -, edxorg_courseroles as ( - select * from {{ ref('stg__edxorg__s3__user_courseaccessrole') }} -) - -, edxorg_users as ( - select * from {{ ref('stg__edxorg__s3__user') }} -) - -, edxorg_profiles as ( - select * from {{ ref('stg__edxorg__s3__user_profile') }} -) - -, user_course_role_seed_file as ( - select * from {{ ref('user_course_roles') }} -) - -, combined_courseroles as ( - select - '{{ var("mitxonline") }}' as platform - , mitxonline_openedx_users.user_username - , mitxonline_openedx_users.user_email - , mitxonline_app_users.user_full_name - , mitxonline_courseroles.courserun_readable_id - , mitxonline_courseroles.organization - , mitxonline_courseroles.courseaccess_role - , {{ generate_hash_id('mitxonline_openedx_users.user_email') }} as hashed_user_email - from mitxonline_courseroles - inner join mitxonline_openedx_users - on mitxonline_courseroles.openedx_user_id = mitxonline_openedx_users.openedx_user_id - left join mitxonline_openedxuser_mapping - on mitxonline_openedx_users.user_username = mitxonline_openedxuser_mapping.openedxuser_username - left join mitxonline_app_users - on mitxonline_openedxuser_mapping.user_id = mitxonline_app_users.user_id - - union all - - select - '{{ var("edxorg") }}' as platform - , edxorg_users.user_username - , edxorg_users.user_email - , edxorg_profiles.user_full_name - , edxorg_courseroles.courserun_edx_readable_id as courserun_readable_id - , edxorg_courseroles.organization - , edxorg_courseroles.courseaccess_role - , {{ generate_hash_id('edxorg_users.user_email') }} as hashed_user_email - from edxorg_courseroles - inner join edxorg_users - on edxorg_courseroles.user_id = edxorg_users.user_id - left join edxorg_profiles - on edxorg_courseroles.user_id = edxorg_profiles.user_id +with + mitxonline_courseroles as (select * from {{ ref("stg__mitxonline__openedx__mysql__user_courseaccessrole") }}), + mitxonline_app_users as (select * from {{ ref("stg__mitxonline__app__postgres__users_user") }}), + mitxonline_openedxuser_mapping as (select * from {{ ref("stg__mitxonline__app__postgres__openedx_openedxuser") }}), + mitxonline_openedx_users as (select * from {{ ref("stg__mitxonline__openedx__mysql__auth_user") }}), + mitxpro_courseroles as (select * from {{ ref("stg__mitxpro__openedx__mysql__user_courseaccessrole") }}), + mitxpro_app_users as (select * from {{ ref("stg__mitxpro__app__postgres__users_user") }}), + mitxpro_openedx_users as (select * from {{ ref("stg__mitxpro__openedx__mysql__auth_user") }}), + residential_courseroles as (select * from {{ ref("stg__mitxresidential__openedx__user_courseaccessrole") }}), + residential_openedx_users as (select * from {{ ref("stg__mitxresidential__openedx__auth_user") }}), + edxorg_courseroles as (select * from {{ ref("stg__edxorg__s3__user_courseaccessrole") }}), + edxorg_users as (select * from {{ ref("stg__edxorg__s3__user") }}), + edxorg_profiles as (select * from {{ ref("stg__edxorg__s3__user_profile") }}), + user_course_role_seed_file as (select * from {{ ref("user_course_roles") }}), + combined_courseroles as ( + select + '{{ var("mitxonline") }}' as platform, + mitxonline_openedx_users.user_username, + mitxonline_openedx_users.user_email, + mitxonline_app_users.user_full_name, + mitxonline_courseroles.courserun_readable_id, + mitxonline_courseroles.organization, + mitxonline_courseroles.courseaccess_role, + {{ generate_hash_id("mitxonline_openedx_users.user_email") }} as hashed_user_email + from mitxonline_courseroles + inner join + mitxonline_openedx_users + on mitxonline_courseroles.openedx_user_id = mitxonline_openedx_users.openedx_user_id + left join + mitxonline_openedxuser_mapping + on mitxonline_openedx_users.user_username = mitxonline_openedxuser_mapping.openedxuser_username + left join mitxonline_app_users on mitxonline_openedxuser_mapping.user_id = mitxonline_app_users.user_id + + union all - union all + select + '{{ var("edxorg") }}' as platform, + edxorg_users.user_username, + edxorg_users.user_email, + edxorg_profiles.user_full_name, + edxorg_courseroles.courserun_edx_readable_id as courserun_readable_id, + edxorg_courseroles.organization, + edxorg_courseroles.courseaccess_role, + {{ generate_hash_id("edxorg_users.user_email") }} as hashed_user_email + from edxorg_courseroles + inner join edxorg_users on edxorg_courseroles.user_id = edxorg_users.user_id + left join edxorg_profiles on edxorg_courseroles.user_id = edxorg_profiles.user_id + + union all - select - '{{ var("mitxpro") }}' as platform - , mitxpro_openedx_users.user_username - , mitxpro_openedx_users.user_email - , mitxpro_app_users.user_full_name - , mitxpro_courseroles.courserun_readable_id - , mitxpro_courseroles.organization - , mitxpro_courseroles.courseaccess_role - , {{ generate_hash_id('mitxpro_openedx_users.user_email') }} as hashed_user_email - from mitxpro_courseroles - inner join mitxpro_openedx_users - on mitxpro_courseroles.openedx_user_id = mitxpro_openedx_users.openedx_user_id - left join mitxpro_app_users - on - mitxpro_openedx_users.user_username = mitxpro_app_users.user_username + select + '{{ var("mitxpro") }}' as platform, + mitxpro_openedx_users.user_username, + mitxpro_openedx_users.user_email, + mitxpro_app_users.user_full_name, + mitxpro_courseroles.courserun_readable_id, + mitxpro_courseroles.organization, + mitxpro_courseroles.courseaccess_role, + {{ generate_hash_id("mitxpro_openedx_users.user_email") }} as hashed_user_email + from mitxpro_courseroles + inner join mitxpro_openedx_users on mitxpro_courseroles.openedx_user_id = mitxpro_openedx_users.openedx_user_id + left join + mitxpro_app_users + on mitxpro_openedx_users.user_username = mitxpro_app_users.user_username or mitxpro_openedx_users.user_email = mitxpro_app_users.user_email - union all - - select - '{{ var("residential") }}' as platform - , residential_openedx_users.user_username - , residential_openedx_users.user_email - , residential_openedx_users.user_full_name - , residential_courseroles.courserun_readable_id - , residential_courseroles.organization - , residential_courseroles.courseaccess_role - , {{ generate_hash_id('residential_openedx_users.user_email') }} as hashed_user_email - from residential_courseroles - inner join residential_openedx_users on residential_courseroles.user_id = residential_openedx_users.user_id -) + union all -, seed_file_users as ( - select * from ( select - combined_courseroles.platform - , combined_courseroles.user_username - , combined_courseroles.user_email - , combined_courseroles.hashed_user_email - , combined_courseroles.user_full_name - , row_number() over ( - partition by combined_courseroles.hashed_user_email, combined_courseroles.platform - order by combined_courseroles.user_email desc - ) as row_num - from combined_courseroles - inner join user_course_role_seed_file - on combined_courseroles.hashed_user_email = user_course_role_seed_file.hashed_user_email + '{{ var("residential") }}' as platform, + residential_openedx_users.user_username, + residential_openedx_users.user_email, + residential_openedx_users.user_full_name, + residential_courseroles.courserun_readable_id, + residential_courseroles.organization, + residential_courseroles.courseaccess_role, + {{ generate_hash_id("residential_openedx_users.user_email") }} as hashed_user_email + from residential_courseroles + inner join residential_openedx_users on residential_courseroles.user_id = residential_openedx_users.user_id + ), + seed_file_users as ( + select * + from + ( + select + combined_courseroles.platform, + combined_courseroles.user_username, + combined_courseroles.user_email, + combined_courseroles.hashed_user_email, + combined_courseroles.user_full_name, + row_number() over ( + partition by combined_courseroles.hashed_user_email, combined_courseroles.platform + order by combined_courseroles.user_email desc + ) as row_num + from combined_courseroles + inner join + user_course_role_seed_file + on combined_courseroles.hashed_user_email = user_course_role_seed_file.hashed_user_email + ) + where row_num = 1 ) - where row_num = 1 -) select - platform - , combined_courseroles.user_username - , combined_courseroles.user_email - , combined_courseroles.user_full_name - , combined_courseroles.courserun_readable_id - , organization - , courseaccess_role - , hashed_user_email + platform, + combined_courseroles.user_username, + combined_courseroles.user_email, + combined_courseroles.user_full_name, + combined_courseroles.courserun_readable_id, + organization, + courseaccess_role, + hashed_user_email from combined_courseroles union all select - user_course_role_seed_file.platform - , seed_file_users.user_username - , seed_file_users.user_email - , seed_file_users.user_full_name - , user_course_role_seed_file.courserun_readable_id - , user_course_role_seed_file.organization - , user_course_role_seed_file.courseaccess_role - , user_course_role_seed_file.hashed_user_email + user_course_role_seed_file.platform, + seed_file_users.user_username, + seed_file_users.user_email, + seed_file_users.user_full_name, + user_course_role_seed_file.courserun_readable_id, + user_course_role_seed_file.organization, + user_course_role_seed_file.courseaccess_role, + user_course_role_seed_file.hashed_user_email from user_course_role_seed_file -left join combined_courseroles - on - user_course_role_seed_file.platform = combined_courseroles.platform - and user_course_role_seed_file.hashed_user_email = combined_courseroles.hashed_user_email - and user_course_role_seed_file.courserun_readable_id = combined_courseroles.courserun_readable_id - and user_course_role_seed_file.courseaccess_role = combined_courseroles.courseaccess_role -left join seed_file_users - on user_course_role_seed_file.hashed_user_email = seed_file_users.hashed_user_email +left join + combined_courseroles + on user_course_role_seed_file.platform = combined_courseroles.platform + and user_course_role_seed_file.hashed_user_email = combined_courseroles.hashed_user_email + and user_course_role_seed_file.courserun_readable_id = combined_courseroles.courserun_readable_id + and user_course_role_seed_file.courseaccess_role = combined_courseroles.courseaccess_role +left join seed_file_users on user_course_role_seed_file.hashed_user_email = seed_file_users.hashed_user_email where combined_courseroles.hashed_user_email is null diff --git a/src/ol_dbt/models/intermediate/combined/int__combined__users.sql b/src/ol_dbt/models/intermediate/combined/int__combined__users.sql index ea7246948..f99912ea7 100644 --- a/src/ol_dbt/models/intermediate/combined/int__combined__users.sql +++ b/src/ol_dbt/models/intermediate/combined/int__combined__users.sql @@ -1,259 +1,238 @@ ---- This model combines intermediate users from different platforms, it contains duplicates for --- MITx Online and edX.org users, deduplication is handled in int__mitx__users -{{ config(materialized='view') }} - -with mitxonline_users as ( - select * from {{ ref('int__mitxonline__users') }} -) - -, mitxpro_users as ( - select * from {{ ref('int__mitxpro__users') }} -) - -, bootcamps_users as ( - select * from {{ ref('int__bootcamps__users') }} -) - -, edxorg_users as ( - select * from {{ ref('int__edxorg__mitx_users') }} -) - -, residential_users as ( - select * from {{ ref('int__mitxresidential__users') }} -) - -, micromasters_users as ( - select * from {{ ref('int__micromasters__users') }} -) - -, emeritus_users as ( - select - user_id - , user_email - , user_full_name - , user_address_country - , user_gender - , user_company - , user_job_title - , user_industry - , user_address_street - , user_address_city - , user_address_state - , user_address_postal_code - , row_number() over ( - partition by coalesce(user_id, user_email, user_full_name) - order by user_gdpr_consent_date desc, enrollment_created_on desc - ) as row_num - from {{ ref('stg__emeritus__api__bigquery__user_enrollments') }} -) - -, global_alumni_users as ( - select - user_id - , user_email - , user_full_name - , user_address_country - , user_gender - , user_company - , user_job_title - , user_industry - , user_address_street - , user_address_city - , user_address_state - , user_address_postal_code - , row_number() over ( - partition by user_email - order by user_gdpr_consent_date desc, courserun_start_on desc - ) as row_num - from {{ ref('stg__global_alumni__api__bigquery__user_enrollments') }} -) - -, combined_users as ( - select - '{{ var("mitxonline") }}' as platform - , cast(user_id as varchar) as user_id - , openedx_user_id - , user_username - , user_email - , user_full_name - , user_address_country - , user_highest_education - , user_gender - , user_birth_year - , user_company - , user_job_title - , user_industry - , user_joined_on - , user_last_login - , user_is_active - , null as user_street_address - , null as user_address_city - , user_address_state as user_address_state_or_territory - , null as user_address_postal_code - from mitxonline_users - - union all - - select - '{{ var("mitxpro") }}' as platform - , cast(user_id as varchar) as user_id - , openedx_user_id - , user_username - , user_email - , user_full_name - , user_address_country - , user_highest_education - , user_gender - , user_birth_year - , user_company - , user_job_title - , user_industry - , user_joined_on - , user_last_login - , user_is_active - , user_street_address - , user_address_city - , user_address_state_or_territory - , user_address_postal_code - from mitxpro_users - - union all - - select - '{{ var("emeritus") }}' as platform - , user_id - , null as openedx_user_id - , null as user_username - , user_email - , user_full_name - , user_address_country - , null as user_highest_education - , user_gender - , null as user_birth_year - , user_company - , user_job_title - , user_industry - , null as user_joined_on - , null as user_last_login - , null as user_is_active - , user_address_street as user_street_address - , user_address_city - , user_address_state as user_address_state_or_territory - , user_address_postal_code - from emeritus_users - where row_num = 1 - - union all - - select - '{{ var("global_alumni") }}' as platform - , user_id - , null as openedx_user_id - , null as user_username - , user_email - , user_full_name - , user_address_country - , null as user_highest_education - , user_gender - , null as user_birth_year - , user_company - , user_job_title - , user_industry - , null as user_joined_on - , null as user_last_login - , null as user_is_active - , user_address_street as user_street_address - , user_address_city - , user_address_state as user_address_state_or_territory - , user_address_postal_code - from global_alumni_users - where row_num = 1 - - union all - - select - '{{ var("bootcamps") }}' as platform - , cast(user_id as varchar) as user_id - , null as openedx_user_id - , user_username - , user_email - , user_full_name - , user_address_country - , user_highest_education - , user_gender - , user_birth_year - , user_company - , user_job_title - , user_industry - , user_joined_on - , user_last_login - , user_is_active - , user_street_address - , user_address_city - , user_address_state_or_territory - , user_address_postal_code - from bootcamps_users - - union all - - select - '{{ var("edxorg") }}' as platform - , cast(edxorg_users.user_id as varchar) as user_id - , edxorg_users.user_id as openedx_user_id - , edxorg_users.user_username - , edxorg_users.user_email - , edxorg_users.user_full_name - , edxorg_users.user_country as user_address_country - , edxorg_users.user_highest_education - , edxorg_users.user_gender - , edxorg_users.user_birth_year - , micromasters_users.user_company_name as user_company - , micromasters_users.user_job_position as user_job_title - , micromasters_users.user_company_industry as user_industry - , edxorg_users.user_joined_on - , edxorg_users.user_last_login - , edxorg_users.user_is_active - , micromasters_users.user_street_address - , micromasters_users.user_address_city - , micromasters_users.user_address_state_or_territory - , micromasters_users.user_address_postal_code - from edxorg_users - left join micromasters_users on edxorg_users.user_username = micromasters_users.user_edxorg_username - - union all - - select - '{{ var("residential") }}' as platform - , cast(user_id as varchar) as user_id - , user_id as openedx_user_id - , user_username - , user_email - , user_full_name - , user_address_country - , user_highest_education - , user_gender - , user_birth_year - , null as user_company - , null as user_job_title - , null as user_industry - , user_joined_on - , user_last_login - , user_is_active - , null as user_street_address - , user_address_city - , null as user_address_state_or_territory - , null as user_address_postal_code - from residential_users -) +-- - This model combines intermediate users from different platforms, it contains duplicates for +-- MITx Online and edX.org users, deduplication is handled in int__mitx__users +{{ config(materialized="view") }} + +with + mitxonline_users as (select * from {{ ref("int__mitxonline__users") }}), + mitxpro_users as (select * from {{ ref("int__mitxpro__users") }}), + bootcamps_users as (select * from {{ ref("int__bootcamps__users") }}), + edxorg_users as (select * from {{ ref("int__edxorg__mitx_users") }}), + residential_users as (select * from {{ ref("int__mitxresidential__users") }}), + micromasters_users as (select * from {{ ref("int__micromasters__users") }}), + emeritus_users as ( + select + user_id, + user_email, + user_full_name, + user_address_country, + user_gender, + user_company, + user_job_title, + user_industry, + user_address_street, + user_address_city, + user_address_state, + user_address_postal_code, + row_number() over ( + partition by coalesce(user_id, user_email, user_full_name) + order by user_gdpr_consent_date desc, enrollment_created_on desc + ) as row_num + from {{ ref("stg__emeritus__api__bigquery__user_enrollments") }} + ), + global_alumni_users as ( + select + user_id, + user_email, + user_full_name, + user_address_country, + user_gender, + user_company, + user_job_title, + user_industry, + user_address_street, + user_address_city, + user_address_state, + user_address_postal_code, + row_number() over ( + partition by user_email order by user_gdpr_consent_date desc, courserun_start_on desc + ) as row_num + from {{ ref("stg__global_alumni__api__bigquery__user_enrollments") }} + ), + combined_users as ( + select + '{{ var("mitxonline") }}' as platform, + cast(user_id as varchar) as user_id, + openedx_user_id, + user_username, + user_email, + user_full_name, + user_address_country, + user_highest_education, + user_gender, + user_birth_year, + user_company, + user_job_title, + user_industry, + user_joined_on, + user_last_login, + user_is_active, + null as user_street_address, + null as user_address_city, + user_address_state as user_address_state_or_territory, + null as user_address_postal_code + from mitxonline_users + + union all + + select + '{{ var("mitxpro") }}' as platform, + cast(user_id as varchar) as user_id, + openedx_user_id, + user_username, + user_email, + user_full_name, + user_address_country, + user_highest_education, + user_gender, + user_birth_year, + user_company, + user_job_title, + user_industry, + user_joined_on, + user_last_login, + user_is_active, + user_street_address, + user_address_city, + user_address_state_or_territory, + user_address_postal_code + from mitxpro_users + + union all + + select + '{{ var("emeritus") }}' as platform, + user_id, + null as openedx_user_id, + null as user_username, + user_email, + user_full_name, + user_address_country, + null as user_highest_education, + user_gender, + null as user_birth_year, + user_company, + user_job_title, + user_industry, + null as user_joined_on, + null as user_last_login, + null as user_is_active, + user_address_street as user_street_address, + user_address_city, + user_address_state as user_address_state_or_territory, + user_address_postal_code + from emeritus_users + where row_num = 1 + + union all + + select + '{{ var("global_alumni") }}' as platform, + user_id, + null as openedx_user_id, + null as user_username, + user_email, + user_full_name, + user_address_country, + null as user_highest_education, + user_gender, + null as user_birth_year, + user_company, + user_job_title, + user_industry, + null as user_joined_on, + null as user_last_login, + null as user_is_active, + user_address_street as user_street_address, + user_address_city, + user_address_state as user_address_state_or_territory, + user_address_postal_code + from global_alumni_users + where row_num = 1 + + union all + + select + '{{ var("bootcamps") }}' as platform, + cast(user_id as varchar) as user_id, + null as openedx_user_id, + user_username, + user_email, + user_full_name, + user_address_country, + user_highest_education, + user_gender, + user_birth_year, + user_company, + user_job_title, + user_industry, + user_joined_on, + user_last_login, + user_is_active, + user_street_address, + user_address_city, + user_address_state_or_territory, + user_address_postal_code + from bootcamps_users + + union all + + select + '{{ var("edxorg") }}' as platform, + cast(edxorg_users.user_id as varchar) as user_id, + edxorg_users.user_id as openedx_user_id, + edxorg_users.user_username, + edxorg_users.user_email, + edxorg_users.user_full_name, + edxorg_users.user_country as user_address_country, + edxorg_users.user_highest_education, + edxorg_users.user_gender, + edxorg_users.user_birth_year, + micromasters_users.user_company_name as user_company, + micromasters_users.user_job_position as user_job_title, + micromasters_users.user_company_industry as user_industry, + edxorg_users.user_joined_on, + edxorg_users.user_last_login, + edxorg_users.user_is_active, + micromasters_users.user_street_address, + micromasters_users.user_address_city, + micromasters_users.user_address_state_or_territory, + micromasters_users.user_address_postal_code + from edxorg_users + left join micromasters_users on edxorg_users.user_username = micromasters_users.user_edxorg_username + + union all + + select + '{{ var("residential") }}' as platform, + cast(user_id as varchar) as user_id, + user_id as openedx_user_id, + user_username, + user_email, + user_full_name, + user_address_country, + user_highest_education, + user_gender, + user_birth_year, + null as user_company, + null as user_job_title, + null as user_industry, + user_joined_on, + user_last_login, + user_is_active, + null as user_street_address, + user_address_city, + null as user_address_state_or_territory, + null as user_address_postal_code + from residential_users + ) select case when user_id is not null - then {{ generate_hash_id('user_id || platform') }} + then {{ generate_hash_id("user_id || platform") }} when user_email is not null - then {{ generate_hash_id('user_email || platform') }} - else - {{ generate_hash_id('user_full_name || platform') }} - end as user_hashed_id - , * + then {{ generate_hash_id("user_email || platform") }} + else {{ generate_hash_id("user_full_name || platform") }} + end as user_hashed_id, + * from combined_users diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_course_structure.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_course_structure.sql index a5d43958a..69cbce245 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_course_structure.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_course_structure.sql @@ -1,65 +1,59 @@ -with course_structure as ( - select * from {{ ref('stg__edxorg__s3__course_structure') }} - order by courserun_readable_id, coursestructure_retrieved_at, coursestructure_block_index +with + course_structure as ( + select * + from {{ ref("stg__edxorg__s3__course_structure") }} + order by courserun_readable_id, coursestructure_retrieved_at, coursestructure_block_index -) - -, latest_course_structure_date as ( - select - courserun_readable_id - , max(coursestructure_retrieved_at) as max_retrieved_date - from course_structure - group by courserun_readable_id -) - -, chapters as ( - select * from course_structure - where coursestructure_block_category = 'chapter' -) - -, course_structure_with_chapters as ( - select - course_structure.* - , chapters.coursestructure_block_id as coursestructure_chapter_id - , chapters.coursestructure_block_title as coursestructure_chapter_title - , row_number() over ( - partition by - course_structure.courserun_readable_id - , course_structure.coursestructure_block_index - , course_structure.coursestructure_retrieved_at - order by chapters.coursestructure_block_index desc - ) as row_num - from course_structure - inner join chapters - on - course_structure.courserun_readable_id = chapters.courserun_readable_id + ), + latest_course_structure_date as ( + select courserun_readable_id, max(coursestructure_retrieved_at) as max_retrieved_date + from course_structure + group by courserun_readable_id + ), + chapters as (select * from course_structure where coursestructure_block_category = 'chapter'), + course_structure_with_chapters as ( + select + course_structure.*, + chapters.coursestructure_block_id as coursestructure_chapter_id, + chapters.coursestructure_block_title as coursestructure_chapter_title, + row_number() over ( + partition by + course_structure.courserun_readable_id, + course_structure.coursestructure_block_index, + course_structure.coursestructure_retrieved_at + order by chapters.coursestructure_block_index desc + ) as row_num + from course_structure + inner join + chapters + on course_structure.courserun_readable_id = chapters.courserun_readable_id and course_structure.coursestructure_retrieved_at = chapters.coursestructure_retrieved_at and course_structure.coursestructure_block_index >= chapters.coursestructure_block_index -) + ) select - course_structure.courserun_readable_id - , course_structure.courserun_title - , course_structure.coursestructure_block_index - , course_structure.coursestructure_block_id - , course_structure.coursestructure_parent_block_id - , course_structure.coursestructure_block_category - , course_structure.coursestructure_block_title - , course_structure.coursestructure_content_hash - , course_structure.coursestructure_block_content_hash - , course_structure.coursestructure_block_metadata - , course_structure.coursestructure_retrieved_at - , course_structure_with_chapters.coursestructure_chapter_id - , course_structure_with_chapters.coursestructure_chapter_title - , if(latest_course_structure_date.max_retrieved_date is not null, true, false) as coursestructure_is_latest + course_structure.courserun_readable_id, + course_structure.courserun_title, + course_structure.coursestructure_block_index, + course_structure.coursestructure_block_id, + course_structure.coursestructure_parent_block_id, + course_structure.coursestructure_block_category, + course_structure.coursestructure_block_title, + course_structure.coursestructure_content_hash, + course_structure.coursestructure_block_content_hash, + course_structure.coursestructure_block_metadata, + course_structure.coursestructure_retrieved_at, + course_structure_with_chapters.coursestructure_chapter_id, + course_structure_with_chapters.coursestructure_chapter_title, + if(latest_course_structure_date.max_retrieved_date is not null, true, false) as coursestructure_is_latest from course_structure -left join latest_course_structure_date - on - course_structure.courserun_readable_id = latest_course_structure_date.courserun_readable_id - and course_structure.coursestructure_retrieved_at = latest_course_structure_date.max_retrieved_date -left join course_structure_with_chapters - on - course_structure.courserun_readable_id = course_structure_with_chapters.courserun_readable_id - and course_structure.coursestructure_block_id = course_structure_with_chapters.coursestructure_block_id - and course_structure.coursestructure_retrieved_at = course_structure_with_chapters.coursestructure_retrieved_at - and course_structure_with_chapters.row_num = 1 +left join + latest_course_structure_date + on course_structure.courserun_readable_id = latest_course_structure_date.courserun_readable_id + and course_structure.coursestructure_retrieved_at = latest_course_structure_date.max_retrieved_date +left join + course_structure_with_chapters + on course_structure.courserun_readable_id = course_structure_with_chapters.courserun_readable_id + and course_structure.coursestructure_block_id = course_structure_with_chapters.coursestructure_block_id + and course_structure.coursestructure_retrieved_at = course_structure_with_chapters.coursestructure_retrieved_at + and course_structure_with_chapters.row_num = 1 diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_certificates.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_certificates.sql index 11a63b81d..be2fafb49 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_certificates.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_certificates.sql @@ -1,182 +1,159 @@ -- Course Certificate information for edx.org -- for DEDP courses that were ran on edX.org, certificates are pull from MicroMasters. - -with person_courses as ( - select * - from {{ ref('stg__edxorg__bigquery__mitx_person_course') }} - where - courserun_platform = '{{ var("edxorg") }}' - and courseruncertificate_is_earned = true -) - -, user_info_combo as ( - select * - from {{ ref('stg__edxorg__bigquery__mitx_user_info_combo') }} - where - courserun_platform = '{{ var("edxorg") }}' - and courseruncertificate_id is not null -) - -, certificates as ( - select - user_info_combo.courseruncertificate_id - , user_info_combo.courseruncertificate_courserun_readable_id as courserun_readable_id - , user_info_combo.courseruncertificate_mode - , user_info_combo.courseruncertificate_grade - , user_info_combo.courseruncertificate_download_url - , user_info_combo.courseruncertificate_download_uuid - , user_info_combo.courseruncertificate_verify_uuid - , user_info_combo.courseruncertificate_name - , user_info_combo.courseruncertificate_status - , user_info_combo.courseruncertificate_created_on - , user_info_combo.courseruncertificate_updated_on - , user_info_combo.user_id - , user_info_combo.user_email - , user_info_combo.user_username - from user_info_combo - inner join person_courses -- this join is needed to filter out courseruncertificate that are not earned - on - user_info_combo.user_id = person_courses.user_id +with + person_courses as ( + select * + from {{ ref("stg__edxorg__bigquery__mitx_person_course") }} + where courserun_platform = '{{ var("edxorg") }}' and courseruncertificate_is_earned = true + ), + user_info_combo as ( + select * + from {{ ref("stg__edxorg__bigquery__mitx_user_info_combo") }} + where courserun_platform = '{{ var("edxorg") }}' and courseruncertificate_id is not null + ), + certificates as ( + select + user_info_combo.courseruncertificate_id, + user_info_combo.courseruncertificate_courserun_readable_id as courserun_readable_id, + user_info_combo.courseruncertificate_mode, + user_info_combo.courseruncertificate_grade, + user_info_combo.courseruncertificate_download_url, + user_info_combo.courseruncertificate_download_uuid, + user_info_combo.courseruncertificate_verify_uuid, + user_info_combo.courseruncertificate_name, + user_info_combo.courseruncertificate_status, + user_info_combo.courseruncertificate_created_on, + user_info_combo.courseruncertificate_updated_on, + user_info_combo.user_id, + user_info_combo.user_email, + user_info_combo.user_username + from user_info_combo + inner join + person_courses -- this join is needed to filter out courseruncertificate that are not earned + on user_info_combo.user_id = person_courses.user_id and user_info_combo.courseruncertificate_courserun_readable_id = person_courses.courserun_readable_id -) - -, users as ( - select * from {{ ref('int__edxorg__mitx_users') }} -) - -, runs as ( - select * from {{ ref('int__edxorg__mitx_courseruns') }} -) - -, micromasters_users as ( - select * from {{ ref('__micromasters__users') }} -) - -, edxorg_enrollments as ( - select * from {{ ref('int__edxorg__mitx_courserun_enrollments') }} -) - -, dedp_edxorg_certificates_from_micromasters as ( - select * from {{ ref('__micromasters_course_certificates_dedp_from_micromasters') }} - where courserun_platform = '{{ var("edxorg") }}' -) - -, dedp_edxorg_grades_from_micromasters as ( - select * from {{ ref('__micromasters_course_grades_dedp_from_micromasters') }} - where courserun_platform = '{{ var("edxorg") }}' -) - -, edxorg_dedp_certificates_from_micromasters as ( - select - dedp_edxorg_certificates_from_micromasters.courserun_edxorg_readable_id as courserun_readable_id - , edxorg_enrollments.courserunenrollment_enrollment_mode as courseruncertificate_mode - , dedp_edxorg_grades_from_micromasters.coursegrade_grade as courseruncertificate_grade - , dedp_edxorg_certificates_from_micromasters.coursecertificate_url as courseruncertificate_download_url - , dedp_edxorg_certificates_from_micromasters.coursecertificate_hash as courseruncertificate_download_uuid - , dedp_edxorg_certificates_from_micromasters.coursecertificate_hash as courseruncertificate_verify_uuid - , micromasters_users.user_full_name as courseruncertificate_name - , 'downloadable' as courseruncertificate_status - , dedp_edxorg_certificates_from_micromasters.coursecertificate_created_on as courseruncertificate_created_on - , dedp_edxorg_certificates_from_micromasters.coursecertificate_updated_on as courseruncertificate_updated_on - , users.user_id - , users.user_email - , users.user_username - , edxorg_enrollments.user_mitxonline_username - , runs.courserun_title - , runs.course_number - , coalesce(micromasters_users.user_full_name, users.user_full_name) as user_full_name - from dedp_edxorg_certificates_from_micromasters - inner join micromasters_users - on dedp_edxorg_certificates_from_micromasters.user_micromasters_id = micromasters_users.user_id - inner join users on micromasters_users.user_edxorg_username = users.user_username - left join - runs - on dedp_edxorg_certificates_from_micromasters.courserun_edxorg_readable_id = runs.courserun_readable_id - left join dedp_edxorg_grades_from_micromasters - on - dedp_edxorg_certificates_from_micromasters.courserun_readable_id + ), + users as (select * from {{ ref("int__edxorg__mitx_users") }}), + runs as (select * from {{ ref("int__edxorg__mitx_courseruns") }}), + micromasters_users as (select * from {{ ref("__micromasters__users") }}), + edxorg_enrollments as (select * from {{ ref("int__edxorg__mitx_courserun_enrollments") }}), + dedp_edxorg_certificates_from_micromasters as ( + select * + from {{ ref("__micromasters_course_certificates_dedp_from_micromasters") }} + where courserun_platform = '{{ var("edxorg") }}' + ), + dedp_edxorg_grades_from_micromasters as ( + select * + from {{ ref("__micromasters_course_grades_dedp_from_micromasters") }} + where courserun_platform = '{{ var("edxorg") }}' + ), + edxorg_dedp_certificates_from_micromasters as ( + select + dedp_edxorg_certificates_from_micromasters.courserun_edxorg_readable_id as courserun_readable_id, + edxorg_enrollments.courserunenrollment_enrollment_mode as courseruncertificate_mode, + dedp_edxorg_grades_from_micromasters.coursegrade_grade as courseruncertificate_grade, + dedp_edxorg_certificates_from_micromasters.coursecertificate_url as courseruncertificate_download_url, + dedp_edxorg_certificates_from_micromasters.coursecertificate_hash as courseruncertificate_download_uuid, + dedp_edxorg_certificates_from_micromasters.coursecertificate_hash as courseruncertificate_verify_uuid, + micromasters_users.user_full_name as courseruncertificate_name, + 'downloadable' as courseruncertificate_status, + dedp_edxorg_certificates_from_micromasters.coursecertificate_created_on as courseruncertificate_created_on, + dedp_edxorg_certificates_from_micromasters.coursecertificate_updated_on as courseruncertificate_updated_on, + users.user_id, + users.user_email, + users.user_username, + edxorg_enrollments.user_mitxonline_username, + runs.courserun_title, + runs.course_number, + coalesce(micromasters_users.user_full_name, users.user_full_name) as user_full_name + from dedp_edxorg_certificates_from_micromasters + inner join + micromasters_users + on dedp_edxorg_certificates_from_micromasters.user_micromasters_id = micromasters_users.user_id + inner join users on micromasters_users.user_edxorg_username = users.user_username + left join + runs on dedp_edxorg_certificates_from_micromasters.courserun_edxorg_readable_id = runs.courserun_readable_id + left join + dedp_edxorg_grades_from_micromasters + on dedp_edxorg_certificates_from_micromasters.courserun_readable_id = dedp_edxorg_grades_from_micromasters.courserun_readable_id and dedp_edxorg_certificates_from_micromasters.user_micromasters_id = dedp_edxorg_grades_from_micromasters.user_micromasters_id - left join edxorg_enrollments - on - dedp_edxorg_certificates_from_micromasters.courserun_edxorg_readable_id + left join + edxorg_enrollments + on dedp_edxorg_certificates_from_micromasters.courserun_edxorg_readable_id = edxorg_enrollments.courserun_readable_id and micromasters_users.user_edxorg_username = edxorg_enrollments.user_username -) - -, edxorg_non_dedp_certificates as ( - select - certificates.courserun_readable_id - , certificates.courseruncertificate_mode - , certificates.courseruncertificate_grade - , certificates.courseruncertificate_download_url - , certificates.courseruncertificate_download_uuid - , certificates.courseruncertificate_verify_uuid - , certificates.courseruncertificate_name - , certificates.courseruncertificate_status - , certificates.courseruncertificate_created_on - , certificates.courseruncertificate_updated_on - , users.user_id - , users.user_email - , users.user_username - , micromasters_users.user_mitxonline_username - , runs.courserun_title - , runs.course_number - , runs.micromasters_program_id - , coalesce(users.user_full_name, micromasters_users.user_full_name) as user_full_name - from certificates - inner join users - on certificates.user_id = users.user_id - left join runs - on certificates.courserun_readable_id = runs.courserun_readable_id - left join micromasters_users - on certificates.user_username = micromasters_users.user_edxorg_username - where - runs.micromasters_program_id != {{ var("dedp_micromasters_program_id") }} - or runs.micromasters_program_id is null -) + ), + edxorg_non_dedp_certificates as ( + select + certificates.courserun_readable_id, + certificates.courseruncertificate_mode, + certificates.courseruncertificate_grade, + certificates.courseruncertificate_download_url, + certificates.courseruncertificate_download_uuid, + certificates.courseruncertificate_verify_uuid, + certificates.courseruncertificate_name, + certificates.courseruncertificate_status, + certificates.courseruncertificate_created_on, + certificates.courseruncertificate_updated_on, + users.user_id, + users.user_email, + users.user_username, + micromasters_users.user_mitxonline_username, + runs.courserun_title, + runs.course_number, + runs.micromasters_program_id, + coalesce(users.user_full_name, micromasters_users.user_full_name) as user_full_name + from certificates + inner join users on certificates.user_id = users.user_id + left join runs on certificates.courserun_readable_id = runs.courserun_readable_id + left join micromasters_users on certificates.user_username = micromasters_users.user_edxorg_username + where + runs.micromasters_program_id != {{ var("dedp_micromasters_program_id") }} + or runs.micromasters_program_id is null + ) select - courserun_readable_id - , courseruncertificate_mode - , courseruncertificate_grade - , courseruncertificate_download_url - , courseruncertificate_download_uuid - , courseruncertificate_verify_uuid - , courseruncertificate_name - , courseruncertificate_status - , courseruncertificate_created_on - , courseruncertificate_updated_on - , user_id - , user_email - , user_username - , user_full_name - , user_mitxonline_username - , courserun_title - , course_number - , micromasters_program_id + courserun_readable_id, + courseruncertificate_mode, + courseruncertificate_grade, + courseruncertificate_download_url, + courseruncertificate_download_uuid, + courseruncertificate_verify_uuid, + courseruncertificate_name, + courseruncertificate_status, + courseruncertificate_created_on, + courseruncertificate_updated_on, + user_id, + user_email, + user_username, + user_full_name, + user_mitxonline_username, + courserun_title, + course_number, + micromasters_program_id from edxorg_non_dedp_certificates union all select - courserun_readable_id - , courseruncertificate_mode - , courseruncertificate_grade - , courseruncertificate_download_url - , courseruncertificate_download_uuid - , courseruncertificate_verify_uuid - , courseruncertificate_name - , courseruncertificate_status - , courseruncertificate_created_on - , courseruncertificate_updated_on - , user_id - , user_email - , user_username - , user_full_name - , user_mitxonline_username - , courserun_title - , course_number - , {{ var("dedp_micromasters_program_id") }} as micromasters_program_id + courserun_readable_id, + courseruncertificate_mode, + courseruncertificate_grade, + courseruncertificate_download_url, + courseruncertificate_download_uuid, + courseruncertificate_verify_uuid, + courseruncertificate_name, + courseruncertificate_status, + courseruncertificate_created_on, + courseruncertificate_updated_on, + user_id, + user_email, + user_username, + user_full_name, + user_mitxonline_username, + courserun_title, + course_number, + {{ var("dedp_micromasters_program_id") }} as micromasters_program_id from edxorg_dedp_certificates_from_micromasters diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_enrollments.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_enrollments.sql index f6d340969..1576d6cd1 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_enrollments.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_enrollments.sql @@ -1,124 +1,94 @@ -- Course Run Enrollment information from edx.org -- For DEDP courses that run on edX.org, enrollments are verified via their purchased orders in MicroMasters - -with person_courses as ( - select - user_id - , user_username - , courserun_readable_id - , courserunenrollment_created_on - , courserunenrollment_enrollment_mode - , courserunenrollment_is_active - from {{ ref('stg__edxorg__bigquery__mitx_person_course') }} - where courserun_platform = '{{ var("edxorg") }}' -) - -, user_info_combo as ( - select - user_id - , courserunenrollment_courserun_readable_id - from {{ ref('stg__edxorg__bigquery__mitx_user_info_combo') }} - where - courserun_platform = '{{ var("edxorg") }}' - and courserunenrollment_courserun_readable_id is not null -) - -, micromasters_runs as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_courserun') }} -) - -, edxorg_enrollments as ( - select - person_courses.user_id - , person_courses.user_username - , person_courses.courserun_readable_id - , person_courses.courserunenrollment_created_on - , person_courses.courserunenrollment_enrollment_mode - , person_courses.courserunenrollment_is_active - , micromasters_runs.courserun_upgrade_deadline - from - person_courses - inner join user_info_combo - on - person_courses.user_id = user_info_combo.user_id +with + person_courses as ( + select + user_id, + user_username, + courserun_readable_id, + courserunenrollment_created_on, + courserunenrollment_enrollment_mode, + courserunenrollment_is_active + from {{ ref("stg__edxorg__bigquery__mitx_person_course") }} + where courserun_platform = '{{ var("edxorg") }}' + ), + user_info_combo as ( + select user_id, courserunenrollment_courserun_readable_id + from {{ ref("stg__edxorg__bigquery__mitx_user_info_combo") }} + where courserun_platform = '{{ var("edxorg") }}' and courserunenrollment_courserun_readable_id is not null + ), + micromasters_runs as (select * from {{ ref("stg__micromasters__app__postgres__courses_courserun") }}), + edxorg_enrollments as ( + select + person_courses.user_id, + person_courses.user_username, + person_courses.courserun_readable_id, + person_courses.courserunenrollment_created_on, + person_courses.courserunenrollment_enrollment_mode, + person_courses.courserunenrollment_is_active, + micromasters_runs.courserun_upgrade_deadline + from person_courses + inner join + user_info_combo + on person_courses.user_id = user_info_combo.user_id and person_courses.courserun_readable_id = user_info_combo.courserunenrollment_courserun_readable_id - left join micromasters_runs - on person_courses.courserun_readable_id = micromasters_runs.courserun_edxorg_readable_id -) - -, edxorg_runs as ( - select * - from {{ ref('stg__edxorg__bigquery__mitx_courserun') }} -) - -, edxorg_users as ( - select * from {{ ref('int__edxorg__mitx_users') }} -) - -, micromasters_orders as ( - select * from {{ ref('stg__micromasters__app__postgres__ecommerce_order') }} -) - -, micromasters_lines as ( - select * from {{ ref('stg__micromasters__app__postgres__ecommerce_line') }} -) - -, micromasters_courses as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_course') }} -) - -, micromasters_users as ( - select * from {{ ref('__micromasters__users') }} -) - -, dedp_edxorg_enrollments_verified as ( - select distinct - edxorg_enrollments.user_id - , edxorg_enrollments.courserun_readable_id - from edxorg_enrollments - inner join edxorg_users on edxorg_enrollments.user_id = edxorg_users.user_id - inner join micromasters_users on edxorg_users.user_username = micromasters_users.user_edxorg_username - inner join micromasters_orders on micromasters_users.user_id = micromasters_orders.user_id - inner join micromasters_lines - on - edxorg_enrollments.courserun_readable_id = micromasters_lines.courserun_edxorg_readable_id + left join + micromasters_runs on person_courses.courserun_readable_id = micromasters_runs.courserun_edxorg_readable_id + ), + edxorg_runs as (select * from {{ ref("stg__edxorg__bigquery__mitx_courserun") }}), + edxorg_users as (select * from {{ ref("int__edxorg__mitx_users") }}), + micromasters_orders as (select * from {{ ref("stg__micromasters__app__postgres__ecommerce_order") }}), + micromasters_lines as (select * from {{ ref("stg__micromasters__app__postgres__ecommerce_line") }}), + micromasters_courses as (select * from {{ ref("stg__micromasters__app__postgres__courses_course") }}), + micromasters_users as (select * from {{ ref("__micromasters__users") }}), + dedp_edxorg_enrollments_verified as ( + select distinct edxorg_enrollments.user_id, edxorg_enrollments.courserun_readable_id + from edxorg_enrollments + inner join edxorg_users on edxorg_enrollments.user_id = edxorg_users.user_id + inner join micromasters_users on edxorg_users.user_username = micromasters_users.user_edxorg_username + inner join micromasters_orders on micromasters_users.user_id = micromasters_orders.user_id + inner join + micromasters_lines + on edxorg_enrollments.courserun_readable_id = micromasters_lines.courserun_edxorg_readable_id and micromasters_orders.order_id = micromasters_lines.order_id - inner join micromasters_runs on micromasters_lines.courserun_readable_id = micromasters_runs.courserun_readable_id - inner join micromasters_courses on micromasters_runs.course_id = micromasters_courses.course_id - where - micromasters_courses.program_id = {{ var("dedp_micromasters_program_id") }} - and micromasters_orders.order_state = 'fulfilled' -) - -, enrollments as ( - select - edxorg_enrollments.courserun_readable_id - , edxorg_runs.course_number - , edxorg_enrollments.courserunenrollment_created_on - , edxorg_enrollments.courserunenrollment_is_active - , edxorg_users.user_id - , edxorg_users.user_email - , edxorg_users.user_username - , micromasters_users.user_mitxonline_username - , edxorg_runs.courserun_title - , edxorg_runs.courserun_start_date as courserun_start_on - , edxorg_enrollments.courserun_upgrade_deadline - , edxorg_users.user_country as user_address_country - , coalesce(edxorg_users.user_full_name, micromasters_users.user_full_name) as user_full_name - , case - when - dedp_edxorg_enrollments_verified.user_id is not null - and dedp_edxorg_enrollments_verified.courserun_readable_id is not null then 'verified' - else edxorg_enrollments.courserunenrollment_enrollment_mode - end as courserunenrollment_enrollment_mode - from edxorg_enrollments - inner join edxorg_users on edxorg_enrollments.user_id = edxorg_users.user_id - left join micromasters_users on edxorg_users.user_username = micromasters_users.user_edxorg_username - left join edxorg_runs on edxorg_enrollments.courserun_readable_id = edxorg_runs.courserun_readable_id - left join dedp_edxorg_enrollments_verified - on - edxorg_enrollments.user_id = dedp_edxorg_enrollments_verified.user_id + inner join + micromasters_runs on micromasters_lines.courserun_readable_id = micromasters_runs.courserun_readable_id + inner join micromasters_courses on micromasters_runs.course_id = micromasters_courses.course_id + where + micromasters_courses.program_id = {{ var("dedp_micromasters_program_id") }} + and micromasters_orders.order_state = 'fulfilled' + ), + enrollments as ( + select + edxorg_enrollments.courserun_readable_id, + edxorg_runs.course_number, + edxorg_enrollments.courserunenrollment_created_on, + edxorg_enrollments.courserunenrollment_is_active, + edxorg_users.user_id, + edxorg_users.user_email, + edxorg_users.user_username, + micromasters_users.user_mitxonline_username, + edxorg_runs.courserun_title, + edxorg_runs.courserun_start_date as courserun_start_on, + edxorg_enrollments.courserun_upgrade_deadline, + edxorg_users.user_country as user_address_country, + coalesce(edxorg_users.user_full_name, micromasters_users.user_full_name) as user_full_name, + case + when + dedp_edxorg_enrollments_verified.user_id is not null + and dedp_edxorg_enrollments_verified.courserun_readable_id is not null + then 'verified' + else edxorg_enrollments.courserunenrollment_enrollment_mode + end as courserunenrollment_enrollment_mode + from edxorg_enrollments + inner join edxorg_users on edxorg_enrollments.user_id = edxorg_users.user_id + left join micromasters_users on edxorg_users.user_username = micromasters_users.user_edxorg_username + left join edxorg_runs on edxorg_enrollments.courserun_readable_id = edxorg_runs.courserun_readable_id + left join + dedp_edxorg_enrollments_verified + on edxorg_enrollments.user_id = dedp_edxorg_enrollments_verified.user_id and edxorg_enrollments.courserun_readable_id = dedp_edxorg_enrollments_verified.courserun_readable_id -) + ) -select * from enrollments +select * +from enrollments diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_grades.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_grades.sql index cb4c695d7..de7aaed9f 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_grades.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_grades.sql @@ -1,48 +1,37 @@ -- Course Run Grades information from edx.org +with + grades as ( + select + user_id, + courserun_readable_id, + courserungrade_passing_grade, + courserungrade_user_grade, + courserungrade_is_passing + from {{ ref("stg__edxorg__bigquery__mitx_person_course") }} + where courserun_platform = '{{ var("edxorg") }}' and courserungrade_user_grade is not null + ), + runs as (select * from {{ ref("int__edxorg__mitx_courseruns") }}), + users as (select * from {{ ref("int__edxorg__mitx_users") }}), + micromasters_users as (select * from {{ ref("__micromasters__users") }}), + edxorg_grades as ( + select + grades.courserun_readable_id, + grades.courserungrade_passing_grade, + grades.courserungrade_user_grade, + grades.courserungrade_is_passing, + users.user_id, + users.user_email, + users.user_username, + users.user_full_name, + micromasters_users.user_mitxonline_username, + runs.courserun_title, + runs.course_number, + runs.micromasters_program_id + from grades + inner join users on grades.user_id = users.user_id + left join runs on grades.courserun_readable_id = runs.courserun_readable_id + left join micromasters_users on users.user_username = micromasters_users.user_edxorg_username + ) -with grades as ( - select - user_id - , courserun_readable_id - , courserungrade_passing_grade - , courserungrade_user_grade - , courserungrade_is_passing - from {{ ref('stg__edxorg__bigquery__mitx_person_course') }} - where - courserun_platform = '{{ var("edxorg") }}' - and courserungrade_user_grade is not null -) - -, runs as ( - select * from {{ ref('int__edxorg__mitx_courseruns') }} -) - -, users as ( - select * from {{ ref('int__edxorg__mitx_users') }} -) - -, micromasters_users as ( - select * from {{ ref('__micromasters__users') }} -) - -, edxorg_grades as ( - select - grades.courserun_readable_id - , grades.courserungrade_passing_grade - , grades.courserungrade_user_grade - , grades.courserungrade_is_passing - , users.user_id - , users.user_email - , users.user_username - , users.user_full_name - , micromasters_users.user_mitxonline_username - , runs.courserun_title - , runs.course_number - , runs.micromasters_program_id - from grades - inner join users on grades.user_id = users.user_id - left join runs on grades.courserun_readable_id = runs.courserun_readable_id - left join micromasters_users on users.user_username = micromasters_users.user_edxorg_username -) - -select * from edxorg_grades +select * +from edxorg_grades diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courseruns.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courseruns.sql index 44be4d66d..b55225a77 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courseruns.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courseruns.sql @@ -1,135 +1,115 @@ -- MITx Course Runs from edx.org ----It also adds a field micromaster_program_id so that we could use it to get program requirements from MicroMaster - - -with runs_from_bigquery as ( - select * - from {{ ref('stg__edxorg__bigquery__mitx_courserun') }} - where courserun_platform = '{{ var("edxorg") }}' -) - -, courseruns as ( - select * from {{ ref('stg__edxorg__api__courserun') }} -) - -, courses as ( - select * from {{ ref('stg__edxorg__api__course') }} -) - -, instructors as ( - select - courseruns.courserun_readable_id - , array_join( - array_agg( - concat( - json_extract_scalar(t.instructor, '$.first_name') - , ' ' - , json_extract_scalar(t.instructor, '$.last_name') - ) - ) - , ', ' - ) as instructor_names - from courseruns - cross join unnest(cast(json_parse(courseruns.courserun_instructors) as array (json))) as t (instructor) -- noqa - group by courseruns.courserun_readable_id -) - -, runs_from_api as ( - select - courseruns.* - , courses.course_organizations - , instructors.instructor_names - , array_join(courses.course_topics, ', ') as course_topics - from courseruns - inner join courses - on courseruns.course_readable_id = courses.course_readable_id - inner join instructors - on courseruns.courserun_readable_id = instructors.courserun_readable_id -) - -, runs as ( - select - runs_from_bigquery.course_number - , runs_from_bigquery.courserun_semester - , runs_from_bigquery.coursedepartment_name - , runs_from_bigquery.coursedepartment_number - , runs_from_api.courserun_enrollment_end_on as courserun_enrollment_end_date - , runs_from_api.courserun_short_description as courserun_description - , runs_from_api.course_topics - , runs_from_api.courserun_pace - , runs_from_api.courserun_time_commitment - , runs_from_api.courserun_estimated_hours - , runs_from_api.courserun_duration - , runs_from_api.courserun_enrollment_mode - , runs_from_api.courserun_availability - , runs_from_api.courserun_is_published - , coalesce( - replace(replace(runs_from_api.courserun_readable_id, 'course-v1:', ''), '+', '/') - , runs_from_bigquery.courserun_readable_id - ) as courserun_readable_id - , coalesce( - runs_from_api.courserun_is_self_paced, runs_from_bigquery.courserun_is_self_paced - ) as courserun_is_self_paced - , coalesce(runs_from_api.course_readable_id, runs_from_bigquery.course_readable_id) as course_readable_id - , coalesce(runs_from_api.courserun_title, runs_from_bigquery.courserun_title) as courserun_title - , coalesce(runs_from_api.courserun_marketing_url, runs_from_bigquery.courserun_url) as courserun_url - , coalesce( - runs_from_api.course_organizations, runs_from_bigquery.courserun_institution - ) as courserun_institution - , coalesce(runs_from_api.instructor_names, runs_from_bigquery.courserun_instructors) as courserun_instructors - , coalesce(runs_from_api.courserun_enrollment_start_on, runs_from_bigquery.courserun_enrollment_start_date) - as courserun_enrollment_start_date - , coalesce(runs_from_api.courserun_start_on, runs_from_bigquery.courserun_start_date) as courserun_start_date - , coalesce(runs_from_api.courserun_end_on, runs_from_bigquery.courserun_end_date) as courserun_end_date - from runs_from_api - full outer join runs_from_bigquery - on - replace(replace(runs_from_api.courserun_readable_id, 'course-v1:', ''), '+', '/') +-- -It also adds a field micromaster_program_id so that we could use it to get program requirements from MicroMaster +with + runs_from_bigquery as ( + select * + from {{ ref("stg__edxorg__bigquery__mitx_courserun") }} + where courserun_platform = '{{ var("edxorg") }}' + ), + courseruns as (select * from {{ ref("stg__edxorg__api__courserun") }}), + courses as (select * from {{ ref("stg__edxorg__api__course") }}), + instructors as ( + select + courseruns.courserun_readable_id, + array_join( + array_agg( + concat( + json_extract_scalar(t.instructor, '$.first_name'), + ' ', + json_extract_scalar(t.instructor, '$.last_name') + ) + ), + ', ' + ) as instructor_names + from courseruns + cross join unnest(cast(json_parse(courseruns.courserun_instructors) as array(json))) as t(instructor) -- noqa + group by courseruns.courserun_readable_id + ), + runs_from_api as ( + select + courseruns.*, + courses.course_organizations, + instructors.instructor_names, + array_join(courses.course_topics, ', ') as course_topics + from courseruns + inner join courses on courseruns.course_readable_id = courses.course_readable_id + inner join instructors on courseruns.courserun_readable_id = instructors.courserun_readable_id + ), + runs as ( + select + runs_from_bigquery.course_number, + runs_from_bigquery.courserun_semester, + runs_from_bigquery.coursedepartment_name, + runs_from_bigquery.coursedepartment_number, + runs_from_api.courserun_enrollment_end_on as courserun_enrollment_end_date, + runs_from_api.courserun_short_description as courserun_description, + runs_from_api.course_topics, + runs_from_api.courserun_pace, + runs_from_api.courserun_time_commitment, + runs_from_api.courserun_estimated_hours, + runs_from_api.courserun_duration, + runs_from_api.courserun_enrollment_mode, + runs_from_api.courserun_availability, + runs_from_api.courserun_is_published, + coalesce( + replace(replace(runs_from_api.courserun_readable_id, 'course-v1:', ''), '+', '/'), + runs_from_bigquery.courserun_readable_id + ) as courserun_readable_id, + coalesce( + runs_from_api.courserun_is_self_paced, runs_from_bigquery.courserun_is_self_paced + ) as courserun_is_self_paced, + coalesce(runs_from_api.course_readable_id, runs_from_bigquery.course_readable_id) as course_readable_id, + coalesce(runs_from_api.courserun_title, runs_from_bigquery.courserun_title) as courserun_title, + coalesce(runs_from_api.courserun_marketing_url, runs_from_bigquery.courserun_url) as courserun_url, + coalesce( + runs_from_api.course_organizations, runs_from_bigquery.courserun_institution + ) as courserun_institution, + coalesce(runs_from_api.instructor_names, runs_from_bigquery.courserun_instructors) as courserun_instructors, + coalesce( + runs_from_api.courserun_enrollment_start_on, runs_from_bigquery.courserun_enrollment_start_date + ) as courserun_enrollment_start_date, + coalesce(runs_from_api.courserun_start_on, runs_from_bigquery.courserun_start_date) as courserun_start_date, + coalesce(runs_from_api.courserun_end_on, runs_from_bigquery.courserun_end_date) as courserun_end_date + from runs_from_api + full outer join + runs_from_bigquery + on replace(replace(runs_from_api.courserun_readable_id, 'course-v1:', ''), '+', '/') = runs_from_bigquery.courserun_readable_id -) - ---- MicroMasters's course_edx_key can either be {org}+{course_number} or course-v1:{org}+{course_number}, so it --- can't be directly used to link courses between edx and MM, it needs to be formatted as {org}/{course_number} -, micromasters_courses as ( - select - course_id - , program_id - , course_edx_key - , course_number - , course_edx_key as course_readable_id - from {{ ref('stg__micromasters__app__postgres__courses_course') }} -) + ), + -- - MicroMasters's course_edx_key can either be {org}+{course_number} or course-v1:{org}+{course_number}, so it + -- can't be directly used to link courses between edx and MM, it needs to be formatted as {org}/{course_number} + micromasters_courses as ( + select course_id, program_id, course_edx_key, course_number, course_edx_key as course_readable_id + from {{ ref("stg__micromasters__app__postgres__courses_course") }} + ) select - runs.courserun_readable_id - , runs.course_number - , runs.course_readable_id - , runs.coursedepartment_name - , runs.coursedepartment_number - , runs.courserun_title - , runs.courserun_semester - , runs.courserun_url - , runs.courserun_institution - , runs.courserun_instructors - , runs.courserun_enrollment_start_date - , runs.courserun_enrollment_end_date - , runs.courserun_start_date - , runs.courserun_end_date - , runs.courserun_is_self_paced - , runs.courserun_description - , runs.course_topics - , runs.courserun_is_published - , runs.courserun_pace - , runs.courserun_time_commitment - , runs.courserun_estimated_hours - , runs.courserun_duration - , runs.courserun_enrollment_mode - , runs.courserun_availability - , micromasters_courses.program_id as micromasters_program_id - , micromasters_courses.course_id as micromasters_course_id + runs.courserun_readable_id, + runs.course_number, + runs.course_readable_id, + runs.coursedepartment_name, + runs.coursedepartment_number, + runs.courserun_title, + runs.courserun_semester, + runs.courserun_url, + runs.courserun_institution, + runs.courserun_instructors, + runs.courserun_enrollment_start_date, + runs.courserun_enrollment_end_date, + runs.courserun_start_date, + runs.courserun_end_date, + runs.courserun_is_self_paced, + runs.courserun_description, + runs.course_topics, + runs.courserun_is_published, + runs.courserun_pace, + runs.courserun_time_commitment, + runs.courserun_estimated_hours, + runs.courserun_duration, + runs.courserun_enrollment_mode, + runs.courserun_availability, + micromasters_courses.program_id as micromasters_program_id, + micromasters_courses.course_id as micromasters_course_id -from - runs -left join - micromasters_courses - on runs.course_number = micromasters_courses.course_number +from runs +left join micromasters_courses on runs.course_number = micromasters_courses.course_number diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_product.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_product.sql index 990010488..83c737cd9 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_product.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_product.sql @@ -1,14 +1,12 @@ -with courseruns as ( - select * from {{ ref('stg__edxorg__api__courserun') }} -) +with courseruns as (select * from {{ ref("stg__edxorg__api__courserun") }}) select - {{ format_course_id('courseruns.courserun_readable_id') }} as courserun_readable_id - , cast(json_extract_scalar(t.seat, '$.price') as decimal(38, 2)) as price - , json_extract_scalar(t.seat, '$.type') as courserun_mode - , json_extract_scalar(t.seat, '$.currency') as currency - , json_extract_scalar(t.seat, '$.upgrade_deadline') as upgrade_deadline - , json_extract_scalar(t.seat, '$.credit_provider') as credit_provider - , json_extract_scalar(t.seat, '$.credit_hours') as credit_hours + {{ format_course_id("courseruns.courserun_readable_id") }} as courserun_readable_id, + cast(json_extract_scalar(t.seat, '$.price') as decimal(38, 2)) as price, + json_extract_scalar(t.seat, '$.type') as courserun_mode, + json_extract_scalar(t.seat, '$.currency') as currency, + json_extract_scalar(t.seat, '$.upgrade_deadline') as upgrade_deadline, + json_extract_scalar(t.seat, '$.credit_provider') as credit_provider, + json_extract_scalar(t.seat, '$.credit_hours') as credit_hours from courseruns -cross join unnest(cast(json_parse(courseruns.courserun_enrollment_modes) as array (json))) as t (seat) -- noqa +cross join unnest(cast(json_parse(courseruns.courserun_enrollment_modes) as array(json))) as t(seat) -- noqa diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_program_certificates.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_program_certificates.sql index 73a0756c0..091e60908 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_program_certificates.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_program_certificates.sql @@ -1,41 +1,35 @@ -with completed_program_learners as ( - select - * - , {{ generate_hash_id('cast(user_id as varchar) || program_uuid') }} as program_certificate_hashed_id - from {{ ref('stg__edxorg__s3__program_learner_report') }} - where user_has_completed_program = true -) - -, micromasters_programs as ( - select * from {{ ref('int__mitx__programs') }} - where is_micromasters_program = true -) - -, completed_program_learners_sorted as ( - select - * - , row_number() over ( - partition by user_id, program_uuid - order by program_certificate_awarded_on desc - ) as row_num - from completed_program_learners -) +with + completed_program_learners as ( + select *, {{ generate_hash_id("cast(user_id as varchar) || program_uuid") }} as program_certificate_hashed_id + from {{ ref("stg__edxorg__s3__program_learner_report") }} + where user_has_completed_program = true + ), + micromasters_programs as (select * from {{ ref("int__mitx__programs") }} where is_micromasters_program = true), + completed_program_learners_sorted as ( + select + *, + row_number() over ( + partition by user_id, program_uuid order by program_certificate_awarded_on desc + ) as row_num + from completed_program_learners + ) select - completed_program_learners_sorted.program_certificate_hashed_id - , completed_program_learners_sorted.program_type - , completed_program_learners_sorted.program_uuid - , completed_program_learners_sorted.program_title - , completed_program_learners_sorted.user_id - , completed_program_learners_sorted.user_username - , completed_program_learners_sorted.user_full_name - , completed_program_learners_sorted.user_has_completed_program - , completed_program_learners_sorted.program_certificate_awarded_on - , micromasters_programs.micromasters_program_id + completed_program_learners_sorted.program_certificate_hashed_id, + completed_program_learners_sorted.program_type, + completed_program_learners_sorted.program_uuid, + completed_program_learners_sorted.program_title, + completed_program_learners_sorted.user_id, + completed_program_learners_sorted.user_username, + completed_program_learners_sorted.user_full_name, + completed_program_learners_sorted.user_has_completed_program, + completed_program_learners_sorted.program_certificate_awarded_on, + micromasters_programs.micromasters_program_id from completed_program_learners_sorted -left join micromasters_programs ---- Finance is split into MIT Finance and Finance, ---- Statistics and Data Science is split into Statistics and Data Science (General track) +left join + micromasters_programs + -- - Finance is split into MIT Finance and Finance, + -- - Statistics and Data Science is split into Statistics and Data Science (General track) -- and Statistics and Data Science on ( completed_program_learners_sorted.program_title like micromasters_programs.program_title || '%' diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_program_courses.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_program_courses.sql index 6de6f4031..0b6991026 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_program_courses.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_program_courses.sql @@ -1,29 +1,21 @@ -with programs as ( - select * from {{ ref('stg__edxorg__s3__programs') }} -) - -, program_courses as ( - select * from {{ ref('stg__edxorg__s3__program_courses') }} -) - -, micromasters_programs as ( - select * from {{ ref('int__mitx__programs') }} - where is_micromasters_program = true -) +with + programs as (select * from {{ ref("stg__edxorg__s3__programs") }}), + program_courses as (select * from {{ ref("stg__edxorg__s3__program_courses") }}), + micromasters_programs as (select * from {{ ref("int__mitx__programs") }} where is_micromasters_program = true) select - programs.program_uuid - , programs.program_title - , programs.program_type - , programs.program_status - , program_courses.course_readable_id - , program_courses.course_title - , program_courses.course_type - , coalesce(micromasters_programs.program_title, programs.program_title) as program_name + programs.program_uuid, + programs.program_title, + programs.program_type, + programs.program_status, + program_courses.course_readable_id, + program_courses.course_title, + program_courses.course_type, + coalesce(micromasters_programs.program_title, programs.program_title) as program_name from program_courses -inner join programs - on program_courses.program_uuid = programs.program_uuid -left join micromasters_programs +inner join programs on program_courses.program_uuid = programs.program_uuid +left join + micromasters_programs on ( programs.program_title like micromasters_programs.program_title || '%' or programs.program_title like '%' || micromasters_programs.program_title diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_program_enrollments.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_program_enrollments.sql index 07e79261a..06ab46857 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_program_enrollments.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_program_enrollments.sql @@ -1,36 +1,30 @@ -with program_learners as ( - select * - from {{ ref('stg__edxorg__s3__program_learner_report') }} -) - -, micromasters_programs as ( - select * from {{ ref('int__mitx__programs') }} - where is_micromasters_program = true -) - -, program_learners_sorted as ( - select - * - , row_number() over ( - partition by user_id, program_uuid - order by program_certificate_awarded_on desc, courserunenrollment_created_on desc - ) as row_num - from program_learners -) +with + program_learners as (select * from {{ ref("stg__edxorg__s3__program_learner_report") }}), + micromasters_programs as (select * from {{ ref("int__mitx__programs") }} where is_micromasters_program = true), + program_learners_sorted as ( + select + *, + row_number() over ( + partition by user_id, program_uuid + order by program_certificate_awarded_on desc, courserunenrollment_created_on desc + ) as row_num + from program_learners + ) select - program_learners_sorted.program_type - , program_learners_sorted.program_uuid - , program_learners_sorted.program_title - , program_learners_sorted.user_id - , program_learners_sorted.user_username - , program_learners_sorted.user_full_name - , program_learners_sorted.user_has_completed_program - , micromasters_programs.micromasters_program_id - , program_learners_sorted.program_track - , coalesce(micromasters_programs.program_title, program_learners_sorted.program_title) as program_name + program_learners_sorted.program_type, + program_learners_sorted.program_uuid, + program_learners_sorted.program_title, + program_learners_sorted.user_id, + program_learners_sorted.user_username, + program_learners_sorted.user_full_name, + program_learners_sorted.user_has_completed_program, + micromasters_programs.micromasters_program_id, + program_learners_sorted.program_track, + coalesce(micromasters_programs.program_title, program_learners_sorted.program_title) as program_name from program_learners_sorted -left join micromasters_programs +left join + micromasters_programs on ( program_learners_sorted.program_title like micromasters_programs.program_title || '%' or program_learners_sorted.program_title like '%' || micromasters_programs.program_title diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_activity.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_activity.sql index 6d51c7a78..f9730177c 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_activity.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_activity.sql @@ -1,16 +1,16 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} select - user_username - , user_id - , useractivity_path - , useractivity_context_object - , useractivity_event_source - , useractivity_event_type - , useractivity_event_object - , useractivity_event_name - , useractivity_page_url - , useractivity_timestamp - , {{ format_course_id('courserun_readable_id') }} as courserun_readable_id -from {{ ref('stg__edxorg__s3__tracking_logs__user_activity') }} + user_username, + user_id, + useractivity_path, + useractivity_context_object, + useractivity_event_source, + useractivity_event_type, + useractivity_event_object, + useractivity_event_name, + useractivity_page_url, + useractivity_timestamp, + {{ format_course_id("courserun_readable_id") }} as courserun_readable_id +from {{ ref("stg__edxorg__s3__tracking_logs__user_activity") }} where courserun_readable_id is not null diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivities.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivities.sql index 9a0cd6db6..f6e461c24 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivities.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivities.sql @@ -1,68 +1,57 @@ -with course_activities as ( - select * from {{ ref('int__edxorg__mitx_user_activity') }} -) - -, course_activities_video as ( - select * from {{ ref('int__edxorg__mitx_user_courseactivity_video') }} -) - -, problem_check as ( - select * from {{ ref('int__edxorg__mitx_user_courseactivity_problemcheck') }} -) - -, problem_check_stats as ( - select - user_username - , courserun_readable_id - , max(useractivity_timestamp) as courseactivity_last_problem_check_timestamp - from problem_check - group by user_username, courserun_readable_id -) - -, play_video_stats as ( - select - user_username - , courserun_readable_id - , count(distinct useractivity_video_id) as courseactivity_num_unique_play_video - , count(*) as courseactivity_num_play_video - , max(useractivity_timestamp) as courseactivity_last_play_video_timestamp - from course_activities_video - where useractivity_event_type = 'play_video' - group by user_username, courserun_readable_id -) - -, all_course_activities_stats as ( - select - course_activities.user_username - , course_activities.courserun_readable_id - , count(distinct date(from_iso8601_timestamp(course_activities.useractivity_timestamp))) - as courseactivity_num_days_activity - , count(*) as courseactivity_num_events - , min(course_activities.useractivity_timestamp) as courseactivity_first_event_timestamp - , max(course_activities.useractivity_timestamp) as courseactivity_last_event_timestamp - from course_activities - group by - course_activities.user_username - , course_activities.courserun_readable_id -) +with + course_activities as (select * from {{ ref("int__edxorg__mitx_user_activity") }}), + course_activities_video as (select * from {{ ref("int__edxorg__mitx_user_courseactivity_video") }}), + problem_check as (select * from {{ ref("int__edxorg__mitx_user_courseactivity_problemcheck") }}), + problem_check_stats as ( + select + user_username, + courserun_readable_id, + max(useractivity_timestamp) as courseactivity_last_problem_check_timestamp + from problem_check + group by user_username, courserun_readable_id + ), + play_video_stats as ( + select + user_username, + courserun_readable_id, + count(distinct useractivity_video_id) as courseactivity_num_unique_play_video, + count(*) as courseactivity_num_play_video, + max(useractivity_timestamp) as courseactivity_last_play_video_timestamp + from course_activities_video + where useractivity_event_type = 'play_video' + group by user_username, courserun_readable_id + ), + all_course_activities_stats as ( + select + course_activities.user_username, + course_activities.courserun_readable_id, + count( + distinct date(from_iso8601_timestamp(course_activities.useractivity_timestamp)) + ) as courseactivity_num_days_activity, + count(*) as courseactivity_num_events, + min(course_activities.useractivity_timestamp) as courseactivity_first_event_timestamp, + max(course_activities.useractivity_timestamp) as courseactivity_last_event_timestamp + from course_activities + group by course_activities.user_username, course_activities.courserun_readable_id + ) select - all_course_activities_stats.user_username - , all_course_activities_stats.courserun_readable_id - , all_course_activities_stats.courseactivity_num_days_activity - , all_course_activities_stats.courseactivity_num_events - , play_video_stats.courseactivity_num_unique_play_video - , play_video_stats.courseactivity_num_play_video - , play_video_stats.courseactivity_last_play_video_timestamp - , problem_check_stats.courseactivity_last_problem_check_timestamp - , all_course_activities_stats.courseactivity_first_event_timestamp - , all_course_activities_stats.courseactivity_last_event_timestamp + all_course_activities_stats.user_username, + all_course_activities_stats.courserun_readable_id, + all_course_activities_stats.courseactivity_num_days_activity, + all_course_activities_stats.courseactivity_num_events, + play_video_stats.courseactivity_num_unique_play_video, + play_video_stats.courseactivity_num_play_video, + play_video_stats.courseactivity_last_play_video_timestamp, + problem_check_stats.courseactivity_last_problem_check_timestamp, + all_course_activities_stats.courseactivity_first_event_timestamp, + all_course_activities_stats.courseactivity_last_event_timestamp from all_course_activities_stats -left join play_video_stats - on - all_course_activities_stats.user_username = play_video_stats.user_username - and all_course_activities_stats.courserun_readable_id = play_video_stats.courserun_readable_id -left join problem_check_stats - on - all_course_activities_stats.user_username = problem_check_stats.user_username - and all_course_activities_stats.courserun_readable_id = problem_check_stats.courserun_readable_id +left join + play_video_stats + on all_course_activities_stats.user_username = play_video_stats.user_username + and all_course_activities_stats.courserun_readable_id = play_video_stats.courserun_readable_id +left join + problem_check_stats + on all_course_activities_stats.user_username = problem_check_stats.user_username + and all_course_activities_stats.courserun_readable_id = problem_check_stats.courserun_readable_id diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivities_daily.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivities_daily.sql index 5f5f53389..dba3df545 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivities_daily.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivities_daily.sql @@ -1,18 +1,14 @@ -with course_activities as ( - select * from {{ ref('int__edxorg__mitx_user_activity') }} -) +with + course_activities as (select * from {{ ref("int__edxorg__mitx_user_activity") }}), + daily_activities_stats as ( + select + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date, + count(*) as courseactivity_num_events + from course_activities + group by user_username, courserun_readable_id, date(from_iso8601_timestamp(useractivity_timestamp)) + ) -, daily_activities_stats as ( - select - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - , count(*) as courseactivity_num_events - from course_activities - group by - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) -) - -select * from daily_activities_stats +select * +from daily_activities_stats diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_discussion.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_discussion.sql index ac250c161..fc14be6b3 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_discussion.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_discussion.sql @@ -1,23 +1,21 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('int__edxorg__mitx_user_activity') }} -) +with course_activities as (select * from {{ ref("int__edxorg__mitx_user_activity") }}) select - user_username - , courserun_readable_id - , user_id - , useractivity_event_source - , useractivity_event_type - , useractivity_path - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_discussion_post_id - , json_query(useractivity_event_object, 'lax $.title' omit quotes) as useractivity_discussion_post_title - , json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as useractivity_discussion_block_id - , json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as useractivity_discussion_block_name - , json_query(useractivity_event_object, 'lax $.url' omit quotes) as useractivity_discussion_page_url - , json_query(useractivity_event_object, 'lax $.query' omit quotes) as useractivity_discussion_search_query - , json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as useractivity_discussion_roles + user_username, + courserun_readable_id, + user_id, + useractivity_event_source, + useractivity_event_type, + useractivity_path, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_discussion_post_id, + json_query(useractivity_event_object, 'lax $.title' omit quotes) as useractivity_discussion_post_title, + json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as useractivity_discussion_block_id, + json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as useractivity_discussion_block_name, + json_query(useractivity_event_object, 'lax $.url' omit quotes) as useractivity_discussion_page_url, + json_query(useractivity_event_object, 'lax $.query' omit quotes) as useractivity_discussion_search_query, + json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as useractivity_discussion_roles from course_activities where useractivity_event_type like 'edx.forum.%' diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_problemcheck.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_problemcheck.sql index 7d291e5ba..0b8dccf9b 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_problemcheck.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_problemcheck.sql @@ -1,24 +1,23 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('int__edxorg__mitx_user_activity') }} -) +with course_activities as (select * from {{ ref("int__edxorg__mitx_user_activity") }}) select - user_username - , courserun_readable_id - , user_id - , useractivity_event_type - , useractivity_timestamp - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id - , json_query(useractivity_event_object, 'lax $.answers' omit quotes) as useractivity_problem_student_answers - , json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as useractivity_problem_attempts - , json_query(useractivity_event_object, 'lax $.success' omit quotes) as useractivity_problem_success - , json_query(useractivity_event_object, 'lax $.grade' omit quotes) as useractivity_problem_current_grade - , json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as useractivity_problem_max_grade + user_username, + courserun_readable_id, + user_id, + useractivity_event_type, + useractivity_timestamp, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id, + json_query(useractivity_event_object, 'lax $.answers' omit quotes) as useractivity_problem_student_answers, + json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as useractivity_problem_attempts, + json_query(useractivity_event_object, 'lax $.success' omit quotes) as useractivity_problem_success, + json_query(useractivity_event_object, 'lax $.grade' omit quotes) as useractivity_problem_current_grade, + json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as useractivity_problem_max_grade from course_activities -where useractivity_event_type = 'problem_check' ---- This event emitted by the browser contain all of the GET parameters, --- only events emitted by the server are useful -and useractivity_event_source = 'server' +where + useractivity_event_type = 'problem_check' + -- - This event emitted by the browser contain all of the GET parameters, + -- only events emitted by the server are useful + and useractivity_event_source = 'server' diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_problemsubmitted.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_problemsubmitted.sql index a16fb6834..129403b01 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_problemsubmitted.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_problemsubmitted.sql @@ -1,22 +1,20 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('int__edxorg__mitx_user_activity') }} -) +with course_activities as (select * from {{ ref("int__edxorg__mitx_user_activity") }}) select - user_username - , courserun_readable_id - , user_id - , useractivity_event_source - , useractivity_event_type - , useractivity_path - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.event_transaction_id' omit quotes) as useractivity_event_id - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id - , json_query(useractivity_event_object, 'lax $.weight' omit quotes) as useractivity_problem_weight - , json_query(useractivity_event_object, 'lax $.weighted_earned' omit quotes) as useractivity_problem_earned_score - , json_query(useractivity_event_object, 'lax $.weighted_possible' omit quotes) as useractivity_problem_max_score + user_username, + courserun_readable_id, + user_id, + useractivity_event_source, + useractivity_event_type, + useractivity_path, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.event_transaction_id' omit quotes) as useractivity_event_id, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id, + json_query(useractivity_event_object, 'lax $.weight' omit quotes) as useractivity_problem_weight, + json_query(useractivity_event_object, 'lax $.weighted_earned' omit quotes) as useractivity_problem_earned_score, + json_query(useractivity_event_object, 'lax $.weighted_possible' omit quotes) as useractivity_problem_max_score from course_activities where useractivity_event_type = 'edx.grades.problem.submitted' diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_video.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_video.sql index 0972e7912..fcb295bc5 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_video.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_user_courseactivity_video.sql @@ -1,30 +1,29 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('int__edxorg__mitx_user_activity') }} -) +with course_activities as (select * from {{ ref("int__edxorg__mitx_user_activity") }}) select - user_username - , courserun_readable_id - , user_id - , useractivity_event_source - , useractivity_event_type - , useractivity_page_url - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_video_id - , case - when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' then null + user_username, + courserun_readable_id, + user_id, + useractivity_event_source, + useractivity_event_type, + useractivity_page_url, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_video_id, + case + when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' + then null else cast(json_query(useractivity_event_object, 'lax $.duration' omit quotes) as decimal(38, 4)) - end as useractivity_video_duration - , json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as useractivity_video_currenttime - , json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as useractivity_video_old_time - , json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as useractivity_video_new_time - , json_query(useractivity_event_object, 'lax $.new_speed' omit quotes) as useractivity_video_new_speed - , json_query(useractivity_event_object, 'lax $.old_speed' omit quotes) as useractivity_video_old_speed + end as useractivity_video_duration, + json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as useractivity_video_currenttime, + json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as useractivity_video_old_time, + json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as useractivity_video_new_time, + json_query(useractivity_event_object, 'lax $.new_speed' omit quotes) as useractivity_video_new_speed, + json_query(useractivity_event_object, 'lax $.old_speed' omit quotes) as useractivity_video_old_speed from course_activities ---- Some events have url as useractivity_event_type that should be filtered as we want video events listed in ---- https://edx.readthedocs.io/projects/devdata/en/latest/internal_data_formats/tracking_logs/student_event_types.html +-- - Some events have url as useractivity_event_type that should be filtered as we want video events listed in +-- - https://edx.readthedocs.io/projects/devdata/en/latest/internal_data_formats/tracking_logs/student_event_types.html -- #video-interaction-events where regexp_like(useractivity_event_type, '(^[\w]+)_video') = true diff --git a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_users.sql b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_users.sql index 41d61a1a8..60be78c6f 100644 --- a/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_users.sql +++ b/src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_users.sql @@ -1,84 +1,68 @@ ----Intermediate MITx users from edx.org ----User data mostly comes from user_info_combo but email or username could be blank in user_info_combo for some older +-- -Intermediate MITx users from edx.org +-- -User data mostly comes from user_info_combo but email or username could be blank in user_info_combo for some older -- courses. For these cases, we use email or username from email_opt_in. +with + user_info_combo as ( + select * + from {{ ref("stg__edxorg__bigquery__mitx_user_info_combo") }} + where courserun_platform = '{{ var("edxorg") }}' + ), + -- - email_opt_in is one row per course_id with the same user_* data, use window function to pick one per user_id + email_opt_in as ( + select + user_id, + user_email, + user_username, + user_full_name, + row_number() over (partition by user_id order by user_email_opt_in_updated_on desc) as row_num + from {{ ref("stg__edxorg__bigquery__mitx_user_email_opt_in") }} + where courserun_platform = '{{ var("edxorg") }}' + ), + most_recent_user_info as ( + select user_id, user_email, user_username, user_full_name from email_opt_in where row_num = 1 + ), + combined_user_info as ( + select + user_info_combo.user_id, + user_info_combo.user_full_name, + user_info_combo.user_country, + user_info_combo.user_city, + user_info_combo.user_birth_year, + user_info_combo.user_mailing_address, + user_info_combo.user_highest_education, + user_info_combo.user_profile_goals, + user_info_combo.user_profile_meta, + user_info_combo.user_joined_on, + user_info_combo.user_gender, + user_info_combo.user_last_login, + coalesce(user_info_combo.user_email, most_recent_user_info.user_email) as user_email, + coalesce(user_info_combo.user_username, most_recent_user_info.user_username) as user_username, + row_number() over ( + partition by user_info_combo.user_id order by user_info_combo.user_last_login desc + ) as row_num + from user_info_combo + left join most_recent_user_info on user_info_combo.user_id = most_recent_user_info.user_id + ), + users as ( + select + user_id, + user_email, + user_username, + user_full_name, + user_country, + user_city, + user_mailing_address, + user_highest_education, + user_birth_year, + user_profile_goals, + user_profile_meta, + user_joined_on, + user_gender, + user_last_login, + if(user_email like 'retired__user%' or user_username like 'retired__user%', false, true) as user_is_active + from combined_user_info + where row_num = 1 + ) -with user_info_combo as ( - select * - from {{ ref('stg__edxorg__bigquery__mitx_user_info_combo') }} - where courserun_platform = '{{ var("edxorg") }}' -) - ---- email_opt_in is one row per course_id with the same user_* data, use window function to pick one per user_id -, email_opt_in as ( - select - user_id - , user_email - , user_username - , user_full_name - , row_number() over ( - partition by user_id - order by user_email_opt_in_updated_on desc - ) as row_num - from {{ ref('stg__edxorg__bigquery__mitx_user_email_opt_in') }} - where courserun_platform = '{{ var("edxorg") }}' -) - -, most_recent_user_info as ( - select - user_id - , user_email - , user_username - , user_full_name - from email_opt_in - where row_num = 1 -) - -, combined_user_info as ( - select - user_info_combo.user_id - , user_info_combo.user_full_name - , user_info_combo.user_country - , user_info_combo.user_city - , user_info_combo.user_birth_year - , user_info_combo.user_mailing_address - , user_info_combo.user_highest_education - , user_info_combo.user_profile_goals - , user_info_combo.user_profile_meta - , user_info_combo.user_joined_on - , user_info_combo.user_gender - , user_info_combo.user_last_login - , coalesce(user_info_combo.user_email, most_recent_user_info.user_email) as user_email - , coalesce(user_info_combo.user_username, most_recent_user_info.user_username) as user_username - , row_number() - over ( - partition by user_info_combo.user_id - order by user_info_combo.user_last_login desc - ) - as row_num - from user_info_combo - left join most_recent_user_info - on user_info_combo.user_id = most_recent_user_info.user_id -) - -, users as ( - select - user_id - , user_email - , user_username - , user_full_name - , user_country - , user_city - , user_mailing_address - , user_highest_education - , user_birth_year - , user_profile_goals - , user_profile_meta - , user_joined_on - , user_gender - , user_last_login - , if(user_email like 'retired__user%' or user_username like 'retired__user%', false, true) as user_is_active - from combined_user_info - where row_num = 1 -) - -select * from users +select * +from users diff --git a/src/ol_dbt/models/intermediate/learn-ai/int__learn_ai__chatbot.sql b/src/ol_dbt/models/intermediate/learn-ai/int__learn_ai__chatbot.sql index b872c35cd..32516dd2e 100644 --- a/src/ol_dbt/models/intermediate/learn-ai/int__learn_ai__chatbot.sql +++ b/src/ol_dbt/models/intermediate/learn-ai/int__learn_ai__chatbot.sql @@ -1,71 +1,63 @@ -with chatsession as ( - select * from {{ ref('stg__learn_ai__app__postgres__chatbots_userchatsession') }} -) - -, djangocheckpoint as ( - select * from {{ ref('stg__learn_ai__app__postgres__chatbots_djangocheckpoint') }} -) - -, users as ( - select * from {{ ref('stg__learn_ai__app__postgres__users_user') }} -) - -, video as ( - select distinct - courserun_readable_id - , retrieved_at - , json_query(block_metadata, 'lax $.transcripts.en' omit quotes) as transcript_id - from {{ ref('dim_course_content') }} - where block_category = 'video' -) - -, videos_with_ranking as ( - select - video.courserun_readable_id - , chatsession.chatsession_object_id - , row_number() over ( - partition by chatsession.chatsession_object_id - order by video.retrieved_at asc - ) as row_num - from chatsession - inner join video - on - chatsession.chatsession_object_id like '%' || video.transcript_id - and video.courserun_readable_id like '%' +with + chatsession as (select * from {{ ref("stg__learn_ai__app__postgres__chatbots_userchatsession") }}), + djangocheckpoint as (select * from {{ ref("stg__learn_ai__app__postgres__chatbots_djangocheckpoint") }}), + users as (select * from {{ ref("stg__learn_ai__app__postgres__users_user") }}), + video as ( + select distinct + courserun_readable_id, + retrieved_at, + json_query(block_metadata, 'lax $.transcripts.en' omit quotes) as transcript_id + from {{ ref("dim_course_content") }} + where block_category = 'video' + ), + videos_with_ranking as ( + select + video.courserun_readable_id, + chatsession.chatsession_object_id, + row_number() over ( + partition by chatsession.chatsession_object_id order by video.retrieved_at asc + ) as row_num + from chatsession + inner join + video + on chatsession.chatsession_object_id like '%' || video.transcript_id + and video.courserun_readable_id + like '%' || substring( - replace(chatsession.chatsession_object_id, 'asset-v1:', ''), 1 - , strpos(replace(chatsession.chatsession_object_id, 'asset-v1:', ''), '+type@asset+block@') - 1 + replace(chatsession.chatsession_object_id, 'asset-v1:', ''), + 1, + strpos(replace(chatsession.chatsession_object_id, 'asset-v1:', ''), '+type@asset+block@') - 1 ) -) + ) select - djangocheckpoint.djangocheckpoint_id - , djangocheckpoint.checkpoint_id - , djangocheckpoint.chatsession_thread_id - , chatsession.chatsession_agent - , chatsession.chatsession_title - , chatsession.chatsession_object_id - , videos_with_ranking.courserun_readable_id - , chatsession.user_id - , users.user_email - , users.user_full_name - , users.user_username - , users.user_global_id - , djangocheckpoint.checkpoint_source - , djangocheckpoint.checkpoint_step - , djangocheckpoint.human_message - , djangocheckpoint.agent_message - , djangocheckpoint.checkpoint_json - , djangocheckpoint.checkpoint_metadata - , djangocheckpoint.parent_checkpoint_id - , djangocheckpoint.checkpoint_namespace - , djangocheckpoint.checkpoint_type - , chatsession.chatsession_created_on - , chatsession.chatsession_updated_on + djangocheckpoint.djangocheckpoint_id, + djangocheckpoint.checkpoint_id, + djangocheckpoint.chatsession_thread_id, + chatsession.chatsession_agent, + chatsession.chatsession_title, + chatsession.chatsession_object_id, + videos_with_ranking.courserun_readable_id, + chatsession.user_id, + users.user_email, + users.user_full_name, + users.user_username, + users.user_global_id, + djangocheckpoint.checkpoint_source, + djangocheckpoint.checkpoint_step, + djangocheckpoint.human_message, + djangocheckpoint.agent_message, + djangocheckpoint.checkpoint_json, + djangocheckpoint.checkpoint_metadata, + djangocheckpoint.parent_checkpoint_id, + djangocheckpoint.checkpoint_namespace, + djangocheckpoint.checkpoint_type, + chatsession.chatsession_created_on, + chatsession.chatsession_updated_on from djangocheckpoint inner join chatsession on djangocheckpoint.chatsession_thread_id = chatsession.chatsession_thread_id left join users on chatsession.user_id = users.user_id -left join videos_with_ranking - on - chatsession.chatsession_object_id = videos_with_ranking.chatsession_object_id - and videos_with_ranking.row_num = 1 +left join + videos_with_ranking + on chatsession.chatsession_object_id = videos_with_ranking.chatsession_object_id + and videos_with_ranking.row_num = 1 diff --git a/src/ol_dbt/models/intermediate/learn-ai/int__learn_ai__tutorbot.sql b/src/ol_dbt/models/intermediate/learn-ai/int__learn_ai__tutorbot.sql index c71109cb1..ac1c9331c 100644 --- a/src/ol_dbt/models/intermediate/learn-ai/int__learn_ai__tutorbot.sql +++ b/src/ol_dbt/models/intermediate/learn-ai/int__learn_ai__tutorbot.sql @@ -1,34 +1,24 @@ -with chatsession as ( - select * from {{ ref('stg__learn_ai__app__postgres__chatbots_userchatsession') }} -) - -, tutorbotoutput as ( - select * from {{ ref("stg__learn_ai__app__postgres__chatbots_tutorbotoutput") }} -) - -, users as ( - select * from {{ ref('stg__learn_ai__app__postgres__users_user') }} -) - -, problem as ( - select * from {{ ref('dim_problem') }} -) +with + chatsession as (select * from {{ ref("stg__learn_ai__app__postgres__chatbots_userchatsession") }}), + tutorbotoutput as (select * from {{ ref("stg__learn_ai__app__postgres__chatbots_tutorbotoutput") }}), + users as (select * from {{ ref("stg__learn_ai__app__postgres__users_user") }}), + problem as (select * from {{ ref("dim_problem") }}) select - tutorbotoutput.tutorbotoutput_id - , tutorbotoutput.chatsession_thread_id - , tutorbotoutput.tutorbot_chat_json - , chatsession.chatsession_agent - , chatsession.chatsession_title - , chatsession.chatsession_object_id as edx_module_id - , problem.courserun_readable_id - , chatsession.user_id - , users.user_email - , users.user_full_name - , users.user_username - , users.user_global_id - , chatsession.chatsession_created_on - , chatsession.chatsession_updated_on + tutorbotoutput.tutorbotoutput_id, + tutorbotoutput.chatsession_thread_id, + tutorbotoutput.tutorbot_chat_json, + chatsession.chatsession_agent, + chatsession.chatsession_title, + chatsession.chatsession_object_id as edx_module_id, + problem.courserun_readable_id, + chatsession.user_id, + users.user_email, + users.user_full_name, + users.user_username, + users.user_global_id, + chatsession.chatsession_created_on, + chatsession.chatsession_updated_on from tutorbotoutput inner join chatsession on tutorbotoutput.chatsession_thread_id = chatsession.chatsession_thread_id left join users on chatsession.user_id = users.user_id diff --git a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_certificates.sql b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_certificates.sql index 8caa160ac..4d6b3a0b7 100644 --- a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_certificates.sql +++ b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_certificates.sql @@ -1,173 +1,154 @@ -with course_certificates_dedp_from_micromasters as ( - select * - from {{ ref('__micromasters_course_certificates_dedp_from_micromasters') }} -) +with + course_certificates_dedp_from_micromasters as ( + select * from {{ ref("__micromasters_course_certificates_dedp_from_micromasters") }} + ), + course_certificates_dedp_from_mitxonline as ( + select * from {{ ref("__micromasters_course_certificates_dedp_from_mitxonline") }} + ), + course_certificates_non_dedp_program as ( + select * from {{ ref("__micromasters_course_certificates_non_dedp_from_edxorg") }} + ), + mitx_users as (select * from {{ ref("int__mitx__users") }}), + -- DEDP course certificates come from MicroMasters and MITxOnline. We've migrated some learners data from + -- MicroMasters to MITxOnline around Oct 2022, but only for those users who have MITxOnline account. + -- To avoid data overlapping, we deduplicate based on their social auth account linked on MicroMasters. + -- for old DEDP courses on edx.org, then we use certificates from MicroMasters + -- for new DEDP course on MITx Online, then we use certificates from MITx Online + dedp_course_certificates_combined as ( + select + course_certificates_dedp_from_micromasters.program_title, + course_certificates_dedp_from_micromasters.micromasters_program_id, + course_certificates_dedp_from_micromasters.mitxonline_program_id, + course_certificates_dedp_from_micromasters.courserun_title, + course_certificates_dedp_from_micromasters.courserun_readable_id, + course_certificates_dedp_from_micromasters.courserun_platform, + course_certificates_dedp_from_micromasters.course_number, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_full_name, + mitx_users.user_address_country as user_country, + mitx_users.user_micromasters_email as user_email, + course_certificates_dedp_from_micromasters.coursecertificate_hash as courseruncertificate_uuid, + course_certificates_dedp_from_micromasters.coursecertificate_url as courseruncertificate_url, + course_certificates_dedp_from_micromasters.coursecertificate_created_on as courseruncertificate_created_on + from course_certificates_dedp_from_micromasters + left join + mitx_users + on course_certificates_dedp_from_micromasters.user_micromasters_id = mitx_users.user_micromasters_id + where course_certificates_dedp_from_micromasters.courserun_platform = '{{ var("edxorg") }}' -, course_certificates_dedp_from_mitxonline as ( - select * - from {{ ref('__micromasters_course_certificates_dedp_from_mitxonline') }} -) + union all -, course_certificates_non_dedp_program as ( - select * - from {{ ref('__micromasters_course_certificates_non_dedp_from_edxorg') }} -) + select + course_certificates_dedp_from_mitxonline.program_title, + course_certificates_dedp_from_mitxonline.micromasters_program_id, + course_certificates_dedp_from_mitxonline.mitxonline_program_id, + course_certificates_dedp_from_mitxonline.courserun_title, + course_certificates_dedp_from_mitxonline.courserun_readable_id, + course_certificates_dedp_from_mitxonline.courserun_platform, + course_certificates_dedp_from_mitxonline.course_number, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_full_name, + mitx_users.user_address_country as user_country, + mitx_users.user_mitxonline_email as user_email, + course_certificates_dedp_from_mitxonline.courseruncertificate_uuid, + course_certificates_dedp_from_mitxonline.courseruncertificate_url, + course_certificates_dedp_from_mitxonline.courseruncertificate_created_on + from course_certificates_dedp_from_mitxonline + left join + mitx_users on course_certificates_dedp_from_mitxonline.user_mitxonline_id = mitx_users.user_mitxonline_id + where course_certificates_dedp_from_mitxonline.courserun_platform = '{{ var("mitxonline") }}' -, mitx_users as ( - select * from {{ ref('int__mitx__users') }} -) - --- DEDP course certificates come from MicroMasters and MITxOnline. We've migrated some learners data from --- MicroMasters to MITxOnline around Oct 2022, but only for those users who have MITxOnline account. --- To avoid data overlapping, we deduplicate based on their social auth account linked on MicroMasters. --- for old DEDP courses on edx.org, then we use certificates from MicroMasters --- for new DEDP course on MITx Online, then we use certificates from MITx Online - -, dedp_course_certificates_combined as ( - select - course_certificates_dedp_from_micromasters.program_title - , course_certificates_dedp_from_micromasters.micromasters_program_id - , course_certificates_dedp_from_micromasters.mitxonline_program_id - , course_certificates_dedp_from_micromasters.courserun_title - , course_certificates_dedp_from_micromasters.courserun_readable_id - , course_certificates_dedp_from_micromasters.courserun_platform - , course_certificates_dedp_from_micromasters.course_number - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_full_name - , mitx_users.user_address_country as user_country - , mitx_users.user_micromasters_email as user_email - , course_certificates_dedp_from_micromasters.coursecertificate_hash as courseruncertificate_uuid - , course_certificates_dedp_from_micromasters.coursecertificate_url as courseruncertificate_url - , course_certificates_dedp_from_micromasters.coursecertificate_created_on as courseruncertificate_created_on - from course_certificates_dedp_from_micromasters - left join mitx_users - on course_certificates_dedp_from_micromasters.user_micromasters_id = mitx_users.user_micromasters_id - where course_certificates_dedp_from_micromasters.courserun_platform = '{{ var("edxorg") }}' - - union all - - select - course_certificates_dedp_from_mitxonline.program_title - , course_certificates_dedp_from_mitxonline.micromasters_program_id - , course_certificates_dedp_from_mitxonline.mitxonline_program_id - , course_certificates_dedp_from_mitxonline.courserun_title - , course_certificates_dedp_from_mitxonline.courserun_readable_id - , course_certificates_dedp_from_mitxonline.courserun_platform - , course_certificates_dedp_from_mitxonline.course_number - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_full_name - , mitx_users.user_address_country as user_country - , mitx_users.user_mitxonline_email as user_email - , course_certificates_dedp_from_mitxonline.courseruncertificate_uuid - , course_certificates_dedp_from_mitxonline.courseruncertificate_url - , course_certificates_dedp_from_mitxonline.courseruncertificate_created_on - from course_certificates_dedp_from_mitxonline - left join mitx_users - on course_certificates_dedp_from_mitxonline.user_mitxonline_id = mitx_users.user_mitxonline_id - where course_certificates_dedp_from_mitxonline.courserun_platform = '{{ var("mitxonline") }}' - -) - -, dedp_course_certificates_sorted as ( - select - program_title - , micromasters_program_id - , mitxonline_program_id - , courserun_title - , courserun_readable_id - , courserun_platform - , course_number - , user_edxorg_username - , user_mitxonline_username - , user_full_name - , user_country - , user_email - , courseruncertificate_uuid - , courseruncertificate_url - , courseruncertificate_created_on - , case - when - user_mitxonline_username is not null + ), + dedp_course_certificates_sorted as ( + select + program_title, + micromasters_program_id, + mitxonline_program_id, + courserun_title, + courserun_readable_id, + courserun_platform, + course_number, + user_edxorg_username, + user_mitxonline_username, + user_full_name, + user_country, + user_email, + courseruncertificate_uuid, + courseruncertificate_url, + courseruncertificate_created_on, + case + when user_mitxonline_username is not null then - row_number() - over ( - partition by courserun_readable_id, user_mitxonline_username, mitxonline_program_id - order by courseruncertificate_created_on desc - ) - else 1 - end as row_num - from dedp_course_certificates_combined -) - -, dedp_course_certificates as ( - select * - from dedp_course_certificates_sorted - where row_num = 1 -) - -, non_dedp_course_certificates as ( - select - course_certificates_non_dedp_program.program_title - , course_certificates_non_dedp_program.mitxonline_program_id - , course_certificates_non_dedp_program.micromasters_program_id - , course_certificates_non_dedp_program.courserun_title - , course_certificates_non_dedp_program.courserun_readable_id - , course_certificates_non_dedp_program.courserun_platform - , course_certificates_non_dedp_program.course_number - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_full_name - , mitx_users.user_address_country as user_country - , course_certificates_non_dedp_program.courseruncertificate_download_uuid as courseruncertificate_uuid - , course_certificates_non_dedp_program.courseruncertificate_download_url as courseruncertificate_url - , course_certificates_non_dedp_program.courseruncertificate_created_on - , coalesce(mitx_users.user_edxorg_email, mitx_users.user_micromasters_email) as user_email - from course_certificates_non_dedp_program - left join mitx_users - on course_certificates_non_dedp_program.user_edxorg_id = mitx_users.user_edxorg_id -) - -, course_certificates as ( - select - program_title - , mitxonline_program_id - , micromasters_program_id - , courserun_title - , courserun_readable_id - , courserun_platform - , course_number - , user_edxorg_username - , user_mitxonline_username - , user_full_name - , user_country - , user_email - , courseruncertificate_uuid - , courseruncertificate_url - , courseruncertificate_created_on - from dedp_course_certificates + row_number() over ( + partition by courserun_readable_id, user_mitxonline_username, mitxonline_program_id + order by courseruncertificate_created_on desc + ) + else 1 + end as row_num + from dedp_course_certificates_combined + ), + dedp_course_certificates as (select * from dedp_course_certificates_sorted where row_num = 1), + non_dedp_course_certificates as ( + select + course_certificates_non_dedp_program.program_title, + course_certificates_non_dedp_program.mitxonline_program_id, + course_certificates_non_dedp_program.micromasters_program_id, + course_certificates_non_dedp_program.courserun_title, + course_certificates_non_dedp_program.courserun_readable_id, + course_certificates_non_dedp_program.courserun_platform, + course_certificates_non_dedp_program.course_number, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_full_name, + mitx_users.user_address_country as user_country, + course_certificates_non_dedp_program.courseruncertificate_download_uuid as courseruncertificate_uuid, + course_certificates_non_dedp_program.courseruncertificate_download_url as courseruncertificate_url, + course_certificates_non_dedp_program.courseruncertificate_created_on, + coalesce(mitx_users.user_edxorg_email, mitx_users.user_micromasters_email) as user_email + from course_certificates_non_dedp_program + left join mitx_users on course_certificates_non_dedp_program.user_edxorg_id = mitx_users.user_edxorg_id + ), + course_certificates as ( + select + program_title, + mitxonline_program_id, + micromasters_program_id, + courserun_title, + courserun_readable_id, + courserun_platform, + course_number, + user_edxorg_username, + user_mitxonline_username, + user_full_name, + user_country, + user_email, + courseruncertificate_uuid, + courseruncertificate_url, + courseruncertificate_created_on + from dedp_course_certificates - union all + union all - select - program_title - , mitxonline_program_id - , micromasters_program_id - , courserun_title - , courserun_readable_id - , courserun_platform - , course_number - , user_edxorg_username - , user_mitxonline_username - , user_full_name - , user_country - , user_email - , courseruncertificate_uuid - , courseruncertificate_url - , courseruncertificate_created_on - from non_dedp_course_certificates -) + select + program_title, + mitxonline_program_id, + micromasters_program_id, + courserun_title, + courserun_readable_id, + courserun_platform, + course_number, + user_edxorg_username, + user_mitxonline_username, + user_full_name, + user_country, + user_email, + courseruncertificate_uuid, + courseruncertificate_url, + courseruncertificate_created_on + from non_dedp_course_certificates + ) select * from course_certificates diff --git a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_enrollments.sql b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_enrollments.sql index 2e0f1997a..6a6b3c3df 100644 --- a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_enrollments.sql +++ b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_enrollments.sql @@ -1,33 +1,28 @@ -with mitx_enrollments as ( - select * from {{ ref('int__mitx__courserun_enrollments_with_programs') }} -) - -, programs as ( - select * from {{ ref('int__mitx__programs') }} - -) +with + mitx_enrollments as (select * from {{ ref("int__mitx__courserun_enrollments_with_programs") }}), + programs as (select * from {{ ref("int__mitx__programs") }}) select - mitx_enrollments.program_title - , mitx_enrollments.micromasters_program_id - , mitx_enrollments.mitxonline_program_id - , mitx_enrollments.user_id - , mitx_enrollments.user_mitxonline_username - , mitx_enrollments.user_edxorg_username - , mitx_enrollments.user_address_country - , mitx_enrollments.user_email - , mitx_enrollments.user_full_name - , mitx_enrollments.courserun_readable_id - , mitx_enrollments.course_number - , mitx_enrollments.platform - , mitx_enrollments.courserunenrollment_created_on - , mitx_enrollments.courserunenrollment_is_active - , mitx_enrollments.courserun_title - , mitx_enrollments.courserunenrollment_enrollment_mode - , programs.is_dedp_program as program_is_dedp + mitx_enrollments.program_title, + mitx_enrollments.micromasters_program_id, + mitx_enrollments.mitxonline_program_id, + mitx_enrollments.user_id, + mitx_enrollments.user_mitxonline_username, + mitx_enrollments.user_edxorg_username, + mitx_enrollments.user_address_country, + mitx_enrollments.user_email, + mitx_enrollments.user_full_name, + mitx_enrollments.courserun_readable_id, + mitx_enrollments.course_number, + mitx_enrollments.platform, + mitx_enrollments.courserunenrollment_created_on, + mitx_enrollments.courserunenrollment_is_active, + mitx_enrollments.courserun_title, + mitx_enrollments.courserunenrollment_enrollment_mode, + programs.is_dedp_program as program_is_dedp from mitx_enrollments -inner join programs - on - mitx_enrollments.micromasters_program_id = programs.micromasters_program_id - or mitx_enrollments.mitxonline_program_id = programs.mitxonline_program_id +inner join + programs + on mitx_enrollments.micromasters_program_id = programs.micromasters_program_id + or mitx_enrollments.mitxonline_program_id = programs.mitxonline_program_id where programs.is_micromasters_program = true diff --git a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_grades.sql b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_grades.sql index 41d9c1f45..c3cc80143 100644 --- a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_grades.sql +++ b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_grades.sql @@ -1,168 +1,146 @@ -with course_grades_dedp_from_micromasters as ( - select * - from {{ ref('__micromasters_course_grades_dedp_from_micromasters') }} -) +with + course_grades_dedp_from_micromasters as ( + select * from {{ ref("__micromasters_course_grades_dedp_from_micromasters") }} + ), + course_grades_dedp_from_mitxonline as ( + select * from {{ ref("__micromasters_course_grades_dedp_from_mitxonline") }} + ), + course_grades_non_dedp_program as (select * from {{ ref("__micromasters_course_grades_non_dedp_from_edxorg") }}), + mitx_users as (select * from {{ ref("int__mitx__users") }}), + -- DEDP course grades come from MicroMasters and MITxOnline. We've migrated some learners data from + -- MicroMasters to MITxOnline around Oct 2022, but only for those users who have MITxOnline account. + -- To avoid data overlapping, we deduplicate based on their social auth account linked on MicroMasters. + -- for old DEDP courses on edx.org, then we use the grade from MicroMasters + -- for new DEDP course on MITx Online, then we use the grade from MITx Online + dedp_course_grades_combined as ( + select + course_grades_dedp_from_micromasters.program_title, + course_grades_dedp_from_micromasters.mitxonline_program_id, + course_grades_dedp_from_micromasters.micromasters_program_id, + course_grades_dedp_from_micromasters.courserun_title, + course_grades_dedp_from_micromasters.courserun_readable_id, + course_grades_dedp_from_micromasters.courserun_platform, + course_grades_dedp_from_micromasters.course_number, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_full_name, + mitx_users.user_address_country as user_country, + mitx_users.user_micromasters_email as user_email, + course_grades_dedp_from_micromasters.coursegrade_grade as grade, + true as is_passing, + course_grades_dedp_from_micromasters.coursegrade_created_on as created_on + from course_grades_dedp_from_micromasters + left join + mitx_users on course_grades_dedp_from_micromasters.user_micromasters_id = mitx_users.user_micromasters_id + where course_grades_dedp_from_micromasters.courserun_platform = '{{ var("edxorg") }}' -, course_grades_dedp_from_mitxonline as ( - select * - from {{ ref('__micromasters_course_grades_dedp_from_mitxonline') }} -) + union all -, course_grades_non_dedp_program as ( - select * - from {{ ref('__micromasters_course_grades_non_dedp_from_edxorg') }} -) + select + course_grades_dedp_from_mitxonline.program_title, + course_grades_dedp_from_mitxonline.mitxonline_program_id, + course_grades_dedp_from_mitxonline.micromasters_program_id, + course_grades_dedp_from_mitxonline.courserun_title, + course_grades_dedp_from_mitxonline.courserun_readable_id, + course_grades_dedp_from_mitxonline.courserun_platform, + course_grades_dedp_from_mitxonline.course_number, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_full_name, + mitx_users.user_address_country as user_country, + mitx_users.user_mitxonline_email as user_email, + course_grades_dedp_from_mitxonline.courserungrade_grade as grade, + course_grades_dedp_from_mitxonline.courserungrade_is_passing as is_passing, + course_grades_dedp_from_mitxonline.courserungrade_created_on as created_on + from course_grades_dedp_from_mitxonline + left join mitx_users on course_grades_dedp_from_mitxonline.user_mitxonline_id = mitx_users.user_mitxonline_id + where course_grades_dedp_from_mitxonline.courserun_platform = '{{ var("mitxonline") }}' -, mitx_users as ( - select * from {{ ref('int__mitx__users') }} -) - --- DEDP course grades come from MicroMasters and MITxOnline. We've migrated some learners data from --- MicroMasters to MITxOnline around Oct 2022, but only for those users who have MITxOnline account. --- To avoid data overlapping, we deduplicate based on their social auth account linked on MicroMasters. --- for old DEDP courses on edx.org, then we use the grade from MicroMasters --- for new DEDP course on MITx Online, then we use the grade from MITx Online - -, dedp_course_grades_combined as ( - select - course_grades_dedp_from_micromasters.program_title - , course_grades_dedp_from_micromasters.mitxonline_program_id - , course_grades_dedp_from_micromasters.micromasters_program_id - , course_grades_dedp_from_micromasters.courserun_title - , course_grades_dedp_from_micromasters.courserun_readable_id - , course_grades_dedp_from_micromasters.courserun_platform - , course_grades_dedp_from_micromasters.course_number - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_full_name - , mitx_users.user_address_country as user_country - , mitx_users.user_micromasters_email as user_email - , course_grades_dedp_from_micromasters.coursegrade_grade as grade - , true as is_passing - , course_grades_dedp_from_micromasters.coursegrade_created_on as created_on - from course_grades_dedp_from_micromasters - left join mitx_users - on course_grades_dedp_from_micromasters.user_micromasters_id = mitx_users.user_micromasters_id - where course_grades_dedp_from_micromasters.courserun_platform = '{{ var("edxorg") }}' - - union all - - select - course_grades_dedp_from_mitxonline.program_title - , course_grades_dedp_from_mitxonline.mitxonline_program_id - , course_grades_dedp_from_mitxonline.micromasters_program_id - , course_grades_dedp_from_mitxonline.courserun_title - , course_grades_dedp_from_mitxonline.courserun_readable_id - , course_grades_dedp_from_mitxonline.courserun_platform - , course_grades_dedp_from_mitxonline.course_number - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_full_name - , mitx_users.user_address_country as user_country - , mitx_users.user_mitxonline_email as user_email - , course_grades_dedp_from_mitxonline.courserungrade_grade as grade - , course_grades_dedp_from_mitxonline.courserungrade_is_passing as is_passing - , course_grades_dedp_from_mitxonline.courserungrade_created_on as created_on - from course_grades_dedp_from_mitxonline - left join mitx_users - on course_grades_dedp_from_mitxonline.user_mitxonline_id = mitx_users.user_mitxonline_id - where course_grades_dedp_from_mitxonline.courserun_platform = '{{ var("mitxonline") }}' - -) - -, dedp_course_grades_sorted as ( - select - program_title - , mitxonline_program_id - , micromasters_program_id - , courserun_title - , courserun_readable_id - , courserun_platform - , course_number - , user_edxorg_username - , user_mitxonline_username - , user_full_name - , user_country - , user_email - , grade - , is_passing - , case - when - user_mitxonline_username is not null + ), + dedp_course_grades_sorted as ( + select + program_title, + mitxonline_program_id, + micromasters_program_id, + courserun_title, + courserun_readable_id, + courserun_platform, + course_number, + user_edxorg_username, + user_mitxonline_username, + user_full_name, + user_country, + user_email, + grade, + is_passing, + case + when user_mitxonline_username is not null then - row_number() - over ( - partition by courserun_readable_id, user_mitxonline_username, mitxonline_program_id - order by created_on desc - ) - else 1 - end as row_num - from dedp_course_grades_combined -) - -, dedp_course_grades as ( - select * - from dedp_course_grades_sorted - where row_num = 1 -) - -, non_dedp_course_grades as ( - select - course_grades_non_dedp_program.program_title - , course_grades_non_dedp_program.mitxonline_program_id - , course_grades_non_dedp_program.micromasters_program_id - , course_grades_non_dedp_program.courserun_title - , course_grades_non_dedp_program.courserun_readable_id - , course_grades_non_dedp_program.courserun_platform - , course_grades_non_dedp_program.course_number - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_full_name - , mitx_users.user_address_country as user_country - , course_grades_non_dedp_program.courserungrade_user_grade as grade - , course_grades_non_dedp_program.courserungrade_is_passing as is_passing - , coalesce(mitx_users.user_edxorg_email, mitx_users.user_micromasters_email) as user_email - from course_grades_non_dedp_program - left join mitx_users - on course_grades_non_dedp_program.user_edxorg_id = mitx_users.user_edxorg_id -) - -, program_course_grades as ( - select - program_title - , mitxonline_program_id - , micromasters_program_id - , courserun_title - , courserun_readable_id - , courserun_platform - , course_number - , user_edxorg_username - , user_mitxonline_username - , user_full_name - , user_country - , user_email - , grade - , is_passing - from dedp_course_grades + row_number() over ( + partition by courserun_readable_id, user_mitxonline_username, mitxonline_program_id + order by created_on desc + ) + else 1 + end as row_num + from dedp_course_grades_combined + ), + dedp_course_grades as (select * from dedp_course_grades_sorted where row_num = 1), + non_dedp_course_grades as ( + select + course_grades_non_dedp_program.program_title, + course_grades_non_dedp_program.mitxonline_program_id, + course_grades_non_dedp_program.micromasters_program_id, + course_grades_non_dedp_program.courserun_title, + course_grades_non_dedp_program.courserun_readable_id, + course_grades_non_dedp_program.courserun_platform, + course_grades_non_dedp_program.course_number, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_full_name, + mitx_users.user_address_country as user_country, + course_grades_non_dedp_program.courserungrade_user_grade as grade, + course_grades_non_dedp_program.courserungrade_is_passing as is_passing, + coalesce(mitx_users.user_edxorg_email, mitx_users.user_micromasters_email) as user_email + from course_grades_non_dedp_program + left join mitx_users on course_grades_non_dedp_program.user_edxorg_id = mitx_users.user_edxorg_id + ), + program_course_grades as ( + select + program_title, + mitxonline_program_id, + micromasters_program_id, + courserun_title, + courserun_readable_id, + courserun_platform, + course_number, + user_edxorg_username, + user_mitxonline_username, + user_full_name, + user_country, + user_email, + grade, + is_passing + from dedp_course_grades - union all + union all - select - program_title - , mitxonline_program_id - , micromasters_program_id - , courserun_title - , courserun_readable_id - , courserun_platform - , course_number - , user_edxorg_username - , user_mitxonline_username - , user_full_name - , user_country - , user_email - , grade - , is_passing - from non_dedp_course_grades -) + select + program_title, + mitxonline_program_id, + micromasters_program_id, + courserun_title, + courserun_readable_id, + courserun_platform, + course_number, + user_edxorg_username, + user_mitxonline_username, + user_full_name, + user_country, + user_email, + grade, + is_passing + from non_dedp_course_grades + ) -select * from program_course_grades +select * +from program_course_grades diff --git a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__dedp_proctored_exam_grades.sql b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__dedp_proctored_exam_grades.sql index 306316fd9..0b50c19c6 100644 --- a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__dedp_proctored_exam_grades.sql +++ b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__dedp_proctored_exam_grades.sql @@ -1,50 +1,33 @@ -with exam_grades as ( - select * - from {{ ref('stg__micromasters__app__postgres__grades_proctoredexamgrade') }} -) - -, examruns as ( - select * from {{ ref('stg__micromasters__app__postgres__exams_examrun') }} -) - -, programs as ( - select * from {{ ref('int__mitx__programs') }} -) - -, courses as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_course') }} -) - -, micromasters_users as ( - select * from {{ ref('__micromasters__users') }} -) - -, mixonline_users as ( - select * from {{ ref('int__mitxonline__users') }} -) +with + exam_grades as (select * from {{ ref("stg__micromasters__app__postgres__grades_proctoredexamgrade") }}), + examruns as (select * from {{ ref("stg__micromasters__app__postgres__exams_examrun") }}), + programs as (select * from {{ ref("int__mitx__programs") }}), + courses as (select * from {{ ref("stg__micromasters__app__postgres__courses_course") }}), + micromasters_users as (select * from {{ ref("__micromasters__users") }}), + mixonline_users as (select * from {{ ref("int__mitxonline__users") }}) select - programs.program_title - , programs.micromasters_program_id - , programs.mitxonline_program_id - , courses.course_title - , courses.course_number - , examruns.examrun_readable_id - , examruns.examrun_semester - , examruns.examrun_passing_grade - , micromasters_users.user_edxorg_username - , micromasters_users.user_mitxonline_username - , micromasters_users.user_full_name - , micromasters_users.user_email as user_micromasters_email - , mixonline_users.user_email as user_mitxonline_email - , exam_grades.proctoredexamgrade_passing_score - , exam_grades.proctoredexamgrade_score - , exam_grades.proctoredexamgrade_percentage_grade - , exam_grades.proctoredexamgrade_is_passing - , exam_grades.proctoredexamgrade_exam_on - , exam_grades.proctoredexamgrade_created_on - , exam_grades.proctoredexamgrade_updated_on - , exam_grades.proctoredexamgrade_id + programs.program_title, + programs.micromasters_program_id, + programs.mitxonline_program_id, + courses.course_title, + courses.course_number, + examruns.examrun_readable_id, + examruns.examrun_semester, + examruns.examrun_passing_grade, + micromasters_users.user_edxorg_username, + micromasters_users.user_mitxonline_username, + micromasters_users.user_full_name, + micromasters_users.user_email as user_micromasters_email, + mixonline_users.user_email as user_mitxonline_email, + exam_grades.proctoredexamgrade_passing_score, + exam_grades.proctoredexamgrade_score, + exam_grades.proctoredexamgrade_percentage_grade, + exam_grades.proctoredexamgrade_is_passing, + exam_grades.proctoredexamgrade_exam_on, + exam_grades.proctoredexamgrade_created_on, + exam_grades.proctoredexamgrade_updated_on, + exam_grades.proctoredexamgrade_id from exam_grades inner join examruns on exam_grades.examrun_id = examruns.examrun_id inner join courses on exam_grades.course_id = courses.course_id diff --git a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__orders.sql b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__orders.sql index a0b235528..14490c91c 100644 --- a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__orders.sql +++ b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__orders.sql @@ -1,88 +1,62 @@ -with orders as ( - select * from {{ ref('stg__micromasters__app__postgres__ecommerce_order') }} -) - -, lines as ( - select * from {{ ref('stg__micromasters__app__postgres__ecommerce_line') }} -) - -, coupons as ( - select * from {{ ref('stg__micromasters__app__postgres__ecommerce_coupon') }} -) - -, redeemedcoupons as ( - select * from {{ ref('stg__micromasters__app__postgres__ecommerce_redeemedcoupon') }} -) - -, receipts as ( - select - * - , row_number() over ( - partition by order_id - order by receipt_created_on desc - ) as row_num - from {{ ref('stg__micromasters__app__postgres__ecommerce_receipt') }} - where receipt_transaction_status != 'ERROR' -) - -, users as ( - select * from {{ ref('__micromasters__users') }} -) - -, courseruns as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_courserun') }} -) - -, edxorg_users as ( - select * from {{ ref('int__edxorg__mitx_users') }} -) +with + orders as (select * from {{ ref("stg__micromasters__app__postgres__ecommerce_order") }}), + lines as (select * from {{ ref("stg__micromasters__app__postgres__ecommerce_line") }}), + coupons as (select * from {{ ref("stg__micromasters__app__postgres__ecommerce_coupon") }}), + redeemedcoupons as (select * from {{ ref("stg__micromasters__app__postgres__ecommerce_redeemedcoupon") }}), + receipts as ( + select *, row_number() over (partition by order_id order by receipt_created_on desc) as row_num + from {{ ref("stg__micromasters__app__postgres__ecommerce_receipt") }} + where receipt_transaction_status != 'ERROR' + ), + users as (select * from {{ ref("__micromasters__users") }}), + courseruns as (select * from {{ ref("stg__micromasters__app__postgres__courses_courserun") }}), + edxorg_users as (select * from {{ ref("int__edxorg__mitx_users") }}) select - orders.order_id - , orders.user_id - , users.user_username - , users.user_mitxonline_username - , users.user_edxorg_username - , edxorg_users.user_id as user_edxorg_id - , edxorg_users.user_email as user_edxorg_email - , users.user_full_name - , users.user_email - , orders.order_state - , orders.order_reference_number - , orders.order_total_price_paid - , orders.order_created_on - , lines.line_id - , lines.line_price - , lines.courserun_readable_id - , lines.courserun_edxorg_readable_id - , courseruns.courserun_platform - , receipts.receipt_reference_number - , receipts.receipt_transaction_uuid - , receipts.receipt_transaction_id - , receipts.receipt_transaction_type - , receipts.receipt_payment_method - , receipts.receipt_authorization_code - , receipts.receipt_bill_to_address_state - , receipts.receipt_bill_to_address_country - , receipts.receipt_payer_name - , receipts.receipt_payer_email - , receipts.receipt_payer_ip_address - , receipts.receipt_payment_amount - , receipts.receipt_payment_currency - , receipts.receipt_payment_card_number - , receipts.receipt_payment_card_type - , receipts.receipt_payment_timestamp - , coupons.coupon_id - , coupons.coupon_type - , coupons.coupon_code - , coupons.coupon_discount_amount_text - , case - when orders.order_state in ('fulfilled', 'refunded') - then redeemedcoupons.redeemedcoupon_created_on - end as redeemedcoupon_created_on - , case + orders.order_id, + orders.user_id, + users.user_username, + users.user_mitxonline_username, + users.user_edxorg_username, + edxorg_users.user_id as user_edxorg_id, + edxorg_users.user_email as user_edxorg_email, + users.user_full_name, + users.user_email, + orders.order_state, + orders.order_reference_number, + orders.order_total_price_paid, + orders.order_created_on, + lines.line_id, + lines.line_price, + lines.courserun_readable_id, + lines.courserun_edxorg_readable_id, + courseruns.courserun_platform, + receipts.receipt_reference_number, + receipts.receipt_transaction_uuid, + receipts.receipt_transaction_id, + receipts.receipt_transaction_type, + receipts.receipt_payment_method, + receipts.receipt_authorization_code, + receipts.receipt_bill_to_address_state, + receipts.receipt_bill_to_address_country, + receipts.receipt_payer_name, + receipts.receipt_payer_email, + receipts.receipt_payer_ip_address, + receipts.receipt_payment_amount, + receipts.receipt_payment_currency, + receipts.receipt_payment_card_number, + receipts.receipt_payment_card_type, + receipts.receipt_payment_timestamp, + coupons.coupon_id, + coupons.coupon_type, + coupons.coupon_code, + coupons.coupon_discount_amount_text, + case + when orders.order_state in ('fulfilled', 'refunded') then redeemedcoupons.redeemedcoupon_created_on + end as redeemedcoupon_created_on, + case when coupons.coupon_amount_type = 'percent-discount' - then cast(lines.line_price * coupons.coupon_amount as decimal(38, 2)) + then cast(lines.line_price * coupons.coupon_amount as decimal(38, 2)) else cast(coupons.coupon_amount as decimal(38, 2)) end as coupon_amount from orders diff --git a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__program_certificates.sql b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__program_certificates.sql index 0676d0795..b4d861b8d 100644 --- a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__program_certificates.sql +++ b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__program_certificates.sql @@ -1,141 +1,127 @@ -with program_certificates_dedp_from_micromasters as ( - select * - from {{ ref('__micromasters_program_certificates_dedp_from_micromasters') }} -) +with + program_certificates_dedp_from_micromasters as ( + select * from {{ ref("__micromasters_program_certificates_dedp_from_micromasters") }} + ), + program_certificates_dedp_from_mitxonline as ( + select * from {{ ref("__micromasters_program_certificates_dedp_from_mitxonline") }} + ), + program_certificates_non_dedp as (select * from {{ ref("__micromasters_program_certificates_non_dedp") }}), + mitx_users as (select * from {{ ref("int__mitx__users") }}), + mitxonline_programs as (select * from {{ ref("int__mitxonline__programs") }}), + -- Some micromasters learners from previous semesters don't have mitxonline logins. The mitxonline + -- database does not include the certificates for those users. However, for future semesters, + -- the mitxonline database will be the source for dedp program certificates and the table in + -- the micromasters database will no longer be updated. Hence we query the micromasters database for + -- dedp certificates earned before 2022-10-01 and the mitxonline database for those earned after + report as ( + select + program_certificates_dedp_from_micromasters.program_certificate_hashed_id, + program_certificates_dedp_from_micromasters.program_title, + program_certificates_dedp_from_micromasters.micromasters_program_id, + program_certificates_dedp_from_micromasters.mitxonline_program_id, + program_certificates_dedp_from_micromasters.program_completion_timestamp, + mitx_users.user_edxorg_id, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_micromasters_email as user_email, + mitx_users.user_first_name, + mitx_users.user_last_name, + mitx_users.user_full_name, + mitx_users.user_gender, + mitx_users.user_birth_year as user_year_of_birth, + mitx_users.user_address_country as user_country, + mitx_users.user_address_state as user_address_state_or_territory, + mitx_users.user_address_city, + mitx_users.user_address_postal_code, + mitx_users.user_street_address, + true as program_is_dedp + from program_certificates_dedp_from_micromasters + left join + mitx_users + on program_certificates_dedp_from_micromasters.user_micromasters_id = mitx_users.user_micromasters_id + where program_certificates_dedp_from_micromasters.program_completion_timestamp < '2022-10-01' -, program_certificates_dedp_from_mitxonline as ( - select * - from {{ ref('__micromasters_program_certificates_dedp_from_mitxonline') }} -) + union all -, program_certificates_non_dedp as ( - select * - from {{ ref('__micromasters_program_certificates_non_dedp') }} -) + select + program_certificates_dedp_from_mitxonline.program_certificate_hashed_id, + program_certificates_dedp_from_mitxonline.program_title, + program_certificates_dedp_from_mitxonline.micromasters_program_id, + program_certificates_dedp_from_mitxonline.mitxonline_program_id, + program_certificates_dedp_from_mitxonline.program_completion_timestamp, + mitx_users.user_edxorg_id, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_mitxonline_email as user_email, + mitx_users.user_first_name, + mitx_users.user_last_name, + mitx_users.user_full_name, + mitx_users.user_gender, + mitx_users.user_birth_year as user_year_of_birth, + mitx_users.user_address_country as user_country, + mitx_users.user_address_state as user_address_state_or_territory, + mitx_users.user_address_city, + mitx_users.user_address_postal_code, + mitx_users.user_street_address, + true as program_is_dedp + from program_certificates_dedp_from_mitxonline + left join + mitx_users on program_certificates_dedp_from_mitxonline.user_mitxonline_id = mitx_users.user_mitxonline_id + where program_certificates_dedp_from_mitxonline.program_completion_timestamp >= '2022-10-01' -, mitx_users as ( - select * from {{ ref('int__mitx__users') }} -) + union all -, mitxonline_programs as ( - select * from {{ ref('int__mitxonline__programs') }} -) --- Some micromasters learners from previous semesters don't have mitxonline logins. The mitxonline --- database does not include the certificates for those users. However, for future semesters, --- the mitxonline database will be the source for dedp program certificates and the table in --- the micromasters database will no longer be updated. Hence we query the micromasters database for --- dedp certificates earned before 2022-10-01 and the mitxonline database for those earned after - -, report as ( - select - program_certificates_dedp_from_micromasters.program_certificate_hashed_id - , program_certificates_dedp_from_micromasters.program_title - , program_certificates_dedp_from_micromasters.micromasters_program_id - , program_certificates_dedp_from_micromasters.mitxonline_program_id - , program_certificates_dedp_from_micromasters.program_completion_timestamp - , mitx_users.user_edxorg_id - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_micromasters_email as user_email - , mitx_users.user_first_name - , mitx_users.user_last_name - , mitx_users.user_full_name - , mitx_users.user_gender - , mitx_users.user_birth_year as user_year_of_birth - , mitx_users.user_address_country as user_country - , mitx_users.user_address_state as user_address_state_or_territory - , mitx_users.user_address_city - , mitx_users.user_address_postal_code - , mitx_users.user_street_address - , true as program_is_dedp - from program_certificates_dedp_from_micromasters - left join mitx_users - on program_certificates_dedp_from_micromasters.user_micromasters_id = mitx_users.user_micromasters_id - where program_certificates_dedp_from_micromasters.program_completion_timestamp < '2022-10-01' - - union all - - select - program_certificates_dedp_from_mitxonline.program_certificate_hashed_id - , program_certificates_dedp_from_mitxonline.program_title - , program_certificates_dedp_from_mitxonline.micromasters_program_id - , program_certificates_dedp_from_mitxonline.mitxonline_program_id - , program_certificates_dedp_from_mitxonline.program_completion_timestamp - , mitx_users.user_edxorg_id - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_mitxonline_email as user_email - , mitx_users.user_first_name - , mitx_users.user_last_name - , mitx_users.user_full_name - , mitx_users.user_gender - , mitx_users.user_birth_year as user_year_of_birth - , mitx_users.user_address_country as user_country - , mitx_users.user_address_state as user_address_state_or_territory - , mitx_users.user_address_city - , mitx_users.user_address_postal_code - , mitx_users.user_street_address - , true as program_is_dedp - from program_certificates_dedp_from_mitxonline - left join mitx_users - on program_certificates_dedp_from_mitxonline.user_mitxonline_id = mitx_users.user_mitxonline_id - where program_certificates_dedp_from_mitxonline.program_completion_timestamp >= '2022-10-01' - - union all - - select - program_certificates_non_dedp.program_certificate_hashed_id - , program_certificates_non_dedp.program_title - , program_certificates_non_dedp.micromasters_program_id - , program_certificates_non_dedp.mitxonline_program_id - , program_certificates_non_dedp.program_completion_timestamp - , mitx_users.user_edxorg_id - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_edxorg_email as user_email - , mitx_users.user_first_name - , mitx_users.user_last_name - , mitx_users.user_full_name - , mitx_users.user_gender - , mitx_users.user_birth_year as user_year_of_birth - , mitx_users.user_address_country as user_country - , mitx_users.user_address_state as user_address_state_or_territory - , mitx_users.user_address_city - , mitx_users.user_address_postal_code - , mitx_users.user_street_address - , false as program_is_dedp - from program_certificates_non_dedp - left join mitx_users - on program_certificates_non_dedp.user_edxorg_id = mitx_users.user_edxorg_id -) + select + program_certificates_non_dedp.program_certificate_hashed_id, + program_certificates_non_dedp.program_title, + program_certificates_non_dedp.micromasters_program_id, + program_certificates_non_dedp.mitxonline_program_id, + program_certificates_non_dedp.program_completion_timestamp, + mitx_users.user_edxorg_id, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_edxorg_email as user_email, + mitx_users.user_first_name, + mitx_users.user_last_name, + mitx_users.user_full_name, + mitx_users.user_gender, + mitx_users.user_birth_year as user_year_of_birth, + mitx_users.user_address_country as user_country, + mitx_users.user_address_state as user_address_state_or_territory, + mitx_users.user_address_city, + mitx_users.user_address_postal_code, + mitx_users.user_street_address, + false as program_is_dedp + from program_certificates_non_dedp + left join mitx_users on program_certificates_non_dedp.user_edxorg_id = mitx_users.user_edxorg_id + ) select - report.program_certificate_hashed_id - , report.user_edxorg_username - , report.user_mitxonline_username - , report.user_email - , report.program_title - , report.micromasters_program_id - , report.mitxonline_program_id - , report.user_edxorg_id - , report.program_completion_timestamp - , report.user_gender - , report.user_first_name - , report.user_last_name - , report.user_full_name - , report.user_year_of_birth - , report.user_country - , report.user_address_postal_code - , report.user_address_city - , report.user_street_address - , report.user_address_state_or_territory - , report.program_is_dedp - , if( - mitxonline_programs.program_id is not null - , mitxonline_programs.program_readable_id - , {{ generate_micromasters_program_readable_id('report.micromasters_program_id','report.program_title') }} + report.program_certificate_hashed_id, + report.user_edxorg_username, + report.user_mitxonline_username, + report.user_email, + report.program_title, + report.micromasters_program_id, + report.mitxonline_program_id, + report.user_edxorg_id, + report.program_completion_timestamp, + report.user_gender, + report.user_first_name, + report.user_last_name, + report.user_full_name, + report.user_year_of_birth, + report.user_country, + report.user_address_postal_code, + report.user_address_city, + report.user_street_address, + report.user_address_state_or_territory, + report.program_is_dedp, + if( + mitxonline_programs.program_id is not null, + mitxonline_programs.program_readable_id, + {{ generate_micromasters_program_readable_id("report.micromasters_program_id", "report.program_title") }} ) as program_readable_id from report -left join mitxonline_programs - on report.mitxonline_program_id = mitxonline_programs.program_id +left join mitxonline_programs on report.mitxonline_program_id = mitxonline_programs.program_id order by report.program_completion_timestamp desc diff --git a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__program_enrollments.sql b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__program_enrollments.sql index b663e38a2..4203309b2 100644 --- a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__program_enrollments.sql +++ b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__program_enrollments.sql @@ -1,198 +1,170 @@ -- MicroMasters Program Enrollment Information - -with micromasters_program_enrollments as ( - --- There are learners who received both 'Statistics and Data Science (General track)' and 'Statistics and Data - -- Science' from 2U data, but we only count them once in 'Statistics and Data Science' for MM program enrollments - -- report. - select - * - , row_number() over ( - partition by user_id, micromasters_program_id - order by program_title - ) as row_num - from {{ ref('int__edxorg__mitx_program_enrollments') }} - where micromasters_program_id is not null -) - -, mm_program_enrollments as ( - select * from {{ ref('stg__micromasters__app__postgres__dashboard_programenrollment') }} -) - -, mitxonline_programenrollments as ( - select * - from {{ ref('int__mitxonline__programenrollments') }} -) - -, programs as ( - select * - from {{ ref('int__mitx__programs') }} -) - -, mitx_users as ( - select * from {{ ref('int__mitx__users') }} -) - -, mitxonline_dedp_records as ( - select - '{{ var("mitxonline") }}' as platform_name - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitxonline_programenrollments.user_email - , programs.micromasters_program_id - , programs.program_title - , mitxonline_programenrollments.program_id as mitxonline_program_id - , mitx_users.user_edxorg_id - , mitx_users.user_address_city - , mitx_users.user_first_name - , mitx_users.user_last_name - , mitx_users.user_address_postal_code - , mitx_users.user_street_address - , mitx_users.user_full_name - , mitx_users.user_micromasters_id as micromasters_user_id - , mitx_users.user_birth_year as user_year_of_birth - , mitx_users.user_gender - , mitx_users.user_address_country as user_country - , mitx_users.user_address_state as user_address_state_or_territory - from mitxonline_programenrollments - inner join programs - on mitxonline_programenrollments.program_id = programs.mitxonline_program_id - inner join mitx_users - on mitxonline_programenrollments.user_id = mitx_users.user_mitxonline_id - where programs.is_dedp_program = true -) - -, mm_dedp_records as ( - select - mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_micromasters_email as user_email - , programs.micromasters_program_id - , programs.program_title - , programs.mitxonline_program_id - , mitx_users.user_edxorg_id - , mitx_users.user_gender - , mitx_users.user_address_country as user_country - , mitx_users.user_address_city - , mitx_users.user_first_name - , mitx_users.user_last_name - , mitx_users.user_address_postal_code - , mitx_users.user_street_address - , mitx_users.user_address_state as user_address_state_or_territory - , mitx_users.user_full_name - , mitx_users.user_micromasters_id as micromasters_user_id - , mitx_users.user_birth_year as user_year_of_birth - , if(mitx_users.is_mitxonline_user = true, '{{ var("mitxonline") }}', '{{ var("edxorg") }}') as platform_name - from mm_program_enrollments - inner join programs - on mm_program_enrollments.program_id = programs.micromasters_program_id - inner join mitx_users - on mm_program_enrollments.user_id = mitx_users.user_micromasters_id - left join mitxonline_dedp_records - on - mitx_users.user_micromasters_email = mitxonline_dedp_records.user_email +with + micromasters_program_enrollments as ( + -- - There are learners who received both 'Statistics and Data Science (General track)' and 'Statistics and Data + -- Science' from 2U data, but we only count them once in 'Statistics and Data Science' for MM program + -- enrollments + -- report. + select *, row_number() over (partition by user_id, micromasters_program_id order by program_title) as row_num + from {{ ref("int__edxorg__mitx_program_enrollments") }} + where micromasters_program_id is not null + ), + mm_program_enrollments as ( + select * from {{ ref("stg__micromasters__app__postgres__dashboard_programenrollment") }} + ), + mitxonline_programenrollments as (select * from {{ ref("int__mitxonline__programenrollments") }}), + programs as (select * from {{ ref("int__mitx__programs") }}), + mitx_users as (select * from {{ ref("int__mitx__users") }}), + mitxonline_dedp_records as ( + select + '{{ var("mitxonline") }}' as platform_name, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitxonline_programenrollments.user_email, + programs.micromasters_program_id, + programs.program_title, + mitxonline_programenrollments.program_id as mitxonline_program_id, + mitx_users.user_edxorg_id, + mitx_users.user_address_city, + mitx_users.user_first_name, + mitx_users.user_last_name, + mitx_users.user_address_postal_code, + mitx_users.user_street_address, + mitx_users.user_full_name, + mitx_users.user_micromasters_id as micromasters_user_id, + mitx_users.user_birth_year as user_year_of_birth, + mitx_users.user_gender, + mitx_users.user_address_country as user_country, + mitx_users.user_address_state as user_address_state_or_territory + from mitxonline_programenrollments + inner join programs on mitxonline_programenrollments.program_id = programs.mitxonline_program_id + inner join mitx_users on mitxonline_programenrollments.user_id = mitx_users.user_mitxonline_id + where programs.is_dedp_program = true + ), + mm_dedp_records as ( + select + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_micromasters_email as user_email, + programs.micromasters_program_id, + programs.program_title, + programs.mitxonline_program_id, + mitx_users.user_edxorg_id, + mitx_users.user_gender, + mitx_users.user_address_country as user_country, + mitx_users.user_address_city, + mitx_users.user_first_name, + mitx_users.user_last_name, + mitx_users.user_address_postal_code, + mitx_users.user_street_address, + mitx_users.user_address_state as user_address_state_or_territory, + mitx_users.user_full_name, + mitx_users.user_micromasters_id as micromasters_user_id, + mitx_users.user_birth_year as user_year_of_birth, + if(mitx_users.is_mitxonline_user = true, '{{ var("mitxonline") }}', '{{ var("edxorg") }}') as platform_name + from mm_program_enrollments + inner join programs on mm_program_enrollments.program_id = programs.micromasters_program_id + inner join mitx_users on mm_program_enrollments.user_id = mitx_users.user_micromasters_id + left join + mitxonline_dedp_records + on mitx_users.user_micromasters_email = mitxonline_dedp_records.user_email and programs.program_title = mitxonline_dedp_records.program_title - where - programs.is_dedp_program = true - and mitxonline_dedp_records.user_email is null -) - -, non_dedp_records as ( - select - '{{ var("edxorg") }}' as platform_name - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_edxorg_email as user_email - , programs.micromasters_program_id - , programs.program_title - , programs.mitxonline_program_id - , mitx_users.user_edxorg_id - , mitx_users.user_gender - , mitx_users.user_address_country as user_country - , mitx_users.user_address_city - , mitx_users.user_first_name - , mitx_users.user_last_name - , mitx_users.user_address_postal_code - , mitx_users.user_street_address - , mitx_users.user_address_state as user_address_state_or_territory - , mitx_users.user_full_name - , mitx_users.user_micromasters_id as micromasters_user_id - , mitx_users.user_birth_year as user_year_of_birth - from micromasters_program_enrollments - inner join mitx_users - on micromasters_program_enrollments.user_id = mitx_users.user_edxorg_id - inner join programs - on micromasters_program_enrollments.micromasters_program_id = programs.micromasters_program_id - where - micromasters_program_enrollments.row_num = 1 - and programs.is_dedp_program = false -) + where programs.is_dedp_program = true and mitxonline_dedp_records.user_email is null + ), + non_dedp_records as ( + select + '{{ var("edxorg") }}' as platform_name, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_edxorg_email as user_email, + programs.micromasters_program_id, + programs.program_title, + programs.mitxonline_program_id, + mitx_users.user_edxorg_id, + mitx_users.user_gender, + mitx_users.user_address_country as user_country, + mitx_users.user_address_city, + mitx_users.user_first_name, + mitx_users.user_last_name, + mitx_users.user_address_postal_code, + mitx_users.user_street_address, + mitx_users.user_address_state as user_address_state_or_territory, + mitx_users.user_full_name, + mitx_users.user_micromasters_id as micromasters_user_id, + mitx_users.user_birth_year as user_year_of_birth + from micromasters_program_enrollments + inner join mitx_users on micromasters_program_enrollments.user_id = mitx_users.user_edxorg_id + inner join + programs on micromasters_program_enrollments.micromasters_program_id = programs.micromasters_program_id + where micromasters_program_enrollments.row_num = 1 and programs.is_dedp_program = false + ) select - user_edxorg_username - , user_mitxonline_username - , user_email - , micromasters_program_id - , program_title - , mitxonline_program_id - , user_edxorg_id - , user_gender - , user_country - , user_address_city - , user_first_name - , user_last_name - , user_address_postal_code - , user_street_address - , user_address_state_or_territory - , user_full_name - , micromasters_user_id - , user_year_of_birth - , platform_name + user_edxorg_username, + user_mitxonline_username, + user_email, + micromasters_program_id, + program_title, + mitxonline_program_id, + user_edxorg_id, + user_gender, + user_country, + user_address_city, + user_first_name, + user_last_name, + user_address_postal_code, + user_street_address, + user_address_state_or_territory, + user_full_name, + micromasters_user_id, + user_year_of_birth, + platform_name from mitxonline_dedp_records union distinct select - user_edxorg_username - , user_mitxonline_username - , user_email - , micromasters_program_id - , program_title - , mitxonline_program_id - , user_edxorg_id - , user_gender - , user_country - , user_address_city - , user_first_name - , user_last_name - , user_address_postal_code - , user_street_address - , user_address_state_or_territory - , user_full_name - , micromasters_user_id - , user_year_of_birth - , platform_name + user_edxorg_username, + user_mitxonline_username, + user_email, + micromasters_program_id, + program_title, + mitxonline_program_id, + user_edxorg_id, + user_gender, + user_country, + user_address_city, + user_first_name, + user_last_name, + user_address_postal_code, + user_street_address, + user_address_state_or_territory, + user_full_name, + micromasters_user_id, + user_year_of_birth, + platform_name from mm_dedp_records union distinct select - user_edxorg_username - , user_mitxonline_username - , user_email - , micromasters_program_id - , program_title - , mitxonline_program_id - , user_edxorg_id - , user_gender - , user_country - , user_address_city - , user_first_name - , user_last_name - , user_address_postal_code - , user_street_address - , user_address_state_or_territory - , user_full_name - , micromasters_user_id - , user_year_of_birth - , platform_name + user_edxorg_username, + user_mitxonline_username, + user_email, + micromasters_program_id, + program_title, + mitxonline_program_id, + user_edxorg_id, + user_gender, + user_country, + user_address_city, + user_first_name, + user_last_name, + user_address_postal_code, + user_street_address, + user_address_state_or_territory, + user_full_name, + micromasters_user_id, + user_year_of_birth, + platform_name from non_dedp_records diff --git a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__program_requirements.sql b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__program_requirements.sql index 1d6dd9df5..6fa8edcc2 100644 --- a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__program_requirements.sql +++ b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__program_requirements.sql @@ -1,84 +1,62 @@ -- Program Requirement for MicroMaster -- courses are categorized as "Core" and "Elective" -- e.g. On production, program requirement for "Statistics and Data Science" (SDS) looks like this: ---- (program_num_required_courses = electiveset_required_number + # of Core courses) - +-- - (program_num_required_courses = electiveset_required_number + # of Core courses) -- program_id course_id Type electiveset_required_number program_num_required_courses ------------------------------------------------------------------------------------------------------ --- 4 27 Elective 1 5 --- 4 37 Elective 1 5 --- 4 30 Core 5 --- 4 29 Core 5 --- 4 26 Core 5 --- 4 28 Core 5 - - -with programs as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_program') }} -) - -, courses as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_course') }} -) - -, electiveset as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_electiveset') }} -) - -, electiveset_to_course as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_electiveset_to_course') }} -) - -, elective_courses as ( - select - electiveset_to_course.course_id - , electiveset.electiveset_id - , electiveset.program_id - , electiveset.electiveset_required_number - from electiveset_to_course - inner join electiveset - on electiveset_to_course.electiveset_id = electiveset.electiveset_id -) - ---- courses that are not in elective_courses are core -, core_courses as ( - select - courses.course_id - , courses.program_id - from courses - left join elective_courses - on - courses.program_id = elective_courses.program_id +-- --------------------------------------------------------------------------------------------------- +-- 4 27 Elective 1 5 +-- 4 37 Elective 1 5 +-- 4 30 Core 5 +-- 4 29 Core 5 +-- 4 26 Core 5 +-- 4 28 Core 5 +with + programs as (select * from {{ ref("stg__micromasters__app__postgres__courses_program") }}), + courses as (select * from {{ ref("stg__micromasters__app__postgres__courses_course") }}), + electiveset as (select * from {{ ref("stg__micromasters__app__postgres__courses_electiveset") }}), + electiveset_to_course as ( + select * from {{ ref("stg__micromasters__app__postgres__courses_electiveset_to_course") }} + ), + elective_courses as ( + select + electiveset_to_course.course_id, + electiveset.electiveset_id, + electiveset.program_id, + electiveset.electiveset_required_number + from electiveset_to_course + inner join electiveset on electiveset_to_course.electiveset_id = electiveset.electiveset_id + ), + -- - courses that are not in elective_courses are core + core_courses as ( + select courses.course_id, courses.program_id + from courses + left join + elective_courses + on courses.program_id = elective_courses.program_id and courses.course_id = elective_courses.course_id - where elective_courses.course_id is null -) - -, combined_courses as ( - select - null as electiveset_id - , program_id - , null as electiveset_required_number - , course_id - , 'Core' as programrequirement_type - from core_courses - - union all - - select - electiveset_id - , program_id - , electiveset_required_number - , course_id - , 'Elective' as programrequirement_type - from elective_courses -) + where elective_courses.course_id is null + ), + combined_courses as ( + select + null as electiveset_id, + program_id, + null as electiveset_required_number, + course_id, + 'Core' as programrequirement_type + from core_courses + + union all + + select electiveset_id, program_id, electiveset_required_number, course_id, 'Elective' as programrequirement_type + from elective_courses + ) select - combined_courses.program_id - , combined_courses.course_id - , combined_courses.electiveset_id - , combined_courses.programrequirement_type - , combined_courses.electiveset_required_number - , programs.program_num_required_courses + combined_courses.program_id, + combined_courses.course_id, + combined_courses.electiveset_id, + combined_courses.programrequirement_type, + combined_courses.electiveset_required_number, + programs.program_num_required_courses from combined_courses inner join programs on combined_courses.program_id = programs.program_id diff --git a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__programs.sql b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__programs.sql index 9183dd53d..d441e6360 100644 --- a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__programs.sql +++ b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__programs.sql @@ -1,13 +1,5 @@ -- MicroMasters Program Information +with programs as (select * from {{ ref("stg__micromasters__app__postgres__courses_program") }}) -with programs as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_program') }} -) - -select - program_id - , program_title - , program_description - , program_is_live - , program_num_required_courses +select program_id, program_title, program_description, program_is_live, program_num_required_courses from programs diff --git a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__users.sql b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__users.sql index 41611be7c..d00b6a18f 100644 --- a/src/ol_dbt/models/intermediate/micromasters/int__micromasters__users.sql +++ b/src/ol_dbt/models/intermediate/micromasters/int__micromasters__users.sql @@ -1,84 +1,63 @@ -with users as ( - select * from {{ ref('__micromasters__users') }} -) - -, education as ( - select - user_profile_id - , user_education_degree - , user_graduation_date - , row_number() over ( - partition by user_profile_id - order by user_graduation_date desc - ) as row_num - from {{ ref('stg__micromasters__app__postgres__profiles_education') }} -) - -, highest_education as ( - select - user_profile_id - , user_education_degree - from education - where row_num = 1 -) - -, employment as ( - select - user_profile_id - , user_company_name - , user_job_position - , user_company_industry - , row_number() over ( - partition by user_profile_id - order by user_start_date desc - ) as row_num - from {{ ref('stg__micromasters__app__postgres__profiles_employment') }} -) - -, most_recent_employment as ( - select - user_profile_id - , user_company_name - , user_job_position - , user_company_industry - from employment - where row_num = 1 -) +with + users as (select * from {{ ref("__micromasters__users") }}), + education as ( + select + user_profile_id, + user_education_degree, + user_graduation_date, + row_number() over (partition by user_profile_id order by user_graduation_date desc) as row_num + from {{ ref("stg__micromasters__app__postgres__profiles_education") }} + ), + highest_education as (select user_profile_id, user_education_degree from education where row_num = 1), + employment as ( + select + user_profile_id, + user_company_name, + user_job_position, + user_company_industry, + row_number() over (partition by user_profile_id order by user_start_date desc) as row_num + from {{ ref("stg__micromasters__app__postgres__profiles_employment") }} + ), + most_recent_employment as ( + select user_profile_id, user_company_name, user_job_position, user_company_industry + from employment + where row_num = 1 + ) select - users.user_id - , users.user_username - , users.user_email - , users.user_joined_on - , users.user_last_login - , users.user_is_active - , users.user_profile_id - , users.user_full_name - , users.user_first_name - , users.user_last_name - , users.user_birth_country - , users.user_address_country - , users.user_address_city - , users.user_address_state_or_territory - , users.user_address_postal_code - , users.user_street_address - , users.user_mailing_address - , users.user_preferred_name - , users.user_preferred_language - , users.user_nationality - , users.user_phone_number - , users.user_bio - , users.user_about_me - , users.user_edx_name - , users.user_edx_goals - , users.user_birth_date - , users.user_gender - , users.user_mitxonline_username - , users.user_edxorg_username - , most_recent_employment.user_company_name - , most_recent_employment.user_company_industry - , most_recent_employment.user_job_position - , coalesce(highest_education.user_education_degree, users.user_highest_education) as user_highest_education + users.user_id, + users.user_username, + users.user_email, + users.user_joined_on, + users.user_last_login, + users.user_is_active, + users.user_profile_id, + users.user_full_name, + users.user_first_name, + users.user_last_name, + users.user_birth_country, + users.user_address_country, + users.user_address_city, + users.user_address_state_or_territory, + users.user_address_postal_code, + users.user_street_address, + users.user_mailing_address, + users.user_preferred_name, + users.user_preferred_language, + users.user_nationality, + users.user_phone_number, + users.user_bio, + users.user_about_me, + users.user_edx_name, + users.user_edx_goals, + users.user_birth_date, + users.user_gender, + users.user_mitxonline_username, + users.user_edxorg_username, + most_recent_employment.user_company_name, + most_recent_employment.user_company_industry, + most_recent_employment.user_job_position, + coalesce(highest_education.user_education_degree, users.user_highest_education) as user_highest_education from users left join highest_education on users.user_profile_id = highest_education.user_profile_id left join most_recent_employment on users.user_profile_id = most_recent_employment.user_profile_id diff --git a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters__users.sql b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters__users.sql index 8d47f910f..f609de742 100644 --- a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters__users.sql +++ b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters__users.sql @@ -1,92 +1,75 @@ -with users as ( - select * from {{ ref('stg__micromasters__app__postgres__auth_user') }} -) - -, profiles as ( - select * from {{ ref('stg__micromasters__app__postgres__profiles_profile') }} -) - -, mitxonline_auth as ( - select - user_id - , user_username as user_mitxonline_username - from {{ ref('stg__micromasters__app__postgres__auth_usersocialauth') }} - where user_auth_provider = 'mitxonline' -) - -, edxorg_auth as ( - select - user_id - , user_username as user_edxorg_username - from {{ ref('stg__micromasters__app__postgres__auth_usersocialauth') }} - where user_auth_provider = 'edxorg' -) - -, edxorg_users as ( - select - user_username as user_edxorg_username - , user_last_login - from {{ ref('int__edxorg__mitx_users') }} -) - -, most_recent_edx_username as ( - select - user_id - , user_edxorg_username - from ( - select - edxorg_auth.user_id - , edxorg_auth.user_edxorg_username - , row_number() over ( - partition by edxorg_auth.user_id - order by edxorg_users.user_last_login desc - ) as row_num - from edxorg_auth - left join edxorg_users on edxorg_auth.user_edxorg_username = edxorg_users.user_edxorg_username +with + users as (select * from {{ ref("stg__micromasters__app__postgres__auth_user") }}), + profiles as (select * from {{ ref("stg__micromasters__app__postgres__profiles_profile") }}), + mitxonline_auth as ( + select user_id, user_username as user_mitxonline_username + from {{ ref("stg__micromasters__app__postgres__auth_usersocialauth") }} + where user_auth_provider = 'mitxonline' + ), + edxorg_auth as ( + select user_id, user_username as user_edxorg_username + from {{ ref("stg__micromasters__app__postgres__auth_usersocialauth") }} + where user_auth_provider = 'edxorg' + ), + edxorg_users as ( + select user_username as user_edxorg_username, user_last_login from {{ ref("int__edxorg__mitx_users") }} + ), + most_recent_edx_username as ( + select user_id, user_edxorg_username + from + ( + select + edxorg_auth.user_id, + edxorg_auth.user_edxorg_username, + row_number() over ( + partition by edxorg_auth.user_id order by edxorg_users.user_last_login desc + ) as row_num + from edxorg_auth + left join edxorg_users on edxorg_auth.user_edxorg_username = edxorg_users.user_edxorg_username + ) + where row_num = 1 ) - where row_num = 1 -) select - users.user_id - , users.user_username - , users.user_email - , users.user_joined_on - , users.user_last_login - , users.user_is_active - , profiles.user_profile_id - , profiles.user_account_privacy - , profiles.user_full_name - , profiles.user_first_name - , profiles.user_last_name - , profiles.user_birth_country - , profiles.user_address_country - , profiles.user_address_city - , profiles.user_address_state_or_territory - , profiles.user_address_postal_code - , profiles.user_street_address - , profiles.user_preferred_name - , profiles.user_email_is_optin - , profiles.user_profile_is_filled_out - , profiles.user_nationality - , profiles.user_phone_number - , profiles.user_employer - , profiles.user_job_title - , profiles.user_preferred_language - , profiles.user_mailing_address - , profiles.user_is_verified - , profiles.user_has_agreed_to_terms_of_service - , profiles.user_language_proficiencies - , profiles.user_profile_parental_consent_is_required - , profiles.user_bio - , profiles.user_about_me - , profiles.user_edx_name - , profiles.user_edx_goals - , profiles.user_birth_date - , profiles.user_gender - , profiles.user_highest_education - , mitxonline_auth.user_mitxonline_username - , most_recent_edx_username.user_edxorg_username + users.user_id, + users.user_username, + users.user_email, + users.user_joined_on, + users.user_last_login, + users.user_is_active, + profiles.user_profile_id, + profiles.user_account_privacy, + profiles.user_full_name, + profiles.user_first_name, + profiles.user_last_name, + profiles.user_birth_country, + profiles.user_address_country, + profiles.user_address_city, + profiles.user_address_state_or_territory, + profiles.user_address_postal_code, + profiles.user_street_address, + profiles.user_preferred_name, + profiles.user_email_is_optin, + profiles.user_profile_is_filled_out, + profiles.user_nationality, + profiles.user_phone_number, + profiles.user_employer, + profiles.user_job_title, + profiles.user_preferred_language, + profiles.user_mailing_address, + profiles.user_is_verified, + profiles.user_has_agreed_to_terms_of_service, + profiles.user_language_proficiencies, + profiles.user_profile_parental_consent_is_required, + profiles.user_bio, + profiles.user_about_me, + profiles.user_edx_name, + profiles.user_edx_goals, + profiles.user_birth_date, + profiles.user_gender, + profiles.user_highest_education, + mitxonline_auth.user_mitxonline_username, + most_recent_edx_username.user_edxorg_username from users left join profiles on users.user_id = profiles.user_id left join mitxonline_auth on users.user_id = mitxonline_auth.user_id diff --git a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_certificates_dedp_from_micromasters.sql b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_certificates_dedp_from_micromasters.sql index 203edb519..ae8f1446d 100644 --- a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_certificates_dedp_from_micromasters.sql +++ b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_certificates_dedp_from_micromasters.sql @@ -1,73 +1,56 @@ ---- Unlike other platforms, MicroMasters DEDP course certificates are based on course not run, we try to find the run ---- with highest grade for the course. If there are multiple runs with the highest grade, pick the latest grade from --- runs before DEDP course certificates were generated - -with dedp_course_certificates as ( - select * from {{ ref('stg__micromasters__app__postgres__grades_coursecertificate') }} -) - -, courserun_grades as ( - select * from {{ ref('stg__micromasters__app__postgres__grades_courserungrade') }} - where courserungrade_is_passing = true -) - -, courseruns as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_courserun') }} -) - -, courserun_grades_sorted as ( - select - courserun_grades.user_id - , courserun_grades.courserun_id - , courseruns.course_id - , row_number() over ( - partition by courserun_grades.user_id, courseruns.course_id - --- in case of multiple highest grades, use secondary sorting to ensure the consistent result - order by courserun_grades.courserungrade_grade desc, courserun_grades.coursegrade_created_on desc - ) as row_num - from courserun_grades - inner join courseruns on courserun_grades.courserun_id = courseruns.courserun_id - inner join dedp_course_certificates - on - courserun_grades.user_id = dedp_course_certificates.user_id +-- - Unlike other platforms, MicroMasters DEDP course certificates are based on course not run, we try to find the run +-- - with highest grade for the course. If there are multiple runs with the highest grade, pick the latest grade from +-- runs before DEDP course certificates were generated +with + dedp_course_certificates as (select * from {{ ref("stg__micromasters__app__postgres__grades_coursecertificate") }}), + courserun_grades as ( + select * + from {{ ref("stg__micromasters__app__postgres__grades_courserungrade") }} + where courserungrade_is_passing = true + ), + courseruns as (select * from {{ ref("stg__micromasters__app__postgres__courses_courserun") }}), + courserun_grades_sorted as ( + select + courserun_grades.user_id, + courserun_grades.courserun_id, + courseruns.course_id, + row_number() over ( + partition by courserun_grades.user_id, courseruns.course_id + -- - in case of multiple highest grades, use secondary sorting to ensure the consistent result + order by courserun_grades.courserungrade_grade desc, courserun_grades.coursegrade_created_on desc + ) as row_num + from courserun_grades + inner join courseruns on courserun_grades.courserun_id = courseruns.courserun_id + inner join + dedp_course_certificates + on courserun_grades.user_id = dedp_course_certificates.user_id and courseruns.course_id = dedp_course_certificates.course_id and courseruns.courserun_start_on < dedp_course_certificates.coursecertificate_created_on -) - -, highest_courserun_grades as ( - select * - from courserun_grades_sorted - where row_num = 1 -) - -, courses as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_course') }} -) - -, programs as ( - select * from {{ ref('int__mitx__programs') }} -) + ), + highest_courserun_grades as (select * from courserun_grades_sorted where row_num = 1), + courses as (select * from {{ ref("stg__micromasters__app__postgres__courses_course") }}), + programs as (select * from {{ ref("int__mitx__programs") }}) select - programs.program_title - , programs.micromasters_program_id - , programs.mitxonline_program_id - , courseruns.courserun_title - , courseruns.courserun_readable_id - , courseruns.courserun_edxorg_readable_id - , courseruns.courserun_platform - , courses.course_number - , dedp_course_certificates.user_id as user_micromasters_id - , dedp_course_certificates.coursecertificate_hash - , dedp_course_certificates.coursecertificate_url - , dedp_course_certificates.coursecertificate_created_on - , dedp_course_certificates.coursecertificate_updated_on + programs.program_title, + programs.micromasters_program_id, + programs.mitxonline_program_id, + courseruns.courserun_title, + courseruns.courserun_readable_id, + courseruns.courserun_edxorg_readable_id, + courseruns.courserun_platform, + courses.course_number, + dedp_course_certificates.user_id as user_micromasters_id, + dedp_course_certificates.coursecertificate_hash, + dedp_course_certificates.coursecertificate_url, + dedp_course_certificates.coursecertificate_created_on, + dedp_course_certificates.coursecertificate_updated_on from dedp_course_certificates -inner join highest_courserun_grades - on - dedp_course_certificates.user_id = highest_courserun_grades.user_id - and dedp_course_certificates.course_id = highest_courserun_grades.course_id +inner join + highest_courserun_grades + on dedp_course_certificates.user_id = highest_courserun_grades.user_id + and dedp_course_certificates.course_id = highest_courserun_grades.course_id inner join courseruns on highest_courserun_grades.courserun_id = courseruns.courserun_id inner join courses on dedp_course_certificates.course_id = courses.course_id inner join programs on courses.program_id = programs.micromasters_program_id diff --git a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_certificates_dedp_from_mitxonline.sql b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_certificates_dedp_from_mitxonline.sql index ac078c1b0..bea96a197 100644 --- a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_certificates_dedp_from_mitxonline.sql +++ b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_certificates_dedp_from_mitxonline.sql @@ -1,40 +1,33 @@ -with courserun_certificates as ( - select * from {{ ref('int__mitxonline__courserun_certificates') }} - --- to be consistent with data from MicroMasters and edX.org, we filter out revoked certificates - where courseruncertificate_is_revoked = false -) - -, courseruns as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, courses as ( - select * from {{ ref('int__mitxonline__courses') }} -) - -, enrollments_with_program as ( - select * from {{ ref('int__mitxonline__courserunenrollments_with_programs') }} -) +with + courserun_certificates as ( + select * + from {{ ref("int__mitxonline__courserun_certificates") }} + -- - to be consistent with data from MicroMasters and edX.org, we filter out revoked certificates + where courseruncertificate_is_revoked = false + ), + courseruns as (select * from {{ ref("int__mitxonline__course_runs") }}), + courses as (select * from {{ ref("int__mitxonline__courses") }}), + enrollments_with_program as (select * from {{ ref("int__mitxonline__courserunenrollments_with_programs") }}) select - enrollments_with_program.program_title - , enrollments_with_program.micromasters_program_id - , enrollments_with_program.mitxonline_program_id - , courseruns.courserun_title - , courseruns.courserun_readable_id - , courseruns.courserun_platform - , courses.course_number - , courserun_certificates.user_username as user_mitxonline_username - , courserun_certificates.user_id as user_mitxonline_id - , courserun_certificates.courseruncertificate_uuid - , courserun_certificates.courseruncertificate_url - , courserun_certificates.courseruncertificate_created_on - , courserun_certificates.courseruncertificate_updated_on + enrollments_with_program.program_title, + enrollments_with_program.micromasters_program_id, + enrollments_with_program.mitxonline_program_id, + courseruns.courserun_title, + courseruns.courserun_readable_id, + courseruns.courserun_platform, + courses.course_number, + courserun_certificates.user_username as user_mitxonline_username, + courserun_certificates.user_id as user_mitxonline_id, + courserun_certificates.courseruncertificate_uuid, + courserun_certificates.courseruncertificate_url, + courserun_certificates.courseruncertificate_created_on, + courserun_certificates.courseruncertificate_updated_on from courserun_certificates inner join courseruns on courserun_certificates.courserun_id = courseruns.courserun_id inner join courses on courserun_certificates.course_id = courses.course_id -inner join enrollments_with_program - on - courseruns.courserun_id = enrollments_with_program.courserun_id - and courserun_certificates.user_id = enrollments_with_program.user_id +inner join + enrollments_with_program + on courseruns.courserun_id = enrollments_with_program.courserun_id + and courserun_certificates.user_id = enrollments_with_program.user_id where enrollments_with_program.is_dedp_program = true diff --git a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_certificates_non_dedp_from_edxorg.sql b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_certificates_non_dedp_from_edxorg.sql index f0e3ae0e0..401a10362 100644 --- a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_certificates_non_dedp_from_edxorg.sql +++ b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_certificates_non_dedp_from_edxorg.sql @@ -1,29 +1,22 @@ -with courserun_certificates as ( - select * from {{ ref('int__edxorg__mitx_courserun_certificates') }} -) - -, courseruns as ( - select * from {{ ref('int__edxorg__mitx_courseruns') }} -) - -, programs as ( - select * from {{ ref('int__mitx__programs') }} -) +with + courserun_certificates as (select * from {{ ref("int__edxorg__mitx_courserun_certificates") }}), + courseruns as (select * from {{ ref("int__edxorg__mitx_courseruns") }}), + programs as (select * from {{ ref("int__mitx__programs") }}) select - programs.program_title - , programs.micromasters_program_id - , programs.mitxonline_program_id - , courseruns.courserun_title - , courseruns.courserun_readable_id - , '{{ var("edxorg") }}' as courserun_platform - , courseruns.course_number - , courserun_certificates.user_username as user_edxorg_username - , courserun_certificates.user_id as user_edxorg_id - , courserun_certificates.courseruncertificate_download_url - , courserun_certificates.courseruncertificate_download_uuid - , courserun_certificates.courseruncertificate_created_on - , courserun_certificates.courseruncertificate_updated_on + programs.program_title, + programs.micromasters_program_id, + programs.mitxonline_program_id, + courseruns.courserun_title, + courseruns.courserun_readable_id, + '{{ var("edxorg") }}' as courserun_platform, + courseruns.course_number, + courserun_certificates.user_username as user_edxorg_username, + courserun_certificates.user_id as user_edxorg_id, + courserun_certificates.courseruncertificate_download_url, + courserun_certificates.courseruncertificate_download_uuid, + courserun_certificates.courseruncertificate_created_on, + courserun_certificates.courseruncertificate_updated_on from courserun_certificates inner join courseruns on courserun_certificates.courserun_readable_id = courseruns.courserun_readable_id inner join programs on courseruns.micromasters_program_id = programs.micromasters_program_id diff --git a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_grades_dedp_from_micromasters.sql b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_grades_dedp_from_micromasters.sql index 7b14ecb35..034494c4c 100644 --- a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_grades_dedp_from_micromasters.sql +++ b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_grades_dedp_from_micromasters.sql @@ -1,77 +1,57 @@ ---- MicroMasters DEDP course combined final grades are based on course, we try to find the run with highest grade --- for the course. If there are multiple runs with the highest grade, pick the latest grade from runs before DEDP --- course certificates were generated - -with dedp_course_grades as ( - select * from {{ ref('stg__micromasters__app__postgres__grades_combinedcoursegrade') }} -) - -, dedp_course_certificates as ( - select * from {{ ref('stg__micromasters__app__postgres__grades_coursecertificate') }} -) - -, courserun_grades as ( - select * from {{ ref('stg__micromasters__app__postgres__grades_courserungrade') }} - where courserungrade_is_passing = true -) - -, courseruns as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_courserun') }} -) - -, courserun_grades_sorted as ( - select - courserun_grades.user_id - , courserun_grades.courserun_id - , courseruns.course_id - , row_number() over ( - partition by courserun_grades.user_id, courseruns.course_id - --- in case of multiple highest grades, use secondary sorting to ensure the consistent result - order by courserun_grades.courserungrade_grade desc, courserun_grades.coursegrade_created_on desc - ) as row_num - from courserun_grades - inner join courseruns on courserun_grades.courserun_id = courseruns.courserun_id - inner join dedp_course_certificates - on - courserun_grades.user_id = dedp_course_certificates.user_id +-- - MicroMasters DEDP course combined final grades are based on course, we try to find the run with highest grade +-- for the course. If there are multiple runs with the highest grade, pick the latest grade from runs before DEDP +-- course certificates were generated +with + dedp_course_grades as (select * from {{ ref("stg__micromasters__app__postgres__grades_combinedcoursegrade") }}), + dedp_course_certificates as (select * from {{ ref("stg__micromasters__app__postgres__grades_coursecertificate") }}), + courserun_grades as ( + select * + from {{ ref("stg__micromasters__app__postgres__grades_courserungrade") }} + where courserungrade_is_passing = true + ), + courseruns as (select * from {{ ref("stg__micromasters__app__postgres__courses_courserun") }}), + courserun_grades_sorted as ( + select + courserun_grades.user_id, + courserun_grades.courserun_id, + courseruns.course_id, + row_number() over ( + partition by courserun_grades.user_id, courseruns.course_id + -- - in case of multiple highest grades, use secondary sorting to ensure the consistent result + order by courserun_grades.courserungrade_grade desc, courserun_grades.coursegrade_created_on desc + ) as row_num + from courserun_grades + inner join courseruns on courserun_grades.courserun_id = courseruns.courserun_id + inner join + dedp_course_certificates + on courserun_grades.user_id = dedp_course_certificates.user_id and courseruns.course_id = dedp_course_certificates.course_id and courseruns.courserun_start_on < dedp_course_certificates.coursecertificate_created_on - inner join dedp_course_grades - on - dedp_course_certificates.user_id = dedp_course_grades.user_id + inner join + dedp_course_grades + on dedp_course_certificates.user_id = dedp_course_grades.user_id and dedp_course_certificates.course_id = dedp_course_grades.course_id -) - -, highest_courserun_grades as ( - select * - from courserun_grades_sorted - where row_num = 1 -) - -, courses as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_course') }} -) - -, programs as ( - select * from {{ ref('int__mitx__programs') }} -) + ), + highest_courserun_grades as (select * from courserun_grades_sorted where row_num = 1), + courses as (select * from {{ ref("stg__micromasters__app__postgres__courses_course") }}), + programs as (select * from {{ ref("int__mitx__programs") }}) select - programs.program_title - , programs.micromasters_program_id - , programs.mitxonline_program_id - , courseruns.courserun_title - , courseruns.courserun_readable_id - , courseruns.courserun_platform - , courses.course_number - , dedp_course_grades.user_id as user_micromasters_id - , cast(dedp_course_grades.coursegrade_grade / 100 as decimal(5, 3)) as coursegrade_grade - , dedp_course_grades.coursegrade_created_on + programs.program_title, + programs.micromasters_program_id, + programs.mitxonline_program_id, + courseruns.courserun_title, + courseruns.courserun_readable_id, + courseruns.courserun_platform, + courses.course_number, + dedp_course_grades.user_id as user_micromasters_id, + cast(dedp_course_grades.coursegrade_grade / 100 as decimal(5, 3)) as coursegrade_grade, + dedp_course_grades.coursegrade_created_on from dedp_course_grades -inner join highest_courserun_grades - on - dedp_course_grades.user_id = highest_courserun_grades.user_id - and dedp_course_grades.course_id = highest_courserun_grades.course_id +inner join + highest_courserun_grades + on dedp_course_grades.user_id = highest_courserun_grades.user_id + and dedp_course_grades.course_id = highest_courserun_grades.course_id inner join courseruns on highest_courserun_grades.courserun_id = courseruns.courserun_id inner join courses on dedp_course_grades.course_id = courses.course_id inner join programs on courses.program_id = programs.micromasters_program_id diff --git a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_grades_dedp_from_mitxonline.sql b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_grades_dedp_from_mitxonline.sql index 3c1d7cb25..c5a3523c6 100644 --- a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_grades_dedp_from_mitxonline.sql +++ b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_grades_dedp_from_mitxonline.sql @@ -1,37 +1,27 @@ -with courserun_grades as ( - select * from {{ ref('int__mitxonline__courserun_grades') }} -) - -, courseruns as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, courses as ( - select * from {{ ref('int__mitxonline__courses') }} -) - -, enrollments_with_program as ( - select * from {{ ref('int__mitxonline__courserunenrollments_with_programs') }} -) +with + courserun_grades as (select * from {{ ref("int__mitxonline__courserun_grades") }}), + courseruns as (select * from {{ ref("int__mitxonline__course_runs") }}), + courses as (select * from {{ ref("int__mitxonline__courses") }}), + enrollments_with_program as (select * from {{ ref("int__mitxonline__courserunenrollments_with_programs") }}) select - enrollments_with_program.program_title - , enrollments_with_program.micromasters_program_id - , enrollments_with_program.mitxonline_program_id - , courseruns.courserun_title - , courseruns.courserun_readable_id - , courseruns.courserun_platform - , courses.course_number - , courserun_grades.user_username as user_mitxonline_username - , courserun_grades.user_id as user_mitxonline_id - , courserun_grades.courserungrade_grade - , courserun_grades.courserungrade_is_passing - , courserun_grades.courserungrade_created_on + enrollments_with_program.program_title, + enrollments_with_program.micromasters_program_id, + enrollments_with_program.mitxonline_program_id, + courseruns.courserun_title, + courseruns.courserun_readable_id, + courseruns.courserun_platform, + courses.course_number, + courserun_grades.user_username as user_mitxonline_username, + courserun_grades.user_id as user_mitxonline_id, + courserun_grades.courserungrade_grade, + courserun_grades.courserungrade_is_passing, + courserun_grades.courserungrade_created_on from courserun_grades inner join courseruns on courserun_grades.courserun_id = courseruns.courserun_id inner join courses on courserun_grades.course_id = courses.course_id -inner join enrollments_with_program - on - courseruns.courserun_id = enrollments_with_program.courserun_id - and courserun_grades.user_id = enrollments_with_program.user_id +inner join + enrollments_with_program + on courseruns.courserun_id = enrollments_with_program.courserun_id + and courserun_grades.user_id = enrollments_with_program.user_id where enrollments_with_program.is_dedp_program = true diff --git a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_grades_non_dedp_from_edxorg.sql b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_grades_non_dedp_from_edxorg.sql index f82f0e6f0..cfac5101a 100644 --- a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_grades_non_dedp_from_edxorg.sql +++ b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_course_grades_non_dedp_from_edxorg.sql @@ -1,27 +1,20 @@ -with courserun_grades as ( - select * from {{ ref('int__edxorg__mitx_courserun_grades') }} -) - -, courseruns as ( - select * from {{ ref('int__edxorg__mitx_courseruns') }} -) - -, programs as ( - select * from {{ ref('int__mitx__programs') }} -) +with + courserun_grades as (select * from {{ ref("int__edxorg__mitx_courserun_grades") }}), + courseruns as (select * from {{ ref("int__edxorg__mitx_courseruns") }}), + programs as (select * from {{ ref("int__mitx__programs") }}) select - programs.program_title - , programs.mitxonline_program_id - , programs.micromasters_program_id - , courseruns.courserun_title - , courseruns.courserun_readable_id - , '{{ var("edxorg") }}' as courserun_platform - , courseruns.course_number - , courserun_grades.user_username as user_edxorg_username - , courserun_grades.user_id as user_edxorg_id - , courserun_grades.courserungrade_is_passing - , courserun_grades.courserungrade_user_grade + programs.program_title, + programs.mitxonline_program_id, + programs.micromasters_program_id, + courseruns.courserun_title, + courseruns.courserun_readable_id, + '{{ var("edxorg") }}' as courserun_platform, + courseruns.course_number, + courserun_grades.user_username as user_edxorg_username, + courserun_grades.user_id as user_edxorg_id, + courserun_grades.courserungrade_is_passing, + courserun_grades.courserungrade_user_grade from courserun_grades inner join courseruns on courserun_grades.courserun_readable_id = courseruns.courserun_readable_id inner join programs on courseruns.micromasters_program_id = programs.micromasters_program_id diff --git a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_program_certificates_dedp_from_micromasters.sql b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_program_certificates_dedp_from_micromasters.sql index ff43ce2a0..dbbb52fda 100644 --- a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_program_certificates_dedp_from_micromasters.sql +++ b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_program_certificates_dedp_from_micromasters.sql @@ -1,23 +1,16 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with mm_program_certificates as ( - select * - from {{ ref('stg__micromasters__app__postgres__grades_programcertificate') }} -) - -, programs as ( - select * - from {{ ref('int__mitx__programs') }} -) +with + mm_program_certificates as (select * from {{ ref("stg__micromasters__app__postgres__grades_programcertificate") }}), + programs as (select * from {{ ref("int__mitx__programs") }}) select - programs.micromasters_program_id - , programs.program_title - , programs.mitxonline_program_id - , {{ generate_hash_id('mm_program_certificates.programcertificate_hash') }} as program_certificate_hashed_id - , mm_program_certificates.programcertificate_created_on as program_completion_timestamp - , mm_program_certificates.user_id as user_micromasters_id + programs.micromasters_program_id, + programs.program_title, + programs.mitxonline_program_id, + {{ generate_hash_id("mm_program_certificates.programcertificate_hash") }} as program_certificate_hashed_id, + mm_program_certificates.programcertificate_created_on as program_completion_timestamp, + mm_program_certificates.user_id as user_micromasters_id from mm_program_certificates -left join programs - on mm_program_certificates.program_id = programs.micromasters_program_id +left join programs on mm_program_certificates.program_id = programs.micromasters_program_id where programs.is_dedp_program = true diff --git a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_program_certificates_dedp_from_mitxonline.sql b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_program_certificates_dedp_from_mitxonline.sql index 7e79598a1..0e597c427 100644 --- a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_program_certificates_dedp_from_mitxonline.sql +++ b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_program_certificates_dedp_from_mitxonline.sql @@ -1,25 +1,17 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with mitxonline_program_certificates as ( - select * - from {{ ref('int__mitxonline__program_certificates') }} -) - -, mitx_programs as ( - select * - from {{ ref('int__mitx__programs') }} -) +with + mitxonline_program_certificates as (select * from {{ ref("int__mitxonline__program_certificates") }}), + mitx_programs as (select * from {{ ref("int__mitx__programs") }}) select - mitxonline_program_certificates.user_username as user_mitxonline_username - , mitxonline_program_certificates.user_id as user_mitxonline_id - , mitx_programs.micromasters_program_id - , mitx_programs.program_title - , mitx_programs.mitxonline_program_id - , {{ generate_hash_id('mitxonline_program_certificates.programcertificate_uuid') }} as program_certificate_hashed_id - , mitxonline_program_certificates.programcertificate_created_on as program_completion_timestamp + mitxonline_program_certificates.user_username as user_mitxonline_username, + mitxonline_program_certificates.user_id as user_mitxonline_id, + mitx_programs.micromasters_program_id, + mitx_programs.program_title, + mitx_programs.mitxonline_program_id, + {{ generate_hash_id("mitxonline_program_certificates.programcertificate_uuid") }} as program_certificate_hashed_id, + mitxonline_program_certificates.programcertificate_created_on as program_completion_timestamp from mitxonline_program_certificates left join mitx_programs on mitxonline_program_certificates.program_id = mitx_programs.mitxonline_program_id -where - mitx_programs.is_dedp_program = true - and mitxonline_program_certificates.programcertificate_is_revoked = false +where mitx_programs.is_dedp_program = true and mitxonline_program_certificates.programcertificate_is_revoked = false diff --git a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_program_certificates_non_dedp.sql b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_program_certificates_non_dedp.sql index 8a30058b3..71d54e499 100644 --- a/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_program_certificates_non_dedp.sql +++ b/src/ol_dbt/models/intermediate/micromasters/subqueries/__micromasters_program_certificates_non_dedp.sql @@ -1,78 +1,59 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with micromasters_program_certificates as ( - --- There are learners who received both 'Statistics and Data Science (General track)' and 'Statistics and Data - -- Science' from 2U data, but we only count them once in 'Statistics and Data Science' for MM program certificates - -- report. - select - * - , row_number() over ( - partition by user_id, micromasters_program_id - order by program_title - ) as row_num - from {{ ref('int__edxorg__mitx_program_certificates') }} - where program_type = 'MicroMasters' -) - -, edx_users as ( - select * - from {{ ref('int__edxorg__mitx_users') }} -) - -, programs as ( - select * - from {{ ref('int__mitx__programs') }} - where is_dedp_program = false -) - -, program_certificates_override_list as ( - select * - from {{ ref('stg__micromasters__app__user_program_certificate_override_list') }} -) - -, non_dedp_certificates as ( - select - edx_users.user_username as user_edxorg_username - , programs.micromasters_program_id - , micromasters_program_certificates.program_title - , programs.mitxonline_program_id - , micromasters_program_certificates.user_id as user_edxorg_id - , micromasters_program_certificates.program_certificate_hashed_id - , micromasters_program_certificates.program_certificate_awarded_on as program_completion_timestamp - from micromasters_program_certificates - left join edx_users - on micromasters_program_certificates.user_id = edx_users.user_id - left join programs - on micromasters_program_certificates.micromasters_program_id = programs.micromasters_program_id - where micromasters_program_certificates.row_num = 1 -) - --- Some users should recieve a certificate even though they don't fulfill the requirements according --- to the course certificates in the edxorg database. A list of these users' user ids on edx and the micromasters --- ids on production for the program they should earn a certificate for are stored in --- stg__micromasters__app__postgres__courses_course. These ids won't match correctly in qa - -, non_dedp_overides as ( - select - edx_users.user_username as user_edxorg_username - , programs.micromasters_program_id - , programs.program_title - , programs.mitxonline_program_id - , edx_users.user_id as user_edxorg_id - , program_certificates_override_list.program_certificate_hashed_id - , null as program_completion_timestamp - from program_certificates_override_list - inner join edx_users - on program_certificates_override_list.user_edxorg_id = edx_users.user_id - inner join programs - on program_certificates_override_list.micromasters_program_id = programs.micromasters_program_id - left join non_dedp_certificates - on - program_certificates_override_list.user_edxorg_id - = non_dedp_certificates.user_edxorg_id and program_certificates_override_list.micromasters_program_id +with + micromasters_program_certificates as ( + -- - There are learners who received both 'Statistics and Data Science (General track)' and 'Statistics and Data + -- Science' from 2U data, but we only count them once in 'Statistics and Data Science' for MM program + -- certificates + -- report. + select *, row_number() over (partition by user_id, micromasters_program_id order by program_title) as row_num + from {{ ref("int__edxorg__mitx_program_certificates") }} + where program_type = 'MicroMasters' + ), + edx_users as (select * from {{ ref("int__edxorg__mitx_users") }}), + programs as (select * from {{ ref("int__mitx__programs") }} where is_dedp_program = false), + program_certificates_override_list as ( + select * from {{ ref("stg__micromasters__app__user_program_certificate_override_list") }} + ), + non_dedp_certificates as ( + select + edx_users.user_username as user_edxorg_username, + programs.micromasters_program_id, + micromasters_program_certificates.program_title, + programs.mitxonline_program_id, + micromasters_program_certificates.user_id as user_edxorg_id, + micromasters_program_certificates.program_certificate_hashed_id, + micromasters_program_certificates.program_certificate_awarded_on as program_completion_timestamp + from micromasters_program_certificates + left join edx_users on micromasters_program_certificates.user_id = edx_users.user_id + left join + programs on micromasters_program_certificates.micromasters_program_id = programs.micromasters_program_id + where micromasters_program_certificates.row_num = 1 + ), + -- Some users should recieve a certificate even though they don't fulfill the requirements according + -- to the course certificates in the edxorg database. A list of these users' user ids on edx and the micromasters + -- ids on production for the program they should earn a certificate for are stored in + -- stg__micromasters__app__postgres__courses_course. These ids won't match correctly in qa + non_dedp_overides as ( + select + edx_users.user_username as user_edxorg_username, + programs.micromasters_program_id, + programs.program_title, + programs.mitxonline_program_id, + edx_users.user_id as user_edxorg_id, + program_certificates_override_list.program_certificate_hashed_id, + null as program_completion_timestamp + from program_certificates_override_list + inner join edx_users on program_certificates_override_list.user_edxorg_id = edx_users.user_id + inner join + programs on program_certificates_override_list.micromasters_program_id = programs.micromasters_program_id + left join + non_dedp_certificates + on program_certificates_override_list.user_edxorg_id = non_dedp_certificates.user_edxorg_id + and program_certificates_override_list.micromasters_program_id = non_dedp_certificates.micromasters_program_id - where non_dedp_certificates.user_edxorg_username is null -) + where non_dedp_certificates.user_edxorg_username is null + ) select * from non_dedp_certificates diff --git a/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_certificates.sql b/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_certificates.sql index 35680eb08..f70f5c4e8 100644 --- a/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_certificates.sql +++ b/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_certificates.sql @@ -1,96 +1,86 @@ ---- MITx course certificates combined from MITx Online and edX.org ---- The logic is to combine from: ----- Non-DEDP course certificate from MITx Online ----- MicroMasters program courses certificate (combined DEDP and other MM program courses) ----- course certificates from edx.org, excluding DEDP and other MM program course certificates +-- - MITx course certificates combined from MITx Online and edX.org +-- - The logic is to combine from: +-- -- Non-DEDP course certificate from MITx Online +-- -- MicroMasters program courses certificate (combined DEDP and other MM program courses) +-- -- course certificates from edx.org, excluding DEDP and other MM program course certificates +{{ config(materialized="view") }} -{{ config(materialized='view') }} - -with mitxonline_certificates as ( - select * - from {{ ref('int__mitxonline__courserun_certificates') }} - where - courserun_platform = '{{ var("mitxonline") }}' - and courseruncertificate_is_revoked = false -) - -, mitxonline_dedp_courses as ( - select distinct course_id from {{ ref('int__mitxonline__program_requirements') }} - where - program_id in ( - {{ var("dedp_mitxonline_international_development_program_id") }} - , {{ var("dedp_mitxonline_public_policy_program_id") }} - ) -) - -, edxorg_non_program_course_certificates as ( - select * from {{ ref('int__edxorg__mitx_courserun_certificates') }} - where micromasters_program_id is null -) - -, program_course_certificates as ( - select * from {{ ref('int__micromasters__course_certificates') }} -) - -, mitxonline_non_dedp_course_certificates as ( - select mitxonline_certificates.* - from mitxonline_certificates - left join mitxonline_dedp_courses on mitxonline_certificates.course_id = mitxonline_dedp_courses.course_id - left join program_course_certificates - on - mitxonline_certificates.courserun_readable_id = program_course_certificates.courserun_readable_id +with + mitxonline_certificates as ( + select * + from {{ ref("int__mitxonline__courserun_certificates") }} + where courserun_platform = '{{ var("mitxonline") }}' and courseruncertificate_is_revoked = false + ), + mitxonline_dedp_courses as ( + select distinct course_id + from {{ ref("int__mitxonline__program_requirements") }} + where + program_id in ( + {{ var("dedp_mitxonline_international_development_program_id") }}, + {{ var("dedp_mitxonline_public_policy_program_id") }} + ) + ), + edxorg_non_program_course_certificates as ( + select * from {{ ref("int__edxorg__mitx_courserun_certificates") }} where micromasters_program_id is null + ), + program_course_certificates as (select * from {{ ref("int__micromasters__course_certificates") }}), + mitxonline_non_dedp_course_certificates as ( + select mitxonline_certificates.* + from mitxonline_certificates + left join mitxonline_dedp_courses on mitxonline_certificates.course_id = mitxonline_dedp_courses.course_id + left join + program_course_certificates + on mitxonline_certificates.courserun_readable_id = program_course_certificates.courserun_readable_id and mitxonline_certificates.user_username = program_course_certificates.user_mitxonline_username - where - mitxonline_dedp_courses.course_id is null - and program_course_certificates.courserun_readable_id is null -) - -, mitx_certificates as ( - select - '{{ var("mitxonline") }}' as platform - , course_number - , courserun_title - , courserun_readable_id - , courseruncertificate_uuid - , courseruncertificate_url - , courseruncertificate_created_on - , user_username as user_mitxonline_username - , user_edxorg_username - , user_email - , user_full_name - from mitxonline_non_dedp_course_certificates + where mitxonline_dedp_courses.course_id is null and program_course_certificates.courserun_readable_id is null + ), + mitx_certificates as ( + select + '{{ var("mitxonline") }}' as platform, + course_number, + courserun_title, + courserun_readable_id, + courseruncertificate_uuid, + courseruncertificate_url, + courseruncertificate_created_on, + user_username as user_mitxonline_username, + user_edxorg_username, + user_email, + user_full_name + from mitxonline_non_dedp_course_certificates - union all + union all - select - '{{ var("edxorg") }}' as platform - , course_number - , courserun_title - , courserun_readable_id - , courseruncertificate_download_uuid as courseruncertificate_uuid - , courseruncertificate_download_url as courseruncertificate_url - , courseruncertificate_created_on - , user_mitxonline_username - , user_username as user_edxorg_username - , user_email - , user_full_name - from edxorg_non_program_course_certificates + select + '{{ var("edxorg") }}' as platform, + course_number, + courserun_title, + courserun_readable_id, + courseruncertificate_download_uuid as courseruncertificate_uuid, + courseruncertificate_download_url as courseruncertificate_url, + courseruncertificate_created_on, + user_mitxonline_username, + user_username as user_edxorg_username, + user_email, + user_full_name + from edxorg_non_program_course_certificates - union all + union all - select distinct - courserun_platform as platform - , course_number - , courserun_title - , courserun_readable_id - , courseruncertificate_uuid - , courseruncertificate_url - , courseruncertificate_created_on - , user_mitxonline_username - , user_edxorg_username - , user_email - , user_full_name - from program_course_certificates -) + select distinct + courserun_platform as platform, + course_number, + courserun_title, + courserun_readable_id, + courseruncertificate_uuid, + courseruncertificate_url, + courseruncertificate_created_on, + user_mitxonline_username, + user_edxorg_username, + user_email, + user_full_name + from program_course_certificates + ) -select * from mitx_certificates +select * +from mitx_certificates diff --git a/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_enrollments.sql b/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_enrollments.sql index 52317b0e7..e0354f82b 100644 --- a/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_enrollments.sql +++ b/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_enrollments.sql @@ -1,68 +1,65 @@ ---- MITx course enrollments from MITx Online and edX.org with no duplication ---- For DEDP courses, verified enrollments are already handled in MITx Online --- and edX.org enrollment intermediate models +-- - MITx course enrollments from MITx Online and edX.org with no duplication +-- - For DEDP courses, verified enrollments are already handled in MITx Online +-- and edX.org enrollment intermediate models +{{ config(materialized="view") }} -{{ config(materialized='view') }} +with + mitxonline_enrollments as ( + select * + from {{ ref("int__mitxonline__courserunenrollments") }} + -- - to dedup, filter out migrated DEDP course enrollments for courses that run on edX.org + where courserunenrollment_platform = '{{ var("mitxonline") }}' + ), + edxorg_enrollments as (select * from {{ ref("int__edxorg__mitx_courserun_enrollments") }}), + mitx_enrollments as ( + select + '{{ var("mitxonline") }}' as platform, + courserunenrollment_id, + courserunenrollment_is_active, + courserunenrollment_created_on, + courserunenrollment_enrollment_mode, + courserunenrollment_enrollment_status, + courserunenrollment_is_edx_enrolled, + courserun_id, + courserun_title, + courserun_readable_id, + course_number, + courserun_start_on, + courserun_upgrade_deadline, + user_id, + user_email, + user_full_name, + user_username, + user_edxorg_username, + user_username as user_mitxonline_username, + user_address_country + from mitxonline_enrollments -with mitxonline_enrollments as ( - select * - from {{ ref('int__mitxonline__courserunenrollments') }} - --- to dedup, filter out migrated DEDP course enrollments for courses that run on edX.org - where courserunenrollment_platform = '{{ var("mitxonline") }}' -) + union all -, edxorg_enrollments as ( - select * from {{ ref('int__edxorg__mitx_courserun_enrollments') }} -) + select + '{{ var("edxorg") }}' as platform, + null as courserunenrollment_id, + courserunenrollment_is_active, + courserunenrollment_created_on, + courserunenrollment_enrollment_mode, + null as courserunenrollment_enrollment_status, + true as courserunenrollment_is_edx_enrolled, + null as courserun_id, + courserun_title, + courserun_readable_id, + course_number, + courserun_start_on, + courserun_upgrade_deadline, + user_id, + user_email, + user_full_name, + user_username, + user_username as user_edxorg_username, + user_mitxonline_username, + user_address_country + from edxorg_enrollments + ) -, mitx_enrollments as ( - select - '{{ var("mitxonline") }}' as platform - , courserunenrollment_id - , courserunenrollment_is_active - , courserunenrollment_created_on - , courserunenrollment_enrollment_mode - , courserunenrollment_enrollment_status - , courserunenrollment_is_edx_enrolled - , courserun_id - , courserun_title - , courserun_readable_id - , course_number - , courserun_start_on - , courserun_upgrade_deadline - , user_id - , user_email - , user_full_name - , user_username - , user_edxorg_username - , user_username as user_mitxonline_username - , user_address_country - from mitxonline_enrollments - - union all - - select - '{{ var("edxorg") }}' as platform - , null as courserunenrollment_id - , courserunenrollment_is_active - , courserunenrollment_created_on - , courserunenrollment_enrollment_mode - , null as courserunenrollment_enrollment_status - , true as courserunenrollment_is_edx_enrolled - , null as courserun_id - , courserun_title - , courserun_readable_id - , course_number - , courserun_start_on - , courserun_upgrade_deadline - , user_id - , user_email - , user_full_name - , user_username - , user_username as user_edxorg_username - , user_mitxonline_username - , user_address_country - from edxorg_enrollments -) - -select * from mitx_enrollments +select * +from mitx_enrollments diff --git a/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_enrollments_with_programs.sql b/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_enrollments_with_programs.sql index b2dedc159..f6ad0d75b 100644 --- a/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_enrollments_with_programs.sql +++ b/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_enrollments_with_programs.sql @@ -1,75 +1,62 @@ -with edx_enrollments as ( - select * - from {{ ref('int__mitx__courserun_enrollments') }} - where platform = '{{ var("edxorg") }}' -) - -, mitx_users as ( - select * from {{ ref('int__mitx__users') }} -) - -, mitxonline_enrollments_with_program as ( - select * - from {{ ref('int__mitxonline__courserunenrollments_with_programs') }} -) - -, program_requirements as ( - select * from {{ ref('int__mitx__program_requirements') }} -) - -, edx_query as ( - select - edx_enrollments.platform - , edx_enrollments.courserunenrollment_is_active - , edx_enrollments.courserunenrollment_created_on - , edx_enrollments.courserunenrollment_enrollment_mode - , edx_enrollments.courserunenrollment_enrollment_status - , edx_enrollments.courserun_id - , edx_enrollments.courserun_title - , edx_enrollments.courserun_readable_id - , edx_enrollments.course_number - , mitx_users.user_edxorg_id as user_id - , mitx_users.user_edxorg_email as user_email - , mitx_users.user_full_name - , mitx_users.user_edxorg_username as user_username - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_address_country - , program_requirements.micromasters_program_id - , program_requirements.mitxonline_program_id - , program_requirements.program_title - from edx_enrollments - left join mitx_users - on edx_enrollments.user_id = mitx_users.user_edxorg_id - inner join program_requirements on edx_enrollments.course_number = program_requirements.course_number -) - -, mitxonline_query as ( - select - mitxonline_enrollments_with_program.courserunenrollment_platform as platform - , mitxonline_enrollments_with_program.courserunenrollment_is_active - , mitxonline_enrollments_with_program.courserunenrollment_created_on - , mitxonline_enrollments_with_program.courserunenrollment_enrollment_mode - , mitxonline_enrollments_with_program.courserunenrollment_enrollment_status - , mitxonline_enrollments_with_program.courserun_id - , mitxonline_enrollments_with_program.courserun_title - , mitxonline_enrollments_with_program.courserun_readable_id - , mitxonline_enrollments_with_program.course_number - , mitx_users.user_mitxonline_id as user_id - , mitx_users.user_mitxonline_email as user_email - , mitx_users.user_full_name - , mitx_users.user_mitxonline_username as user_username - , mitx_users.user_edxorg_username - , mitx_users.user_mitxonline_username - , mitx_users.user_address_country - , mitxonline_enrollments_with_program.micromasters_program_id - , mitxonline_enrollments_with_program.mitxonline_program_id - , mitxonline_enrollments_with_program.program_title - from mitxonline_enrollments_with_program - left join mitx_users - on mitxonline_enrollments_with_program.user_id = mitx_users.user_mitxonline_id - where mitxonline_enrollments_with_program.courserunenrollment_platform = '{{ var("mitxonline") }}' -) +with + edx_enrollments as ( + select * from {{ ref("int__mitx__courserun_enrollments") }} where platform = '{{ var("edxorg") }}' + ), + mitx_users as (select * from {{ ref("int__mitx__users") }}), + mitxonline_enrollments_with_program as ( + select * from {{ ref("int__mitxonline__courserunenrollments_with_programs") }} + ), + program_requirements as (select * from {{ ref("int__mitx__program_requirements") }}), + edx_query as ( + select + edx_enrollments.platform, + edx_enrollments.courserunenrollment_is_active, + edx_enrollments.courserunenrollment_created_on, + edx_enrollments.courserunenrollment_enrollment_mode, + edx_enrollments.courserunenrollment_enrollment_status, + edx_enrollments.courserun_id, + edx_enrollments.courserun_title, + edx_enrollments.courserun_readable_id, + edx_enrollments.course_number, + mitx_users.user_edxorg_id as user_id, + mitx_users.user_edxorg_email as user_email, + mitx_users.user_full_name, + mitx_users.user_edxorg_username as user_username, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_address_country, + program_requirements.micromasters_program_id, + program_requirements.mitxonline_program_id, + program_requirements.program_title + from edx_enrollments + left join mitx_users on edx_enrollments.user_id = mitx_users.user_edxorg_id + inner join program_requirements on edx_enrollments.course_number = program_requirements.course_number + ), + mitxonline_query as ( + select + mitxonline_enrollments_with_program.courserunenrollment_platform as platform, + mitxonline_enrollments_with_program.courserunenrollment_is_active, + mitxonline_enrollments_with_program.courserunenrollment_created_on, + mitxonline_enrollments_with_program.courserunenrollment_enrollment_mode, + mitxonline_enrollments_with_program.courserunenrollment_enrollment_status, + mitxonline_enrollments_with_program.courserun_id, + mitxonline_enrollments_with_program.courserun_title, + mitxonline_enrollments_with_program.courserun_readable_id, + mitxonline_enrollments_with_program.course_number, + mitx_users.user_mitxonline_id as user_id, + mitx_users.user_mitxonline_email as user_email, + mitx_users.user_full_name, + mitx_users.user_mitxonline_username as user_username, + mitx_users.user_edxorg_username, + mitx_users.user_mitxonline_username, + mitx_users.user_address_country, + mitxonline_enrollments_with_program.micromasters_program_id, + mitxonline_enrollments_with_program.mitxonline_program_id, + mitxonline_enrollments_with_program.program_title + from mitxonline_enrollments_with_program + left join mitx_users on mitxonline_enrollments_with_program.user_id = mitx_users.user_mitxonline_id + where mitxonline_enrollments_with_program.courserunenrollment_platform = '{{ var("mitxonline") }}' + ) select * from edx_query diff --git a/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_grades.sql b/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_grades.sql index bf0f4e197..57a612e76 100644 --- a/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_grades.sql +++ b/src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_grades.sql @@ -1,85 +1,79 @@ ---- MITx course grades combined from MITx Online and edX.org +-- - MITx course grades combined from MITx Online and edX.org +{{ config(materialized="view") }} -{{ config(materialized='view') }} - -with mitxonline_grades as ( - select * - from {{ ref('int__mitxonline__courserun_grades') }} - where courserun_platform = '{{ var("mitxonline") }}' -) - -, mitxonline_dedp_courses as ( - select distinct course_id from {{ ref('int__mitxonline__program_requirements') }} - where - program_id in ( - {{ var("dedp_mitxonline_international_development_program_id") }} - , {{ var("dedp_mitxonline_public_policy_program_id") }} - ) -) - -, edxorg_non_program_course_grades as ( - select * from {{ ref('int__edxorg__mitx_courserun_grades') }} - where micromasters_program_id is null -) - -, all_program_course_grades as ( - select * from {{ ref('int__micromasters__course_grades') }} -) - -, mitxonline_non_dedp_course_grades as ( - select mitxonline_grades.* - from mitxonline_grades - left join mitxonline_dedp_courses on mitxonline_grades.course_id = mitxonline_dedp_courses.course_id - left join all_program_course_grades - on - mitxonline_grades.courserun_readable_id = all_program_course_grades.courserun_readable_id +with + mitxonline_grades as ( + select * + from {{ ref("int__mitxonline__courserun_grades") }} + where courserun_platform = '{{ var("mitxonline") }}' + ), + mitxonline_dedp_courses as ( + select distinct course_id + from {{ ref("int__mitxonline__program_requirements") }} + where + program_id in ( + {{ var("dedp_mitxonline_international_development_program_id") }}, + {{ var("dedp_mitxonline_public_policy_program_id") }} + ) + ), + edxorg_non_program_course_grades as ( + select * from {{ ref("int__edxorg__mitx_courserun_grades") }} where micromasters_program_id is null + ), + all_program_course_grades as (select * from {{ ref("int__micromasters__course_grades") }}), + mitxonline_non_dedp_course_grades as ( + select mitxonline_grades.* + from mitxonline_grades + left join mitxonline_dedp_courses on mitxonline_grades.course_id = mitxonline_dedp_courses.course_id + left join + all_program_course_grades + on mitxonline_grades.courserun_readable_id = all_program_course_grades.courserun_readable_id and mitxonline_grades.user_username = all_program_course_grades.user_mitxonline_username - where mitxonline_dedp_courses.course_id is null and all_program_course_grades.courserun_readable_id is null -) - -, mitx_grades as ( - select - '{{ var("mitxonline") }}' as platform - , course_number - , courserun_title - , courserun_readable_id - , courserungrade_grade - , courserungrade_is_passing - , user_email - , user_full_name - , user_edxorg_username - , user_username as user_mitxonline_username - from mitxonline_non_dedp_course_grades + where mitxonline_dedp_courses.course_id is null and all_program_course_grades.courserun_readable_id is null + ), + mitx_grades as ( + select + '{{ var("mitxonline") }}' as platform, + course_number, + courserun_title, + courserun_readable_id, + courserungrade_grade, + courserungrade_is_passing, + user_email, + user_full_name, + user_edxorg_username, + user_username as user_mitxonline_username + from mitxonline_non_dedp_course_grades - union all + union all - select - '{{ var("edxorg") }}' as platform - , course_number - , courserun_title - , courserun_readable_id - , courserungrade_user_grade as courserungrade_grade - , courserungrade_is_passing - , user_email - , user_full_name - , user_username as user_edxorg_username - , user_mitxonline_username - from edxorg_non_program_course_grades + select + '{{ var("edxorg") }}' as platform, + course_number, + courserun_title, + courserun_readable_id, + courserungrade_user_grade as courserungrade_grade, + courserungrade_is_passing, + user_email, + user_full_name, + user_username as user_edxorg_username, + user_mitxonline_username + from edxorg_non_program_course_grades - union all + union all - select distinct - courserun_platform as platform - , course_number - , courserun_title - , courserun_readable_id - , grade as courserungrade_grade - , is_passing as courserungrade_is_passing - , user_email - , user_full_name - , user_edxorg_username - , user_mitxonline_username - from all_program_course_grades -) + select distinct + courserun_platform as platform, + course_number, + courserun_title, + courserun_readable_id, + grade as courserungrade_grade, + is_passing as courserungrade_is_passing, + user_email, + user_full_name, + user_edxorg_username, + user_mitxonline_username + from all_program_course_grades + ) -select * from mitx_grades +select * +from mitx_grades diff --git a/src/ol_dbt/models/intermediate/mitx/int__mitx__courses.sql b/src/ol_dbt/models/intermediate/mitx/int__mitx__courses.sql index f118f54c8..3e6ad3801 100644 --- a/src/ol_dbt/models/intermediate/mitx/int__mitx__courses.sql +++ b/src/ol_dbt/models/intermediate/mitx/int__mitx__courses.sql @@ -1,46 +1,38 @@ -with mitxonline_courses as ( - select - course_id as mitxonline_course_id - , course_title - , course_is_live - , course_readable_id - , course_number - from {{ ref('int__mitxonline__courses') }} -) +with + mitxonline_courses as ( + select course_id as mitxonline_course_id, course_title, course_is_live, course_readable_id, course_number + from {{ ref("int__mitxonline__courses") }} + ), + edx_courseruns as ( + select + *, + row_number() over ( + partition by trim(course_number) order by courserun_start_date desc + ) as course_recency_rank + from {{ ref("int__edxorg__mitx_courseruns") }} + ), + edx_courses as ( + select + courserun_title as course_title, + micromasters_program_id, + micromasters_course_id, + course_number, + course_readable_id + from edx_courseruns + where course_recency_rank = 1 + ), + mitx_courses as ( + select + mitxonline_courses.mitxonline_course_id, + edx_courses.micromasters_course_id, + coalesce(mitxonline_courses.course_number, edx_courses.course_number) as course_number, + coalesce(mitxonline_courses.course_readable_id, edx_courses.course_readable_id) as course_readable_id, + coalesce(mitxonline_courses.course_title, edx_courses.course_title) as course_title, + coalesce(mitxonline_courses.course_number is not null, false) as is_on_mitxonline, + coalesce(edx_courses.course_number is not null, false) as is_on_edxorg + from mitxonline_courses + full join edx_courses on mitxonline_courses.course_number = edx_courses.course_number + ) -, edx_courseruns as ( - select - * - , row_number() over ( - partition by trim(course_number) - order by courserun_start_date desc - ) as course_recency_rank - from {{ ref('int__edxorg__mitx_courseruns') }} -) - -, edx_courses as ( - select - courserun_title as course_title - , micromasters_program_id - , micromasters_course_id - , course_number - , course_readable_id - from edx_courseruns - where course_recency_rank = 1 -) - -, mitx_courses as ( - select - mitxonline_courses.mitxonline_course_id - , edx_courses.micromasters_course_id - , coalesce(mitxonline_courses.course_number, edx_courses.course_number) as course_number - , coalesce(mitxonline_courses.course_readable_id, edx_courses.course_readable_id) as course_readable_id - , coalesce(mitxonline_courses.course_title, edx_courses.course_title) as course_title - , coalesce(mitxonline_courses.course_number is not null, false) as is_on_mitxonline - , coalesce(edx_courses.course_number is not null, false) as is_on_edxorg - from mitxonline_courses - full join edx_courses - on mitxonline_courses.course_number = edx_courses.course_number -) - -select * from mitx_courses +select * +from mitx_courses diff --git a/src/ol_dbt/models/intermediate/mitx/int__mitx__program_certificates.sql b/src/ol_dbt/models/intermediate/mitx/int__mitx__program_certificates.sql index 3e0593cd0..31b95f75a 100644 --- a/src/ol_dbt/models/intermediate/mitx/int__mitx__program_certificates.sql +++ b/src/ol_dbt/models/intermediate/mitx/int__mitx__program_certificates.sql @@ -1,45 +1,40 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with micromasters_program_certificates as ( - select * from {{ ref('int__micromasters__program_certificates') }} -) +with + micromasters_program_certificates as (select * from {{ ref("int__micromasters__program_certificates") }}), + mitxonline_program_certificates as ( + select * + from {{ ref("int__mitxonline__program_certificates") }} + -- - dedp is handled in micromasters_program_certificates + where program_is_dedp = false + ), + mitx_programs as (select * from {{ ref("int__mitx__programs") }}), + mitx_program_certificates as ( + select + micromasters_program_certificates.program_title, + micromasters_program_certificates.micromasters_program_id, + micromasters_program_certificates.mitxonline_program_id, + micromasters_program_certificates.program_completion_timestamp, + micromasters_program_certificates.user_mitxonline_username, + micromasters_program_certificates.user_edxorg_username, + micromasters_program_certificates.user_email, + micromasters_program_certificates.user_full_name + from micromasters_program_certificates -, mitxonline_program_certificates as ( - select * from {{ ref('int__mitxonline__program_certificates') }} - --- dedp is handled in micromasters_program_certificates - where program_is_dedp = false -) + union all -, mitx_programs as ( - select * from {{ ref('int__mitx__programs') }} -) + select + mitxonline_program_certificates.program_title, + mitx_programs.micromasters_program_id, + mitxonline_program_certificates.program_id as mitxonline_program_id, + mitxonline_program_certificates.programcertificate_created_on as program_completion_timestamp, + mitxonline_program_certificates.user_username as user_mitxonline_username, + mitxonline_program_certificates.user_edxorg_username, + mitxonline_program_certificates.user_email, + mitxonline_program_certificates.user_full_name + from mitxonline_program_certificates + left join mitx_programs on mitxonline_program_certificates.program_id = mitx_programs.mitxonline_program_id + ) -, mitx_program_certificates as ( - select - micromasters_program_certificates.program_title - , micromasters_program_certificates.micromasters_program_id - , micromasters_program_certificates.mitxonline_program_id - , micromasters_program_certificates.program_completion_timestamp - , micromasters_program_certificates.user_mitxonline_username - , micromasters_program_certificates.user_edxorg_username - , micromasters_program_certificates.user_email - , micromasters_program_certificates.user_full_name - from micromasters_program_certificates - - union all - - select - mitxonline_program_certificates.program_title - , mitx_programs.micromasters_program_id - , mitxonline_program_certificates.program_id as mitxonline_program_id - , mitxonline_program_certificates.programcertificate_created_on as program_completion_timestamp - , mitxonline_program_certificates.user_username as user_mitxonline_username - , mitxonline_program_certificates.user_edxorg_username - , mitxonline_program_certificates.user_email - , mitxonline_program_certificates.user_full_name - from mitxonline_program_certificates - left join mitx_programs - on mitxonline_program_certificates.program_id = mitx_programs.mitxonline_program_id -) - -select * from mitx_program_certificates +select * +from mitx_program_certificates diff --git a/src/ol_dbt/models/intermediate/mitx/int__mitx__program_requirements.sql b/src/ol_dbt/models/intermediate/mitx/int__mitx__program_requirements.sql index 513a1c36d..9952ee259 100644 --- a/src/ol_dbt/models/intermediate/mitx/int__mitx__program_requirements.sql +++ b/src/ol_dbt/models/intermediate/mitx/int__mitx__program_requirements.sql @@ -1,59 +1,46 @@ -with micromasters_program_requirements as ( - select * from {{ ref('int__micromasters__program_requirements') }} -) - -, mitxonline_program_requirements as ( - select * from {{ ref('int__mitxonline__program_requirements') }} -) - -, courses as ( - select * from {{ ref('int__mitx__courses') }} -) - -, programs as ( - select * from {{ ref('int__mitx__programs') }} -) +with + micromasters_program_requirements as (select * from {{ ref("int__micromasters__program_requirements") }}), + mitxonline_program_requirements as (select * from {{ ref("int__mitxonline__program_requirements") }}), + courses as (select * from {{ ref("int__mitx__courses") }}), + programs as (select * from {{ ref("int__mitx__programs") }}) select - programs.micromasters_program_id - , programs.mitxonline_program_id - , courses.course_number - , programs.program_title - , mitxonline_program_requirements.programrequirement_requirement_id as mitxonline_programrequirement_requirement_id - , null as micromasters_electiveset_id - , mitxonline_program_requirements.programrequirement_type - , mitxonline_program_requirements.programrequirement_title - , mitxonline_program_requirements.electiveset_required_number - , mitxonline_program_requirements.programrequirement_is_a_nested_requirement - - , mitxonline_program_requirements.programrequirement_parent_requirement_id - as mitxonline_programrequirement_parent_requirement_id - , mitxonline_program_requirements.program_num_required_courses + programs.micromasters_program_id, + programs.mitxonline_program_id, + courses.course_number, + programs.program_title, + mitxonline_program_requirements.programrequirement_requirement_id as mitxonline_programrequirement_requirement_id, + null as micromasters_electiveset_id, + mitxonline_program_requirements.programrequirement_type, + mitxonline_program_requirements.programrequirement_title, + mitxonline_program_requirements.electiveset_required_number, + mitxonline_program_requirements.programrequirement_is_a_nested_requirement, + mitxonline_program_requirements.programrequirement_parent_requirement_id + as mitxonline_programrequirement_parent_requirement_id, + mitxonline_program_requirements.program_num_required_courses from programs -inner join mitxonline_program_requirements - on programs.mitxonline_program_id = mitxonline_program_requirements.program_id -inner join courses - on mitxonline_program_requirements.course_id = courses.mitxonline_course_id +inner join + mitxonline_program_requirements on programs.mitxonline_program_id = mitxonline_program_requirements.program_id +inner join courses on mitxonline_program_requirements.course_id = courses.mitxonline_course_id where programs.mitxonline_program_id is not null union all select - programs.micromasters_program_id - , programs.mitxonline_program_id - , courses.course_number - , programs.program_title - , null as mitxonline_programrequirement_requirement_id - , micromasters_program_requirements.electiveset_id as micromasters_electiveset_id - , micromasters_program_requirements.programrequirement_type - , micromasters_program_requirements.programrequirement_type as programrequirement_title - , micromasters_program_requirements.electiveset_required_number - , false as programrequirement_is_a_nested_requirement - , null as mitxonline_programrequirement_parent_requirement_id - , micromasters_program_requirements.program_num_required_courses + programs.micromasters_program_id, + programs.mitxonline_program_id, + courses.course_number, + programs.program_title, + null as mitxonline_programrequirement_requirement_id, + micromasters_program_requirements.electiveset_id as micromasters_electiveset_id, + micromasters_program_requirements.programrequirement_type, + micromasters_program_requirements.programrequirement_type as programrequirement_title, + micromasters_program_requirements.electiveset_required_number, + false as programrequirement_is_a_nested_requirement, + null as mitxonline_programrequirement_parent_requirement_id, + micromasters_program_requirements.program_num_required_courses from programs -inner join micromasters_program_requirements - on programs.micromasters_program_id = micromasters_program_requirements.program_id -inner join courses - on micromasters_program_requirements.course_id = courses.micromasters_course_id +inner join + micromasters_program_requirements on programs.micromasters_program_id = micromasters_program_requirements.program_id +inner join courses on micromasters_program_requirements.course_id = courses.micromasters_course_id where programs.mitxonline_program_id is null diff --git a/src/ol_dbt/models/intermediate/mitx/int__mitx__programs.sql b/src/ol_dbt/models/intermediate/mitx/int__mitx__programs.sql index 9b51134bf..63682be47 100644 --- a/src/ol_dbt/models/intermediate/mitx/int__mitx__programs.sql +++ b/src/ol_dbt/models/intermediate/mitx/int__mitx__programs.sql @@ -1,31 +1,26 @@ -- MITx Program Information - -with micromasters_programs as ( - select * from {{ ref('int__micromasters__programs') }} -) - -, mitxonline_programs as ( - select * from {{ ref('int__mitxonline__programs') }} -) +with + micromasters_programs as (select * from {{ ref("int__micromasters__programs") }}), + mitxonline_programs as (select * from {{ ref("int__mitxonline__programs") }}) select - micromasters_programs.program_id as micromasters_program_id - , null as mitxonline_program_id - , micromasters_programs.program_description - , micromasters_programs.program_title - , true as is_micromasters_program - , false as is_dedp_program + micromasters_programs.program_id as micromasters_program_id, + null as mitxonline_program_id, + micromasters_programs.program_description, + micromasters_programs.program_title, + true as is_micromasters_program, + false as is_dedp_program from micromasters_programs where micromasters_programs.program_id != {{ var("dedp_micromasters_program_id") }} union all select case when mitxonline_programs.program_id = {{ var("dedp_mitxonline_international_development_program_id") }} - then {{ var("dedp_micromasters_program_id") }} - end as micromasters_program_id - , mitxonline_programs.program_id as mitxonline_program_id - , null as program_description - , mitxonline_programs.program_title - , mitxonline_programs.program_is_micromasters as is_micromasters_program - , mitxonline_programs.program_is_dedp as is_dedp_program + then {{ var("dedp_micromasters_program_id") }} + end as micromasters_program_id, + mitxonline_programs.program_id as mitxonline_program_id, + null as program_description, + mitxonline_programs.program_title, + mitxonline_programs.program_is_micromasters as is_micromasters_program, + mitxonline_programs.program_is_dedp as is_dedp_program from mitxonline_programs diff --git a/src/ol_dbt/models/intermediate/mitx/int__mitx__users.sql b/src/ol_dbt/models/intermediate/mitx/int__mitx__users.sql index f0d973be2..702c6c4bd 100644 --- a/src/ol_dbt/models/intermediate/mitx/int__mitx__users.sql +++ b/src/ol_dbt/models/intermediate/mitx/int__mitx__users.sql @@ -1,224 +1,215 @@ ----MITx users from MITx Online and edX with dedup ----For users exist on both MITx Online and edX.org, profile data are COALESCE from MITx Online, MicroMasters or edX.org - -with users as ( - select * from {{ ref('int__mitxonline__users') }} -) -----deduplicate users based on their openedx user_username, prioritizing those with an openedx_user_id --- and the most recent joined date - -, mitxonline_users as ( - select * from ( +-- -MITx users from MITx Online and edX with dedup +-- -For users exist on both MITx Online and edX.org, profile data are COALESCE from MITx Online, MicroMasters or edX.org +with + users as (select * from {{ ref("int__mitxonline__users") }}), + -- --deduplicate users based on their openedx user_username, prioritizing those with an openedx_user_id + -- and the most recent joined date + mitxonline_users as ( + select * + from + ( + select + *, + row_number() over ( + partition by user_username order by openedx_user_id asc nulls last, user_joined_on desc + ) as row_num + from users + ) + where row_num = 1 + ), + edxorg_users as (select * from {{ ref("int__edxorg__mitx_users") }}), + micromasters_users as (select * from {{ ref("int__micromasters__users") }}), + mitxonline_users_view as ( + select + mitxonline_users.user_id as user_mitxonline_id, + mitxonline_users.user_global_id, + mitxonline_users.user_username as user_mitxonline_username, + mitxonline_users.user_edxorg_username, + mitxonline_users.user_email as user_mitxonline_email, + micromasters_users.user_email as user_micromasters_email, + micromasters_users.user_id as user_micromasters_id, + mitxonline_users.user_full_name, + mitxonline_users.user_first_name, + mitxonline_users.user_last_name, + mitxonline_users.user_joined_on, + mitxonline_users.user_last_login, + mitxonline_users.user_is_active, + true as is_mitxonline_user, + coalesce(mitxonline_users.user_company, micromasters_users.user_company_name) as user_company, + coalesce(mitxonline_users.user_industry, micromasters_users.user_company_industry) as user_industry, + coalesce(mitxonline_users.user_job_title, micromasters_users.user_job_position) as user_job_title, + coalesce( + mitxonline_users.user_address_country, micromasters_users.user_address_country + ) as user_address_country, + coalesce( + mitxonline_users.user_address_state, micromasters_users.user_address_state_or_territory + ) as user_address_state, + coalesce( + mitxonline_users.user_highest_education, micromasters_users.user_highest_education + ) as user_highest_education, + coalesce(mitxonline_users.user_gender, micromasters_users.user_gender) as user_gender, + coalesce( + mitxonline_users.user_birth_year, cast(substring(micromasters_users.user_birth_date, 1, 4) as int) + ) as user_birth_year + from mitxonline_users + left join micromasters_users on mitxonline_users.user_username = micromasters_users.user_mitxonline_username + ), + -- -augment edxorg users profile with MicroMasters + edxorg_users_view as ( select - * - , row_number() over ( - partition by user_username - order by openedx_user_id asc nulls last, user_joined_on desc - ) as row_num - from users + edxorg_users.user_id as user_edxorg_id, + edxorg_users.user_username as user_edxorg_username, + edxorg_users.user_email as user_edxorg_email, + micromasters_users.user_email as user_micromasters_email, + micromasters_users.user_id as user_micromasters_id, + micromasters_users.user_first_name, + micromasters_users.user_last_name, + edxorg_users.user_joined_on, + edxorg_users.user_last_login, + edxorg_users.user_is_active, + micromasters_users.user_company_name, + micromasters_users.user_company_industry, + micromasters_users.user_job_position, + micromasters_users.user_address_state_or_territory, + micromasters_users.user_address_postal_code, + micromasters_users.user_street_address, + micromasters_users.user_address_city, + true as is_edxorg_user, + coalesce(micromasters_users.user_full_name, edxorg_users.user_full_name) as user_full_name, + coalesce(micromasters_users.user_address_country, edxorg_users.user_country) as user_address_country, + coalesce( + micromasters_users.user_highest_education, edxorg_users.user_highest_education + ) as user_highest_education, + coalesce(micromasters_users.user_gender, edxorg_users.user_gender) as user_gender, + coalesce( + cast(substring(micromasters_users.user_birth_date, 1, 4) as int), edxorg_users.user_birth_year + ) as user_birth_year + from edxorg_users + left join micromasters_users on edxorg_users.user_username = micromasters_users.user_edxorg_username + ), + mitxonline_edxorg_users as ( + select + mitxonline_users_view.user_mitxonline_id, + edxorg_users_view.user_edxorg_id, + mitxonline_users_view.user_global_id, + mitxonline_users_view.user_mitxonline_username, + edxorg_users_view.user_edxorg_username, + mitxonline_users_view.user_mitxonline_email, + edxorg_users_view.user_edxorg_email, + mitxonline_users_view.user_joined_on as user_joined_on_mitxonline, + edxorg_users_view.user_joined_on as user_joined_on_edxorg, + mitxonline_users_view.user_last_login as user_last_login_on_mitxonline, + edxorg_users_view.user_last_login as user_last_login_on_edxorg, + mitxonline_users_view.user_is_active as user_is_active_on_mitxonline, + edxorg_users_view.user_is_active as user_is_active_on_edxorg, + edxorg_users_view.user_address_postal_code, + edxorg_users_view.user_street_address, + edxorg_users_view.user_address_city, + coalesce(mitxonline_users_view.is_mitxonline_user is not null, false) as is_mitxonline_user, + coalesce(edxorg_users_view.is_edxorg_user is not null, false) as is_edxorg_user, + coalesce(mitxonline_users_view.user_full_name, edxorg_users_view.user_full_name) as user_full_name, + coalesce(mitxonline_users_view.user_first_name, edxorg_users_view.user_first_name) as user_first_name, + coalesce(mitxonline_users_view.user_last_name, edxorg_users_view.user_last_name) as user_last_name, + coalesce( + mitxonline_users_view.user_address_country, edxorg_users_view.user_address_country + ) as user_address_country, + coalesce( + mitxonline_users_view.user_address_state, edxorg_users_view.user_address_state_or_territory + ) as user_address_state, + coalesce( + mitxonline_users_view.user_highest_education, edxorg_users_view.user_highest_education + ) as user_highest_education, + coalesce(mitxonline_users_view.user_gender, edxorg_users_view.user_gender) as user_gender, + coalesce(mitxonline_users_view.user_birth_year, edxorg_users_view.user_birth_year) as user_birth_year, + coalesce(mitxonline_users_view.user_company, edxorg_users_view.user_company_name) as user_company, + coalesce(mitxonline_users_view.user_job_title, edxorg_users_view.user_job_position) as user_job_title, + coalesce(mitxonline_users_view.user_industry, edxorg_users_view.user_company_industry) as user_industry, + coalesce( + mitxonline_users_view.user_micromasters_id, edxorg_users_view.user_micromasters_id + ) as user_micromasters_id, + coalesce( + mitxonline_users_view.user_micromasters_email, edxorg_users_view.user_micromasters_email + ) as user_micromasters_email + from mitxonline_users_view + full outer join + edxorg_users_view on mitxonline_users_view.user_edxorg_username = edxorg_users_view.user_edxorg_username ) - where row_num = 1 -) - -, edxorg_users as ( - select * from {{ ref('int__edxorg__mitx_users') }} -) - -, micromasters_users as ( - select * from {{ ref('int__micromasters__users') }} -) - -, mitxonline_users_view as ( - select - mitxonline_users.user_id as user_mitxonline_id - , mitxonline_users.user_global_id - , mitxonline_users.user_username as user_mitxonline_username - , mitxonline_users.user_edxorg_username - , mitxonline_users.user_email as user_mitxonline_email - , micromasters_users.user_email as user_micromasters_email - , micromasters_users.user_id as user_micromasters_id - , mitxonline_users.user_full_name - , mitxonline_users.user_first_name - , mitxonline_users.user_last_name - , mitxonline_users.user_joined_on - , mitxonline_users.user_last_login - , mitxonline_users.user_is_active - , true as is_mitxonline_user - , coalesce(mitxonline_users.user_company, micromasters_users.user_company_name) as user_company - , coalesce(mitxonline_users.user_industry, micromasters_users.user_company_industry) as user_industry - , coalesce(mitxonline_users.user_job_title, micromasters_users.user_job_position) as user_job_title - , coalesce( - mitxonline_users.user_address_country, micromasters_users.user_address_country - ) as user_address_country - , coalesce( - mitxonline_users.user_address_state, micromasters_users.user_address_state_or_territory - ) as user_address_state - , coalesce( - mitxonline_users.user_highest_education, micromasters_users.user_highest_education - ) as user_highest_education - , coalesce(mitxonline_users.user_gender, micromasters_users.user_gender) as user_gender - , coalesce( - mitxonline_users.user_birth_year, cast(substring(micromasters_users.user_birth_date, 1, 4) as int) - ) as user_birth_year - from mitxonline_users - left join micromasters_users on mitxonline_users.user_username = micromasters_users.user_mitxonline_username -) - ----augment edxorg users profile with MicroMasters -, edxorg_users_view as ( - select - edxorg_users.user_id as user_edxorg_id - , edxorg_users.user_username as user_edxorg_username - , edxorg_users.user_email as user_edxorg_email - , micromasters_users.user_email as user_micromasters_email - , micromasters_users.user_id as user_micromasters_id - , micromasters_users.user_first_name - , micromasters_users.user_last_name - , edxorg_users.user_joined_on - , edxorg_users.user_last_login - , edxorg_users.user_is_active - , micromasters_users.user_company_name - , micromasters_users.user_company_industry - , micromasters_users.user_job_position - , micromasters_users.user_address_state_or_territory - , micromasters_users.user_address_postal_code - , micromasters_users.user_street_address - , micromasters_users.user_address_city - , true as is_edxorg_user - , coalesce(micromasters_users.user_full_name, edxorg_users.user_full_name) as user_full_name - , coalesce(micromasters_users.user_address_country, edxorg_users.user_country) as user_address_country - , coalesce( - micromasters_users.user_highest_education, edxorg_users.user_highest_education - ) as user_highest_education - , coalesce(micromasters_users.user_gender, edxorg_users.user_gender) as user_gender - , coalesce( - cast(substring(micromasters_users.user_birth_date, 1, 4) as int), edxorg_users.user_birth_year - ) as user_birth_year - from edxorg_users - left join micromasters_users on edxorg_users.user_username = micromasters_users.user_edxorg_username -) - -, mitxonline_edxorg_users as ( - select - mitxonline_users_view.user_mitxonline_id - , edxorg_users_view.user_edxorg_id - , mitxonline_users_view.user_global_id - , mitxonline_users_view.user_mitxonline_username - , edxorg_users_view.user_edxorg_username - , mitxonline_users_view.user_mitxonline_email - , edxorg_users_view.user_edxorg_email - , mitxonline_users_view.user_joined_on as user_joined_on_mitxonline - , edxorg_users_view.user_joined_on as user_joined_on_edxorg - , mitxonline_users_view.user_last_login as user_last_login_on_mitxonline - , edxorg_users_view.user_last_login as user_last_login_on_edxorg - , mitxonline_users_view.user_is_active as user_is_active_on_mitxonline - , edxorg_users_view.user_is_active as user_is_active_on_edxorg - , edxorg_users_view.user_address_postal_code - , edxorg_users_view.user_street_address - , edxorg_users_view.user_address_city - , coalesce(mitxonline_users_view.is_mitxonline_user is not null, false) as is_mitxonline_user - , coalesce(edxorg_users_view.is_edxorg_user is not null, false) as is_edxorg_user - , coalesce(mitxonline_users_view.user_full_name, edxorg_users_view.user_full_name) as user_full_name - , coalesce(mitxonline_users_view.user_first_name, edxorg_users_view.user_first_name) as user_first_name - , coalesce(mitxonline_users_view.user_last_name, edxorg_users_view.user_last_name) as user_last_name - , coalesce(mitxonline_users_view.user_address_country, edxorg_users_view.user_address_country) - as user_address_country - , coalesce(mitxonline_users_view.user_address_state, edxorg_users_view.user_address_state_or_territory) - as user_address_state - , coalesce(mitxonline_users_view.user_highest_education, edxorg_users_view.user_highest_education) - as user_highest_education - , coalesce(mitxonline_users_view.user_gender, edxorg_users_view.user_gender) as user_gender - , coalesce(mitxonline_users_view.user_birth_year, edxorg_users_view.user_birth_year) as user_birth_year - , coalesce(mitxonline_users_view.user_company, edxorg_users_view.user_company_name) as user_company - , coalesce(mitxonline_users_view.user_job_title, edxorg_users_view.user_job_position) as user_job_title - , coalesce(mitxonline_users_view.user_industry, edxorg_users_view.user_company_industry) as user_industry - , coalesce( - mitxonline_users_view.user_micromasters_id, edxorg_users_view.user_micromasters_id - ) as user_micromasters_id - , coalesce( - mitxonline_users_view.user_micromasters_email, edxorg_users_view.user_micromasters_email - ) as user_micromasters_email - from mitxonline_users_view - full outer join edxorg_users_view - on mitxonline_users_view.user_edxorg_username = edxorg_users_view.user_edxorg_username -) select - is_mitxonline_user - , is_edxorg_user - , user_global_id - , user_mitxonline_id - , user_edxorg_id - , user_micromasters_id - , user_mitxonline_username - , user_edxorg_username - , user_mitxonline_email - , user_edxorg_email - , user_micromasters_email - , user_joined_on_mitxonline - , user_joined_on_edxorg - , user_last_login_on_mitxonline - , user_last_login_on_edxorg - , user_is_active_on_mitxonline - , user_is_active_on_edxorg - , user_full_name - , user_first_name - , user_last_name - , user_address_country - , user_address_state - , user_address_city - , user_address_postal_code - , user_street_address - , user_highest_education - , user_gender - , user_birth_year - , user_company - , user_industry - , user_job_title - , case + is_mitxonline_user, + is_edxorg_user, + user_global_id, + user_mitxonline_id, + user_edxorg_id, + user_micromasters_id, + user_mitxonline_username, + user_edxorg_username, + user_mitxonline_email, + user_edxorg_email, + user_micromasters_email, + user_joined_on_mitxonline, + user_joined_on_edxorg, + user_last_login_on_mitxonline, + user_last_login_on_edxorg, + user_is_active_on_mitxonline, + user_is_active_on_edxorg, + user_full_name, + user_first_name, + user_last_name, + user_address_country, + user_address_state, + user_address_city, + user_address_postal_code, + user_street_address, + user_highest_education, + user_gender, + user_birth_year, + user_company, + user_industry, + user_job_title, + case when is_mitxonline_user = true - then {{ generate_hash_id("cast(user_mitxonline_id as varchar) || 'MITx Online'") }} + then {{ generate_hash_id("cast(user_mitxonline_id as varchar) || 'MITx Online'") }} when is_edxorg_user = true - then {{ generate_hash_id("cast(user_edxorg_id as varchar) || 'edX.org'") }} + then {{ generate_hash_id("cast(user_edxorg_id as varchar) || 'edX.org'") }} end as user_hashed_id from mitxonline_edxorg_users union distinct ---- append micromasters users who don't exist in mitxonline_edxorg_users +-- - append micromasters users who don't exist in mitxonline_edxorg_users select - if(micromasters_users.user_mitxonline_username is not null, true, false) as is_mitxonline_user - , if(micromasters_users.user_edxorg_username is not null, true, false) as is_edxorg_user - , null as user_global_id - , null as user_mitxonline_id - , null as user_edxorg_id - , micromasters_users.user_id as user_micromasters_id - , micromasters_users.user_mitxonline_username - , micromasters_users.user_edxorg_username - , null as user_mitxonline_email - , null as user_edxorg_email - , micromasters_users.user_email as user_micromasters_email - , null as user_joined_on_mitxonline - , null as user_joined_on_edxorg - , null as user_last_login_on_mitxonline - , null as user_last_login_on_edxorg - , null as user_is_active_on_mitxonline - , null as user_is_active_on_edxorg - , micromasters_users.user_full_name - , micromasters_users.user_first_name - , micromasters_users.user_last_name - , micromasters_users.user_address_country - , micromasters_users.user_address_state_or_territory as user_address_state - , micromasters_users.user_address_city - , micromasters_users.user_address_postal_code - , micromasters_users.user_street_address - , micromasters_users.user_highest_education - , micromasters_users.user_gender - , cast(substring(micromasters_users.user_birth_date, 1, 4) as int) as user_birth_year - , micromasters_users.user_company_name as user_company - , micromasters_users.user_company_industry as user_industry - , micromasters_users.user_job_position as user_job_title - , {{ generate_hash_id("cast(user_micromasters_id as varchar) || 'MicroMasters'") }} as user_hashed_id + if(micromasters_users.user_mitxonline_username is not null, true, false) as is_mitxonline_user, + if(micromasters_users.user_edxorg_username is not null, true, false) as is_edxorg_user, + null as user_global_id, + null as user_mitxonline_id, + null as user_edxorg_id, + micromasters_users.user_id as user_micromasters_id, + micromasters_users.user_mitxonline_username, + micromasters_users.user_edxorg_username, + null as user_mitxonline_email, + null as user_edxorg_email, + micromasters_users.user_email as user_micromasters_email, + null as user_joined_on_mitxonline, + null as user_joined_on_edxorg, + null as user_last_login_on_mitxonline, + null as user_last_login_on_edxorg, + null as user_is_active_on_mitxonline, + null as user_is_active_on_edxorg, + micromasters_users.user_full_name, + micromasters_users.user_first_name, + micromasters_users.user_last_name, + micromasters_users.user_address_country, + micromasters_users.user_address_state_or_territory as user_address_state, + micromasters_users.user_address_city, + micromasters_users.user_address_postal_code, + micromasters_users.user_street_address, + micromasters_users.user_highest_education, + micromasters_users.user_gender, + cast(substring(micromasters_users.user_birth_date, 1, 4) as int) as user_birth_year, + micromasters_users.user_company_name as user_company, + micromasters_users.user_company_industry as user_industry, + micromasters_users.user_job_position as user_job_title, + {{ generate_hash_id("cast(user_micromasters_id as varchar) || 'MicroMasters'") }} as user_hashed_id from micromasters_users -left join mitxonline_edxorg_users - on micromasters_users.user_id = mitxonline_edxorg_users.user_micromasters_id +left join mitxonline_edxorg_users on micromasters_users.user_id = mitxonline_edxorg_users.user_micromasters_id where mitxonline_edxorg_users.user_micromasters_id is null diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__bulk_email_optin.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__bulk_email_optin.sql index 894bde18a..d38b8f25a 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__bulk_email_optin.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__bulk_email_optin.sql @@ -1,31 +1,18 @@ -- Email optin information for MITx Online openededx - -with email_optout as ( - select * - from {{ ref('stg__mitxonline__openedx__mysql__bulk_email_optout') }} -) - -, courserun as ( - select * - from {{ ref('stg__mitxonline__app__postgres__courses_courserun') }} -) - -, users as ( - select * - from {{ ref('int__mitxonline__users') }} -) +with + email_optout as (select * from {{ ref("stg__mitxonline__openedx__mysql__bulk_email_optout") }}), + courserun as (select * from {{ ref("stg__mitxonline__app__postgres__courses_courserun") }}), + users as (select * from {{ ref("int__mitxonline__users") }}) select - users.openedx_user_id - , users.user_full_name - , users.user_username - , users.user_email - , email_optout.courserun_readable_id - , courserun.courserun_title - , case when email_optout.email_optout_id is null then 1 else 0 end as email_opted_in + users.openedx_user_id, + users.user_full_name, + users.user_username, + users.user_email, + email_optout.courserun_readable_id, + courserun.courserun_title, + case when email_optout.email_optout_id is null then 1 else 0 end as email_opted_in from users -left join email_optout - on users.openedx_user_id = email_optout.openedx_user_id -left join courserun - on email_optout.courserun_readable_id = courserun.courserun_readable_id +left join email_optout on users.openedx_user_id = email_optout.openedx_user_id +left join courserun on email_optout.courserun_readable_id = courserun.courserun_readable_id where users.openedx_user_id is not null diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_blockedcountries.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_blockedcountries.sql index a76bfffb3..4b231d706 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_blockedcountries.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_blockedcountries.sql @@ -1,12 +1,6 @@ -- Course's Blocked countries information for MITx Online -- Keep it as separate model for flexibility to satisfy different use cases +with blockedcountries as (select * from {{ ref("stg__mitxonline__app__postgres__courses_blockedcountry") }}) -with blockedcountries as ( - select * - from {{ ref('stg__mitxonline__app__postgres__courses_blockedcountry') }} -) - -select - course_id - , blockedcountry_code +select course_id, blockedcountry_code from blockedcountries diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_instructors.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_instructors.sql index 70747dfef..cdf34230f 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_instructors.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_instructors.sql @@ -1,21 +1,14 @@ -with instructors as ( - select * from {{ ref('stg__mitxonline__app__postgres__cms_instructorpage') }} -) - -, instructor_pagelinks as ( - select * from {{ ref('stg__mitxonline__app__postgres__cms_instructorpagelink') }} -) - -, course_pages as ( - select * from {{ ref('stg__mitxonline__app__postgres__cms_coursepage') }} -) +with + instructors as (select * from {{ ref("stg__mitxonline__app__postgres__cms_instructorpage") }}), + instructor_pagelinks as (select * from {{ ref("stg__mitxonline__app__postgres__cms_instructorpagelink") }}), + course_pages as (select * from {{ ref("stg__mitxonline__app__postgres__cms_coursepage") }}) select distinct - course_pages.course_id - , instructors.instructor_name - , instructors.instructor_title - , instructors.instructor_bio_short - , instructors.instructor_bio_long + course_pages.course_id, + instructors.instructor_name, + instructors.instructor_title, + instructors.instructor_bio_short, + instructors.instructor_bio_long from course_pages inner join instructor_pagelinks on course_pages.wagtail_page_id = instructor_pagelinks.wagtail_page_id inner join instructors on instructor_pagelinks.instructor_wagtail_page_id = instructors.wagtail_page_id diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_runs.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_runs.sql index 9b853027a..b8908d5b0 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_runs.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_runs.sql @@ -1,30 +1,24 @@ -- Course Runs information for MITx Online - -with runs as ( - select * - from {{ ref('stg__mitxonline__app__postgres__courses_courserun') }} -) - -, courses as ( - select * from {{ ref('stg__mitxonline__app__postgres__courses_course') }} -) +with + runs as (select * from {{ ref("stg__mitxonline__app__postgres__courses_courserun") }}), + courses as (select * from {{ ref("stg__mitxonline__app__postgres__courses_course") }}) select - runs.courserun_id - , runs.course_id - , courses.course_number - , runs.courserun_title - , runs.courserun_readable_id - , runs.courserun_edx_readable_id - , runs.courserun_tag - , runs.courserun_url - , runs.courserun_start_on - , runs.courserun_end_on - , runs.courserun_enrollment_start_on - , runs.courserun_enrollment_end_on - , runs.courserun_upgrade_deadline - , runs.courserun_is_self_paced - , runs.courserun_is_live - , runs.courserun_platform + runs.courserun_id, + runs.course_id, + courses.course_number, + runs.courserun_title, + runs.courserun_readable_id, + runs.courserun_edx_readable_id, + runs.courserun_tag, + runs.courserun_url, + runs.courserun_start_on, + runs.courserun_end_on, + runs.courserun_enrollment_start_on, + runs.courserun_enrollment_end_on, + runs.courserun_upgrade_deadline, + runs.courserun_is_self_paced, + runs.courserun_is_live, + runs.courserun_platform from runs inner join courses on runs.course_id = courses.course_id diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_structure.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_structure.sql index cbd4c338c..a298a362f 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_structure.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_structure.sql @@ -1,66 +1,60 @@ -with course_structure as ( - select * from {{ ref('stg__mitxonline__openedx__api__course_structure') }} - order by courserun_readable_id, coursestructure_retrieved_at, coursestructure_block_index +with + course_structure as ( + select * + from {{ ref("stg__mitxonline__openedx__api__course_structure") }} + order by courserun_readable_id, coursestructure_retrieved_at, coursestructure_block_index -) - -, latest_course_structure_date as ( - select - courserun_readable_id - , max(coursestructure_retrieved_at) as max_retrieved_date - from course_structure - group by courserun_readable_id -) - -, chapters as ( - select * from course_structure - where coursestructure_block_category = 'chapter' -) - -, course_structure_with_chapters as ( - select - course_structure.* - , chapters.coursestructure_block_id as coursestructure_chapter_id - , chapters.coursestructure_block_title as coursestructure_chapter_title - , row_number() over ( - partition by - course_structure.courserun_readable_id - , course_structure.coursestructure_block_index - , course_structure.coursestructure_retrieved_at - order by chapters.coursestructure_block_index desc - ) as row_num - from course_structure - inner join chapters - on - course_structure.courserun_readable_id = chapters.courserun_readable_id + ), + latest_course_structure_date as ( + select courserun_readable_id, max(coursestructure_retrieved_at) as max_retrieved_date + from course_structure + group by courserun_readable_id + ), + chapters as (select * from course_structure where coursestructure_block_category = 'chapter'), + course_structure_with_chapters as ( + select + course_structure.*, + chapters.coursestructure_block_id as coursestructure_chapter_id, + chapters.coursestructure_block_title as coursestructure_chapter_title, + row_number() over ( + partition by + course_structure.courserun_readable_id, + course_structure.coursestructure_block_index, + course_structure.coursestructure_retrieved_at + order by chapters.coursestructure_block_index desc + ) as row_num + from course_structure + inner join + chapters + on course_structure.courserun_readable_id = chapters.courserun_readable_id and course_structure.coursestructure_retrieved_at = chapters.coursestructure_retrieved_at and course_structure.coursestructure_block_index >= chapters.coursestructure_block_index -) + ) select - course_structure.courserun_readable_id - , course_structure.courserun_title - , course_structure.coursestructure_block_index - , course_structure.coursestructure_block_id - , course_structure.coursestructure_parent_block_id - , course_structure.coursestructure_block_category - , course_structure.coursestructure_block_title - , course_structure.coursestructure_content_hash - , course_structure.coursestructure_block_content_hash - , course_structure.coursestructure_block_metadata - , course_structure.courserun_start_on - , course_structure.coursestructure_retrieved_at - , course_structure_with_chapters.coursestructure_chapter_id - , course_structure_with_chapters.coursestructure_chapter_title - , if(latest_course_structure_date.max_retrieved_date is not null, true, false) as coursestructure_is_latest + course_structure.courserun_readable_id, + course_structure.courserun_title, + course_structure.coursestructure_block_index, + course_structure.coursestructure_block_id, + course_structure.coursestructure_parent_block_id, + course_structure.coursestructure_block_category, + course_structure.coursestructure_block_title, + course_structure.coursestructure_content_hash, + course_structure.coursestructure_block_content_hash, + course_structure.coursestructure_block_metadata, + course_structure.courserun_start_on, + course_structure.coursestructure_retrieved_at, + course_structure_with_chapters.coursestructure_chapter_id, + course_structure_with_chapters.coursestructure_chapter_title, + if(latest_course_structure_date.max_retrieved_date is not null, true, false) as coursestructure_is_latest from course_structure -left join latest_course_structure_date - on - course_structure.courserun_readable_id = latest_course_structure_date.courserun_readable_id - and course_structure.coursestructure_retrieved_at = latest_course_structure_date.max_retrieved_date -left join course_structure_with_chapters - on - course_structure.courserun_readable_id = course_structure_with_chapters.courserun_readable_id - and course_structure.coursestructure_block_id = course_structure_with_chapters.coursestructure_block_id - and course_structure.coursestructure_retrieved_at = course_structure_with_chapters.coursestructure_retrieved_at - and course_structure_with_chapters.row_num = 1 +left join + latest_course_structure_date + on course_structure.courserun_readable_id = latest_course_structure_date.courserun_readable_id + and course_structure.coursestructure_retrieved_at = latest_course_structure_date.max_retrieved_date +left join + course_structure_with_chapters + on course_structure.courserun_readable_id = course_structure_with_chapters.courserun_readable_id + and course_structure.coursestructure_block_id = course_structure_with_chapters.coursestructure_block_id + and course_structure.coursestructure_retrieved_at = course_structure_with_chapters.coursestructure_retrieved_at + and course_structure_with_chapters.row_num = 1 diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_to_departments.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_to_departments.sql index 4c087f902..0bf900132 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_to_departments.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_to_departments.sql @@ -1,18 +1,9 @@ -- MITx Online Course to Department Information -- Keep it as separate model for flexibility to satisfy different use cases +with + departments as (select * from {{ ref("stg__mitxonline__app__postgres__courses_department") }}), + course_to_departments as (select * from {{ ref("stg__mitxonline__app__postgres__courses_course_to_department") }}) -with departments as ( - select * - from {{ ref('stg__mitxonline__app__postgres__courses_department') }} -) - -, course_to_departments as ( - select * - from {{ ref('stg__mitxonline__app__postgres__courses_course_to_department') }} -) - -select - departments.coursedepartment_name - , course_to_departments.course_id +select departments.coursedepartment_name, course_to_departments.course_id from course_to_departments inner join departments on course_to_departments.coursedepartment_id = departments.coursedepartment_id diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_to_topics.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_to_topics.sql index 0cd5a219d..079879124 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_to_topics.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__course_to_topics.sql @@ -1,22 +1,9 @@ -with topics as ( - select * from {{ ref('stg__mitxonline__app__postgres__courses_coursetopic') }} -) +with + topics as (select * from {{ ref("stg__mitxonline__app__postgres__courses_coursetopic") }}), + course_topics as (select * from {{ ref("stg__mitxonline__app__postgres__cms_coursepage_topics") }}), + course_pages as (select * from {{ ref("stg__mitxonline__app__postgres__cms_coursepage") }}) -, course_topics as ( - select * from {{ ref('stg__mitxonline__app__postgres__cms_coursepage_topics') }} -) - -, course_pages as ( - select * from {{ ref('stg__mitxonline__app__postgres__cms_coursepage') }} -) - -select - course_pages.course_id - , course_topics.coursetopic_id - , topics.coursetopic_parent_id - , topics.coursetopic_name +select course_pages.course_id, course_topics.coursetopic_id, topics.coursetopic_parent_id, topics.coursetopic_name from course_pages -inner join course_topics - on course_pages.wagtail_page_id = course_topics.wagtail_page_id -inner join topics - on course_topics.coursetopic_id = topics.coursetopic_id +inner join course_topics on course_pages.wagtail_page_id = course_topics.wagtail_page_id +inner join topics on course_topics.coursetopic_id = topics.coursetopic_id diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_certificates.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_certificates.sql index 4e24f959e..59eb74972 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_certificates.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_certificates.sql @@ -1,40 +1,32 @@ -- Course Certificate information for MITx Online +with + certificates as (select * from {{ ref("stg__mitxonline__app__postgres__courses_courseruncertificate") }}), + runs as (select * from {{ ref("int__mitxonline__course_runs") }}), + users as (select * from {{ ref("int__mitxonline__users") }}), + courserun_certificates as ( + select + certificates.courseruncertificate_id, + certificates.courseruncertificate_uuid, + certificates.courserun_id, + runs.courserun_title, + runs.course_number, + runs.courserun_readable_id, + runs.courserun_url, + runs.courserun_platform, + runs.course_id, + certificates.courseruncertificate_url, + certificates.courseruncertificate_is_revoked, + certificates.courseruncertificate_created_on, + certificates.courseruncertificate_updated_on, + certificates.user_id, + users.user_username, + users.user_edxorg_username, + users.user_email, + users.user_full_name + from certificates + inner join runs on certificates.courserun_id = runs.courserun_id + inner join users on certificates.user_id = users.user_id + ) -with certificates as ( - select * from {{ ref('stg__mitxonline__app__postgres__courses_courseruncertificate') }} -) - -, runs as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} -) - -, courserun_certificates as ( - select - certificates.courseruncertificate_id - , certificates.courseruncertificate_uuid - , certificates.courserun_id - , runs.courserun_title - , runs.course_number - , runs.courserun_readable_id - , runs.courserun_url - , runs.courserun_platform - , runs.course_id - , certificates.courseruncertificate_url - , certificates.courseruncertificate_is_revoked - , certificates.courseruncertificate_created_on - , certificates.courseruncertificate_updated_on - , certificates.user_id - , users.user_username - , users.user_edxorg_username - , users.user_email - , users.user_full_name - from certificates - inner join runs on certificates.courserun_id = runs.courserun_id - inner join users on certificates.user_id = users.user_id -) - -select * from courserun_certificates +select * +from courserun_certificates diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_grades.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_grades.sql index 6a3ec3d79..b34f1dc1c 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_grades.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_grades.sql @@ -1,40 +1,32 @@ -- Course Grade information for MITx Online +with + grades as (select * from {{ ref("stg__mitxonline__app__postgres__courses_courserungrade") }}), + runs as (select * from {{ ref("int__mitxonline__course_runs") }}), + users as (select * from {{ ref("int__mitxonline__users") }}), + courserun_grades as ( + select + grades.courserungrade_id, + grades.courserun_id, + runs.course_id, + runs.course_number, + runs.courserun_title, + runs.courserun_readable_id, + runs.courserun_platform, + runs.courserun_url, + grades.courserungrade_grade, + grades.courserungrade_letter_grade, + grades.courserungrade_is_passing, + grades.courserungrade_created_on, + grades.courserungrade_updated_on, + grades.user_id, + users.user_username, + users.user_edxorg_username, + users.user_email, + users.user_full_name + from grades + inner join runs on grades.courserun_id = runs.courserun_id + inner join users on grades.user_id = users.user_id + ) -with grades as ( - select * from {{ ref('stg__mitxonline__app__postgres__courses_courserungrade') }} -) - -, runs as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} -) - -, courserun_grades as ( - select - grades.courserungrade_id - , grades.courserun_id - , runs.course_id - , runs.course_number - , runs.courserun_title - , runs.courserun_readable_id - , runs.courserun_platform - , runs.courserun_url - , grades.courserungrade_grade - , grades.courserungrade_letter_grade - , grades.courserungrade_is_passing - , grades.courserungrade_created_on - , grades.courserungrade_updated_on - , grades.user_id - , users.user_username - , users.user_edxorg_username - , users.user_email - , users.user_full_name - from grades - inner join runs on grades.courserun_id = runs.courserun_id - inner join users on grades.user_id = users.user_id -) - -select * from courserun_grades +select * +from courserun_grades diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_subsection_grades.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_subsection_grades.sql index ce3a50d12..e26b4fb39 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_subsection_grades.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_subsection_grades.sql @@ -1,55 +1,53 @@ -with subsection_grades as ( - select * from {{ ref('stg__mitxonline__openedx__mysql__grades_subsectiongrade') }} -) +with + subsection_grades as (select * from {{ ref("stg__mitxonline__openedx__mysql__grades_subsectiongrade") }}), + subsection_grade_overrides as ( + select * from {{ ref("stg__mitxonline__openedx__mysql__grades_subsectiongradeoverride") }} + ), + course_structure as ( + select + *, + row_number() over ( + partition by courserun_readable_id, coursestructure_block_id order by coursestructure_retrieved_at desc + ) as row_num + from {{ ref("int__mitxonline__course_structure") }} + ), + subsection_grades_joined as ( + select + subsection_grades.courserun_readable_id, + subsection_grades.coursestructure_block_id, + subsection_grades.visibleblocks_hash, + course_structure.coursestructure_block_title, + course_structure.coursestructure_chapter_id, + subsection_grades.openedx_user_id, + subsection_grades.subsectiongrade_first_attempted_on, + subsection_grades.subsectiongrade_created_on, + coalesce( + subsection_grade_overrides.subsectiongradeoverride_total_score, + subsection_grades.subsectiongrade_total_score + ) as subsectiongrade_total_score, + coalesce( + subsection_grade_overrides.subsectiongradeoverride_total_earned_score, + subsection_grades.subsectiongrade_total_earned_score + ) as subsectiongrade_total_earned_score, + coalesce( + subsection_grade_overrides.subsectiongradeoverride_total_graded_score, + subsection_grades.subsectiongrade_total_graded_score + ) as subsectiongrade_total_graded_score, + coalesce( + subsection_grade_overrides.subsectiongradeoverride_total_earned_graded_score, + subsection_grades.subsectiongrade_total_earned_graded_score + ) as subsectiongrade_total_earned_graded_score, + if( + subsection_grade_overrides.subsectiongradeoverride_id is not null, true, false + ) as subsectiongrade_is_overridden + from subsection_grades + inner join + course_structure on subsection_grades.coursestructure_block_id = course_structure.coursestructure_block_id + left join + subsection_grade_overrides + on subsection_grades.subsectiongrade_id = subsection_grade_overrides.subsectiongrade_id + where course_structure.row_num = 1 + ) -, subsection_grade_overrides as ( - select * from {{ ref('stg__mitxonline__openedx__mysql__grades_subsectiongradeoverride') }} -) - -, course_structure as ( - select - * - , row_number() over ( - partition by courserun_readable_id, coursestructure_block_id - order by coursestructure_retrieved_at desc - ) as row_num - from {{ ref('int__mitxonline__course_structure') }} -) - -, subsection_grades_joined as ( - select - subsection_grades.courserun_readable_id - , subsection_grades.coursestructure_block_id - , subsection_grades.visibleblocks_hash - , course_structure.coursestructure_block_title - , course_structure.coursestructure_chapter_id - , subsection_grades.openedx_user_id - , subsection_grades.subsectiongrade_first_attempted_on - , subsection_grades.subsectiongrade_created_on - , coalesce( - subsection_grade_overrides.subsectiongradeoverride_total_score - , subsection_grades.subsectiongrade_total_score - ) as subsectiongrade_total_score - , coalesce( - subsection_grade_overrides.subsectiongradeoverride_total_earned_score - , subsection_grades.subsectiongrade_total_earned_score - ) as subsectiongrade_total_earned_score - , coalesce( - subsection_grade_overrides.subsectiongradeoverride_total_graded_score - , subsection_grades.subsectiongrade_total_graded_score - ) as subsectiongrade_total_graded_score - , coalesce( - subsection_grade_overrides.subsectiongradeoverride_total_earned_graded_score - , subsection_grades.subsectiongrade_total_earned_graded_score - ) as subsectiongrade_total_earned_graded_score - , if(subsection_grade_overrides.subsectiongradeoverride_id is not null, true, false) - as subsectiongrade_is_overridden - from subsection_grades - inner join course_structure - on subsection_grades.coursestructure_block_id = course_structure.coursestructure_block_id - left join subsection_grade_overrides - on subsection_grades.subsectiongrade_id = subsection_grade_overrides.subsectiongrade_id - where course_structure.row_num = 1 -) - -select * from subsection_grades_joined +select * +from subsection_grades_joined diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_videos.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_videos.sql index 173d0bfd4..8ae6dfe7c 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_videos.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserun_videos.sql @@ -1,32 +1,21 @@ -with mitxonline_coursevideos as ( - select * - from {{ ref('stg__mitxonline__openedx__mysql__edxval_coursevideo') }} -) - -, mitxonline_videos as ( - select * from {{ ref('stg__mitxonline__openedx__mysql__edxval_video') }} -) - -, ovs_videos as ( - select - * - , row_number() over ( - partition by video_title - order by video_created_on desc - ) as video_rank - from {{ ref('int__ovs__videos') }} - where platform = '{{ var("mitxonline") }}' -) +with + mitxonline_coursevideos as (select * from {{ ref("stg__mitxonline__openedx__mysql__edxval_coursevideo") }}), + mitxonline_videos as (select * from {{ ref("stg__mitxonline__openedx__mysql__edxval_video") }}), + ovs_videos as ( + select *, row_number() over (partition by video_title order by video_created_on desc) as video_rank + from {{ ref("int__ovs__videos") }} + where platform = '{{ var("mitxonline") }}' + ) select - mitxonline_coursevideos.courserun_readable_id - , mitxonline_coursevideos.coursevideo_is_hidden - , mitxonline_videos.video_edx_uuid - , mitxonline_videos.video_client_id - , mitxonline_videos.video_status - , coalesce(cast(ovs_videos.video_duration as decimal(38, 4)), mitxonline_videos.video_duration) as video_duration + mitxonline_coursevideos.courserun_readable_id, + mitxonline_coursevideos.coursevideo_is_hidden, + mitxonline_videos.video_edx_uuid, + mitxonline_videos.video_client_id, + mitxonline_videos.video_status, + coalesce(cast(ovs_videos.video_duration as decimal(38, 4)), mitxonline_videos.video_duration) as video_duration from mitxonline_coursevideos inner join mitxonline_videos on mitxonline_coursevideos.video_id = mitxonline_videos.video_id ---- the relationship between OVS videos and MITx Online videos are linked via video title and client video ID +-- - the relationship between OVS videos and MITx Online videos are linked via video title and client video ID -- e.g. 14.310x_Lect9_Seg8_Final.mp4. If there are multiple videos with same title on OVS, use the latest one. left join ovs_videos on mitxonline_videos.video_client_id = ovs_videos.video_title and ovs_videos.video_rank = 1 diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserunenrollments.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserunenrollments.sql index 1136c827c..b0d8abcfe 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserunenrollments.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserunenrollments.sql @@ -3,136 +3,101 @@ -- or MicroMasters. We migrated some DEDP orders from MicroMasters for those who have accounts on MITx Online, but due -- to migration timing issue (orders could be modified on MM after migration), we should use MM orders to verify -- enrollments for DEDP runs in '3T2021', '1T2022', '2T2022' - -with enrollments as ( - select * from {{ ref('stg__mitxonline__app__postgres__courses_courserunenrollment') }} -) - -, mitxonline_users as ( - select * from {{ ref('int__mitxonline__users') }} -) - -, mitxonline_runs as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, mitxonline_programs as ( - select * from {{ ref('int__mitxonline__program_requirements') }} -) - -, mitxonline_orders as ( - select * from {{ ref('int__mitxonline__ecommerce_order') }} -) - -, micromasters_orders as ( - select * from {{ ref('stg__micromasters__app__postgres__ecommerce_order') }} -) - -, micromasters_lines as ( - select * from {{ ref('stg__micromasters__app__postgres__ecommerce_line') }} -) - -, micromasters_users as ( - select * from {{ ref('__micromasters__users') }} -) - -, dedp_enrollments_verified_in_micromasters as ( - select distinct - enrollments.user_id - , enrollments.courserun_id - from enrollments - inner join mitxonline_runs on enrollments.courserun_id = mitxonline_runs.courserun_id - inner join mitxonline_programs on mitxonline_runs.course_id = mitxonline_programs.course_id - inner join mitxonline_users on enrollments.user_id = mitxonline_users.user_id - inner join micromasters_users - on mitxonline_users.user_micromasters_profile_id = micromasters_users.user_profile_id - inner join micromasters_orders on micromasters_users.user_id = micromasters_orders.user_id - inner join micromasters_lines - on - mitxonline_runs.courserun_readable_id = micromasters_lines.courserun_readable_id +with + enrollments as (select * from {{ ref("stg__mitxonline__app__postgres__courses_courserunenrollment") }}), + mitxonline_users as (select * from {{ ref("int__mitxonline__users") }}), + mitxonline_runs as (select * from {{ ref("int__mitxonline__course_runs") }}), + mitxonline_programs as (select * from {{ ref("int__mitxonline__program_requirements") }}), + mitxonline_orders as (select * from {{ ref("int__mitxonline__ecommerce_order") }}), + micromasters_orders as (select * from {{ ref("stg__micromasters__app__postgres__ecommerce_order") }}), + micromasters_lines as (select * from {{ ref("stg__micromasters__app__postgres__ecommerce_line") }}), + micromasters_users as (select * from {{ ref("__micromasters__users") }}), + dedp_enrollments_verified_in_micromasters as ( + select distinct enrollments.user_id, enrollments.courserun_id + from enrollments + inner join mitxonline_runs on enrollments.courserun_id = mitxonline_runs.courserun_id + inner join mitxonline_programs on mitxonline_runs.course_id = mitxonline_programs.course_id + inner join mitxonline_users on enrollments.user_id = mitxonline_users.user_id + inner join + micromasters_users on mitxonline_users.user_micromasters_profile_id = micromasters_users.user_profile_id + inner join micromasters_orders on micromasters_users.user_id = micromasters_orders.user_id + inner join + micromasters_lines + on mitxonline_runs.courserun_readable_id = micromasters_lines.courserun_readable_id and micromasters_orders.order_id = micromasters_lines.order_id - where - mitxonline_runs.courserun_tag in ('2T2022', '1T2022', '3T2021') - and mitxonline_programs.program_id in ( - {{ var("dedp_mitxonline_international_development_program_id") }} - , {{ var("dedp_mitxonline_public_policy_program_id") }} - ) - and micromasters_orders.order_state = 'fulfilled' -) - -, dedp_enrollments_verified_in_mitxonline as ( - select distinct - enrollments.user_id - , enrollments.courserun_id - from enrollments - inner join mitxonline_orders - on - enrollments.user_id = mitxonline_orders.user_id + where + mitxonline_runs.courserun_tag in ('2T2022', '1T2022', '3T2021') + and mitxonline_programs.program_id in ( + {{ var("dedp_mitxonline_international_development_program_id") }}, + {{ var("dedp_mitxonline_public_policy_program_id") }} + ) + and micromasters_orders.order_state = 'fulfilled' + ), + dedp_enrollments_verified_in_mitxonline as ( + select distinct enrollments.user_id, enrollments.courserun_id + from enrollments + inner join + mitxonline_orders + on enrollments.user_id = mitxonline_orders.user_id and enrollments.courserun_id = mitxonline_orders.courserun_id - inner join mitxonline_runs on enrollments.courserun_id = mitxonline_runs.courserun_id - inner join mitxonline_programs on mitxonline_runs.course_id = mitxonline_programs.course_id - where - mitxonline_runs.courserun_tag not in ('2T2022', '1T2022', '3T2021') - and mitxonline_programs.program_id in ( - {{ var("dedp_mitxonline_international_development_program_id") }} - , {{ var("dedp_mitxonline_public_policy_program_id") }} - ) - and mitxonline_orders.order_state = 'fulfilled' - -) - -, dedp_enrollments_verified as ( - select - user_id - , courserun_id - from dedp_enrollments_verified_in_micromasters - - union distinct - - select - user_id - , courserun_id - from dedp_enrollments_verified_in_mitxonline -) - -, mitxonline_enrollments as ( - select - enrollments.courserunenrollment_id - , enrollments.courserunenrollment_is_active - , enrollments.user_id - , enrollments.courserun_id - , enrollments.courserunenrollment_created_on - , enrollments.courserunenrollment_enrollment_status - , enrollments.courserunenrollment_is_edx_enrolled - , mitxonline_runs.courserun_platform as courserunenrollment_platform - , mitxonline_runs.courserun_title - , mitxonline_runs.courserun_readable_id - , mitxonline_runs.course_number - , mitxonline_runs.course_id - , mitxonline_runs.courserun_start_on - , mitxonline_runs.courserun_upgrade_deadline - , mitxonline_users.user_username - , mitxonline_users.user_email - , mitxonline_users.user_edxorg_username - , mitxonline_users.user_full_name - , mitxonline_users.user_address_country - , case - when - dedp_enrollments_verified.user_id is not null - and dedp_enrollments_verified.courserun_id is not null - and enrollments.courserunenrollment_enrollment_status is null + inner join mitxonline_runs on enrollments.courserun_id = mitxonline_runs.courserun_id + inner join mitxonline_programs on mitxonline_runs.course_id = mitxonline_programs.course_id + where + mitxonline_runs.courserun_tag not in ('2T2022', '1T2022', '3T2021') + and mitxonline_programs.program_id in ( + {{ var("dedp_mitxonline_international_development_program_id") }}, + {{ var("dedp_mitxonline_public_policy_program_id") }} + ) + and mitxonline_orders.order_state = 'fulfilled' + + ), + dedp_enrollments_verified as ( + select user_id, courserun_id + from dedp_enrollments_verified_in_micromasters + + union distinct + + select user_id, courserun_id + from dedp_enrollments_verified_in_mitxonline + ), + mitxonline_enrollments as ( + select + enrollments.courserunenrollment_id, + enrollments.courserunenrollment_is_active, + enrollments.user_id, + enrollments.courserun_id, + enrollments.courserunenrollment_created_on, + enrollments.courserunenrollment_enrollment_status, + enrollments.courserunenrollment_is_edx_enrolled, + mitxonline_runs.courserun_platform as courserunenrollment_platform, + mitxonline_runs.courserun_title, + mitxonline_runs.courserun_readable_id, + mitxonline_runs.course_number, + mitxonline_runs.course_id, + mitxonline_runs.courserun_start_on, + mitxonline_runs.courserun_upgrade_deadline, + mitxonline_users.user_username, + mitxonline_users.user_email, + mitxonline_users.user_edxorg_username, + mitxonline_users.user_full_name, + mitxonline_users.user_address_country, + case + when + dedp_enrollments_verified.user_id is not null + and dedp_enrollments_verified.courserun_id is not null + and enrollments.courserunenrollment_enrollment_status is null then 'verified' - else enrollments.courserunenrollment_enrollment_mode - end as courserunenrollment_enrollment_mode - from enrollments - inner join mitxonline_runs on enrollments.courserun_id = mitxonline_runs.courserun_id - inner join mitxonline_users on enrollments.user_id = mitxonline_users.user_id - left join dedp_enrollments_verified - on - enrollments.user_id = dedp_enrollments_verified.user_id + else enrollments.courserunenrollment_enrollment_mode + end as courserunenrollment_enrollment_mode + from enrollments + inner join mitxonline_runs on enrollments.courserun_id = mitxonline_runs.courserun_id + inner join mitxonline_users on enrollments.user_id = mitxonline_users.user_id + left join + dedp_enrollments_verified + on enrollments.user_id = dedp_enrollments_verified.user_id and enrollments.courserun_id = dedp_enrollments_verified.courserun_id -) + ) select * from mitxonline_enrollments diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserunenrollments_with_programs.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserunenrollments_with_programs.sql index 4d97871f9..f422b9177 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserunenrollments_with_programs.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserunenrollments_with_programs.sql @@ -1,84 +1,74 @@ -with enrollments as ( - select * - from {{ ref('int__mitxonline__courserunenrollments') }} -) - -, program_requirements as ( - select * from {{ ref('int__mitx__program_requirements') }} -) - -, mitxonline_good_economics_for_hard_times_programs as ( - select * from {{ ref('__mitxonline_good_economics_for_hard_times_program') }} -) - -, programs as ( - select * from {{ ref('int__mitx__programs') }} -) - -, main_query as ( - select - enrollments.courserunenrollment_id - , enrollments.courserunenrollment_platform - , enrollments.courserunenrollment_is_active - , enrollments.courserunenrollment_created_on - , enrollments.courserunenrollment_enrollment_mode - , enrollments.courserunenrollment_enrollment_status - , enrollments.courserun_id - , enrollments.course_id - , enrollments.courserun_title - , enrollments.courserun_readable_id - , enrollments.course_number - , enrollments.user_id - , enrollments.user_email - , enrollments.user_full_name - , enrollments.user_username - , enrollments.user_edxorg_username - , enrollments.user_address_country - , enrollments.courserunenrollment_is_edx_enrolled - , program_requirements.micromasters_program_id - , program_requirements.mitxonline_program_id - , program_requirements.program_title - , programs.is_micromasters_program - , programs.is_dedp_program - from enrollments - inner join program_requirements on enrollments.course_number = program_requirements.course_number - inner join programs - on program_requirements.mitxonline_program_id = programs.mitxonline_program_id - where enrollments.course_number != '{{ "dedp_mitxonline_good_economics_for_hard_times_course_number" }}' -) - -, good_economics_for_hard_times_query as ( - select - enrollments.courserunenrollment_id - , enrollments.courserunenrollment_platform - , enrollments.courserunenrollment_is_active - , enrollments.courserunenrollment_created_on - , enrollments.courserunenrollment_enrollment_mode - , enrollments.courserunenrollment_enrollment_status - , enrollments.courserun_id - , enrollments.course_id - , enrollments.courserun_title - , enrollments.courserun_readable_id - , enrollments.course_number - , enrollments.user_id - , enrollments.user_email - , enrollments.user_full_name - , enrollments.user_username - , enrollments.user_edxorg_username - , enrollments.user_address_country - , enrollments.courserunenrollment_is_edx_enrolled - , programs.micromasters_program_id - , programs.mitxonline_program_id - , programs.program_title - , programs.is_micromasters_program - , programs.is_dedp_program - from enrollments - inner join mitxonline_good_economics_for_hard_times_programs - on enrollments.courserunenrollment_id = mitxonline_good_economics_for_hard_times_programs.courserunenrollment_id - inner join programs - on mitxonline_good_economics_for_hard_times_programs.program_id = programs.mitxonline_program_id - where enrollments.course_number = '{{ "dedp_mitxonline_good_economics_for_hard_times_course_number" }}' -) +with + enrollments as (select * from {{ ref("int__mitxonline__courserunenrollments") }}), + program_requirements as (select * from {{ ref("int__mitx__program_requirements") }}), + mitxonline_good_economics_for_hard_times_programs as ( + select * from {{ ref("__mitxonline_good_economics_for_hard_times_program") }} + ), + programs as (select * from {{ ref("int__mitx__programs") }}), + main_query as ( + select + enrollments.courserunenrollment_id, + enrollments.courserunenrollment_platform, + enrollments.courserunenrollment_is_active, + enrollments.courserunenrollment_created_on, + enrollments.courserunenrollment_enrollment_mode, + enrollments.courserunenrollment_enrollment_status, + enrollments.courserun_id, + enrollments.course_id, + enrollments.courserun_title, + enrollments.courserun_readable_id, + enrollments.course_number, + enrollments.user_id, + enrollments.user_email, + enrollments.user_full_name, + enrollments.user_username, + enrollments.user_edxorg_username, + enrollments.user_address_country, + enrollments.courserunenrollment_is_edx_enrolled, + program_requirements.micromasters_program_id, + program_requirements.mitxonline_program_id, + program_requirements.program_title, + programs.is_micromasters_program, + programs.is_dedp_program + from enrollments + inner join program_requirements on enrollments.course_number = program_requirements.course_number + inner join programs on program_requirements.mitxonline_program_id = programs.mitxonline_program_id + where enrollments.course_number != '{{ "dedp_mitxonline_good_economics_for_hard_times_course_number" }}' + ), + good_economics_for_hard_times_query as ( + select + enrollments.courserunenrollment_id, + enrollments.courserunenrollment_platform, + enrollments.courserunenrollment_is_active, + enrollments.courserunenrollment_created_on, + enrollments.courserunenrollment_enrollment_mode, + enrollments.courserunenrollment_enrollment_status, + enrollments.courserun_id, + enrollments.course_id, + enrollments.courserun_title, + enrollments.courserun_readable_id, + enrollments.course_number, + enrollments.user_id, + enrollments.user_email, + enrollments.user_full_name, + enrollments.user_username, + enrollments.user_edxorg_username, + enrollments.user_address_country, + enrollments.courserunenrollment_is_edx_enrolled, + programs.micromasters_program_id, + programs.mitxonline_program_id, + programs.program_title, + programs.is_micromasters_program, + programs.is_dedp_program + from enrollments + inner join + mitxonline_good_economics_for_hard_times_programs + on enrollments.courserunenrollment_id + = mitxonline_good_economics_for_hard_times_programs.courserunenrollment_id + inner join + programs on mitxonline_good_economics_for_hard_times_programs.program_id = programs.mitxonline_program_id + where enrollments.course_number = '{{ "dedp_mitxonline_good_economics_for_hard_times_course_number" }}' + ) select * from main_query diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courses.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courses.sql index 0e0841f50..b1de79666 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courses.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courses.sql @@ -1,79 +1,58 @@ -- Course information for MITx Online - -with courses as ( - select * - from {{ ref('stg__mitxonline__app__postgres__courses_course') }} -) - -, course_pages as ( - select * from {{ ref('stg__mitxonline__app__postgres__cms_coursepage') }} -) - -, wagtail_page as ( - select * from {{ ref('stg__mitxonline__app__postgres__cms_wagtail_page') }} -) - -, program_requirements as ( - select * from {{ ref('int__mitxonline__program_requirements') }} -) - -, programs as ( - select * from {{ ref('int__mitxonline__programs') }} -) - -, course_certification_type as ( - select * from ( - select - program_requirements.course_id - , program_requirements.program_id - , programs.program_certification_type - , row_number() over ( - partition by program_requirements.course_id - order by programs.program_is_micromasters - ) as row_num - from program_requirements - inner join programs on program_requirements.program_id = programs.program_id +with + courses as (select * from {{ ref("stg__mitxonline__app__postgres__courses_course") }}), + course_pages as (select * from {{ ref("stg__mitxonline__app__postgres__cms_coursepage") }}), + wagtail_page as (select * from {{ ref("stg__mitxonline__app__postgres__cms_wagtail_page") }}), + program_requirements as (select * from {{ ref("int__mitxonline__program_requirements") }}), + programs as (select * from {{ ref("int__mitxonline__programs") }}), + course_certification_type as ( + select * + from + ( + select + program_requirements.course_id, + program_requirements.program_id, + programs.program_certification_type, + row_number() over ( + partition by program_requirements.course_id order by programs.program_is_micromasters + ) as row_num + from program_requirements + inner join programs on program_requirements.program_id = programs.program_id + ) + where row_num = 1 + ), + course_topics as ( + select course_id, array_join(array_agg(coursetopic_name), ', ') as course_topics + from {{ ref("int__mitxonline__course_to_topics") }} + group by course_id + ), + course_instructors as ( + select course_id, array_join(array_agg(instructor_name), ', ') as course_instructors + from {{ ref("int__mitxonline__course_instructors") }} + group by course_id ) - where row_num = 1 -) - -, course_topics as ( - select - course_id - , array_join(array_agg(coursetopic_name), ', ') as course_topics - from {{ ref('int__mitxonline__course_to_topics') }} - group by course_id -) - -, course_instructors as ( - select - course_id - , array_join(array_agg(instructor_name), ', ') as course_instructors - from {{ ref('int__mitxonline__course_instructors') }} - group by course_id -) select - courses.course_id - , courses.course_title - , courses.course_is_live - , courses.course_readable_id - , courses.course_number - , course_pages.course_description - , course_pages.course_price - , course_pages.course_length - , course_pages.course_effort - , course_pages.course_prerequisites - , course_pages.course_about - , course_pages.course_what_you_learn - , course_topics.course_topics - , course_instructors.course_instructors - , wagtail_page.wagtail_page_slug as course_page_slug - , wagtail_page.wagtail_page_url_path as course_page_url_path - , wagtail_page.wagtail_page_is_live as course_page_is_live - , wagtail_page.wagtail_page_first_published_on as course_page_first_published_on - , wagtail_page.wagtail_page_last_published_on as course_page_last_published_on - , coalesce( + courses.course_id, + courses.course_title, + courses.course_is_live, + courses.course_readable_id, + courses.course_number, + course_pages.course_description, + course_pages.course_price, + course_pages.course_length, + course_pages.course_effort, + course_pages.course_prerequisites, + course_pages.course_about, + course_pages.course_what_you_learn, + course_topics.course_topics, + course_instructors.course_instructors, + wagtail_page.wagtail_page_slug as course_page_slug, + wagtail_page.wagtail_page_url_path as course_page_url_path, + wagtail_page.wagtail_page_is_live as course_page_is_live, + wagtail_page.wagtail_page_first_published_on as course_page_first_published_on, + wagtail_page.wagtail_page_last_published_on as course_page_last_published_on, + coalesce( course_certification_type.program_certification_type, 'Certificate of Completion' ) as course_certification_type from courses diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_basket.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_basket.sql index a989765f3..bd49f318a 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_basket.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_basket.sql @@ -1,11 +1,4 @@ -with source as ( - select * - from {{ ref('stg__mitxonline__app__postgres__ecommerce_basket') }} -) +with source as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_basket") }}) -select - basket_id - , user_id - , basket_created_on - , basket_updated_on +select basket_id, user_id, basket_created_on, basket_updated_on from source diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_basketdiscount.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_basketdiscount.sql index 0a1ca3e54..73c719eb4 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_basketdiscount.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_basketdiscount.sql @@ -1,14 +1,11 @@ -with source as ( - select * - from {{ ref('stg__mitxonline__app__postgres__ecommerce_basketdiscount') }} -) +with source as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_basketdiscount") }}) select - basketdiscount_id - , basketdiscount_created_on - , basketdiscount_updated_on - , user_id - , basketdiscount_applied_on - , basket_id - , discount_id + basketdiscount_id, + basketdiscount_created_on, + basketdiscount_updated_on, + user_id, + basketdiscount_applied_on, + basket_id, + discount_id from source diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_basketitem.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_basketitem.sql index 442432b9f..a0164d33e 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_basketitem.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_basketitem.sql @@ -1,13 +1,4 @@ -with source as ( - select * - from {{ ref('stg__mitxonline__app__postgres__ecommerce_basketitem') }} -) +with source as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_basketitem") }}) -select - basketitem_id - , basketitem_quantity - , basket_id - , basketitem_created_on - , product_id - , basketitem_updated_on +select basketitem_id, basketitem_quantity, basket_id, basketitem_created_on, product_id, basketitem_updated_on from source diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_discount.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_discount.sql index 80cb157b8..300ec78f9 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_discount.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_discount.sql @@ -1,17 +1,15 @@ -with discount as ( - select * from {{ ref('stg__mitxonline__app__postgres__ecommerce_discount') }} -) +with discount as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_discount") }}) select - discount_id - , discount_amount - , discount_created_on - , discount_updated_on - , discount_code - , discount_type - , discount_activated_on - , discount_expires_on - , discount_max_redemptions - , discount_redemption_type - , discount_source + discount_id, + discount_amount, + discount_created_on, + discount_updated_on, + discount_code, + discount_type, + discount_activated_on, + discount_expires_on, + discount_max_redemptions, + discount_redemption_type, + discount_source from discount diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_discountproduct.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_discountproduct.sql index 363ad28c5..e546fab79 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_discountproduct.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_discountproduct.sql @@ -1,11 +1,4 @@ -with discountproduct as ( - select * from {{ ref('stg__mitxonline__app__postgres__ecommerce_discountproduct') }} -) +with discountproduct as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_discountproduct") }}) -select - discountproduct_id - , discountproduct_created_on - , discountproduct_updated_on - , product_id - , discount_id +select discountproduct_id, discountproduct_created_on, discountproduct_updated_on, product_id, discount_id from discountproduct diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_discountredemption.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_discountredemption.sql index e5d42c503..aaaf5997c 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_discountredemption.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_discountredemption.sql @@ -1,11 +1,4 @@ -with discountredemption as ( - select * from {{ ref('stg__mitxonline__app__postgres__ecommerce_discountredemption') }} -) +with discountredemption as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_discountredemption") }}) -select - discountredemption_id - , user_id - , discountredemption_timestamp - , order_id - , discount_id +select discountredemption_id, user_id, discountredemption_timestamp, order_id, discount_id from discountredemption diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_order.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_order.sql index 31bfc25be..8d1b3a493 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_order.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_order.sql @@ -1,103 +1,70 @@ -with lines as ( - select * from {{ ref('stg__mitxonline__app__postgres__ecommerce_line') }} -) +with + lines as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_line") }}), + contenttypes as (select * from {{ ref("stg__mitxonline__app__postgres__django_contenttype") }}), + versions as ( + select * + from {{ ref("stg__mitxonline__app__postgres__reversion_version") }} + where + contenttype_id in ( + select contenttypes.contenttype_id + from contenttypes + where contenttypes.contenttype_full_name = 'ecommerce_product' + ) + ), + orders as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_order") }}), + users as (select * from {{ ref("int__mitxonline__users") }}), + intermediate_products_view as (select * from {{ ref("int__mitxonline__ecommerce_product") }}), + discounts as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_discount") }}), + -- -- this table doesn't have constraint so apply additional logic here as there should be only one discount + -- -- that's actually applied to an order + discountredemptions as ( + select *, row_number() over (partition by order_id order by discountredemption_timestamp desc) as row_num + from {{ ref("stg__mitxonline__app__postgres__ecommerce_discountredemption") }} -, contenttypes as ( - select * from {{ ref('stg__mitxonline__app__postgres__django_contenttype') }} -) - -, versions as ( - select * - from {{ ref('stg__mitxonline__app__postgres__reversion_version') }} - where - contenttype_id in ( - select contenttypes.contenttype_id - from - contenttypes - where contenttypes.contenttype_full_name = 'ecommerce_product' - ) -) - -, orders as ( - select * - from {{ ref('stg__mitxonline__app__postgres__ecommerce_order') }} -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} -) - -, intermediate_products_view as ( - select * from {{ ref('int__mitxonline__ecommerce_product') }} -) - -, discounts as ( - select * from {{ ref('stg__mitxonline__app__postgres__ecommerce_discount') }} -) - ----- this table doesn't have constraint so apply additional logic here as there should be only one discount ----- that's actually applied to an order - -, discountredemptions as ( - select - * - , row_number() over ( - partition by order_id - order by discountredemption_timestamp desc - ) as row_num - from {{ ref('stg__mitxonline__app__postgres__ecommerce_discountredemption') }} - -) - ----- small amounts of duplicated payments for the same order. For those, we pick the most recent one -, payments as ( - select - * - , row_number() over ( - partition by order_id - order by transaction_created_on desc - ) as row_num - from {{ ref('int__mitxonline__ecommerce_transaction') }} - where transaction_type = 'payment' -) + ), + -- -- small amounts of duplicated payments for the same order. For those, we pick the most recent one + payments as ( + select *, row_number() over (partition by order_id order by transaction_created_on desc) as row_num + from {{ ref("int__mitxonline__ecommerce_transaction") }} + where transaction_type = 'payment' + ) select - orders.order_id - , orders.order_state - , orders.order_created_on - , orders.order_reference_number - , orders.order_total_price_paid - , users.user_id - , users.user_username - , users.user_full_name - , users.user_email - , lines.line_id - , lines.product_version_id - , intermediate_products_view.product_price - , intermediate_products_view.product_type - , intermediate_products_view.product_id - , intermediate_products_view.courserun_id - , intermediate_products_view.programrun_id - , intermediate_products_view.courserun_readable_id - , intermediate_products_view.program_readable_id - , discounts.discount_source - , discounts.discount_redemption_type - , discounts.discount_code - , discounts.discount_amount_text - , payments.transaction_id - , payments.transaction_authorization_code as payment_authorization_code - , payments.transaction_payment_method as payment_method - , payments.transaction_readable_identifier as payment_transaction_id - , payments.transaction_reference_number as payment_req_reference_number - , payments.transaction_bill_to_address_state as payment_bill_to_address_state - , payments.transaction_bill_to_address_country as payment_bill_to_address_country - , case - when orders.order_state in ('fulfilled', 'refunded') - then discountredemptions.discountredemption_timestamp - end as discountredemption_timestamp - , case + orders.order_id, + orders.order_state, + orders.order_created_on, + orders.order_reference_number, + orders.order_total_price_paid, + users.user_id, + users.user_username, + users.user_full_name, + users.user_email, + lines.line_id, + lines.product_version_id, + intermediate_products_view.product_price, + intermediate_products_view.product_type, + intermediate_products_view.product_id, + intermediate_products_view.courserun_id, + intermediate_products_view.programrun_id, + intermediate_products_view.courserun_readable_id, + intermediate_products_view.program_readable_id, + discounts.discount_source, + discounts.discount_redemption_type, + discounts.discount_code, + discounts.discount_amount_text, + payments.transaction_id, + payments.transaction_authorization_code as payment_authorization_code, + payments.transaction_payment_method as payment_method, + payments.transaction_readable_identifier as payment_transaction_id, + payments.transaction_reference_number as payment_req_reference_number, + payments.transaction_bill_to_address_state as payment_bill_to_address_state, + payments.transaction_bill_to_address_country as payment_bill_to_address_country, + case + when orders.order_state in ('fulfilled', 'refunded') then discountredemptions.discountredemption_timestamp + end as discountredemption_timestamp, + case when discounts.discount_type = 'percent-off' - then cast(intermediate_products_view.product_price * (discounts.discount_amount / 100) as decimal(38, 2)) + then cast(intermediate_products_view.product_price * (discounts.discount_amount / 100) as decimal(38, 2)) else cast(discounts.discount_amount as decimal(38, 2)) end as discount_amount from lines @@ -105,8 +72,6 @@ inner join orders on lines.order_id = orders.order_id inner join users on orders.order_purchaser_user_id = users.user_id inner join versions on lines.product_version_id = versions.version_id inner join intermediate_products_view on versions.version_object_id = intermediate_products_view.product_id -left join payments - on orders.order_id = payments.order_id and payments.row_num = 1 -left join discountredemptions - on orders.order_id = discountredemptions.order_id and discountredemptions.row_num = 1 +left join payments on orders.order_id = payments.order_id and payments.row_num = 1 +left join discountredemptions on orders.order_id = discountredemptions.order_id and discountredemptions.row_num = 1 left join discounts on discountredemptions.discount_id = discounts.discount_id diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_product.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_product.sql index 764e044a0..401568e5d 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_product.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_product.sql @@ -1,54 +1,39 @@ -with products as ( - select * - from {{ ref('stg__mitxonline__app__postgres__ecommerce_product') }} -) - -, contenttypes as ( - select * - from {{ ref('stg__mitxonline__app__postgres__django_contenttype') }} -) - -, courseruns as ( - select * - from {{ ref('stg__mitxonline__app__postgres__courses_courserun') }} -) - -, programruns as ( - select * - from {{ ref('stg__mitxonline__app__postgres__courses_programrun') }} -) - -, programs as ( - select * from {{ ref('stg__mitxonline__app__postgres__courses_program') }} -) - -, product_subquery as ( - select - products.product_id - , products.product_price - , products.product_is_active - , products.product_created_on - , products.product_description - , case contenttypes.contenttype_full_name - when 'courses_courserun' then products.product_object_id - end as courserun_id - , case contenttypes.contenttype_full_name - when 'courses_programrun' then products.product_object_id - end as programrun_id - , case contenttypes.contenttype_full_name - when 'courses_courserun' then 'course run' - when 'courses_programrun' then 'program run' - end as product_type - from products - inner join contenttypes on products.contenttype_id = contenttypes.contenttype_id -) +with + products as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_product") }}), + contenttypes as (select * from {{ ref("stg__mitxonline__app__postgres__django_contenttype") }}), + courseruns as (select * from {{ ref("stg__mitxonline__app__postgres__courses_courserun") }}), + programruns as (select * from {{ ref("stg__mitxonline__app__postgres__courses_programrun") }}), + programs as (select * from {{ ref("stg__mitxonline__app__postgres__courses_program") }}), + product_subquery as ( + select + products.product_id, + products.product_price, + products.product_is_active, + products.product_created_on, + products.product_description, + case + contenttypes.contenttype_full_name when 'courses_courserun' then products.product_object_id + end as courserun_id, + case + contenttypes.contenttype_full_name when 'courses_programrun' then products.product_object_id + end as programrun_id, + case + contenttypes.contenttype_full_name + when 'courses_courserun' + then 'course run' + when 'courses_programrun' + then 'program run' + end as product_type + from products + inner join contenttypes on products.contenttype_id = contenttypes.contenttype_id + ) select - product_subquery.* - , courseruns.course_id - , programruns.program_id - , courseruns.courserun_readable_id - , programs.program_readable_id + product_subquery.*, + courseruns.course_id, + programruns.program_id, + courseruns.courserun_readable_id, + programs.program_readable_id from product_subquery left join courseruns on product_subquery.courserun_id = courseruns.courserun_id left join programruns on product_subquery.programrun_id = programruns.programrun_id diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_transaction.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_transaction.sql index 2f8c9c702..94444e817 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_transaction.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_transaction.sql @@ -1,42 +1,38 @@ -with transactions as ( - select * - from {{ ref('stg__mitxonline__app__postgres__ecommerce_transaction') }} -) +with transactions as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_transaction") }}) select - transaction_id - , transaction_data - , transaction_amount - , order_id - , transaction_created_on - , transaction_readable_identifier - , transaction_type - , cast(json_query(transaction_data, 'lax $.req_amount' omit quotes) as decimal(38, 2)) as transaction_payment_amount - , json_query(transaction_data, 'lax $.req_transaction_uuid' omit quotes) as transaction_uuid - , json_query(transaction_data, 'lax $.decision' omit quotes) as transaction_status - , json_query(transaction_data, 'lax $.req_payment_method' omit quotes) as transaction_payment_method - , json_query(transaction_data, 'lax $.req_reference_number' omit quotes) as transaction_reference_number - , json_query(transaction_data, 'lax $.req_bill_to_address_state' omit quotes) as transaction_bill_to_address_state - , json_query( - transaction_data, 'lax $.req_bill_to_address_country' omit quotes - ) as transaction_bill_to_address_country - , json_query(transaction_data, 'lax $.req_transaction_type' omit quotes) as transaction_req_type - , json_query(transaction_data, 'lax $.req_currency' omit quotes) as transaction_payment_currency - , json_query(transaction_data, 'lax $.req_bill_to_email' omit quotes) as transaction_payer_email - , json_query(transaction_data, 'lax $.req_card_number' omit quotes) as transaction_payment_card_number - , json_query(transaction_data, 'lax $.req_customer_ip_address' omit quotes) as transaction_payer_ip_address - , json_query(transaction_data, 'lax $.card_type_name' omit quotes) as transaction_payment_card_type - , concat( - json_query(transaction_data, 'lax $.req_bill_to_forename' omit quotes) - , ' ' - , json_query(transaction_data, 'lax $.req_bill_to_surname' omit quotes) - ) as transaction_payer_name - , coalesce( - json_query(transaction_data, 'lax $.auth_code' omit quotes) - , json_query(transaction_data, 'lax $.processorInformation.approvalCode' omit quotes) - ) as transaction_authorization_code - , coalesce( - json_query(transaction_data, 'lax $.signed_date_time' omit quotes) - , json_query(transaction_data, 'lax $.submitTimeUtc' omit quotes) + transaction_id, + transaction_data, + transaction_amount, + order_id, + transaction_created_on, + transaction_readable_identifier, + transaction_type, + cast(json_query(transaction_data, 'lax $.req_amount' omit quotes) as decimal(38, 2)) as transaction_payment_amount, + json_query(transaction_data, 'lax $.req_transaction_uuid' omit quotes) as transaction_uuid, + json_query(transaction_data, 'lax $.decision' omit quotes) as transaction_status, + json_query(transaction_data, 'lax $.req_payment_method' omit quotes) as transaction_payment_method, + json_query(transaction_data, 'lax $.req_reference_number' omit quotes) as transaction_reference_number, + json_query(transaction_data, 'lax $.req_bill_to_address_state' omit quotes) as transaction_bill_to_address_state, + json_query(transaction_data, 'lax $.req_bill_to_address_country' omit quotes) as transaction_bill_to_address_country + , + json_query(transaction_data, 'lax $.req_transaction_type' omit quotes) as transaction_req_type, + json_query(transaction_data, 'lax $.req_currency' omit quotes) as transaction_payment_currency, + json_query(transaction_data, 'lax $.req_bill_to_email' omit quotes) as transaction_payer_email, + json_query(transaction_data, 'lax $.req_card_number' omit quotes) as transaction_payment_card_number, + json_query(transaction_data, 'lax $.req_customer_ip_address' omit quotes) as transaction_payer_ip_address, + json_query(transaction_data, 'lax $.card_type_name' omit quotes) as transaction_payment_card_type, + concat( + json_query(transaction_data, 'lax $.req_bill_to_forename' omit quotes), + ' ', + json_query(transaction_data, 'lax $.req_bill_to_surname' omit quotes) + ) as transaction_payer_name, + coalesce( + json_query(transaction_data, 'lax $.auth_code' omit quotes), + json_query(transaction_data, 'lax $.processorInformation.approvalCode' omit quotes) + ) as transaction_authorization_code, + coalesce( + json_query(transaction_data, 'lax $.signed_date_time' omit quotes), + json_query(transaction_data, 'lax $.submitTimeUtc' omit quotes) ) as transaction_timestamp from transactions diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_userdiscount.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_userdiscount.sql index 7810abe1a..2d34c5b03 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_userdiscount.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_userdiscount.sql @@ -1,11 +1,4 @@ -with userdiscount as ( - select * from {{ ref('stg__mitxonline__app__postgres__ecommerce_userdiscount') }} -) +with userdiscount as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_userdiscount") }}) -select - userdiscount_id - , userdiscount_created_on - , userdiscount_updated_on - , user_id - , discount_id +select userdiscount_id, userdiscount_created_on, userdiscount_updated_on, user_id, discount_id from userdiscount diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_countryincomethreshold.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_countryincomethreshold.sql index e7e9e0457..bcb39a645 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_countryincomethreshold.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_countryincomethreshold.sql @@ -1,11 +1,12 @@ -with countryincomethreshold as ( - select * from {{ ref('stg__mitxonline__app__postgres__flexiblepricing_countryincomethreshold') }} -) +with + countryincomethreshold as ( + select * from {{ ref("stg__mitxonline__app__postgres__flexiblepricing_countryincomethreshold") }} + ) select - countryincomethreshold_id - , countryincomethreshold_created_on - , countryincomethreshold_updated_on - , countryincomethreshold_country_code - , countryincomethreshold_income_threshold + countryincomethreshold_id, + countryincomethreshold_created_on, + countryincomethreshold_updated_on, + countryincomethreshold_country_code, + countryincomethreshold_income_threshold from countryincomethreshold diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_currencyexchangerate.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_currencyexchangerate.sql index 3c6dfdbd8..1700eb7d5 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_currencyexchangerate.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_currencyexchangerate.sql @@ -1,12 +1,10 @@ -with source as ( - select * from {{ ref('stg__mitxonline__app__postgres__flexiblepricing_currencyexchangerate') }} -) +with source as (select * from {{ ref("stg__mitxonline__app__postgres__flexiblepricing_currencyexchangerate") }}) select - currencyexchangerate_id - , currencyexchangerate_created_on - , currencyexchangerate_updated_on - , currencyexchangerate_description - , currencyexchangerate_currency_code - , currencyexchangerate_exchange_rate + currencyexchangerate_id, + currencyexchangerate_created_on, + currencyexchangerate_updated_on, + currencyexchangerate_description, + currencyexchangerate_currency_code, + currencyexchangerate_exchange_rate from source diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_flexiblepriceapplication.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_flexiblepriceapplication.sql index b24219119..718700504 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_flexiblepriceapplication.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_flexiblepriceapplication.sql @@ -1,58 +1,42 @@ -with flexiblepriceapplication as ( - select * from {{ ref('stg__mitxonline__app__postgres__flexiblepricing_flexiblepriceapplication') }} -) - -, flexiblepricetier as ( - select * from {{ ref('stg__mitxonline__app__postgres__flexiblepricing_flexiblepricetier') }} -) - -, discount as ( - select * from {{ ref('stg__mitxonline__app__postgres__ecommerce_discount') }} -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} -) - -, contenttypes as ( - select * - from {{ ref('stg__mitxonline__app__postgres__django_contenttype') }} -) +with + flexiblepriceapplication as ( + select * from {{ ref("stg__mitxonline__app__postgres__flexiblepricing_flexiblepriceapplication") }} + ), + flexiblepricetier as (select * from {{ ref("stg__mitxonline__app__postgres__flexiblepricing_flexiblepricetier") }}), + discount as (select * from {{ ref("stg__mitxonline__app__postgres__ecommerce_discount") }}), + users as (select * from {{ ref("int__mitxonline__users") }}), + contenttypes as (select * from {{ ref("stg__mitxonline__app__postgres__django_contenttype") }}) select - flexiblepriceapplication.flexiblepriceapplication_id - , flexiblepriceapplication.flexiblepriceapplication_status - , flexiblepriceapplication.flexiblepricetier_id - , flexiblepricetier.discount_id - , discount.discount_type - , discount.discount_amount - , flexiblepriceapplication.user_id - , users.user_username - , users.user_full_name - , users.user_email - , users.user_address_country - , flexiblepriceapplication.flexiblepriceapplication_created_on - , flexiblepriceapplication.flexiblepriceapplication_income_usd - , flexiblepriceapplication.flexiblepriceapplication_updated_on - , flexiblepriceapplication.flexiblepriceapplication_justification - , flexiblepriceapplication.flexiblepriceapplication_original_income - , flexiblepriceapplication.flexiblepriceapplication_country_of_income - , flexiblepriceapplication.flexiblepriceapplication_original_currency - , flexiblepriceapplication.flexiblepriceapplication_exchange_rate_timestamp - , flexiblepriceapplication.flexiblepriceapplication_date_documents_sent - , flexiblepriceapplication.flexiblepriceapplication_country_of_residence - - , case contenttypes.contenttype_full_name - when 'courses_course' then flexiblepriceapplication.courseware_object_id - end as course_id - - , case contenttypes.contenttype_full_name - when 'courses_program' then flexiblepriceapplication.courseware_object_id - end as program_id - - , case contenttypes.contenttype_full_name - when 'courses_course' then 'course' - when 'courses_program' then 'program' + flexiblepriceapplication.flexiblepriceapplication_id, + flexiblepriceapplication.flexiblepriceapplication_status, + flexiblepriceapplication.flexiblepricetier_id, + flexiblepricetier.discount_id, + discount.discount_type, + discount.discount_amount, + flexiblepriceapplication.user_id, + users.user_username, + users.user_full_name, + users.user_email, + users.user_address_country, + flexiblepriceapplication.flexiblepriceapplication_created_on, + flexiblepriceapplication.flexiblepriceapplication_income_usd, + flexiblepriceapplication.flexiblepriceapplication_updated_on, + flexiblepriceapplication.flexiblepriceapplication_justification, + flexiblepriceapplication.flexiblepriceapplication_original_income, + flexiblepriceapplication.flexiblepriceapplication_country_of_income, + flexiblepriceapplication.flexiblepriceapplication_original_currency, + flexiblepriceapplication.flexiblepriceapplication_exchange_rate_timestamp, + flexiblepriceapplication.flexiblepriceapplication_date_documents_sent, + flexiblepriceapplication.flexiblepriceapplication_country_of_residence, + case + contenttypes.contenttype_full_name when 'courses_course' then flexiblepriceapplication.courseware_object_id + end as course_id, + case + contenttypes.contenttype_full_name when 'courses_program' then flexiblepriceapplication.courseware_object_id + end as program_id, + case + contenttypes.contenttype_full_name when 'courses_course' then 'course' when 'courses_program' then 'program' end as courseware_type from flexiblepriceapplication inner join contenttypes on flexiblepriceapplication.contenttype_id = contenttypes.contenttype_id diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_flexiblepricetier.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_flexiblepricetier.sql index 498c5e910..76643236a 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_flexiblepricetier.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__flexiblepricing_flexiblepricetier.sql @@ -1,15 +1,13 @@ -with source as ( - select * from {{ ref('stg__mitxonline__app__postgres__flexiblepricing_flexiblepricetier') }} -) +with source as (select * from {{ ref("stg__mitxonline__app__postgres__flexiblepricing_flexiblepricetier") }}) select - flexiblepricetier_id - , flexiblepricetier_is_current - , flexiblepricetier_created_on - , flexiblepricetier_updated_on - , discount_id - , courseware_object_id - , flexiblepricetier_income_threshold_usd - , contenttype_id + flexiblepricetier_id, + flexiblepricetier_is_current, + flexiblepricetier_created_on, + flexiblepricetier_updated_on, + discount_id, + courseware_object_id, + flexiblepricetier_income_threshold_usd, + contenttype_id from source diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__proctored_exam_grades.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__proctored_exam_grades.sql index 470153fba..0ca026443 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__proctored_exam_grades.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__proctored_exam_grades.sql @@ -1,59 +1,41 @@ -with exam_unit_grades as ( - select * from {{ ref('int__mitxonline__courserun_subsection_grades') }} - where - lower(coursestructure_block_title) = 'proctored exam' - and subsectiongrade_total_graded_score > 0 -) - -, exam_courserun_grades as ( - select * from {{ ref('int__mitxonline__courserun_grades') }} - where lower(courserun_title) like '%proctored exam' -) - -, exam_courseruns_from_micromasters as ( - --- pulling exam courseruns from MicroMasters to associate a exam run with a course - --- since all these MITxT exam runs and their associations to courses exist in MicroMasters. - select - examrun_readable_id - , examrun_semester - , examrun_passing_grade - , course_id - from {{ ref('stg__micromasters__app__postgres__exams_examrun') }} -) - -, courses_from_micromasters as ( - select * from {{ ref('stg__micromasters__app__postgres__courses_course') }} -) - -, courseruns_from_mitxonline as ( - select * from {{ ref('stg__mitxonline__app__postgres__courses_courserun') }} -) - -, courses_from_mitxonline as ( - select * from {{ ref('stg__mitxonline__app__postgres__courses_course') }} -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} -) +with + exam_unit_grades as ( + select * + from {{ ref("int__mitxonline__courserun_subsection_grades") }} + where lower(coursestructure_block_title) = 'proctored exam' and subsectiongrade_total_graded_score > 0 + ), + exam_courserun_grades as ( + select * from {{ ref("int__mitxonline__courserun_grades") }} where lower(courserun_title) like '%proctored exam' + ), + exam_courseruns_from_micromasters as ( + -- - pulling exam courseruns from MicroMasters to associate a exam run with a course + -- - since all these MITxT exam runs and their associations to courses exist in MicroMasters. + select examrun_readable_id, examrun_semester, examrun_passing_grade, course_id + from {{ ref("stg__micromasters__app__postgres__exams_examrun") }} + ), + courses_from_micromasters as (select * from {{ ref("stg__micromasters__app__postgres__courses_course") }}), + courseruns_from_mitxonline as (select * from {{ ref("stg__mitxonline__app__postgres__courses_courserun") }}), + courses_from_mitxonline as (select * from {{ ref("stg__mitxonline__app__postgres__courses_course") }}), + users as (select * from {{ ref("int__mitxonline__users") }}) select - courses_from_mitxonline.course_number - , courses_from_mitxonline.course_title - , exam_courseruns_from_micromasters.examrun_semester as semester - , exam_courserun_grades.courserun_readable_id - , users.user_id - , users.openedx_user_id - , users.user_edxorg_username - , users.user_username - , users.user_full_name - , users.user_email - , exam_courseruns_from_micromasters.examrun_passing_grade as proctoredexamgrade_passing_grade - , exam_courserun_grades.courserungrade_grade as proctoredexamgrade_grade - , exam_courserun_grades.courserungrade_created_on as proctoredexamgrade_created_on + courses_from_mitxonline.course_number, + courses_from_mitxonline.course_title, + exam_courseruns_from_micromasters.examrun_semester as semester, + exam_courserun_grades.courserun_readable_id, + users.user_id, + users.openedx_user_id, + users.user_edxorg_username, + users.user_username, + users.user_full_name, + users.user_email, + exam_courseruns_from_micromasters.examrun_passing_grade as proctoredexamgrade_passing_grade, + exam_courserun_grades.courserungrade_grade as proctoredexamgrade_grade, + exam_courserun_grades.courserungrade_created_on as proctoredexamgrade_created_on from exam_courserun_grades inner join users on exam_courserun_grades.user_id = users.user_id -left join exam_courseruns_from_micromasters +left join + exam_courseruns_from_micromasters on exam_courserun_grades.courserun_readable_id = exam_courseruns_from_micromasters.examrun_readable_id left join courses_from_micromasters on exam_courseruns_from_micromasters.course_id = courses_from_micromasters.course_id left join courses_from_mitxonline on courses_from_micromasters.course_number = courses_from_mitxonline.course_number @@ -61,28 +43,28 @@ left join courses_from_mitxonline on courses_from_micromasters.course_number = c union all select - courses_from_mitxonline.course_number - , courses_from_mitxonline.course_title - , courseruns_from_mitxonline.courserun_tag as semester - , exam_unit_grades.courserun_readable_id - , users.user_id - , users.openedx_user_id - , users.user_edxorg_username - , users.user_username - , users.user_full_name - , users.user_email - , null as proctoredexamgrade_passing_grade - , exam_unit_grades.subsectiongrade_total_earned_graded_score / exam_unit_grades.subsectiongrade_total_graded_score - as proctoredexamgrade_grade - , exam_unit_grades.subsectiongrade_created_on as proctoredexamgrade_created_on + courses_from_mitxonline.course_number, + courses_from_mitxonline.course_title, + courseruns_from_mitxonline.courserun_tag as semester, + exam_unit_grades.courserun_readable_id, + users.user_id, + users.openedx_user_id, + users.user_edxorg_username, + users.user_username, + users.user_full_name, + users.user_email, + null as proctoredexamgrade_passing_grade, + exam_unit_grades.subsectiongrade_total_earned_graded_score + / exam_unit_grades.subsectiongrade_total_graded_score as proctoredexamgrade_grade, + exam_unit_grades.subsectiongrade_created_on as proctoredexamgrade_created_on from exam_unit_grades inner join users on exam_unit_grades.openedx_user_id = users.openedx_user_id inner join courseruns_from_mitxonline on exam_unit_grades.courserun_readable_id = courseruns_from_mitxonline.courserun_readable_id inner join courses_from_mitxonline on courseruns_from_mitxonline.course_id = courses_from_mitxonline.course_id -left join exam_courserun_grades - on - users.user_username = exam_courserun_grades.user_username - and exam_unit_grades.courserun_readable_id = exam_courserun_grades.courserun_readable_id +left join + exam_courserun_grades + on users.user_username = exam_courserun_grades.user_username + and exam_unit_grades.courserun_readable_id = exam_courserun_grades.courserun_readable_id where exam_courserun_grades.courserungrade_id is null diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__program_certificates.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__program_certificates.sql index e431a2042..e4acf4b86 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__program_certificates.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__program_certificates.sql @@ -1,43 +1,35 @@ -- Program Certificate information for MITx Online +with + certificates as (select * from {{ ref("stg__mitxonline__app__postgres__courses_programcertificate") }}), + programs as (select * from {{ ref("stg__mitxonline__app__postgres__courses_program") }}), + users as (select * from {{ ref("int__mitxonline__users") }}), + program_certificates as ( + select + certificates.programcertificate_id, + certificates.programcertificate_uuid, + certificates.program_id, + programs.program_title, + programs.program_readable_id, + programs.program_type, + programs.program_is_dedp, + programs.program_is_micromasters, + certificates.programcertificate_is_revoked, + certificates.programcertificate_created_on, + certificates.programcertificate_updated_on, + certificates.user_id, + users.user_username, + users.user_edxorg_username, + users.user_full_name, + users.user_email, + if( + certificates.programcertificate_is_revoked = false, + concat('https://mitxonline.mit.edu/certificate/program/', certificates.programcertificate_uuid), + null + ) as programcertificate_url + from certificates + inner join programs on certificates.program_id = programs.program_id + inner join users on certificates.user_id = users.user_id + ) -with certificates as ( - select * from {{ ref('stg__mitxonline__app__postgres__courses_programcertificate') }} -) - -, programs as ( - select * from {{ ref('stg__mitxonline__app__postgres__courses_program') }} -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} -) - -, program_certificates as ( - select - certificates.programcertificate_id - , certificates.programcertificate_uuid - , certificates.program_id - , programs.program_title - , programs.program_readable_id - , programs.program_type - , programs.program_is_dedp - , programs.program_is_micromasters - , certificates.programcertificate_is_revoked - , certificates.programcertificate_created_on - , certificates.programcertificate_updated_on - , certificates.user_id - , users.user_username - , users.user_edxorg_username - , users.user_full_name - , users.user_email - , if( - certificates.programcertificate_is_revoked = false - , concat('https://mitxonline.mit.edu/certificate/program/', certificates.programcertificate_uuid) - , null - ) as programcertificate_url - from certificates - inner join programs on certificates.program_id = programs.program_id - inner join users on certificates.user_id = users.user_id -) - -select * from program_certificates +select * +from program_certificates diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__program_instructors.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__program_instructors.sql index 0204f6af4..bf3001d96 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__program_instructors.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__program_instructors.sql @@ -1,21 +1,14 @@ -with instructors as ( - select * from {{ ref('stg__mitxonline__app__postgres__cms_instructorpage') }} -) - -, instructor_pagelinks as ( - select * from {{ ref('stg__mitxonline__app__postgres__cms_instructorpagelink') }} -) - -, program_pages as ( - select * from {{ ref('stg__mitxonline__app__postgres__cms_programpage') }} -) +with + instructors as (select * from {{ ref("stg__mitxonline__app__postgres__cms_instructorpage") }}), + instructor_pagelinks as (select * from {{ ref("stg__mitxonline__app__postgres__cms_instructorpagelink") }}), + program_pages as (select * from {{ ref("stg__mitxonline__app__postgres__cms_programpage") }}) select - program_pages.program_id - , instructors.instructor_name - , instructors.instructor_title - , instructors.instructor_bio_short - , instructors.instructor_bio_long + program_pages.program_id, + instructors.instructor_name, + instructors.instructor_title, + instructors.instructor_bio_short, + instructors.instructor_bio_long from program_pages inner join instructor_pagelinks on program_pages.wagtail_page_id = instructor_pagelinks.wagtail_page_id inner join instructors on instructor_pagelinks.instructor_wagtail_page_id = instructors.wagtail_page_id diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__program_requirements.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__program_requirements.sql index 1845e991b..5f76284b1 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__program_requirements.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__program_requirements.sql @@ -1,124 +1,99 @@ -- Program Requirement for MITx Online -with program_requirements as ( - select * - from {{ ref('stg__mitxonline__app__postgres__courses_programrequirement') }} -) - -, operator_nodes as ( - select * - from program_requirements - where programrequirement_node_type = 'operator' -) - -, course_nodes as ( - select * - from program_requirements - where programrequirement_node_type = 'course' -) - -, all_requirements as ( - select - operator_nodes.programrequirement_id as programrequirement_requirement_id - , course_nodes.course_id - , course_nodes.program_id - , operator_nodes.programrequirement_title - , operator_nodes.programrequirement_path - , operator_nodes.programrequirement_operator_value - as electiveset_required_number - , case operator_nodes.programrequirement_operator - when 'min_number_of' then 'Elective' - when 'all_of' then 'Core' - end - as programrequirement_type - - , row_number() - over ( +with + program_requirements as (select * from {{ ref("stg__mitxonline__app__postgres__courses_programrequirement") }}), + operator_nodes as (select * from program_requirements where programrequirement_node_type = 'operator'), + course_nodes as (select * from program_requirements where programrequirement_node_type = 'course'), + all_requirements as ( + select + operator_nodes.programrequirement_id as programrequirement_requirement_id, + course_nodes.course_id, + course_nodes.program_id, + operator_nodes.programrequirement_title, + operator_nodes.programrequirement_path, + operator_nodes.programrequirement_operator_value as electiveset_required_number, + case + operator_nodes.programrequirement_operator + when 'min_number_of' + then 'Elective' + when 'all_of' + then 'Core' + end as programrequirement_type, + row_number() over ( partition by course_nodes.course_id, course_nodes.program_id order by length(operator_nodes.programrequirement_path) desc - ) - as requirement_nesting - from course_nodes - left join operator_nodes - on course_nodes.programrequirement_path like operator_nodes.programrequirement_path || '%' -) - -, child_requirements as ( - select - programrequirement_requirement_id - , course_id - , program_id - , programrequirement_type - , programrequirement_title - , electiveset_required_number - , programrequirement_path - from all_requirements - where all_requirements.requirement_nesting = 1 -) - -, parent_requirements as ( - select - programrequirement_requirement_id - , course_id - , program_id - , programrequirement_type - , programrequirement_title - , electiveset_required_number - , programrequirement_path - from all_requirements - where all_requirements.requirement_nesting = 2 -) - -, combined_requirements as ( - select - child_requirements.programrequirement_requirement_id - , child_requirements.course_id - , child_requirements.program_id - , child_requirements.programrequirement_type - , child_requirements.programrequirement_title - , child_requirements.electiveset_required_number - , parent_requirements.programrequirement_requirement_id as programrequirement_parent_requirement_id - , parent_requirements.programrequirement_requirement_id is not null - as programrequirement_is_a_nested_requirement - from child_requirements - left join parent_requirements - on - child_requirements.programrequirement_path like parent_requirements.programrequirement_path || '%' + ) as requirement_nesting + from course_nodes + left join + operator_nodes on course_nodes.programrequirement_path like operator_nodes.programrequirement_path || '%' + ), + child_requirements as ( + select + programrequirement_requirement_id, + course_id, + program_id, + programrequirement_type, + programrequirement_title, + electiveset_required_number, + programrequirement_path + from all_requirements + where all_requirements.requirement_nesting = 1 + ), + parent_requirements as ( + select + programrequirement_requirement_id, + course_id, + program_id, + programrequirement_type, + programrequirement_title, + electiveset_required_number, + programrequirement_path + from all_requirements + where all_requirements.requirement_nesting = 2 + ), + combined_requirements as ( + select + child_requirements.programrequirement_requirement_id, + child_requirements.course_id, + child_requirements.program_id, + child_requirements.programrequirement_type, + child_requirements.programrequirement_title, + child_requirements.electiveset_required_number, + parent_requirements.programrequirement_requirement_id as programrequirement_parent_requirement_id, + parent_requirements.programrequirement_requirement_id + is not null as programrequirement_is_a_nested_requirement + from child_requirements + left join + parent_requirements + on child_requirements.programrequirement_path like parent_requirements.programrequirement_path || '%' and child_requirements.course_id = parent_requirements.course_id and child_requirements.program_id = parent_requirements.program_id -) - -, core_courses_count as ( - select - program_id - , count_if(programrequirement_type = 'Core') as program_num_core_courses - from combined_requirements - group by program_id -) - -, elective_courses_count as ( - select - program_id - , sum(electiveset_required_number) as program_num_elective_courses - from ( - select - combined_requirements.program_id - , combined_requirements.programrequirement_requirement_id - , avg(combined_requirements.electiveset_required_number) as electiveset_required_number + ), + core_courses_count as ( + select program_id, count_if(programrequirement_type = 'Core') as program_num_core_courses from combined_requirements - where - combined_requirements.programrequirement_type = 'Elective' - and combined_requirements.programrequirement_is_a_nested_requirement = false - group by - combined_requirements.program_id - , combined_requirements.programrequirement_requirement_id - ) - group by program_id + group by program_id + ), + elective_courses_count as ( + select program_id, sum(electiveset_required_number) as program_num_elective_courses + from + ( + select + combined_requirements.program_id, + combined_requirements.programrequirement_requirement_id, + avg(combined_requirements.electiveset_required_number) as electiveset_required_number + from combined_requirements + where + combined_requirements.programrequirement_type = 'Elective' + and combined_requirements.programrequirement_is_a_nested_requirement = false + group by combined_requirements.program_id, combined_requirements.programrequirement_requirement_id + ) + group by program_id -) + ) select - combined_requirements.* - , coalesce(core_courses_count.program_num_core_courses, 0) + combined_requirements.*, + coalesce(core_courses_count.program_num_core_courses, 0) + coalesce(elective_courses_count.program_num_elective_courses, 0) as program_num_required_courses from combined_requirements left join core_courses_count on combined_requirements.program_id = core_courses_count.program_id diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__programenrollments.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__programenrollments.sql index 424641982..0d2d9a0b7 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__programenrollments.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__programenrollments.sql @@ -1,39 +1,24 @@ -- Enrollment information for MITx Online +with + enrollments as (select * from {{ ref("stg__mitxonline__app__postgres__courses_programenrollment") }}), + programs as (select program_id, program_title from {{ ref("stg__mitxonline__app__postgres__courses_program") }}), + users as (select user_id, user_username, user_email from {{ ref("int__mitxonline__users") }}), + mitxonline_enrollments as ( + select + enrollments.programenrollment_id, + enrollments.programenrollment_is_active, + enrollments.user_id, + enrollments.program_id, + enrollments.programenrollment_created_on, + enrollments.programenrollment_enrollment_mode, + enrollments.programenrollment_enrollment_status, + programs.program_title, + users.user_username, + users.user_email + from enrollments + inner join programs on enrollments.program_id = programs.program_id + inner join users on enrollments.user_id = users.user_id + ) -with enrollments as ( - select * from {{ ref('stg__mitxonline__app__postgres__courses_programenrollment') }} -) - -, programs as ( - select - program_id - , program_title - from {{ ref('stg__mitxonline__app__postgres__courses_program') }} -) - -, users as ( - select - user_id - , user_username - , user_email - from {{ ref('int__mitxonline__users') }} -) - -, mitxonline_enrollments as ( - select - enrollments.programenrollment_id - , enrollments.programenrollment_is_active - , enrollments.user_id - , enrollments.program_id - , enrollments.programenrollment_created_on - , enrollments.programenrollment_enrollment_mode - , enrollments.programenrollment_enrollment_status - , programs.program_title - , users.user_username - , users.user_email - from enrollments - inner join programs on enrollments.program_id = programs.program_id - inner join users on enrollments.user_id = users.user_id -) - -select * from mitxonline_enrollments +select * +from mitxonline_enrollments diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__programs.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__programs.sql index 853b01043..7da5f91a0 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__programs.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__programs.sql @@ -1,71 +1,50 @@ -- Program information for MITx Online - -with programs as ( - select * from {{ ref('stg__mitxonline__app__postgres__courses_program') }} -) - -, program_pages as ( - select * from {{ ref('stg__mitxonline__app__postgres__cms_programpage') }} -) - -, program_requirements as ( - select - program_id - , course_id - from {{ ref('int__mitxonline__program_requirements') }} -) - -, wagtail_page as ( - select * from {{ ref('stg__mitxonline__app__postgres__cms_wagtail_page') }} -) - -, course_topics as ( - select * from {{ ref('int__mitxonline__course_to_topics') }} -) - -, program_topics as ( - select - program_requirements.program_id - , array_join(array_distinct(array_agg(course_topics.coursetopic_name)), ', ') as program_topics - from course_topics - inner join program_requirements on course_topics.course_id = program_requirements.course_id - group by program_requirements.program_id -) - -, program_instructors as ( - select - program_id - , array_join(array_agg(instructor_name), ', ') as program_instructors - from {{ ref('int__mitxonline__program_instructors') }} - group by program_id -) +with + programs as (select * from {{ ref("stg__mitxonline__app__postgres__courses_program") }}), + program_pages as (select * from {{ ref("stg__mitxonline__app__postgres__cms_programpage") }}), + program_requirements as (select program_id, course_id from {{ ref("int__mitxonline__program_requirements") }}), + wagtail_page as (select * from {{ ref("stg__mitxonline__app__postgres__cms_wagtail_page") }}), + course_topics as (select * from {{ ref("int__mitxonline__course_to_topics") }}), + program_topics as ( + select + program_requirements.program_id, + array_join(array_distinct(array_agg(course_topics.coursetopic_name)), ', ') as program_topics + from course_topics + inner join program_requirements on course_topics.course_id = program_requirements.course_id + group by program_requirements.program_id + ), + program_instructors as ( + select program_id, array_join(array_agg(instructor_name), ', ') as program_instructors + from {{ ref("int__mitxonline__program_instructors") }} + group by program_id + ) select - programs.program_id - , programs.program_title - , programs.program_name - , programs.program_track - , programs.program_is_live - , programs.program_readable_id - , programs.program_type - , programs.program_availability - , programs.program_certification_type - , programs.program_is_dedp - , programs.program_is_micromasters - , program_pages.program_description - , program_pages.program_price - , program_pages.program_length - , program_pages.program_effort - , program_pages.program_prerequisites - , program_pages.program_about - , program_pages.program_what_you_learn - , program_topics.program_topics - , program_instructors.program_instructors - , wagtail_page.wagtail_page_slug as program_page_slug - , wagtail_page.wagtail_page_url_path as program_page_url_path - , wagtail_page.wagtail_page_is_live as program_page_is_live - , wagtail_page.wagtail_page_first_published_on as program_page_first_published_on - , wagtail_page.wagtail_page_last_published_on as program_page_last_published_on + programs.program_id, + programs.program_title, + programs.program_name, + programs.program_track, + programs.program_is_live, + programs.program_readable_id, + programs.program_type, + programs.program_availability, + programs.program_certification_type, + programs.program_is_dedp, + programs.program_is_micromasters, + program_pages.program_description, + program_pages.program_price, + program_pages.program_length, + program_pages.program_effort, + program_pages.program_prerequisites, + program_pages.program_about, + program_pages.program_what_you_learn, + program_topics.program_topics, + program_instructors.program_instructors, + wagtail_page.wagtail_page_slug as program_page_slug, + wagtail_page.wagtail_page_url_path as program_page_url_path, + wagtail_page.wagtail_page_is_live as program_page_is_live, + wagtail_page.wagtail_page_first_published_on as program_page_first_published_on, + wagtail_page.wagtail_page_last_published_on as program_page_last_published_on from programs left join program_pages on programs.program_id = program_pages.program_id left join wagtail_page on program_pages.wagtail_page_id = wagtail_page.wagtail_page_id diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__uai_customer_courserun_mapping.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__uai_customer_courserun_mapping.sql index 6c4244fd0..5164a5c9f 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__uai_customer_courserun_mapping.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__uai_customer_courserun_mapping.sql @@ -1,11 +1,7 @@ -with courseruns as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) +with courseruns as (select * from {{ ref("int__mitxonline__course_runs") }}) select - courserun_readable_id - , split_part(replace(replace(courserun_readable_id, 'course-v1:', ''), 'UAI_', ''), '+', 1) as organization + courserun_readable_id, + split_part(replace(replace(courserun_readable_id, 'course-v1:', ''), 'UAI_', ''), '+', 1) as organization from courseruns -where - courserun_readable_id like 'course-v1:UAI_%' - and courserun_readable_id not like 'course-v1:UAI_SOURCE%' +where courserun_readable_id like 'course-v1:UAI_%' and courserun_readable_id not like 'course-v1:UAI_SOURCE%' diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivities.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivities.sql index 819d84e1f..9148412ff 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivities.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivities.sql @@ -1,103 +1,87 @@ -with course_activities as ( - select * from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} -) - -, course_activities_video as ( - select * from {{ ref('int__mitxonline__user_courseactivity_video') }} -) - -, problem_check as ( - select * from {{ ref('int__mitxonline__user_courseactivity_problemcheck') }} -) - -, course_state as ( - select * from {{ ref('stg__mitxonline__openedx__mysql__courseware_studentmodule') }} -) - -, course_structure as ( - select * from {{ ref('int__mitxonline__course_structure') }} -) - -, course_chapters_stats as ( - select - course_state.openedx_user_id - , course_state.courserun_readable_id - , count(distinct course_structure.coursestructure_chapter_id) as courseactivity_num_chapters_visited - from course_state - inner join course_structure - on - course_state.courserun_readable_id = course_structure.courserun_readable_id +with + course_activities as ( + select * + from {{ ref("stg__mitxonline__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ), + users as (select * from {{ ref("int__mitxonline__users") }}), + course_activities_video as (select * from {{ ref("int__mitxonline__user_courseactivity_video") }}), + problem_check as (select * from {{ ref("int__mitxonline__user_courseactivity_problemcheck") }}), + course_state as (select * from {{ ref("stg__mitxonline__openedx__mysql__courseware_studentmodule") }}), + course_structure as (select * from {{ ref("int__mitxonline__course_structure") }}), + course_chapters_stats as ( + select + course_state.openedx_user_id, + course_state.courserun_readable_id, + count(distinct course_structure.coursestructure_chapter_id) as courseactivity_num_chapters_visited + from course_state + inner join + course_structure + on course_state.courserun_readable_id = course_structure.courserun_readable_id and course_state.coursestructure_block_id = course_structure.coursestructure_block_id - group by course_state.openedx_user_id, course_state.courserun_readable_id -) - -, problem_check_stats as ( - select - user_username - , courserun_readable_id - , max(useractivity_timestamp) as courseactivity_last_problem_check_timestamp - from problem_check - group by user_username, courserun_readable_id -) - -, play_video_stats as ( - select - user_username - , courserun_readable_id - , count(distinct useractivity_video_id) as courseactivity_num_unique_play_video - , count(*) as courseactivity_num_play_video - , max(useractivity_timestamp) as courseactivity_last_play_video_timestamp - from course_activities_video - where useractivity_event_type = 'play_video' - group by user_username, courserun_readable_id -) - -, all_course_activities_stats as ( - select - course_activities.user_username - , course_activities.courserun_readable_id - , coalesce(users.openedx_user_id, course_activities.openedx_user_id) as openedx_user_id - , count(distinct date(from_iso8601_timestamp(course_activities.useractivity_timestamp))) - as courseactivity_num_days_activity - , count(*) as courseactivity_num_events - , min(course_activities.useractivity_timestamp) as courseactivity_first_event_timestamp - , max(course_activities.useractivity_timestamp) as courseactivity_last_event_timestamp - from course_activities - left join users on course_activities.user_username = users.user_username - group by - course_activities.user_username - , coalesce(users.openedx_user_id, course_activities.openedx_user_id) - , course_activities.courserun_readable_id -) + group by course_state.openedx_user_id, course_state.courserun_readable_id + ), + problem_check_stats as ( + select + user_username, + courserun_readable_id, + max(useractivity_timestamp) as courseactivity_last_problem_check_timestamp + from problem_check + group by user_username, courserun_readable_id + ), + play_video_stats as ( + select + user_username, + courserun_readable_id, + count(distinct useractivity_video_id) as courseactivity_num_unique_play_video, + count(*) as courseactivity_num_play_video, + max(useractivity_timestamp) as courseactivity_last_play_video_timestamp + from course_activities_video + where useractivity_event_type = 'play_video' + group by user_username, courserun_readable_id + ), + all_course_activities_stats as ( + select + course_activities.user_username, + course_activities.courserun_readable_id, + coalesce(users.openedx_user_id, course_activities.openedx_user_id) as openedx_user_id, + count( + distinct date(from_iso8601_timestamp(course_activities.useractivity_timestamp)) + ) as courseactivity_num_days_activity, + count(*) as courseactivity_num_events, + min(course_activities.useractivity_timestamp) as courseactivity_first_event_timestamp, + max(course_activities.useractivity_timestamp) as courseactivity_last_event_timestamp + from course_activities + left join users on course_activities.user_username = users.user_username + group by + course_activities.user_username, + coalesce(users.openedx_user_id, course_activities.openedx_user_id), + course_activities.courserun_readable_id + ) select - all_course_activities_stats.openedx_user_id - , all_course_activities_stats.user_username - , all_course_activities_stats.courserun_readable_id - , all_course_activities_stats.courseactivity_num_days_activity - , all_course_activities_stats.courseactivity_num_events - , play_video_stats.courseactivity_num_unique_play_video - , play_video_stats.courseactivity_num_play_video - , play_video_stats.courseactivity_last_play_video_timestamp - , problem_check_stats.courseactivity_last_problem_check_timestamp - , course_chapters_stats.courseactivity_num_chapters_visited - , all_course_activities_stats.courseactivity_first_event_timestamp - , all_course_activities_stats.courseactivity_last_event_timestamp + all_course_activities_stats.openedx_user_id, + all_course_activities_stats.user_username, + all_course_activities_stats.courserun_readable_id, + all_course_activities_stats.courseactivity_num_days_activity, + all_course_activities_stats.courseactivity_num_events, + play_video_stats.courseactivity_num_unique_play_video, + play_video_stats.courseactivity_num_play_video, + play_video_stats.courseactivity_last_play_video_timestamp, + problem_check_stats.courseactivity_last_problem_check_timestamp, + course_chapters_stats.courseactivity_num_chapters_visited, + all_course_activities_stats.courseactivity_first_event_timestamp, + all_course_activities_stats.courseactivity_last_event_timestamp from all_course_activities_stats -left join play_video_stats - on - all_course_activities_stats.user_username = play_video_stats.user_username - and all_course_activities_stats.courserun_readable_id = play_video_stats.courserun_readable_id -left join problem_check_stats - on - all_course_activities_stats.user_username = problem_check_stats.user_username - and all_course_activities_stats.courserun_readable_id = problem_check_stats.courserun_readable_id -left join course_chapters_stats - on - all_course_activities_stats.openedx_user_id = course_chapters_stats.openedx_user_id - and all_course_activities_stats.courserun_readable_id = course_chapters_stats.courserun_readable_id +left join + play_video_stats + on all_course_activities_stats.user_username = play_video_stats.user_username + and all_course_activities_stats.courserun_readable_id = play_video_stats.courserun_readable_id +left join + problem_check_stats + on all_course_activities_stats.user_username = problem_check_stats.user_username + and all_course_activities_stats.courserun_readable_id = problem_check_stats.courserun_readable_id +left join + course_chapters_stats + on all_course_activities_stats.openedx_user_id = course_chapters_stats.openedx_user_id + and all_course_activities_stats.courserun_readable_id = course_chapters_stats.courserun_readable_id diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivities_daily.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivities_daily.sql index 9fe9d8028..f78bf8952 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivities_daily.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivities_daily.sql @@ -1,19 +1,18 @@ -with course_activities as ( - select * from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxonline__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ), + daily_activities_stats as ( + select + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date, + count(*) as courseactivity_num_events + from course_activities + group by user_username, courserun_readable_id, date(from_iso8601_timestamp(useractivity_timestamp)) + ) -, daily_activities_stats as ( - select - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - , count(*) as courseactivity_num_events - from course_activities - group by - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) -) - -select * from daily_activities_stats +select * +from daily_activities_stats diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_discussion.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_discussion.sql index acd138cac..d718fe647 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_discussion.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_discussion.sql @@ -1,24 +1,26 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxonline__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , openedx_user_id - , useractivity_event_source - , useractivity_event_type - , useractivity_path - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_discussion_post_id - , json_query(useractivity_event_object, 'lax $.title' omit quotes) as useractivity_discussion_post_title - , json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as useractivity_discussion_block_id - , json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as useractivity_discussion_block_name - , json_query(useractivity_event_object, 'lax $.url' omit quotes) as useractivity_discussion_page_url - , json_query(useractivity_event_object, 'lax $.query' omit quotes) as useractivity_discussion_search_query - , json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as useractivity_discussion_roles + user_username, + courserun_readable_id, + openedx_user_id, + useractivity_event_source, + useractivity_event_type, + useractivity_path, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_discussion_post_id, + json_query(useractivity_event_object, 'lax $.title' omit quotes) as useractivity_discussion_post_title, + json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as useractivity_discussion_block_id, + json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as useractivity_discussion_block_name, + json_query(useractivity_event_object, 'lax $.url' omit quotes) as useractivity_discussion_page_url, + json_query(useractivity_event_object, 'lax $.query' omit quotes) as useractivity_discussion_search_query, + json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as useractivity_discussion_roles from course_activities where useractivity_event_type like 'edx.forum.%' diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_problemcheck.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_problemcheck.sql index c5eb92aa8..d0bebe744 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_problemcheck.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_problemcheck.sql @@ -1,25 +1,28 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxonline__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , openedx_user_id - , useractivity_event_type - , useractivity_timestamp - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id - , json_query(useractivity_event_object, 'lax $.answers' omit quotes) as useractivity_problem_student_answers - , json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as useractivity_problem_attempts - , json_query(useractivity_event_object, 'lax $.success' omit quotes) as useractivity_problem_success - , json_query(useractivity_event_object, 'lax $.grade' omit quotes) as useractivity_problem_current_grade - , json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as useractivity_problem_max_grade + user_username, + courserun_readable_id, + openedx_user_id, + useractivity_event_type, + useractivity_timestamp, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id, + json_query(useractivity_event_object, 'lax $.answers' omit quotes) as useractivity_problem_student_answers, + json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as useractivity_problem_attempts, + json_query(useractivity_event_object, 'lax $.success' omit quotes) as useractivity_problem_success, + json_query(useractivity_event_object, 'lax $.grade' omit quotes) as useractivity_problem_current_grade, + json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as useractivity_problem_max_grade from course_activities -where useractivity_event_type = 'problem_check' ---- This event emitted by the browser contain all of the GET parameters, --- only events emitted by the server are useful -and useractivity_event_source = 'server' +where + useractivity_event_type = 'problem_check' + -- - This event emitted by the browser contain all of the GET parameters, + -- only events emitted by the server are useful + and useractivity_event_source = 'server' diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_problemsubmitted.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_problemsubmitted.sql index c9e1eb89e..fc177c843 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_problemsubmitted.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_problemsubmitted.sql @@ -1,23 +1,25 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxonline__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , openedx_user_id - , useractivity_event_source - , useractivity_event_type - , useractivity_path - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.event_transaction_id' omit quotes) as useractivity_event_id - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id - , json_query(useractivity_event_object, 'lax $.weight' omit quotes) as useractivity_problem_weight - , json_query(useractivity_event_object, 'lax $.weighted_earned' omit quotes) as useractivity_problem_earned_score - , json_query(useractivity_event_object, 'lax $.weighted_possible' omit quotes) as useractivity_problem_max_score + user_username, + courserun_readable_id, + openedx_user_id, + useractivity_event_source, + useractivity_event_type, + useractivity_path, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.event_transaction_id' omit quotes) as useractivity_event_id, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id, + json_query(useractivity_event_object, 'lax $.weight' omit quotes) as useractivity_problem_weight, + json_query(useractivity_event_object, 'lax $.weighted_earned' omit quotes) as useractivity_problem_earned_score, + json_query(useractivity_event_object, 'lax $.weighted_possible' omit quotes) as useractivity_problem_max_score from course_activities where useractivity_event_type = 'edx.grades.problem.submitted' diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_showanswer.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_showanswer.sql index e46d7185c..b86707959 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_showanswer.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_showanswer.sql @@ -1,16 +1,18 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxonline__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , openedx_user_id - , useractivity_path - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id + user_username, + courserun_readable_id, + openedx_user_id, + useractivity_path, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id from course_activities where useractivity_event_type = 'showanswer' diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_video.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_video.sql index 4f1eaba2d..2d28251f8 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_video.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_video.sql @@ -1,28 +1,31 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxonline__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , openedx_user_id - , useractivity_event_source - , useractivity_event_type - , useractivity_page_url - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_video_id - , case - when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' then null + user_username, + courserun_readable_id, + openedx_user_id, + useractivity_event_source, + useractivity_event_type, + useractivity_page_url, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_video_id, + case + when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' + then null else cast(json_query(useractivity_event_object, 'lax $.duration' omit quotes) as decimal(38, 4)) - end as useractivity_video_duration - , json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as useractivity_video_currenttime - , json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as useractivity_video_old_time - , json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as useractivity_video_new_time - , json_query(useractivity_event_object, 'lax $.new_speed' omit quotes) as useractivity_video_new_speed - , json_query(useractivity_event_object, 'lax $.old_speed' omit quotes) as useractivity_video_old_speed + end as useractivity_video_duration, + json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as useractivity_video_currenttime, + json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as useractivity_video_old_time, + json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as useractivity_video_new_time, + json_query(useractivity_event_object, 'lax $.new_speed' omit quotes) as useractivity_video_new_speed, + json_query(useractivity_event_object, 'lax $.old_speed' omit quotes) as useractivity_video_old_speed from course_activities where regexp_like(useractivity_event_type, '(^[\w]+)_video') = true diff --git a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__users.sql b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__users.sql index 511351edb..cbc5273a4 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__users.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__users.sql @@ -1,87 +1,57 @@ -with users as ( - select * - from {{ ref('stg__mitxonline__app__postgres__users_user') }} -) - -, users_legaladdress as ( - select * - from {{ ref('stg__mitxonline__app__postgres__users_legaladdress') }} -) - -, users_profile as ( - select * from {{ ref('stg__mitxonline__app__postgres__users_userprofile') }} -) - -, micromasters_profiles as ( - select * from {{ ref('stg__micromasters__app__postgres__profiles_profile') }} -) - -, micromasters_auth as ( - select * - from {{ ref('stg__micromasters__app__postgres__auth_usersocialauth') }} - where user_auth_provider = 'mitxonline' -) - ---- this profile is pull from MicroMasters so that we can match MM profile with MITxOnline users based on username -, micromasters_profile as ( - select - micromasters_profiles.user_profile_id - , micromasters_profiles.user_id - , micromasters_auth.user_username - from micromasters_profiles - inner join micromasters_auth on micromasters_profiles.user_id = micromasters_auth.user_id -) - -, micromasters_users as ( - select * from {{ ref('__micromasters__users') }} -) - -, openedx_users as ( - select * from {{ ref('stg__mitxonline__openedx__mysql__auth_user') }} -) - -, openedxuser_mapping as ( - select * from {{ ref('stg__mitxonline__app__postgres__openedx_openedxuser') }} -) +with + users as (select * from {{ ref("stg__mitxonline__app__postgres__users_user") }}), + users_legaladdress as (select * from {{ ref("stg__mitxonline__app__postgres__users_legaladdress") }}), + users_profile as (select * from {{ ref("stg__mitxonline__app__postgres__users_userprofile") }}), + micromasters_profiles as (select * from {{ ref("stg__micromasters__app__postgres__profiles_profile") }}), + micromasters_auth as ( + select * + from {{ ref("stg__micromasters__app__postgres__auth_usersocialauth") }} + where user_auth_provider = 'mitxonline' + ), + -- - this profile is pull from MicroMasters so that we can match MM profile with MITxOnline users based on username + micromasters_profile as ( + select micromasters_profiles.user_profile_id, micromasters_profiles.user_id, micromasters_auth.user_username + from micromasters_profiles + inner join micromasters_auth on micromasters_profiles.user_id = micromasters_auth.user_id + ), + micromasters_users as (select * from {{ ref("__micromasters__users") }}), + openedx_users as (select * from {{ ref("stg__mitxonline__openedx__mysql__auth_user") }}), + openedxuser_mapping as (select * from {{ ref("stg__mitxonline__app__postgres__openedx_openedxuser") }}) select - users.user_id - , users.user_global_id - , openedx_users.openedx_user_id - , users.user_full_name - , users.user_email - , users.user_joined_on - , users.user_last_login - , users.user_is_active - , users_legaladdress.user_address_country - , users_legaladdress.user_address_state - , users_legaladdress.user_first_name - , users_legaladdress.user_last_name - , users_profile.user_birth_year - , users_profile.user_company - , users_profile.user_job_title - , users_profile.user_industry - , users_profile.user_job_function - , users_profile.user_leadership_level - , users_profile.user_highest_education - , users_profile.user_gender - , users_profile.user_company_size - , users_profile.user_years_experience - , users_profile.user_type_is_student - , users_profile.user_type_is_professional - , users_profile.user_type_is_educator - , users_profile.user_type_is_other - , micromasters_profile.user_profile_id as user_micromasters_profile_id - , micromasters_users.user_edxorg_username - , coalesce(openedx_users.user_username, users.user_username) as user_username + users.user_id, + users.user_global_id, + openedx_users.openedx_user_id, + users.user_full_name, + users.user_email, + users.user_joined_on, + users.user_last_login, + users.user_is_active, + users_legaladdress.user_address_country, + users_legaladdress.user_address_state, + users_legaladdress.user_first_name, + users_legaladdress.user_last_name, + users_profile.user_birth_year, + users_profile.user_company, + users_profile.user_job_title, + users_profile.user_industry, + users_profile.user_job_function, + users_profile.user_leadership_level, + users_profile.user_highest_education, + users_profile.user_gender, + users_profile.user_company_size, + users_profile.user_years_experience, + users_profile.user_type_is_student, + users_profile.user_type_is_professional, + users_profile.user_type_is_educator, + users_profile.user_type_is_other, + micromasters_profile.user_profile_id as user_micromasters_profile_id, + micromasters_users.user_edxorg_username, + coalesce(openedx_users.user_username, users.user_username) as user_username from users left join users_legaladdress on users.user_id = users_legaladdress.user_id left join users_profile on users.user_id = users_profile.user_id -left join openedxuser_mapping - on users.user_id = openedxuser_mapping.user_id -left join openedx_users - on openedxuser_mapping.openedxuser_username = openedx_users.user_username -left join micromasters_profile - on openedxuser_mapping.openedxuser_username = micromasters_profile.user_username -left join micromasters_users - on micromasters_profile.user_profile_id = micromasters_users.user_profile_id +left join openedxuser_mapping on users.user_id = openedxuser_mapping.user_id +left join openedx_users on openedxuser_mapping.openedxuser_username = openedx_users.user_username +left join micromasters_profile on openedxuser_mapping.openedxuser_username = micromasters_profile.user_username +left join micromasters_users on micromasters_profile.user_profile_id = micromasters_users.user_profile_id diff --git a/src/ol_dbt/models/intermediate/mitxonline/subqueries/__mitxonline_good_economics_for_hard_times_program.sql b/src/ol_dbt/models/intermediate/mitxonline/subqueries/__mitxonline_good_economics_for_hard_times_program.sql index 2190b99ba..bac06d82f 100644 --- a/src/ol_dbt/models/intermediate/mitxonline/subqueries/__mitxonline_good_economics_for_hard_times_program.sql +++ b/src/ol_dbt/models/intermediate/mitxonline/subqueries/__mitxonline_good_economics_for_hard_times_program.sql @@ -2,50 +2,41 @@ -- 14.009x can only be used to complete the requirements for one program. If the learner is enrolled in the DEDP -- Public Policy program, then we count it for that program. Otherwise, if the learner completed the course before -- Fall 2023 count it as a DEDP Internation Development elective - -with mitxonline_programenrollments as ( - select * from {{ ref('int__mitxonline__programenrollments') }} -) - -, mitxonline_courserunenrollments as ( - select * from {{ ref('int__mitxonline__courserunenrollments') }} -) - -, mitxonline_courseruns as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, query as ( - select - mitxonline_courserunenrollments.courserunenrollment_id - , case - when - count_if( - mitxonline_programenrollments.program_id = {{ var("dedp_mitxonline_public_policy_program_id") }} - ) - > 0 +with + mitxonline_programenrollments as (select * from {{ ref("int__mitxonline__programenrollments") }}), + mitxonline_courserunenrollments as (select * from {{ ref("int__mitxonline__courserunenrollments") }}), + mitxonline_courseruns as (select * from {{ ref("int__mitxonline__course_runs") }}), + query as ( + select + mitxonline_courserunenrollments.courserunenrollment_id, + case + when + count_if( + mitxonline_programenrollments.program_id = {{ var("dedp_mitxonline_public_policy_program_id") }} + ) + > 0 then {{ var("dedp_mitxonline_public_policy_program_id") }} - when - mitxonline_courseruns.courserun_start_on - < to_iso8601( - date '{{ var("dedp_mitxonline_good_economics_for_hard_times_internation_development_cutoff") }}' - ) + when + mitxonline_courseruns.courserun_start_on < to_iso8601( + date '{{ var("dedp_mitxonline_good_economics_for_hard_times_internation_development_cutoff") }}' + ) then {{ var("dedp_mitxonline_international_development_program_id") }} - end as program_id - from mitxonline_courserunenrollments - inner join mitxonline_courseruns - on mitxonline_courserunenrollments.courserun_id = mitxonline_courseruns.courserun_id - left join mitxonline_programenrollments - on mitxonline_courserunenrollments.user_id = mitxonline_programenrollments.user_id - where - mitxonline_programenrollments.program_id in ( - {{ var("dedp_mitxonline_international_development_program_id") }} - , {{ var("dedp_mitxonline_public_policy_program_id") }} - ) - and mitxonline_courseruns.course_number - = '{{ var("dedp_mitxonline_good_economics_for_hard_times_course_number") }}' - group by mitxonline_courserunenrollments.courserunenrollment_id, mitxonline_courseruns.courserun_start_on -) + end as program_id + from mitxonline_courserunenrollments + inner join + mitxonline_courseruns on mitxonline_courserunenrollments.courserun_id = mitxonline_courseruns.courserun_id + left join + mitxonline_programenrollments + on mitxonline_courserunenrollments.user_id = mitxonline_programenrollments.user_id + where + mitxonline_programenrollments.program_id in ( + {{ var("dedp_mitxonline_international_development_program_id") }}, + {{ var("dedp_mitxonline_public_policy_program_id") }} + ) + and mitxonline_courseruns.course_number + = '{{ var("dedp_mitxonline_good_economics_for_hard_times_course_number") }}' + group by mitxonline_courserunenrollments.courserunenrollment_id, mitxonline_courseruns.courserun_start_on + ) select * from query diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2bcoupon.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2bcoupon.sql index 0b9a81535..a40df1c8e 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2bcoupon.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2bcoupon.sql @@ -1,19 +1,16 @@ -with b2bcoupon as ( - select * - from {{ ref('stg__mitxpro__app__postgres__b2becommerce_b2bcoupon') }} -) +with b2bcoupon as (select * from {{ ref("stg__mitxpro__app__postgres__b2becommerce_b2bcoupon") }}) select - b2bcoupon_id - , b2bcoupon_updated_on - , b2bcoupon_created_on - , b2bcoupon_expires_on - , company_id - , product_id - , b2bcoupon_is_enabled - , b2bcoupon_activated_on - , b2bcoupon_discount_percent - , b2bcoupon_name - , b2bcoupon_is_reusable - , b2bcoupon_coupon_code + b2bcoupon_id, + b2bcoupon_updated_on, + b2bcoupon_created_on, + b2bcoupon_expires_on, + company_id, + product_id, + b2bcoupon_is_enabled, + b2bcoupon_activated_on, + b2bcoupon_discount_percent, + b2bcoupon_name, + b2bcoupon_is_reusable, + b2bcoupon_coupon_code from b2bcoupon diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2bcouponredemption.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2bcouponredemption.sql index d8c98d4e6..4f570f25d 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2bcouponredemption.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2bcouponredemption.sql @@ -1,12 +1,4 @@ -with b2bcouponredemption as ( - select * - from {{ ref('stg__mitxpro__app__postgres__b2becommerce_b2bcouponredemption') }} -) +with b2bcouponredemption as (select * from {{ ref("stg__mitxpro__app__postgres__b2becommerce_b2bcouponredemption") }}) -select - b2bcouponredemption_id - , b2border_id - , b2bcoupon_id - , b2bcouponredemption_updated_on - , b2bcouponredemption_created_on +select b2bcouponredemption_id, b2border_id, b2bcoupon_id, b2bcouponredemption_updated_on, b2bcouponredemption_created_on from b2bcouponredemption diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2border.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2border.sql index 50c8436fe..bdedb42d4 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2border.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2border.sql @@ -1,59 +1,40 @@ -with b2borders as ( - select * - from {{ ref('stg__mitxpro__app__postgres__b2becommerce_b2border') }} -) - -, productversions as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_productversion') }} -) - -, products as ( - select * - from {{ ref('int__mitxpro__ecommerce_product') }} -) - -, salesforce_opportunity as ( - select * from {{ ref('int__salesforce__opportunity') }} -) - -, couponpaymentversion as ( - select * - from {{ ref('int__mitxpro__ecommerce_couponpaymentversion') }} -) - -, salesforce_b2border as ( - select - b2borders.b2border_id - , salesforce_opportunity.opportunity_id - from b2borders - inner join couponpaymentversion - on b2borders.couponpaymentversion_id = couponpaymentversion.couponpaymentversion_id - inner join salesforce_opportunity - on b2borders.b2border_contract_number like '%' || salesforce_opportunity.opportunity_id -) +with + b2borders as (select * from {{ ref("stg__mitxpro__app__postgres__b2becommerce_b2border") }}), + productversions as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_productversion") }}), + products as (select * from {{ ref("int__mitxpro__ecommerce_product") }}), + salesforce_opportunity as (select * from {{ ref("int__salesforce__opportunity") }}), + couponpaymentversion as (select * from {{ ref("int__mitxpro__ecommerce_couponpaymentversion") }}), + salesforce_b2border as ( + select b2borders.b2border_id, salesforce_opportunity.opportunity_id + from b2borders + inner join + couponpaymentversion on b2borders.couponpaymentversion_id = couponpaymentversion.couponpaymentversion_id + inner join + salesforce_opportunity + on b2borders.b2border_contract_number like '%' || salesforce_opportunity.opportunity_id + ) select - b2borders.b2border_id - , b2borders.b2border_updated_on - , b2borders.b2border_created_on - , b2borders.b2border_total_price - , b2borders.b2border_status - , b2borders.b2border_per_item_price - , b2borders.b2border_unique_uuid - , b2borders.b2border_num_seats - , b2borders.b2bcoupon_id - , b2borders.productversion_id - , b2borders.couponpaymentversion_id - , b2borders.b2border_contract_number - , b2borders.b2border_discount - , b2borders.programrun_id - , b2borders.b2border_email - , products.product_id - , products.courserun_id - , products.program_id - , products.product_type - , salesforce_b2border.opportunity_id as salesforce_opportunity_id + b2borders.b2border_id, + b2borders.b2border_updated_on, + b2borders.b2border_created_on, + b2borders.b2border_total_price, + b2borders.b2border_status, + b2borders.b2border_per_item_price, + b2borders.b2border_unique_uuid, + b2borders.b2border_num_seats, + b2borders.b2bcoupon_id, + b2borders.productversion_id, + b2borders.couponpaymentversion_id, + b2borders.b2border_contract_number, + b2borders.b2border_discount, + b2borders.programrun_id, + b2borders.b2border_email, + products.product_id, + products.courserun_id, + products.program_id, + products.product_type, + salesforce_b2border.opportunity_id as salesforce_opportunity_id from b2borders inner join productversions on b2borders.productversion_id = productversions.productversion_id inner join products on productversions.product_id = products.product_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2breceipt.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2breceipt.sql index bfb89914f..268761180 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2breceipt.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__b2becommerce_b2breceipt.sql @@ -1,19 +1,16 @@ -with b2breceipts as ( - select * - from {{ ref('stg__mitxpro__app__postgres__b2becommerce_b2breceipt') }} -) +with b2breceipts as (select * from {{ ref("stg__mitxpro__app__postgres__b2becommerce_b2breceipt") }}) select - b2breceipt_id - , b2breceipt_created_on - , b2breceipt_updated_on - , b2breceipt_data - , b2border_id - , b2breceipt_transaction_status - , b2breceipt_transaction_id - , b2breceipt_authorization_code - , b2breceipt_reference_number - , b2breceipt_payment_method - , b2breceipt_bill_to_address_state - , b2breceipt_bill_to_address_country + b2breceipt_id, + b2breceipt_created_on, + b2breceipt_updated_on, + b2breceipt_data, + b2border_id, + b2breceipt_transaction_status, + b2breceipt_transaction_id, + b2breceipt_authorization_code, + b2breceipt_reference_number, + b2breceipt_payment_method, + b2breceipt_bill_to_address_state, + b2breceipt_bill_to_address_country from b2breceipts diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__course_runs.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__course_runs.sql index 1fef5f752..339cbd711 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__course_runs.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__course_runs.sql @@ -1,35 +1,25 @@ -- Course Runs information for MITxPro - -with course_runs as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_courserun') }} -) - -, courses as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_course') }} -) - -, platforms as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_platform') }} -) +with + course_runs as (select * from {{ ref("stg__mitxpro__app__postgres__courses_courserun") }}), + courses as (select * from {{ ref("stg__mitxpro__app__postgres__courses_course") }}), + platforms as (select * from {{ ref("stg__mitxpro__app__postgres__courses_platform") }}) select - course_runs.courserun_id - , course_runs.course_id - , course_runs.courserun_title - , course_runs.courserun_readable_id - , course_runs.courserun_edx_readable_id - , course_runs.courserun_external_readable_id - , course_runs.courserun_tag - , course_runs.courserun_url - , course_runs.courserun_start_on - , course_runs.courserun_end_on - , course_runs.courserun_enrollment_start_on - , course_runs.courserun_enrollment_end_on - , course_runs.courserun_is_live - , platforms.platform_name - , platforms.platform_id + course_runs.courserun_id, + course_runs.course_id, + course_runs.courserun_title, + course_runs.courserun_readable_id, + course_runs.courserun_edx_readable_id, + course_runs.courserun_external_readable_id, + course_runs.courserun_tag, + course_runs.courserun_url, + course_runs.courserun_start_on, + course_runs.courserun_end_on, + course_runs.courserun_enrollment_start_on, + course_runs.courserun_enrollment_end_on, + course_runs.courserun_is_live, + platforms.platform_name, + platforms.platform_id from course_runs -inner join courses - on course_runs.course_id = courses.course_id -left join platforms - on courses.platform_id = platforms.platform_id +inner join courses on course_runs.course_id = courses.course_id +left join platforms on courses.platform_id = platforms.platform_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__course_structure.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__course_structure.sql index 5860a5b66..ab1367681 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__course_structure.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__course_structure.sql @@ -1,66 +1,60 @@ -with course_structure as ( - select * from {{ ref('stg__mitxpro__openedx__api__course_structure') }} - order by courserun_readable_id, coursestructure_retrieved_at, coursestructure_block_index +with + course_structure as ( + select * + from {{ ref("stg__mitxpro__openedx__api__course_structure") }} + order by courserun_readable_id, coursestructure_retrieved_at, coursestructure_block_index -) - -, latest_course_structure_date as ( - select - courserun_readable_id - , max(coursestructure_retrieved_at) as max_retrieved_date - from course_structure - group by courserun_readable_id -) - -, chapters as ( - select * from course_structure - where coursestructure_block_category = 'chapter' -) - -, course_structure_with_chapters as ( - select - course_structure.* - , chapters.coursestructure_block_id as coursestructure_chapter_id - , chapters.coursestructure_block_title as coursestructure_chapter_title - , row_number() over ( - partition by - course_structure.courserun_readable_id - , course_structure.coursestructure_block_index - , course_structure.coursestructure_retrieved_at - order by chapters.coursestructure_block_index desc - ) as row_num - from course_structure - inner join chapters - on - course_structure.courserun_readable_id = chapters.courserun_readable_id + ), + latest_course_structure_date as ( + select courserun_readable_id, max(coursestructure_retrieved_at) as max_retrieved_date + from course_structure + group by courserun_readable_id + ), + chapters as (select * from course_structure where coursestructure_block_category = 'chapter'), + course_structure_with_chapters as ( + select + course_structure.*, + chapters.coursestructure_block_id as coursestructure_chapter_id, + chapters.coursestructure_block_title as coursestructure_chapter_title, + row_number() over ( + partition by + course_structure.courserun_readable_id, + course_structure.coursestructure_block_index, + course_structure.coursestructure_retrieved_at + order by chapters.coursestructure_block_index desc + ) as row_num + from course_structure + inner join + chapters + on course_structure.courserun_readable_id = chapters.courserun_readable_id and course_structure.coursestructure_retrieved_at = chapters.coursestructure_retrieved_at and course_structure.coursestructure_block_index >= chapters.coursestructure_block_index -) + ) select - course_structure.courserun_readable_id - , course_structure.courserun_title - , course_structure.coursestructure_block_index - , course_structure.coursestructure_block_id - , course_structure.coursestructure_parent_block_id - , course_structure.coursestructure_block_category - , course_structure.coursestructure_block_title - , course_structure.coursestructure_content_hash - , course_structure.coursestructure_block_content_hash - , course_structure.coursestructure_block_metadata - , course_structure.courserun_start_on - , course_structure.coursestructure_retrieved_at - , course_structure_with_chapters.coursestructure_chapter_id - , course_structure_with_chapters.coursestructure_chapter_title - , if(latest_course_structure_date.max_retrieved_date is not null, true, false) as coursestructure_is_latest + course_structure.courserun_readable_id, + course_structure.courserun_title, + course_structure.coursestructure_block_index, + course_structure.coursestructure_block_id, + course_structure.coursestructure_parent_block_id, + course_structure.coursestructure_block_category, + course_structure.coursestructure_block_title, + course_structure.coursestructure_content_hash, + course_structure.coursestructure_block_content_hash, + course_structure.coursestructure_block_metadata, + course_structure.courserun_start_on, + course_structure.coursestructure_retrieved_at, + course_structure_with_chapters.coursestructure_chapter_id, + course_structure_with_chapters.coursestructure_chapter_title, + if(latest_course_structure_date.max_retrieved_date is not null, true, false) as coursestructure_is_latest from course_structure -left join latest_course_structure_date - on - course_structure.courserun_readable_id = latest_course_structure_date.courserun_readable_id - and course_structure.coursestructure_retrieved_at = latest_course_structure_date.max_retrieved_date -left join course_structure_with_chapters - on - course_structure.courserun_readable_id = course_structure_with_chapters.courserun_readable_id - and course_structure.coursestructure_block_id = course_structure_with_chapters.coursestructure_block_id - and course_structure.coursestructure_retrieved_at = course_structure_with_chapters.coursestructure_retrieved_at - and course_structure_with_chapters.row_num = 1 +left join + latest_course_structure_date + on course_structure.courserun_readable_id = latest_course_structure_date.courserun_readable_id + and course_structure.coursestructure_retrieved_at = latest_course_structure_date.max_retrieved_date +left join + course_structure_with_chapters + on course_structure.courserun_readable_id = course_structure_with_chapters.courserun_readable_id + and course_structure.coursestructure_block_id = course_structure_with_chapters.coursestructure_block_id + and course_structure.coursestructure_retrieved_at = course_structure_with_chapters.coursestructure_retrieved_at + and course_structure_with_chapters.row_num = 1 diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserun_certificates.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserun_certificates.sql index 7318c5d35..13deaf85b 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserun_certificates.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserun_certificates.sql @@ -1,37 +1,29 @@ -- Course Certificate information for MIT xPro +with + certificates as (select * from {{ ref("stg__mitxpro__app__postgres__courses_courseruncertificate") }}), + runs as (select * from {{ ref("stg__mitxpro__app__postgres__courses_courserun") }}), + users as (select * from {{ ref("stg__mitxpro__app__postgres__users_user") }}), + courserun_certificates as ( + select + certificates.courseruncertificate_id, + certificates.courseruncertificate_uuid, + certificates.courserun_id, + runs.courserun_title, + runs.courserun_readable_id, + runs.courserun_url, + runs.course_id, + certificates.courseruncertificate_is_revoked, + certificates.courseruncertificate_url, + certificates.courseruncertificate_created_on, + certificates.courseruncertificate_updated_on, + certificates.user_id, + users.user_username, + users.user_email, + users.user_full_name + from certificates + inner join runs on certificates.courserun_id = runs.courserun_id + inner join users on certificates.user_id = users.user_id + ) -with certificates as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_courseruncertificate') }} -) - -, runs as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_courserun') }} -) - -, users as ( - select * from {{ ref('stg__mitxpro__app__postgres__users_user') }} -) - -, courserun_certificates as ( - select - certificates.courseruncertificate_id - , certificates.courseruncertificate_uuid - , certificates.courserun_id - , runs.courserun_title - , runs.courserun_readable_id - , runs.courserun_url - , runs.course_id - , certificates.courseruncertificate_is_revoked - , certificates.courseruncertificate_url - , certificates.courseruncertificate_created_on - , certificates.courseruncertificate_updated_on - , certificates.user_id - , users.user_username - , users.user_email - , users.user_full_name - from certificates - inner join runs on certificates.courserun_id = runs.courserun_id - inner join users on certificates.user_id = users.user_id -) - -select * from courserun_certificates +select * +from courserun_certificates diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserun_grades.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserun_grades.sql index 5034addbb..c12195a5c 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserun_grades.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserun_grades.sql @@ -1,36 +1,28 @@ -- Course Grade information for MIT xPro +with + grades as (select * from {{ ref("stg__mitxpro__app__postgres__courses_courserungrade") }}), + runs as (select * from {{ ref("stg__mitxpro__app__postgres__courses_courserun") }}), + users as (select * from {{ ref("stg__mitxpro__app__postgres__users_user") }}), + courserun_grades as ( + select + grades.courserungrade_id, + grades.courserun_id, + runs.course_id, + runs.courserun_title, + runs.courserun_readable_id, + runs.courserun_url, + grades.courserungrade_grade, + grades.courserungrade_letter_grade, + grades.courserungrade_is_passing, + grades.courserungrade_created_on, + grades.courserungrade_updated_on, + grades.user_id, + users.user_username, + users.user_email + from grades + inner join runs on grades.courserun_id = runs.courserun_id + inner join users on grades.user_id = users.user_id + ) -with grades as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_courserungrade') }} -) - -, runs as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_courserun') }} -) - -, users as ( - select * from {{ ref('stg__mitxpro__app__postgres__users_user') }} -) - -, courserun_grades as ( - select - grades.courserungrade_id - , grades.courserun_id - , runs.course_id - , runs.courserun_title - , runs.courserun_readable_id - , runs.courserun_url - , grades.courserungrade_grade - , grades.courserungrade_letter_grade - , grades.courserungrade_is_passing - , grades.courserungrade_created_on - , grades.courserungrade_updated_on - , grades.user_id - , users.user_username - , users.user_email - from grades - inner join runs on grades.courserun_id = runs.courserun_id - inner join users on grades.user_id = users.user_id -) - -select * from courserun_grades +select * +from courserun_grades diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserun_videos.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserun_videos.sql index 3d6a9d305..47ac39aa8 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserun_videos.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserun_videos.sql @@ -1,32 +1,21 @@ -with mitxpro_coursevideos as ( - select * - from {{ ref('stg__mitxpro__openedx__mysql__edxval_coursevideo') }} -) - -, mitxpro_videos as ( - select * from {{ ref('stg__mitxpro__openedx__mysql__edxval_video') }} -) - -, ovs_videos as ( - select - * - , row_number() over ( - partition by video_title - order by video_created_on desc - ) as video_rank - from {{ ref('int__ovs__videos') }} - where platform = '{{ var("mitxpro") }}' -) +with + mitxpro_coursevideos as (select * from {{ ref("stg__mitxpro__openedx__mysql__edxval_coursevideo") }}), + mitxpro_videos as (select * from {{ ref("stg__mitxpro__openedx__mysql__edxval_video") }}), + ovs_videos as ( + select *, row_number() over (partition by video_title order by video_created_on desc) as video_rank + from {{ ref("int__ovs__videos") }} + where platform = '{{ var("mitxpro") }}' + ) select - mitxpro_coursevideos.courserun_readable_id - , mitxpro_coursevideos.coursevideo_is_hidden - , mitxpro_videos.video_edx_uuid - , mitxpro_videos.video_client_id - , mitxpro_videos.video_status - , coalesce(cast(ovs_videos.video_duration as decimal(38, 4)), mitxpro_videos.video_duration) as video_duration + mitxpro_coursevideos.courserun_readable_id, + mitxpro_coursevideos.coursevideo_is_hidden, + mitxpro_videos.video_edx_uuid, + mitxpro_videos.video_client_id, + mitxpro_videos.video_status, + coalesce(cast(ovs_videos.video_duration as decimal(38, 4)), mitxpro_videos.video_duration) as video_duration from mitxpro_coursevideos inner join mitxpro_videos on mitxpro_coursevideos.video_id = mitxpro_videos.video_id ---- the relationship between OVS videos and xPro videos are linked via video title and client video ID +-- - the relationship between OVS videos and xPro videos are linked via video title and client video ID -- e.g. TTSBSV3_Web3_BVB_FINAL.mp4. If there are multiple videos with same title on OVS, use the latest one. left join ovs_videos on mitxpro_videos.video_client_id = ovs_videos.video_title and ovs_videos.video_rank = 1 diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserunenrollments.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserunenrollments.sql index 8087194cd..aabbf19bb 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserunenrollments.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courserunenrollments.sql @@ -1,50 +1,40 @@ -- Enrollment information for MITxPro - -with enrollments as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_courserunenrollment') }} -) - -, openedx_enrollments as ( - select * from {{ ref('stg__mitxpro__openedx__mysql__courserun_enrollment') }} -) - -, runs as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_courserun') }} -) - -, users as ( - select * from {{ ref('int__mitxpro__users') }} -) - -, mitxpro_enrollments as ( - select - enrollments.courserunenrollment_id - , enrollments.courserunenrollment_is_active - , enrollments.user_id - , enrollments.courserun_id - , enrollments.courserunenrollment_created_on - , enrollments.courserunenrollment_enrollment_status - , enrollments.courserunenrollment_is_edx_enrolled - , runs.courserun_readable_id - , runs.courserun_title - , runs.courserun_start_on - , users.user_username - , users.user_email - , users.user_full_name - , users.user_address_country - , enrollments.ecommerce_company_id - , enrollments.ecommerce_order_id - , case - when enrollments.ecommerce_order_id is not null then 'no-id-professional' - else openedx_enrollments.courserunenrollment_enrollment_mode - end as courserunenrollment_enrollment_mode - from enrollments - left join runs on enrollments.courserun_id = runs.courserun_id - left join users on enrollments.user_id = users.user_id - left join openedx_enrollments - on - runs.courserun_readable_id = openedx_enrollments.courserun_readable_id +with + enrollments as (select * from {{ ref("stg__mitxpro__app__postgres__courses_courserunenrollment") }}), + openedx_enrollments as (select * from {{ ref("stg__mitxpro__openedx__mysql__courserun_enrollment") }}), + runs as (select * from {{ ref("stg__mitxpro__app__postgres__courses_courserun") }}), + users as (select * from {{ ref("int__mitxpro__users") }}), + mitxpro_enrollments as ( + select + enrollments.courserunenrollment_id, + enrollments.courserunenrollment_is_active, + enrollments.user_id, + enrollments.courserun_id, + enrollments.courserunenrollment_created_on, + enrollments.courserunenrollment_enrollment_status, + enrollments.courserunenrollment_is_edx_enrolled, + runs.courserun_readable_id, + runs.courserun_title, + runs.courserun_start_on, + users.user_username, + users.user_email, + users.user_full_name, + users.user_address_country, + enrollments.ecommerce_company_id, + enrollments.ecommerce_order_id, + case + when enrollments.ecommerce_order_id is not null + then 'no-id-professional' + else openedx_enrollments.courserunenrollment_enrollment_mode + end as courserunenrollment_enrollment_mode + from enrollments + left join runs on enrollments.courserun_id = runs.courserun_id + left join users on enrollments.user_id = users.user_id + left join + openedx_enrollments + on runs.courserun_readable_id = openedx_enrollments.courserun_readable_id and users.openedx_user_id = openedx_enrollments.openedx_user_id -) + ) -select * from mitxpro_enrollments +select * +from mitxpro_enrollments diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courses.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courses.sql index f9f52b55b..22facdaf7 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courses.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courses.sql @@ -1,82 +1,53 @@ -- Course information for MITxPro - -with courses as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_course') }} -) - -, cms_courses as ( - select * from {{ ref('stg__mitxpro__app__postgres__cms_coursepage') }} -) - -, wagtail_page as ( - select * from {{ ref('stg__mitxpro__app__postgres__wagtail_page') }} -) - -, platform as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_platform') }} -) - -, certificate_page as ( - select * from {{ ref('stg__mitxpro__app__postgres__cms_certificatepage') }} -) - -, certificate_page_path as ( - select - certificate_page.wagtail_page_id - , certificate_page.cms_certificate_ceus - , wagtail_page.wagtail_page_path - from certificate_page - inner join wagtail_page - on certificate_page.wagtail_page_id = wagtail_page.wagtail_page_id -) - -, course_topics as ( - select - course_id - , array_join(array_agg(coursetopic_name), ', ') as course_topics - from {{ ref('int__mitxpro__courses_to_topics') }} - group by course_id -) - -, course_instructors as ( - select - course_id - , array_join(array_agg(cms_facultymemberspage_facultymember_name), ', ') as course_instructors - from {{ ref('int__mitxpro__coursesfaculty') }} - group by course_id -) +with + courses as (select * from {{ ref("stg__mitxpro__app__postgres__courses_course") }}), + cms_courses as (select * from {{ ref("stg__mitxpro__app__postgres__cms_coursepage") }}), + wagtail_page as (select * from {{ ref("stg__mitxpro__app__postgres__wagtail_page") }}), + platform as (select * from {{ ref("stg__mitxpro__app__postgres__courses_platform") }}), + certificate_page as (select * from {{ ref("stg__mitxpro__app__postgres__cms_certificatepage") }}), + certificate_page_path as ( + select certificate_page.wagtail_page_id, certificate_page.cms_certificate_ceus, wagtail_page.wagtail_page_path + from certificate_page + inner join wagtail_page on certificate_page.wagtail_page_id = wagtail_page.wagtail_page_id + ), + course_topics as ( + select course_id, array_join(array_agg(coursetopic_name), ', ') as course_topics + from {{ ref("int__mitxpro__courses_to_topics") }} + group by course_id + ), + course_instructors as ( + select course_id, array_join(array_agg(cms_facultymemberspage_facultymember_name), ', ') as course_instructors + from {{ ref("int__mitxpro__coursesfaculty") }} + group by course_id + ) select - courses.course_id - , courses.program_id - , courses.course_title - , courses.course_is_live - , courses.course_readable_id - , courses.course_is_external - , courses.short_program_code - , platform.platform_name - , cms_courses.cms_coursepage_description - , cms_courses.cms_coursepage_subhead - , cms_courses.cms_coursepage_catalog_details - , cms_courses.cms_coursepage_duration - , cms_courses.cms_coursepage_format - , cms_courses.cms_coursepage_time_commitment - , certificate_page_path.cms_certificate_ceus - , course_topics.course_topics - , course_instructors.course_instructors - , wagtail_page.wagtail_page_slug as cms_coursepage_slug - , wagtail_page.wagtail_page_url_path as cms_coursepage_url_path - , wagtail_page.wagtail_page_is_live as cms_coursepage_is_live - , wagtail_page.wagtail_page_first_published_on as cms_coursepage_first_published_on - , wagtail_page.wagtail_page_last_published_on as cms_coursepage_last_published_on + courses.course_id, + courses.program_id, + courses.course_title, + courses.course_is_live, + courses.course_readable_id, + courses.course_is_external, + courses.short_program_code, + platform.platform_name, + cms_courses.cms_coursepage_description, + cms_courses.cms_coursepage_subhead, + cms_courses.cms_coursepage_catalog_details, + cms_courses.cms_coursepage_duration, + cms_courses.cms_coursepage_format, + cms_courses.cms_coursepage_time_commitment, + certificate_page_path.cms_certificate_ceus, + course_topics.course_topics, + course_instructors.course_instructors, + wagtail_page.wagtail_page_slug as cms_coursepage_slug, + wagtail_page.wagtail_page_url_path as cms_coursepage_url_path, + wagtail_page.wagtail_page_is_live as cms_coursepage_is_live, + wagtail_page.wagtail_page_first_published_on as cms_coursepage_first_published_on, + wagtail_page.wagtail_page_last_published_on as cms_coursepage_last_published_on from courses -left join cms_courses - on courses.course_id = cms_courses.course_id -left join wagtail_page - on cms_courses.wagtail_page_id = wagtail_page.wagtail_page_id -left join platform - on courses.platform_id = platform.platform_id -left join certificate_page_path - on wagtail_page.wagtail_page_path like certificate_page_path.wagtail_page_path || '%' +left join cms_courses on courses.course_id = cms_courses.course_id +left join wagtail_page on cms_courses.wagtail_page_id = wagtail_page.wagtail_page_id +left join platform on courses.platform_id = platform.platform_id +left join certificate_page_path on wagtail_page.wagtail_page_path like certificate_page_path.wagtail_page_path || '%' left join course_topics on courses.course_id = course_topics.course_id left join course_instructors on courses.course_id = course_instructors.course_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courses_to_topics.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courses_to_topics.sql index a19320a23..22173ef1d 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courses_to_topics.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__courses_to_topics.sql @@ -1,24 +1,13 @@ -with coursepagetopic as ( - select * - from {{ ref('stg__mitxpro__app__postgres__cms_coursepage_topics') }} -) - -, course_topics as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_coursetopic') }} -) - -, coursepages as ( - select * - from {{ ref('stg__mitxpro__app__postgres__cms_coursepage') }} -) +with + coursepagetopic as (select * from {{ ref("stg__mitxpro__app__postgres__cms_coursepage_topics") }}), + course_topics as (select * from {{ ref("stg__mitxpro__app__postgres__courses_coursetopic") }}), + coursepages as (select * from {{ ref("stg__mitxpro__app__postgres__cms_coursepage") }}) select - coursepages.course_id - , coursepagetopic.coursetopic_id - , course_topics.coursetopic_parent_coursetopic_id - , course_topics.coursetopic_name + coursepages.course_id, + coursepagetopic.coursetopic_id, + course_topics.coursetopic_parent_coursetopic_id, + course_topics.coursetopic_name from coursepagetopic -inner join coursepages - on coursepagetopic.wagtail_page_id = coursepages.wagtail_page_id -inner join course_topics - on coursepagetopic.coursetopic_id = course_topics.coursetopic_id +inner join coursepages on coursepagetopic.wagtail_page_id = coursepages.wagtail_page_id +inner join course_topics on coursepagetopic.coursetopic_id = course_topics.coursetopic_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__coursesfaculty.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__coursesfaculty.sql index 8600f51ba..90c4af952 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__coursesfaculty.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__coursesfaculty.sql @@ -1,49 +1,34 @@ -with coursepages as ( - select * - from {{ ref('stg__mitxpro__app__postgres__cms_coursepage') }} -) +with + coursepages as (select * from {{ ref("stg__mitxpro__app__postgres__cms_coursepage") }}), + facultymemberspage as ( + select + *, cast(json_parse(cms_facultymemberspage_faculty) as array(json)) as cms_facultymemberspage_faculty_array + from {{ ref("stg__mitxpro__app__postgres__cms_facultymemberspage") }} + ), + wagtailpages as (select * from {{ ref("stg__mitxpro__app__postgres__wagtail_page") }}), + unnestedfacultymemberspage as ( + select + facultymemberspage.wagtail_page_id, + json_format(cms_facultymemberspage_facultymember) as cms_facultymemberspage_facultymember -- noqa -, facultymemberspage as ( - select - * - , cast(json_parse(cms_facultymemberspage_faculty) as array (json)) as cms_facultymemberspage_faculty_array - from {{ ref('stg__mitxpro__app__postgres__cms_facultymemberspage') }} -) - -, wagtailpages as ( - select * - from {{ ref('stg__mitxpro__app__postgres__wagtail_page') }} -) - -, unnestedfacultymemberspage as ( - select - facultymemberspage.wagtail_page_id - , json_format(cms_facultymemberspage_facultymember) as cms_facultymemberspage_facultymember -- noqa - - from facultymemberspage - cross join - unnest(facultymemberspage.cms_facultymemberspage_faculty_array) - as t(cms_facultymemberspage_facultymember) -- noqa -) - -, coursepageswithpath as ( - select - coursepages.wagtail_page_id - , coursepages.course_id - , wagtailpages.wagtail_page_path - from coursepages - inner join wagtailpages - on coursepages.wagtail_page_id = wagtailpages.wagtail_page_id -) + from facultymemberspage + cross join + unnest(facultymemberspage.cms_facultymemberspage_faculty_array) as t(cms_facultymemberspage_facultymember) -- noqa + ), + coursepageswithpath as ( + select coursepages.wagtail_page_id, coursepages.course_id, wagtailpages.wagtail_page_path + from coursepages + inner join wagtailpages on coursepages.wagtail_page_id = wagtailpages.wagtail_page_id + ) select - coursepageswithpath.course_id - , json_query(unnestedfacultymemberspage.cms_facultymemberspage_facultymember, 'lax $.value.name') - as cms_facultymemberspage_facultymember_name - , json_query(unnestedfacultymemberspage.cms_facultymemberspage_facultymember, 'lax $.value.description') - as cms_facultymemberspage_facultymember_description + coursepageswithpath.course_id, + json_query( + unnestedfacultymemberspage.cms_facultymemberspage_facultymember, 'lax $.value.name' + ) as cms_facultymemberspage_facultymember_name, + json_query( + unnestedfacultymemberspage.cms_facultymemberspage_facultymember, 'lax $.value.description' + ) as cms_facultymemberspage_facultymember_description from unnestedfacultymemberspage -inner join wagtailpages - on unnestedfacultymemberspage.wagtail_page_id = wagtailpages.wagtail_page_id -inner join coursepageswithpath - on wagtailpages.wagtail_page_path like coursepageswithpath.wagtail_page_path || '%' +inner join wagtailpages on unnestedfacultymemberspage.wagtail_page_id = wagtailpages.wagtail_page_id +inner join coursepageswithpath on wagtailpages.wagtail_page_path like coursepageswithpath.wagtail_page_path || '%' diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__coursesinprogram.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__coursesinprogram.sql index d084dc2ad..46aaed67b 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__coursesinprogram.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__coursesinprogram.sql @@ -1,52 +1,25 @@ -with coursesinprogram as ( - select * - from {{ ref('stg__mitxpro__app__postgres__cms_coursesinprogrampage') }} -) - -, coursepages as ( - select * - from {{ ref('stg__mitxpro__app__postgres__cms_coursepage') }} -) - -, programpages as ( - select * - from {{ ref('stg__mitxpro__app__postgres__cms_programpage') }} -) - -, wagtailpages as ( - select * - from {{ ref('stg__mitxpro__app__postgres__wagtail_page') }} -) - -, unnestedcoursesinprogram as ( - select - - coursesinprogram.wagtail_page_id as coursesinprogrampage_wagtail_page_id - , coursepage_wagtail_page_id --noqa - - from coursesinprogram - cross join - unnest(coursesinprogram.cms_coursesinprogrampage_coursepage_wagtail_page_ids) - as t(coursepage_wagtail_page_id) --noqa -) - -, programpageswithpath as ( - select - programpages.wagtail_page_id - , programpages.program_id - , wagtailpages.wagtail_page_path - from programpages - inner join wagtailpages - on programpages.wagtail_page_id = wagtailpages.wagtail_page_id -) - -select - coursepages.course_id - , programpageswithpath.program_id +with + coursesinprogram as (select * from {{ ref("stg__mitxpro__app__postgres__cms_coursesinprogrampage") }}), + coursepages as (select * from {{ ref("stg__mitxpro__app__postgres__cms_coursepage") }}), + programpages as (select * from {{ ref("stg__mitxpro__app__postgres__cms_programpage") }}), + wagtailpages as (select * from {{ ref("stg__mitxpro__app__postgres__wagtail_page") }}), + unnestedcoursesinprogram as ( + select coursesinprogram.wagtail_page_id as coursesinprogrampage_wagtail_page_id, coursepage_wagtail_page_id -- noqa + + from coursesinprogram + cross join + unnest(coursesinprogram.cms_coursesinprogrampage_coursepage_wagtail_page_ids) as t( + coursepage_wagtail_page_id + ) -- noqa + ), + programpageswithpath as ( + select programpages.wagtail_page_id, programpages.program_id, wagtailpages.wagtail_page_path + from programpages + inner join wagtailpages on programpages.wagtail_page_id = wagtailpages.wagtail_page_id + ) + +select coursepages.course_id, programpageswithpath.program_id from unnestedcoursesinprogram -left join coursepages - on unnestedcoursesinprogram.coursepage_wagtail_page_id = coursepages.wagtail_page_id -left join wagtailpages - on unnestedcoursesinprogram.coursesinprogrampage_wagtail_page_id = wagtailpages.wagtail_page_id -inner join programpageswithpath - on wagtailpages.wagtail_page_path like programpageswithpath.wagtail_page_path || '%' +left join coursepages on unnestedcoursesinprogram.coursepage_wagtail_page_id = coursepages.wagtail_page_id +left join wagtailpages on unnestedcoursesinprogram.coursesinprogrampage_wagtail_page_id = wagtailpages.wagtail_page_id +inner join programpageswithpath on wagtailpages.wagtail_page_path like programpageswithpath.wagtail_page_path || '%' diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__coursetopic.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__coursetopic.sql index e9277de73..3312f3217 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__coursetopic.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__coursetopic.sql @@ -1,12 +1,5 @@ -with coursetopic as ( - select * - from {{ ref('stg__mitxpro__app__postgres__courses_coursetopic') }} -) +with coursetopic as (select * from {{ ref("stg__mitxpro__app__postgres__courses_coursetopic") }}) select - coursetopic_id - , coursetopic_name - , coursetopic_parent_coursetopic_id - , coursetopic_created_on - , coursetopic_updated_on + coursetopic_id, coursetopic_name, coursetopic_parent_coursetopic_id, coursetopic_created_on, coursetopic_updated_on from coursetopic diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_allcoupons.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_allcoupons.sql index b552b033d..52ead50c6 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_allcoupons.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_allcoupons.sql @@ -1,101 +1,82 @@ -with coupon as ( - select * - from {{ ref('int__mitxpro__ecommerce_coupon') }} -) - -, ecommerce_couponversion as ( - select * - from {{ ref('int__mitxpro__ecommerce_couponversion') }} -) - -, ecommerce_couponpaymentversion as ( - select * - from {{ ref('int__mitxpro__ecommerce_couponpaymentversion') }} -) - -, b2becommerce_b2bcoupon as ( - select * - from {{ ref('int__mitxpro__b2becommerce_b2bcoupon') }} -) - -, latest_couponversion as ( - select - coupon_id - , max(couponversion_updated_on) as max_couponversion_updated_on - from ecommerce_couponversion - group by coupon_id -) - -, reg_coupon_fields as ( - select - coupon.coupon_code - , coupon.couponpayment_name as coupon_name - , coupon.coupon_created_on - , ecommerce_couponpaymentversion.couponpaymentversion_payment_transaction as payment_transaction - , ecommerce_couponpaymentversion.couponpaymentversion_coupon_type as coupon_type - , ecommerce_couponpaymentversion.couponpaymentversion_discount_source as discount_source - , ecommerce_couponpaymentversion.couponpaymentversion_activated_on as activated_on - , ecommerce_couponpaymentversion.couponpaymentversion_expires_on as expires_on - , 'ecommerce_coupon' as coupon_source_table - , null as b2bcoupon_id - , coupon.coupon_id - , ecommerce_couponpaymentversion.couponpaymentversion_discount_amount_text as discount_amount - from coupon - inner join latest_couponversion - on coupon.coupon_id = latest_couponversion.coupon_id - inner join ecommerce_couponversion - on - latest_couponversion.coupon_id = ecommerce_couponversion.coupon_id +with + coupon as (select * from {{ ref("int__mitxpro__ecommerce_coupon") }}), + ecommerce_couponversion as (select * from {{ ref("int__mitxpro__ecommerce_couponversion") }}), + ecommerce_couponpaymentversion as (select * from {{ ref("int__mitxpro__ecommerce_couponpaymentversion") }}), + b2becommerce_b2bcoupon as (select * from {{ ref("int__mitxpro__b2becommerce_b2bcoupon") }}), + latest_couponversion as ( + select coupon_id, max(couponversion_updated_on) as max_couponversion_updated_on + from ecommerce_couponversion + group by coupon_id + ), + reg_coupon_fields as ( + select + coupon.coupon_code, + coupon.couponpayment_name as coupon_name, + coupon.coupon_created_on, + ecommerce_couponpaymentversion.couponpaymentversion_payment_transaction as payment_transaction, + ecommerce_couponpaymentversion.couponpaymentversion_coupon_type as coupon_type, + ecommerce_couponpaymentversion.couponpaymentversion_discount_source as discount_source, + ecommerce_couponpaymentversion.couponpaymentversion_activated_on as activated_on, + ecommerce_couponpaymentversion.couponpaymentversion_expires_on as expires_on, + 'ecommerce_coupon' as coupon_source_table, + null as b2bcoupon_id, + coupon.coupon_id, + ecommerce_couponpaymentversion.couponpaymentversion_discount_amount_text as discount_amount + from coupon + inner join latest_couponversion on coupon.coupon_id = latest_couponversion.coupon_id + inner join + ecommerce_couponversion + on latest_couponversion.coupon_id = ecommerce_couponversion.coupon_id and latest_couponversion.max_couponversion_updated_on = ecommerce_couponversion.couponversion_updated_on - inner join ecommerce_couponpaymentversion - on ecommerce_couponversion.couponpaymentversion_id = ecommerce_couponpaymentversion.couponpaymentversion_id -) - -, b2b_coupon_fields as ( - select - b2bcoupon_coupon_code as coupon_code - , b2bcoupon_name as coupon_name - , b2bcoupon_created_on as coupon_created_on - , null as payment_transaction - , null as coupon_type - , null as discount_source - , b2bcoupon_activated_on as activated_on - , b2bcoupon_expires_on as expires_on - , 'b2bcoupon' as coupon_source_table - , b2bcoupon_id - , null as coupon_id - , cast(cast((b2bcoupon_discount_percent * 100) as integer) as varchar) || '% off' as discount_amount - from b2becommerce_b2bcoupon -) + inner join + ecommerce_couponpaymentversion + on ecommerce_couponversion.couponpaymentversion_id = ecommerce_couponpaymentversion.couponpaymentversion_id + ), + b2b_coupon_fields as ( + select + b2bcoupon_coupon_code as coupon_code, + b2bcoupon_name as coupon_name, + b2bcoupon_created_on as coupon_created_on, + null as payment_transaction, + null as coupon_type, + null as discount_source, + b2bcoupon_activated_on as activated_on, + b2bcoupon_expires_on as expires_on, + 'b2bcoupon' as coupon_source_table, + b2bcoupon_id, + null as coupon_id, + cast(cast((b2bcoupon_discount_percent * 100) as integer) as varchar) || '% off' as discount_amount + from b2becommerce_b2bcoupon + ) select - coupon_code - , coupon_name - , coupon_created_on - , payment_transaction - , discount_amount - , coupon_type - , discount_source - , activated_on - , expires_on - , coupon_source_table - , b2bcoupon_id - , coupon_id + coupon_code, + coupon_name, + coupon_created_on, + payment_transaction, + discount_amount, + coupon_type, + discount_source, + activated_on, + expires_on, + coupon_source_table, + b2bcoupon_id, + coupon_id from reg_coupon_fields union all select - coupon_code - , coupon_name - , coupon_created_on - , payment_transaction - , discount_amount - , coupon_type - , discount_source - , activated_on - , expires_on - , coupon_source_table - , b2bcoupon_id - , coupon_id + coupon_code, + coupon_name, + coupon_created_on, + payment_transaction, + discount_amount, + coupon_type, + discount_source, + activated_on, + expires_on, + coupon_source_table, + b2bcoupon_id, + coupon_id from b2b_coupon_fields diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_allorders.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_allorders.sql index 5baef1b8b..03855ff0b 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_allorders.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_allorders.sql @@ -1,235 +1,178 @@ -with b2becommerce_b2border as ( - select * - from {{ ref('int__mitxpro__b2becommerce_b2border') }} -) - -, b2becommerce_b2breceipt as ( - select * - from {{ ref('int__mitxpro__b2becommerce_b2breceipt') }} -) - -, b2becommerce_b2bcouponredemption as ( - select * - from {{ ref('int__mitxpro__b2becommerce_b2bcouponredemption') }} -) - -, ecommerce_couponpaymentversion as ( - select * - from {{ ref('int__mitxpro__ecommerce_couponpaymentversion') }} -) - -, ecommerce_couponredemption as ( - select * - from {{ ref('int__mitxpro__ecommerce_couponredemption') }} -) - -, ecommerce_order as ( - select * - from {{ ref('int__mitxpro__ecommerce_order') }} -) - -, ecommerce_line as ( - select * - from {{ ref('int__mitxpro__ecommerce_line') }} -) - -, users as ( - select * - from {{ ref('int__mitxpro__users') }} -) - -, course_runs as ( - select * - from {{ ref('int__mitxpro__course_runs') }} -) - -, programs as ( - select * - from {{ ref('int__mitxpro__programs') }} -) - -, productversion as ( - select * - from {{ ref('int__mitxpro__ecommerce_productversion') }} -) - -, reg_order_fields as ( - select - ecommerce_order.order_id - , b2becommerce_b2border.b2border_id - , ecommerce_order.order_state - , ecommerce_line.line_id - , ecommerce_line.product_id - , ecommerce_couponpaymentversion.couponpaymentversion_payment_transaction - , ecommerce_couponpaymentversion.couponpaymentversion_coupon_type - , b2becommerce_b2border.b2border_discount - , users.user_email - , ecommerce_line.product_type - , course_runs.courserun_id - , course_runs.courserun_readable_id - , programs.program_readable_id - , ecommerce_order.coupon_id - , ecommerce_order.order_created_on - , productversion.productversion_readable_id - , b2becommerce_b2border.b2bcoupon_id - , b2becommerce_b2border.b2border_contract_number - , ecommerce_order.receipt_reference_number as req_reference_number - , ecommerce_order.receipt_authorization_code - , ecommerce_order.receipt_transaction_id - , ecommerce_order.receipt_payment_method - , ecommerce_order.receipt_bill_to_address_state - , ecommerce_order.receipt_bill_to_address_country - , case - when ecommerce_order.order_state in ('fulfilled', 'refunded') +with + b2becommerce_b2border as (select * from {{ ref("int__mitxpro__b2becommerce_b2border") }}), + b2becommerce_b2breceipt as (select * from {{ ref("int__mitxpro__b2becommerce_b2breceipt") }}), + b2becommerce_b2bcouponredemption as (select * from {{ ref("int__mitxpro__b2becommerce_b2bcouponredemption") }}), + ecommerce_couponpaymentversion as (select * from {{ ref("int__mitxpro__ecommerce_couponpaymentversion") }}), + ecommerce_couponredemption as (select * from {{ ref("int__mitxpro__ecommerce_couponredemption") }}), + ecommerce_order as (select * from {{ ref("int__mitxpro__ecommerce_order") }}), + ecommerce_line as (select * from {{ ref("int__mitxpro__ecommerce_line") }}), + users as (select * from {{ ref("int__mitxpro__users") }}), + course_runs as (select * from {{ ref("int__mitxpro__course_runs") }}), + programs as (select * from {{ ref("int__mitxpro__programs") }}), + productversion as (select * from {{ ref("int__mitxpro__ecommerce_productversion") }}), + reg_order_fields as ( + select + ecommerce_order.order_id, + b2becommerce_b2border.b2border_id, + ecommerce_order.order_state, + ecommerce_line.line_id, + ecommerce_line.product_id, + ecommerce_couponpaymentversion.couponpaymentversion_payment_transaction, + ecommerce_couponpaymentversion.couponpaymentversion_coupon_type, + b2becommerce_b2border.b2border_discount, + users.user_email, + ecommerce_line.product_type, + course_runs.courserun_id, + course_runs.courserun_readable_id, + programs.program_readable_id, + ecommerce_order.coupon_id, + ecommerce_order.order_created_on, + productversion.productversion_readable_id, + b2becommerce_b2border.b2bcoupon_id, + b2becommerce_b2border.b2border_contract_number, + ecommerce_order.receipt_reference_number as req_reference_number, + ecommerce_order.receipt_authorization_code, + ecommerce_order.receipt_transaction_id, + ecommerce_order.receipt_payment_method, + ecommerce_order.receipt_bill_to_address_state, + ecommerce_order.receipt_bill_to_address_country, + case + when ecommerce_order.order_state in ('fulfilled', 'refunded') then ecommerce_couponredemption.couponredemption_created_on - end as coupon_redeemed_on - , case - when - ecommerce_couponredemption.couponredemption_id is not null - and ecommerce_order.order_state in ('fulfilled', 'refunded') + end as coupon_redeemed_on, + case + when + ecommerce_couponredemption.couponredemption_id is not null + and ecommerce_order.order_state in ('fulfilled', 'refunded') then true - end as redeemed - from ecommerce_order - inner join ecommerce_line - on ecommerce_order.order_id = ecommerce_line.order_id - left join ecommerce_couponpaymentversion - on ecommerce_order.couponpaymentversion_id = ecommerce_couponpaymentversion.couponpaymentversion_id - left join ecommerce_couponredemption - on ecommerce_order.order_id = ecommerce_couponredemption.order_id - left join users - on ecommerce_order.order_purchaser_user_id = users.user_id - left join course_runs - on ecommerce_line.courserun_id = course_runs.courserun_id - left join programs - on ecommerce_line.program_id = programs.program_id - left join productversion - on ecommerce_line.productversion_id = productversion.productversion_id - left join b2becommerce_b2border - on ecommerce_couponpaymentversion.couponpaymentversion_id = b2becommerce_b2border.couponpaymentversion_id -) - -, reg_order_test as ( - select b2border_id - from reg_order_fields - group by b2border_id -) - -, b2b_order_fields as ( - select - ecommerce_order.order_id - , b2becommerce_b2border.b2border_id - , ecommerce_line.line_id - , b2becommerce_b2border.product_id - , ecommerce_couponpaymentversion.couponpaymentversion_payment_transaction - , ecommerce_couponpaymentversion.couponpaymentversion_coupon_type - , b2becommerce_b2border.b2border_discount - , course_runs.courserun_id - , course_runs.courserun_readable_id - , programs.program_readable_id - , null as coupon_id - , b2becommerce_b2border.b2bcoupon_id - , b2becommerce_b2border.b2border_created_on as order_created_on - , b2becommerce_b2border.b2border_status as order_state - , b2becommerce_b2border.product_type - , b2becommerce_b2border.b2border_email as user_email - , b2becommerce_b2border.b2border_contract_number - , productversion.productversion_readable_id - , b2becommerce_b2breceipt.b2breceipt_reference_number as req_reference_number - , b2becommerce_b2breceipt.b2breceipt_authorization_code as receipt_authorization_code - , b2becommerce_b2breceipt.b2breceipt_transaction_id as receipt_transaction_id - , b2becommerce_b2breceipt.b2breceipt_payment_method as receipt_payment_method - , b2becommerce_b2breceipt.b2breceipt_bill_to_address_state as receipt_bill_to_address_state - , b2becommerce_b2breceipt.b2breceipt_bill_to_address_country as receipt_bill_to_address_country - , case - when b2becommerce_b2border.b2border_status in ('fulfilled', 'refunded') + end as redeemed + from ecommerce_order + inner join ecommerce_line on ecommerce_order.order_id = ecommerce_line.order_id + left join + ecommerce_couponpaymentversion + on ecommerce_order.couponpaymentversion_id = ecommerce_couponpaymentversion.couponpaymentversion_id + left join ecommerce_couponredemption on ecommerce_order.order_id = ecommerce_couponredemption.order_id + left join users on ecommerce_order.order_purchaser_user_id = users.user_id + left join course_runs on ecommerce_line.courserun_id = course_runs.courserun_id + left join programs on ecommerce_line.program_id = programs.program_id + left join productversion on ecommerce_line.productversion_id = productversion.productversion_id + left join + b2becommerce_b2border + on ecommerce_couponpaymentversion.couponpaymentversion_id = b2becommerce_b2border.couponpaymentversion_id + ), + reg_order_test as (select b2border_id from reg_order_fields group by b2border_id), + b2b_order_fields as ( + select + ecommerce_order.order_id, + b2becommerce_b2border.b2border_id, + ecommerce_line.line_id, + b2becommerce_b2border.product_id, + ecommerce_couponpaymentversion.couponpaymentversion_payment_transaction, + ecommerce_couponpaymentversion.couponpaymentversion_coupon_type, + b2becommerce_b2border.b2border_discount, + course_runs.courserun_id, + course_runs.courserun_readable_id, + programs.program_readable_id, + null as coupon_id, + b2becommerce_b2border.b2bcoupon_id, + b2becommerce_b2border.b2border_created_on as order_created_on, + b2becommerce_b2border.b2border_status as order_state, + b2becommerce_b2border.product_type, + b2becommerce_b2border.b2border_email as user_email, + b2becommerce_b2border.b2border_contract_number, + productversion.productversion_readable_id, + b2becommerce_b2breceipt.b2breceipt_reference_number as req_reference_number, + b2becommerce_b2breceipt.b2breceipt_authorization_code as receipt_authorization_code, + b2becommerce_b2breceipt.b2breceipt_transaction_id as receipt_transaction_id, + b2becommerce_b2breceipt.b2breceipt_payment_method as receipt_payment_method, + b2becommerce_b2breceipt.b2breceipt_bill_to_address_state as receipt_bill_to_address_state, + b2becommerce_b2breceipt.b2breceipt_bill_to_address_country as receipt_bill_to_address_country, + case + when b2becommerce_b2border.b2border_status in ('fulfilled', 'refunded') then b2becommerce_b2bcouponredemption.b2bcouponredemption_created_on - end as coupon_redeemed_on - , case - when - b2becommerce_b2bcouponredemption.b2bcouponredemption_id is not null - and b2becommerce_b2border.b2border_status in ('fulfilled', 'refunded') + end as coupon_redeemed_on, + case + when + b2becommerce_b2bcouponredemption.b2bcouponredemption_id is not null + and b2becommerce_b2border.b2border_status in ('fulfilled', 'refunded') then true - end as redeemed - from b2becommerce_b2border - left join ecommerce_couponpaymentversion - on b2becommerce_b2border.couponpaymentversion_id = ecommerce_couponpaymentversion.couponpaymentversion_id - left join ecommerce_order - on ecommerce_couponpaymentversion.couponpaymentversion_id = ecommerce_order.couponpaymentversion_id - left join ecommerce_line - on ecommerce_order.order_id = ecommerce_line.order_id - left join course_runs - on ecommerce_line.courserun_id = course_runs.courserun_id - left join programs - on ecommerce_line.program_id = programs.program_id - left join b2becommerce_b2breceipt - on b2becommerce_b2border.b2border_id = b2becommerce_b2breceipt.b2border_id - left join productversion - on b2becommerce_b2border.productversion_id = productversion.productversion_id - left join b2becommerce_b2bcouponredemption - on - b2becommerce_b2border.b2border_id = b2becommerce_b2bcouponredemption.b2border_id + end as redeemed + from b2becommerce_b2border + left join + ecommerce_couponpaymentversion + on b2becommerce_b2border.couponpaymentversion_id = ecommerce_couponpaymentversion.couponpaymentversion_id + left join + ecommerce_order + on ecommerce_couponpaymentversion.couponpaymentversion_id = ecommerce_order.couponpaymentversion_id + left join ecommerce_line on ecommerce_order.order_id = ecommerce_line.order_id + left join course_runs on ecommerce_line.courserun_id = course_runs.courserun_id + left join programs on ecommerce_line.program_id = programs.program_id + left join b2becommerce_b2breceipt on b2becommerce_b2border.b2border_id = b2becommerce_b2breceipt.b2border_id + left join productversion on b2becommerce_b2border.productversion_id = productversion.productversion_id + left join + b2becommerce_b2bcouponredemption + on b2becommerce_b2border.b2border_id = b2becommerce_b2bcouponredemption.b2border_id and b2becommerce_b2border.b2bcoupon_id = b2becommerce_b2bcouponredemption.b2bcoupon_id - left join reg_order_test - on b2becommerce_b2border.b2border_id = reg_order_test.b2border_id - where reg_order_test.b2border_id is null -) + left join reg_order_test on b2becommerce_b2border.b2border_id = reg_order_test.b2border_id + where reg_order_test.b2border_id is null + ) select - order_id - , b2border_id - , order_created_on - , order_state - , line_id - , product_id - , couponpaymentversion_payment_transaction - , couponpaymentversion_coupon_type - , b2border_discount - , redeemed - , coupon_redeemed_on - , user_email - , product_type - , courserun_id - , courserun_readable_id - , program_readable_id - , coupon_id - , b2bcoupon_id - , b2border_contract_number - , req_reference_number - , productversion_readable_id - , receipt_authorization_code - , receipt_transaction_id - , receipt_payment_method - , receipt_bill_to_address_state - , receipt_bill_to_address_country + order_id, + b2border_id, + order_created_on, + order_state, + line_id, + product_id, + couponpaymentversion_payment_transaction, + couponpaymentversion_coupon_type, + b2border_discount, + redeemed, + coupon_redeemed_on, + user_email, + product_type, + courserun_id, + courserun_readable_id, + program_readable_id, + coupon_id, + b2bcoupon_id, + b2border_contract_number, + req_reference_number, + productversion_readable_id, + receipt_authorization_code, + receipt_transaction_id, + receipt_payment_method, + receipt_bill_to_address_state, + receipt_bill_to_address_country from reg_order_fields union distinct select - order_id - , b2border_id - , order_created_on - , order_state - , line_id - , product_id - , couponpaymentversion_payment_transaction - , couponpaymentversion_coupon_type - , b2border_discount - , redeemed - , coupon_redeemed_on - , user_email - , product_type - , courserun_id - , courserun_readable_id - , program_readable_id - , coupon_id - , b2bcoupon_id - , b2border_contract_number - , req_reference_number - , productversion_readable_id - , receipt_authorization_code - , receipt_transaction_id - , receipt_payment_method - , receipt_bill_to_address_state - , receipt_bill_to_address_country + order_id, + b2border_id, + order_created_on, + order_state, + line_id, + product_id, + couponpaymentversion_payment_transaction, + couponpaymentversion_coupon_type, + b2border_discount, + redeemed, + coupon_redeemed_on, + user_email, + product_type, + courserun_id, + courserun_readable_id, + program_readable_id, + coupon_id, + b2bcoupon_id, + b2border_contract_number, + req_reference_number, + productversion_readable_id, + receipt_authorization_code, + receipt_transaction_id, + receipt_payment_method, + receipt_bill_to_address_state, + receipt_bill_to_address_country from b2b_order_fields diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_basket.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_basket.sql index f1fab2ab6..3e86bdea0 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_basket.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_basket.sql @@ -1,18 +1,7 @@ -with basket as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_basket') }} -) +with + basket as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_basket") }}), + couponbasket as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_couponbasket") }}) -, couponbasket as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_couponbasket') }} -) - -select - basket.basket_id - , basket.user_id - , basket.basket_created_on - , basket.basket_updated_on - , couponbasket.coupon_id +select basket.basket_id, basket.user_id, basket.basket_created_on, basket.basket_updated_on, couponbasket.coupon_id from basket left join couponbasket on basket.basket_id = couponbasket.basket_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_basketitem.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_basketitem.sql index cd4a35ecf..715e7cf1c 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_basketitem.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_basketitem.sql @@ -1,21 +1,15 @@ -with basketitem as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_basketitem') }} -) - -, basket as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_basket') }} -) +with + basketitem as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_basketitem") }}), + basket as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_basket") }}) select - basketitem.basketitem_id - , basketitem.basketitem_quantity - , basketitem.basket_id - , basketitem.basketitem_created_on - , basketitem.product_id - , basketitem.basketitem_updated_on - , basketitem.programrun_id - , basket.user_id + basketitem.basketitem_id, + basketitem.basketitem_quantity, + basketitem.basket_id, + basketitem.basketitem_created_on, + basketitem.product_id, + basketitem.basketitem_updated_on, + basketitem.programrun_id, + basket.user_id from basketitem inner join basket on basketitem.basket_id = basket.basket_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_basketrunselection.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_basketrunselection.sql index 8d6f26eef..2f99749a3 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_basketrunselection.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_basketrunselection.sql @@ -1,19 +1,13 @@ -with basketrunselection as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_basketrunselection') }} -) - -, basket as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_basket') }} -) +with + basketrunselection as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_basketrunselection") }}), + basket as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_basket") }}) select - basketrunselection.basketrunselection_id - , basketrunselection.basket_id - , basketrunselection.courserun_id - , basketrunselection.basketrunselection_created_on - , basketrunselection.basketrunselection_updated_on - , basket.user_id + basketrunselection.basketrunselection_id, + basketrunselection.basket_id, + basketrunselection.courserun_id, + basketrunselection.basketrunselection_created_on, + basketrunselection.basketrunselection_updated_on, + basket.user_id from basketrunselection inner join basket on basketrunselection.basket_id = basket.basket_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_company.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_company.sql index ff581081b..83188925d 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_company.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_company.sql @@ -1,11 +1,4 @@ -with company as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_company') }} -) +with company as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_company") }}) -select - company_id - , company_name - , company_updated_on - , company_created_on +select company_id, company_name, company_updated_on, company_created_on from company diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_coupon.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_coupon.sql index 106ad59d6..720408e50 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_coupon.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_coupon.sql @@ -1,21 +1,15 @@ -with coupon as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_coupon') }} -) - -, couponpayment as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_couponpayment') }} -) +with + coupon as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_coupon") }}), + couponpayment as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_couponpayment") }}) select - coupon.coupon_id - , coupon.coupon_code - , couponpayment.couponpayment_name - , coupon.coupon_is_active - , coupon.coupon_applies_to_future_runs - , coupon.coupon_is_global - , coupon.coupon_updated_on - , coupon.coupon_created_on + coupon.coupon_id, + coupon.coupon_code, + couponpayment.couponpayment_name, + coupon.coupon_is_active, + coupon.coupon_applies_to_future_runs, + coupon.coupon_is_global, + coupon.coupon_updated_on, + coupon.coupon_created_on from coupon inner join couponpayment on coupon.couponpayment_id = couponpayment.couponpayment_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponpaymentversion.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponpaymentversion.sql index 148210cfe..c63574e65 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponpaymentversion.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponpaymentversion.sql @@ -1,31 +1,25 @@ -with couponpaymentversion as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_couponpaymentversion') }} -) - -, couponpayment as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_couponpayment') }} -) +with + couponpaymentversion as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_couponpaymentversion") }}), + couponpayment as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_couponpayment") }}) select - couponpaymentversion.couponpaymentversion_id - , couponpaymentversion.couponpaymentversion_expires_on - , couponpaymentversion.company_id - , couponpaymentversion.couponpaymentversion_num_coupon_codes - , couponpaymentversion.couponpaymentversion_activated_on - , couponpaymentversion.couponpaymentversion_coupon_type - , couponpaymentversion.couponpaymentversion_created_on - , couponpaymentversion.couponpaymentversion_discount_amount - , couponpaymentversion.couponpaymentversion_updated_on - , couponpaymentversion.couponpaymentversion_max_redemptions_per_user - , couponpayment.couponpayment_name - , couponpaymentversion.couponpaymentversion_is_automatic - , couponpaymentversion.couponpaymentversion_discount_source - , couponpaymentversion.couponpaymentversion_payment_transaction - , couponpaymentversion.couponpaymentversion_tag - , couponpaymentversion.couponpaymentversion_discount_type - , couponpaymentversion.couponpaymentversion_discount_amount_text - , couponpaymentversion.couponpaymentversion_max_redemptions + couponpaymentversion.couponpaymentversion_id, + couponpaymentversion.couponpaymentversion_expires_on, + couponpaymentversion.company_id, + couponpaymentversion.couponpaymentversion_num_coupon_codes, + couponpaymentversion.couponpaymentversion_activated_on, + couponpaymentversion.couponpaymentversion_coupon_type, + couponpaymentversion.couponpaymentversion_created_on, + couponpaymentversion.couponpaymentversion_discount_amount, + couponpaymentversion.couponpaymentversion_updated_on, + couponpaymentversion.couponpaymentversion_max_redemptions_per_user, + couponpayment.couponpayment_name, + couponpaymentversion.couponpaymentversion_is_automatic, + couponpaymentversion.couponpaymentversion_discount_source, + couponpaymentversion.couponpaymentversion_payment_transaction, + couponpaymentversion.couponpaymentversion_tag, + couponpaymentversion.couponpaymentversion_discount_type, + couponpaymentversion.couponpaymentversion_discount_amount_text, + couponpaymentversion.couponpaymentversion_max_redemptions from couponpaymentversion inner join couponpayment on couponpaymentversion.couponpayment_id = couponpayment.couponpayment_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponproduct.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponproduct.sql index 7ac6e4cbb..24ddc222c 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponproduct.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponproduct.sql @@ -1,13 +1,4 @@ -with couponproduct as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_couponproduct') }} -) +with couponproduct as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_couponproduct") }}) -select - couponproduct_id - , product_id - , coupon_id - , couponproduct_created_on - , couponproduct_updated_on - , programrun_id +select couponproduct_id, product_id, coupon_id, couponproduct_created_on, couponproduct_updated_on, programrun_id from couponproduct diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponredemption.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponredemption.sql index 63c35ef23..b6216f72b 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponredemption.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponredemption.sql @@ -1,12 +1,4 @@ -with ecommerce_couponredemption as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_couponredemption') }} -) +with ecommerce_couponredemption as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_couponredemption") }}) -select - couponredemption_id - , order_id - , couponversion_id - , couponredemption_created_on - , couponredemption_updated_on +select couponredemption_id, order_id, couponversion_id, couponredemption_created_on, couponredemption_updated_on from ecommerce_couponredemption diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponversion.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponversion.sql index 775d336fd..5a7b42dae 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponversion.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_couponversion.sql @@ -1,31 +1,22 @@ -with ecommerce_couponversion as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_couponversion') }} -) - -, latest_ecommerce_couponversion as ( - select - coupon_id - , max(couponversion_updated_on) as max_couponversion_updated_on - from ecommerce_couponversion - group by coupon_id -) +with + ecommerce_couponversion as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_couponversion") }}), + latest_ecommerce_couponversion as ( + select coupon_id, max(couponversion_updated_on) as max_couponversion_updated_on + from ecommerce_couponversion + group by coupon_id + ) select - ecommerce_couponversion.couponversion_id - , ecommerce_couponversion.coupon_id - , ecommerce_couponversion.couponpaymentversion_id - , ecommerce_couponversion.couponversion_updated_on - , ecommerce_couponversion.couponversion_created_on - , case - when - latest_ecommerce_couponversion.max_couponversion_updated_on - is not null then 'Y' - else 'N' + ecommerce_couponversion.couponversion_id, + ecommerce_couponversion.coupon_id, + ecommerce_couponversion.couponpaymentversion_id, + ecommerce_couponversion.couponversion_updated_on, + ecommerce_couponversion.couponversion_created_on, + case + when latest_ecommerce_couponversion.max_couponversion_updated_on is not null then 'Y' else 'N' end as is_latest_couponversion from ecommerce_couponversion -left join latest_ecommerce_couponversion - on - ecommerce_couponversion.coupon_id = latest_ecommerce_couponversion.coupon_id - and ecommerce_couponversion.couponversion_updated_on - = latest_ecommerce_couponversion.max_couponversion_updated_on +left join + latest_ecommerce_couponversion + on ecommerce_couponversion.coupon_id = latest_ecommerce_couponversion.coupon_id + and ecommerce_couponversion.couponversion_updated_on = latest_ecommerce_couponversion.max_couponversion_updated_on diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_line.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_line.sql index b9afb5728..f7dd04365 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_line.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_line.sql @@ -1,37 +1,23 @@ -with lines as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_line') }} -) - -, productversions as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_productversion') }} -) - -, products as ( - select * - from {{ ref('int__mitxpro__ecommerce_product') }} -) - -, programrunlines as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_programrunline') }} -) +with + lines as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_line") }}), + productversions as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_productversion") }}), + products as (select * from {{ ref("int__mitxpro__ecommerce_product") }}), + programrunlines as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_programrunline") }}) select - lines.line_id - , lines.order_id - , lines.productversion_id - , productversions.productversion_price as product_price - , lines.line_created_on - , lines.line_updated_on - , products.product_id - , products.courserun_id - , products.program_id - , products.product_type - , products.courserun_readable_id - , products.program_readable_id - , programrunlines.programrun_id + lines.line_id, + lines.order_id, + lines.productversion_id, + productversions.productversion_price as product_price, + lines.line_created_on, + lines.line_updated_on, + products.product_id, + products.courserun_id, + products.program_id, + products.product_type, + products.courserun_readable_id, + products.program_readable_id, + programrunlines.programrun_id from lines inner join productversions on lines.productversion_id = productversions.productversion_id inner join products on productversions.product_id = products.product_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_linerunselection.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_linerunselection.sql index cf64674ff..afc54d082 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_linerunselection.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_linerunselection.sql @@ -1,12 +1,4 @@ -with linerunselection as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_linerunselection') }} -) +with linerunselection as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_linerunselection") }}) -select - linerunselection_id - , line_id - , courserun_id - , linerunselection_created_on - , linerunselection_updated_on +select linerunselection_id, line_id, courserun_id, linerunselection_created_on, linerunselection_updated_on from linerunselection diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_order.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_order.sql index d8833a0f1..08abf6693 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_order.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_order.sql @@ -1,64 +1,40 @@ -with orders as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_order') }} -) - -, couponredemption as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_couponredemption') }} -) - -, couponversion as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_couponversion') }} -) - -, coupon as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_coupon') }} -) - -, couponpaymentversion as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_couponpaymentversion') }} -) - -, receipts as ( - select * - from {{ ref('int__mitxpro__ecommerce_receipt') }} - where receipt_transaction_status != 'ERROR' -) +with + orders as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_order") }}), + couponredemption as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_couponredemption") }}), + couponversion as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_couponversion") }}), + coupon as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_coupon") }}), + couponpaymentversion as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_couponpaymentversion") }}), + receipts as (select * from {{ ref("int__mitxpro__ecommerce_receipt") }} where receipt_transaction_status != 'ERROR') select - orders.order_id - , orders.order_state - , orders.order_purchaser_user_id - , orders.order_total_price_paid - , orders.order_created_on - , orders.order_updated_on - , couponversion.coupon_id - , couponversion.couponpaymentversion_id - , orders.order_tax_country_code - , orders.order_tax_rate - , orders.order_tax_rate_name - , coupon.coupon_code - , couponpaymentversion.couponpaymentversion_payment_transaction - , couponpaymentversion.couponpaymentversion_coupon_type - , couponpaymentversion.couponpaymentversion_discount_source - , couponpaymentversion.couponpaymentversion_discount_amount_text - , couponpaymentversion.couponpaymentversion_discount_type - , couponpaymentversion.couponpaymentversion_discount_amount - , receipts.receipt_reference_number - , receipts.receipt_authorization_code - , receipts.receipt_payment_method - , receipts.receipt_transaction_id - , receipts.receipt_bill_to_address_state - , receipts.receipt_bill_to_address_country - , orders.order_tax_amount - , orders.order_total_price_paid_plus_tax - , case - when orders.order_state in ('fulfilled', 'refunded') - then couponredemption.couponredemption_created_on + orders.order_id, + orders.order_state, + orders.order_purchaser_user_id, + orders.order_total_price_paid, + orders.order_created_on, + orders.order_updated_on, + couponversion.coupon_id, + couponversion.couponpaymentversion_id, + orders.order_tax_country_code, + orders.order_tax_rate, + orders.order_tax_rate_name, + coupon.coupon_code, + couponpaymentversion.couponpaymentversion_payment_transaction, + couponpaymentversion.couponpaymentversion_coupon_type, + couponpaymentversion.couponpaymentversion_discount_source, + couponpaymentversion.couponpaymentversion_discount_amount_text, + couponpaymentversion.couponpaymentversion_discount_type, + couponpaymentversion.couponpaymentversion_discount_amount, + receipts.receipt_reference_number, + receipts.receipt_authorization_code, + receipts.receipt_payment_method, + receipts.receipt_transaction_id, + receipts.receipt_bill_to_address_state, + receipts.receipt_bill_to_address_country, + orders.order_tax_amount, + orders.order_total_price_paid_plus_tax, + case + when orders.order_state in ('fulfilled', 'refunded') then couponredemption.couponredemption_created_on end as couponredemption_created_on from orders left join couponredemption on orders.order_id = couponredemption.order_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_product.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_product.sql index 3cab812ee..22629958a 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_product.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_product.sql @@ -1,66 +1,53 @@ -with products as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_product') }} -) - -, contenttypes as ( - select * - from {{ ref('stg__mitxpro__app__postgres__django_contenttype') }} -) - -, courseruns as ( - select * - from {{ ref('stg__mitxpro__app__postgres__courses_courserun') }} -) - -, programs as ( - select * - from {{ ref('stg__mitxpro__app__postgres__courses_program') }} -) - -, latest_productversion as ( - select * from - ( - select - productversion.* - , row_number() over ( - partition by productversion.product_id - order by productversion.productversion_updated_on desc - ) as row_num - from {{ ref('int__mitxpro__ecommerce_productversion') }} as productversion - ) as product - where row_num = 1 -) - -, product_subquery as ( - select - products.product_id - , products.product_is_active - , products.product_created_on - , products.product_updated_on - , products.product_is_private - , case contenttypes.contenttype_full_name - when 'courses_courserun' then products.product_object_id - end as courserun_id - , case contenttypes.contenttype_full_name - when 'courses_program' then products.product_object_id - end as program_id - , case contenttypes.contenttype_full_name - when 'courses_courserun' then 'course run' - when 'courses_program' then 'program' - else contenttypes.contenttype_full_name - end as product_type - from products - inner join contenttypes on products.contenttype_id = contenttypes.contenttype_id -) +with + products as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_product") }}), + contenttypes as (select * from {{ ref("stg__mitxpro__app__postgres__django_contenttype") }}), + courseruns as (select * from {{ ref("stg__mitxpro__app__postgres__courses_courserun") }}), + programs as (select * from {{ ref("stg__mitxpro__app__postgres__courses_program") }}), + latest_productversion as ( + select * + from + ( + select + productversion.*, + row_number() over ( + partition by productversion.product_id order by productversion.productversion_updated_on desc + ) as row_num + from {{ ref("int__mitxpro__ecommerce_productversion") }} as productversion + ) as product + where row_num = 1 + ), + product_subquery as ( + select + products.product_id, + products.product_is_active, + products.product_created_on, + products.product_updated_on, + products.product_is_private, + case + contenttypes.contenttype_full_name when 'courses_courserun' then products.product_object_id + end as courserun_id, + case + contenttypes.contenttype_full_name when 'courses_program' then products.product_object_id + end as program_id, + case + contenttypes.contenttype_full_name + when 'courses_courserun' + then 'course run' + when 'courses_program' + then 'program' + else contenttypes.contenttype_full_name + end as product_type + from products + inner join contenttypes on products.contenttype_id = contenttypes.contenttype_id + ) select - product_subquery.* - , latest_productversion.productversion_price as product_list_price - , latest_productversion.productversion_description as product_description - , courseruns.course_id - , courseruns.courserun_readable_id - , programs.program_readable_id + product_subquery.*, + latest_productversion.productversion_price as product_list_price, + latest_productversion.productversion_description as product_description, + courseruns.course_id, + courseruns.courserun_readable_id, + programs.program_readable_id from product_subquery left join latest_productversion on product_subquery.product_id = latest_productversion.product_id left join courseruns on product_subquery.courserun_id = courseruns.courserun_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_productcouponassignment.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_productcouponassignment.sql index d69f143cc..fae345ea2 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_productcouponassignment.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_productcouponassignment.sql @@ -1,24 +1,20 @@ -with productcouponassignment as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_productcouponassignment') }} -) - -, couponproduct as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_couponproduct') }} -) +with + productcouponassignment as ( + select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_productcouponassignment") }} + ), + couponproduct as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_couponproduct") }}) select - productcouponassignment.productcouponassignment_id - , productcouponassignment.productcouponassignment_message_status_updated_on - , productcouponassignment.productcouponassignment_is_redeemed - , productcouponassignment.productcouponassignment_message_status - , productcouponassignment.productcouponassignment_email - , productcouponassignment.productcouponassignment_original_email - , productcouponassignment.productcouponassignment_created_on - , productcouponassignment.productcouponassignment_updated_on - , couponproduct.coupon_id - , couponproduct.product_id - , couponproduct.programrun_id + productcouponassignment.productcouponassignment_id, + productcouponassignment.productcouponassignment_message_status_updated_on, + productcouponassignment.productcouponassignment_is_redeemed, + productcouponassignment.productcouponassignment_message_status, + productcouponassignment.productcouponassignment_email, + productcouponassignment.productcouponassignment_original_email, + productcouponassignment.productcouponassignment_created_on, + productcouponassignment.productcouponassignment_updated_on, + couponproduct.coupon_id, + couponproduct.product_id, + couponproduct.programrun_id from productcouponassignment inner join couponproduct on productcouponassignment.couponproduct_id = couponproduct.couponproduct_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_productversion.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_productversion.sql index f0c8b107e..6f5a71e5f 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_productversion.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_productversion.sql @@ -1,15 +1,12 @@ -with productversions as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_productversion') }} -) +with productversions as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_productversion") }}) select - productversion_id - , productversion_readable_id - , productversion_price - , productversion_description - , product_id - , productversion_requires_enrollment_code - , productversion_updated_on - , productversion_created_on + productversion_id, + productversion_readable_id, + productversion_price, + productversion_description, + product_id, + productversion_requires_enrollment_code, + productversion_updated_on, + productversion_created_on from productversions diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_receipt.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_receipt.sql index e55aa4b77..be6bfb7ae 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_receipt.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__ecommerce_receipt.sql @@ -1,33 +1,30 @@ -with receipts as ( - select * - from {{ ref('stg__mitxpro__app__postgres__ecommerce_receipt') }} -) +with receipts as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_receipt") }}) select - receipt_id - , receipt_created_on - , receipt_updated_on - , receipt_data - , order_id - , cast(json_query(receipt_data, 'lax $.req_amount' omit quotes) as decimal(38, 2)) as receipt_payment_amount - , json_query(receipt_data, 'lax $.req_transaction_uuid' omit quotes) as receipt_transaction_uuid - , json_query(receipt_data, 'lax $.decision' omit quotes) as receipt_transaction_status - , json_query(receipt_data, 'lax $.transaction_id' omit quotes) as receipt_transaction_id - , json_query(receipt_data, 'lax $.auth_code' omit quotes) as receipt_authorization_code - , json_query(receipt_data, 'lax $.req_payment_method' omit quotes) as receipt_payment_method - , json_query(receipt_data, 'lax $.req_reference_number' omit quotes) as receipt_reference_number - , json_query(receipt_data, 'lax $.req_bill_to_address_state' omit quotes) as receipt_bill_to_address_state - , json_query(receipt_data, 'lax $.req_bill_to_address_country' omit quotes) as receipt_bill_to_address_country - , json_query(receipt_data, 'lax $.req_transaction_type' omit quotes) as receipt_transaction_type - , json_query(receipt_data, 'lax $.req_currency' omit quotes) as receipt_payment_currency - , json_query(receipt_data, 'lax $.req_bill_to_email' omit quotes) as receipt_payer_email - , json_query(receipt_data, 'lax $.req_card_number' omit quotes) as receipt_payment_card_number - , json_query(receipt_data, 'lax $.req_customer_ip_address' omit quotes) as receipt_payer_ip_address - , json_query(receipt_data, 'lax $.card_type_name' omit quotes) as receipt_payment_card_type - , concat( - json_query(receipt_data, 'lax $.req_bill_to_forename' omit quotes) - , ' ' - , json_query(receipt_data, 'lax $.req_bill_to_surname' omit quotes) - ) as receipt_payer_name - , json_query(receipt_data, 'lax $.signed_date_time' omit quotes) as receipt_payment_timestamp + receipt_id, + receipt_created_on, + receipt_updated_on, + receipt_data, + order_id, + cast(json_query(receipt_data, 'lax $.req_amount' omit quotes) as decimal(38, 2)) as receipt_payment_amount, + json_query(receipt_data, 'lax $.req_transaction_uuid' omit quotes) as receipt_transaction_uuid, + json_query(receipt_data, 'lax $.decision' omit quotes) as receipt_transaction_status, + json_query(receipt_data, 'lax $.transaction_id' omit quotes) as receipt_transaction_id, + json_query(receipt_data, 'lax $.auth_code' omit quotes) as receipt_authorization_code, + json_query(receipt_data, 'lax $.req_payment_method' omit quotes) as receipt_payment_method, + json_query(receipt_data, 'lax $.req_reference_number' omit quotes) as receipt_reference_number, + json_query(receipt_data, 'lax $.req_bill_to_address_state' omit quotes) as receipt_bill_to_address_state, + json_query(receipt_data, 'lax $.req_bill_to_address_country' omit quotes) as receipt_bill_to_address_country, + json_query(receipt_data, 'lax $.req_transaction_type' omit quotes) as receipt_transaction_type, + json_query(receipt_data, 'lax $.req_currency' omit quotes) as receipt_payment_currency, + json_query(receipt_data, 'lax $.req_bill_to_email' omit quotes) as receipt_payer_email, + json_query(receipt_data, 'lax $.req_card_number' omit quotes) as receipt_payment_card_number, + json_query(receipt_data, 'lax $.req_customer_ip_address' omit quotes) as receipt_payer_ip_address, + json_query(receipt_data, 'lax $.card_type_name' omit quotes) as receipt_payment_card_type, + concat( + json_query(receipt_data, 'lax $.req_bill_to_forename' omit quotes), + ' ', + json_query(receipt_data, 'lax $.req_bill_to_surname' omit quotes) + ) as receipt_payer_name, + json_query(receipt_data, 'lax $.signed_date_time' omit quotes) as receipt_payment_timestamp from receipts diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__platforms.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__platforms.sql index 01f64d477..440216409 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__platforms.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__platforms.sql @@ -1,10 +1,5 @@ -- Platform information for MITxPro +with platforms as (select * from {{ ref("stg__mitxpro__app__postgres__courses_platform") }}) -with platforms as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_platform') }} -) - -select - platform_id - , platform_name +select platform_id, platform_name from platforms diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__program_certificates.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__program_certificates.sql index 6316535cf..ed495540b 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__program_certificates.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__program_certificates.sql @@ -1,38 +1,30 @@ -- Program Certificate information for MIT xPro +with + certificates as (select * from {{ ref("stg__mitxpro__app__postgres__courses_programcertificate") }}), + programs as (select * from {{ ref("stg__mitxpro__app__postgres__courses_program") }}), + users as (select * from {{ ref("stg__mitxpro__app__postgres__users_user") }}), + program_certificates as ( + select + certificates.programcertificate_id, + certificates.programcertificate_uuid, + certificates.program_id, + programs.program_title, + programs.program_readable_id, + certificates.programcertificate_is_revoked, + certificates.programcertificate_created_on, + certificates.programcertificate_updated_on, + certificates.user_id, + users.user_username, + users.user_email, + if( + certificates.programcertificate_is_revoked = false, + concat('https://xpro.mit.edu/certificate/program/', certificates.programcertificate_uuid), + null + ) as programcertificate_url + from certificates + inner join programs on certificates.program_id = programs.program_id + inner join users on certificates.user_id = users.user_id + ) -with certificates as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_programcertificate') }} -) - -, programs as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_program') }} -) - -, users as ( - select * from {{ ref('stg__mitxpro__app__postgres__users_user') }} -) - -, program_certificates as ( - select - certificates.programcertificate_id - , certificates.programcertificate_uuid - , certificates.program_id - , programs.program_title - , programs.program_readable_id - , certificates.programcertificate_is_revoked - , certificates.programcertificate_created_on - , certificates.programcertificate_updated_on - , certificates.user_id - , users.user_username - , users.user_email - , if( - certificates.programcertificate_is_revoked = false - , concat('https://xpro.mit.edu/certificate/program/', certificates.programcertificate_uuid) - , null - ) as programcertificate_url - from certificates - inner join programs on certificates.program_id = programs.program_id - inner join users on certificates.user_id = users.user_id -) - -select * from program_certificates +select * +from program_certificates diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__program_runs.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__program_runs.sql index d70952a4a..b99ffa361 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__program_runs.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__program_runs.sql @@ -1,21 +1,16 @@ -- Program Runs information for MITxPro - -with program_runs as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_programrun') }} -) - -, programs as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_program') }} -) +with + program_runs as (select * from {{ ref("stg__mitxpro__app__postgres__courses_programrun") }}), + programs as (select * from {{ ref("stg__mitxpro__app__postgres__courses_program") }}) select - program_runs.programrun_id - , programs.program_id - , programs.program_title - , program_runs.programrun_tag - , program_runs.programrun_start_on - , program_runs.programrun_end_on - , programs.short_program_code - , concat(programs.program_readable_id, '+', program_runs.programrun_tag) as programrun_readable_id + program_runs.programrun_id, + programs.program_id, + programs.program_title, + program_runs.programrun_tag, + program_runs.programrun_start_on, + program_runs.programrun_end_on, + programs.short_program_code, + concat(programs.program_readable_id, '+', program_runs.programrun_tag) as programrun_readable_id from program_runs inner join programs on program_runs.program_id = programs.program_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__programenrollments.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__programenrollments.sql index f662da28c..fcc6d5993 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__programenrollments.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__programenrollments.sql @@ -1,40 +1,25 @@ -- Enrollment information for MITxPro +with + enrollments as (select * from {{ ref("stg__mitxpro__app__postgres__courses_programenrollment") }}), + programs as (select program_id, program_title from {{ ref("stg__mitxpro__app__postgres__courses_program") }}), + users as (select user_id, user_username, user_email from {{ ref("stg__mitxpro__app__postgres__users_user") }}), + mitxpro_enrollments as ( + select + enrollments.programenrollment_id, + enrollments.programenrollment_is_active, + enrollments.user_id, + enrollments.program_id, + enrollments.programenrollment_created_on, + enrollments.programenrollment_enrollment_status, + programs.program_title, + users.user_username, + users.user_email, + enrollments.ecommerce_company_id, + enrollments.ecommerce_order_id + from enrollments + left join programs on enrollments.program_id = programs.program_id + left join users on enrollments.user_id = users.user_id + ) -with enrollments as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_programenrollment') }} -) - -, programs as ( - select - program_id - , program_title - from {{ ref('stg__mitxpro__app__postgres__courses_program') }} -) - -, users as ( - select - user_id - , user_username - , user_email - from {{ ref('stg__mitxpro__app__postgres__users_user') }} -) - -, mitxpro_enrollments as ( - select - enrollments.programenrollment_id - , enrollments.programenrollment_is_active - , enrollments.user_id - , enrollments.program_id - , enrollments.programenrollment_created_on - , enrollments.programenrollment_enrollment_status - , programs.program_title - , users.user_username - , users.user_email - , enrollments.ecommerce_company_id - , enrollments.ecommerce_order_id - from enrollments - left join programs on enrollments.program_id = programs.program_id - left join users on enrollments.user_id = users.user_id -) - -select * from mitxpro_enrollments +select * +from mitxpro_enrollments diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__programs.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__programs.sql index 448858697..6bce90759 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__programs.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__programs.sql @@ -1,89 +1,54 @@ -- Program information for MITxPro - -with programs as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_program') }} -) - -, cms_programs as ( - select * from {{ ref('stg__mitxpro__app__postgres__cms_programpage') }} -) - -, wagtail_page as ( - select * from {{ ref('stg__mitxpro__app__postgres__wagtail_page') }} -) - -, platform as ( - select * from {{ ref('stg__mitxpro__app__postgres__courses_platform') }} -) - -, course_program as ( - select * from {{ ref('int__mitxpro__coursesinprogram') }} -) - -, course_topics as ( - select * from {{ ref('int__mitxpro__courses_to_topics') }} -) - -, certificate_page as ( - select * from {{ ref('stg__mitxpro__app__postgres__cms_certificatepage') }} -) - -, certificate_page_path as ( - select - certificate_page.wagtail_page_id - , certificate_page.cms_certificate_ceus - , wagtail_page.wagtail_page_path - from certificate_page - inner join wagtail_page - on certificate_page.wagtail_page_id = wagtail_page.wagtail_page_id -) - -, program_topics as ( - select - course_program.program_id - , array_join(array_agg(course_topics.coursetopic_name), ', ') as program_topics - from course_topics - inner join course_program on course_topics.course_id = course_program.course_id - group by course_program.program_id -) - -, program_instructors as ( - select - program_id - , array_join(array_agg(cms_facultymemberspage_facultymember_name), ', ') as program_instructors - from {{ ref('int__mitxpro__programsfaculty') }} - group by program_id -) +with + programs as (select * from {{ ref("stg__mitxpro__app__postgres__courses_program") }}), + cms_programs as (select * from {{ ref("stg__mitxpro__app__postgres__cms_programpage") }}), + wagtail_page as (select * from {{ ref("stg__mitxpro__app__postgres__wagtail_page") }}), + platform as (select * from {{ ref("stg__mitxpro__app__postgres__courses_platform") }}), + course_program as (select * from {{ ref("int__mitxpro__coursesinprogram") }}), + course_topics as (select * from {{ ref("int__mitxpro__courses_to_topics") }}), + certificate_page as (select * from {{ ref("stg__mitxpro__app__postgres__cms_certificatepage") }}), + certificate_page_path as ( + select certificate_page.wagtail_page_id, certificate_page.cms_certificate_ceus, wagtail_page.wagtail_page_path + from certificate_page + inner join wagtail_page on certificate_page.wagtail_page_id = wagtail_page.wagtail_page_id + ), + program_topics as ( + select course_program.program_id, array_join(array_agg(course_topics.coursetopic_name), ', ') as program_topics + from course_topics + inner join course_program on course_topics.course_id = course_program.course_id + group by course_program.program_id + ), + program_instructors as ( + select program_id, array_join(array_agg(cms_facultymemberspage_facultymember_name), ', ') as program_instructors + from {{ ref("int__mitxpro__programsfaculty") }} + group by program_id + ) select - programs.program_id - , programs.program_title - , programs.program_is_live - , programs.program_readable_id - , programs.program_is_external - , platform.platform_name - , cms_programs.cms_programpage_description - , cms_programs.cms_programpage_subhead - , cms_programs.cms_programpage_catalog_details - , cms_programs.cms_programpage_duration - , cms_programs.cms_programpage_format - , cms_programs.cms_programpage_time_commitment - , certificate_page_path.cms_certificate_ceus - , program_topics.program_topics - , program_instructors.program_instructors - , wagtail_page.wagtail_page_slug as cms_programpage_slug - , wagtail_page.wagtail_page_url_path as cms_programpage_url_path - , wagtail_page.wagtail_page_is_live as cms_programpage_is_live - , wagtail_page.wagtail_page_first_published_on as cms_programpage_first_published_on - , wagtail_page.wagtail_page_last_published_on as cms_programpage_last_published_on + programs.program_id, + programs.program_title, + programs.program_is_live, + programs.program_readable_id, + programs.program_is_external, + platform.platform_name, + cms_programs.cms_programpage_description, + cms_programs.cms_programpage_subhead, + cms_programs.cms_programpage_catalog_details, + cms_programs.cms_programpage_duration, + cms_programs.cms_programpage_format, + cms_programs.cms_programpage_time_commitment, + certificate_page_path.cms_certificate_ceus, + program_topics.program_topics, + program_instructors.program_instructors, + wagtail_page.wagtail_page_slug as cms_programpage_slug, + wagtail_page.wagtail_page_url_path as cms_programpage_url_path, + wagtail_page.wagtail_page_is_live as cms_programpage_is_live, + wagtail_page.wagtail_page_first_published_on as cms_programpage_first_published_on, + wagtail_page.wagtail_page_last_published_on as cms_programpage_last_published_on from programs -left join cms_programs - on programs.program_id = cms_programs.program_id -left join wagtail_page - on cms_programs.wagtail_page_id = wagtail_page.wagtail_page_id -left join platform - on programs.platform_id = platform.platform_id -left join certificate_page_path - on wagtail_page.wagtail_page_path like certificate_page_path.wagtail_page_path || '%' +left join cms_programs on programs.program_id = cms_programs.program_id +left join wagtail_page on cms_programs.wagtail_page_id = wagtail_page.wagtail_page_id +left join platform on programs.platform_id = platform.platform_id +left join certificate_page_path on wagtail_page.wagtail_page_path like certificate_page_path.wagtail_page_path || '%' left join program_topics on programs.program_id = program_topics.program_id left join program_instructors on programs.program_id = program_instructors.program_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__programsfaculty.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__programsfaculty.sql index 13af99166..ced05b6a9 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__programsfaculty.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__programsfaculty.sql @@ -1,51 +1,36 @@ -with programpages as ( - select * - from {{ ref('stg__mitxpro__app__postgres__cms_programpage') }} -) - -, facultymemberspage as ( - select - * - , cast(json_parse(cms_facultymemberspage_faculty) as array (json)) as cms_facultymemberspage_faculty_array - from {{ ref('stg__mitxpro__app__postgres__cms_facultymemberspage') }} -) - -, wagtailpages as ( - select * - from {{ ref('stg__mitxpro__app__postgres__wagtail_page') }} -) - -, unnestedfacultymemberspage as ( - select - facultymemberspage.wagtail_page_id - , json_format(cms_facultymemberspage_facultymember) as cms_facultymemberspage_facultymember -- noqa - - from facultymemberspage - cross join - unnest(facultymemberspage.cms_facultymemberspage_faculty_array) - as t(cms_facultymemberspage_facultymember) -- noqa - -) - -, programspageswithpath as ( - select - programpages.wagtail_page_id - , programpages.program_id - , wagtailpages.wagtail_page_path - from programpages - inner join wagtailpages - on programpages.wagtail_page_id = wagtailpages.wagtail_page_id -) +with + programpages as (select * from {{ ref("stg__mitxpro__app__postgres__cms_programpage") }}), + facultymemberspage as ( + select + *, cast(json_parse(cms_facultymemberspage_faculty) as array(json)) as cms_facultymemberspage_faculty_array + from {{ ref("stg__mitxpro__app__postgres__cms_facultymemberspage") }} + ), + wagtailpages as (select * from {{ ref("stg__mitxpro__app__postgres__wagtail_page") }}), + unnestedfacultymemberspage as ( + select + facultymemberspage.wagtail_page_id, + json_format(cms_facultymemberspage_facultymember) as cms_facultymemberspage_facultymember -- noqa + + from facultymemberspage + cross join + unnest(facultymemberspage.cms_facultymemberspage_faculty_array) as t(cms_facultymemberspage_facultymember) -- noqa + + ), + programspageswithpath as ( + select programpages.wagtail_page_id, programpages.program_id, wagtailpages.wagtail_page_path + from programpages + inner join wagtailpages on programpages.wagtail_page_id = wagtailpages.wagtail_page_id + ) select - programspageswithpath.program_id - , json_query(unnestedfacultymemberspage.cms_facultymemberspage_facultymember, 'lax $.value.name') - as cms_facultymemberspage_facultymember_name - , json_query(unnestedfacultymemberspage.cms_facultymemberspage_facultymember, 'lax $.value.description') - as cms_facultymemberspage_facultymember_description + programspageswithpath.program_id, + json_query( + unnestedfacultymemberspage.cms_facultymemberspage_facultymember, 'lax $.value.name' + ) as cms_facultymemberspage_facultymember_name, + json_query( + unnestedfacultymemberspage.cms_facultymemberspage_facultymember, 'lax $.value.description' + ) as cms_facultymemberspage_facultymember_description from unnestedfacultymemberspage -inner join wagtailpages - on unnestedfacultymemberspage.wagtail_page_id = wagtailpages.wagtail_page_id -inner join programspageswithpath - on wagtailpages.wagtail_page_path like programspageswithpath.wagtail_page_path || '%' +inner join wagtailpages on unnestedfacultymemberspage.wagtail_page_id = wagtailpages.wagtail_page_id +inner join programspageswithpath on wagtailpages.wagtail_page_path like programspageswithpath.wagtail_page_path || '%' diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivities.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivities.sql index 59bcc22b6..c5dc6ced1 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivities.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivities.sql @@ -1,104 +1,88 @@ -- xPro user activities from tracking logs -with course_activities as ( - select * from {{ ref('stg__mitxpro__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) - -, course_activities_video as ( - select * from {{ ref('int__mitxpro__user_courseactivity_video') }} -) - -, problem_check as ( - select * from {{ ref('int__mitxpro__user_courseactivity_problemcheck') }} -) - -, users as ( - select * from {{ ref('int__mitxpro__users') }} -) - -, student_module as ( - select * from {{ ref('stg__mitxpro__openedx__mysql__courseware_studentmodule') }} -) - -, course_structure as ( - select * from {{ ref('int__mitxpro__course_structure') }} -) - -, course_chapters_stats as ( - select - student_module.openedx_user_id - , student_module.courserun_readable_id - , count(distinct course_structure.coursestructure_chapter_id) as courseactivity_num_chapters_visited - from student_module - inner join course_structure - on - student_module.courserun_readable_id = course_structure.courserun_readable_id +with + course_activities as ( + select * + from {{ ref("stg__mitxpro__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ), + course_activities_video as (select * from {{ ref("int__mitxpro__user_courseactivity_video") }}), + problem_check as (select * from {{ ref("int__mitxpro__user_courseactivity_problemcheck") }}), + users as (select * from {{ ref("int__mitxpro__users") }}), + student_module as (select * from {{ ref("stg__mitxpro__openedx__mysql__courseware_studentmodule") }}), + course_structure as (select * from {{ ref("int__mitxpro__course_structure") }}), + course_chapters_stats as ( + select + student_module.openedx_user_id, + student_module.courserun_readable_id, + count(distinct course_structure.coursestructure_chapter_id) as courseactivity_num_chapters_visited + from student_module + inner join + course_structure + on student_module.courserun_readable_id = course_structure.courserun_readable_id and student_module.coursestructure_block_id = course_structure.coursestructure_block_id - group by student_module.openedx_user_id, student_module.courserun_readable_id -) - -, play_video_stats as ( - select - user_username - , courserun_readable_id - , count(distinct useractivity_video_id) as courseactivity_num_unique_play_video - , count(*) as courseactivity_num_play_video - , max(useractivity_timestamp) as courseactivity_last_play_video_timestamp - from course_activities_video - where useractivity_event_type = 'play_video' - group by user_username, courserun_readable_id -) - -, problem_check_stats as ( - select - user_username - , courserun_readable_id - , max(useractivity_timestamp) as courseactivity_last_problem_check_timestamp - from problem_check - group by user_username, courserun_readable_id -) - -, all_course_activities_stats as ( - select - course_activities.user_username - , course_activities.courserun_readable_id - , coalesce(users.openedx_user_id, course_activities.openedx_user_id) as openedx_user_id - , count(distinct date(from_iso8601_timestamp(course_activities.useractivity_timestamp))) - as courseactivity_num_days_activity - , count(*) as courseactivity_num_events - , min(course_activities.useractivity_timestamp) as courseactivity_first_event_timestamp - , max(course_activities.useractivity_timestamp) as courseactivity_last_event_timestamp - from course_activities - left join users on course_activities.user_username = users.user_username - group by - course_activities.user_username - , coalesce(users.openedx_user_id, course_activities.openedx_user_id) - , course_activities.courserun_readable_id -) + group by student_module.openedx_user_id, student_module.courserun_readable_id + ), + play_video_stats as ( + select + user_username, + courserun_readable_id, + count(distinct useractivity_video_id) as courseactivity_num_unique_play_video, + count(*) as courseactivity_num_play_video, + max(useractivity_timestamp) as courseactivity_last_play_video_timestamp + from course_activities_video + where useractivity_event_type = 'play_video' + group by user_username, courserun_readable_id + ), + problem_check_stats as ( + select + user_username, + courserun_readable_id, + max(useractivity_timestamp) as courseactivity_last_problem_check_timestamp + from problem_check + group by user_username, courserun_readable_id + ), + all_course_activities_stats as ( + select + course_activities.user_username, + course_activities.courserun_readable_id, + coalesce(users.openedx_user_id, course_activities.openedx_user_id) as openedx_user_id, + count( + distinct date(from_iso8601_timestamp(course_activities.useractivity_timestamp)) + ) as courseactivity_num_days_activity, + count(*) as courseactivity_num_events, + min(course_activities.useractivity_timestamp) as courseactivity_first_event_timestamp, + max(course_activities.useractivity_timestamp) as courseactivity_last_event_timestamp + from course_activities + left join users on course_activities.user_username = users.user_username + group by + course_activities.user_username, + coalesce(users.openedx_user_id, course_activities.openedx_user_id), + course_activities.courserun_readable_id + ) select - all_course_activities_stats.user_username - , all_course_activities_stats.openedx_user_id - , all_course_activities_stats.courserun_readable_id - , all_course_activities_stats.courseactivity_num_days_activity - , all_course_activities_stats.courseactivity_num_events - , play_video_stats.courseactivity_num_unique_play_video - , play_video_stats.courseactivity_num_play_video - , play_video_stats.courseactivity_last_play_video_timestamp - , problem_check_stats.courseactivity_last_problem_check_timestamp - , all_course_activities_stats.courseactivity_first_event_timestamp - , all_course_activities_stats.courseactivity_last_event_timestamp - , course_chapters_stats.courseactivity_num_chapters_visited + all_course_activities_stats.user_username, + all_course_activities_stats.openedx_user_id, + all_course_activities_stats.courserun_readable_id, + all_course_activities_stats.courseactivity_num_days_activity, + all_course_activities_stats.courseactivity_num_events, + play_video_stats.courseactivity_num_unique_play_video, + play_video_stats.courseactivity_num_play_video, + play_video_stats.courseactivity_last_play_video_timestamp, + problem_check_stats.courseactivity_last_problem_check_timestamp, + all_course_activities_stats.courseactivity_first_event_timestamp, + all_course_activities_stats.courseactivity_last_event_timestamp, + course_chapters_stats.courseactivity_num_chapters_visited from all_course_activities_stats -left join play_video_stats - on - all_course_activities_stats.user_username = play_video_stats.user_username - and all_course_activities_stats.courserun_readable_id = play_video_stats.courserun_readable_id -left join problem_check_stats - on - all_course_activities_stats.user_username = problem_check_stats.user_username - and all_course_activities_stats.courserun_readable_id = problem_check_stats.courserun_readable_id -left join course_chapters_stats - on - all_course_activities_stats.openedx_user_id = course_chapters_stats.openedx_user_id - and all_course_activities_stats.courserun_readable_id = course_chapters_stats.courserun_readable_id +left join + play_video_stats + on all_course_activities_stats.user_username = play_video_stats.user_username + and all_course_activities_stats.courserun_readable_id = play_video_stats.courserun_readable_id +left join + problem_check_stats + on all_course_activities_stats.user_username = problem_check_stats.user_username + and all_course_activities_stats.courserun_readable_id = problem_check_stats.courserun_readable_id +left join + course_chapters_stats + on all_course_activities_stats.openedx_user_id = course_chapters_stats.openedx_user_id + and all_course_activities_stats.courserun_readable_id = course_chapters_stats.courserun_readable_id diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivities_daily.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivities_daily.sql index b6dc939d4..e64b9abbd 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivities_daily.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivities_daily.sql @@ -1,19 +1,18 @@ -with course_activities as ( - select * from {{ ref('stg__mitxpro__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxpro__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ), + daily_activities_stats as ( + select + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date, + count(*) as courseactivity_num_events + from course_activities + group by user_username, courserun_readable_id, date(from_iso8601_timestamp(useractivity_timestamp)) + ) -, daily_activities_stats as ( - select - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - , count(*) as courseactivity_num_events - from course_activities - group by - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) -) - -select * from daily_activities_stats +select * +from daily_activities_stats diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_discussion.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_discussion.sql index 53a289fa0..ed3b16e5d 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_discussion.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_discussion.sql @@ -1,24 +1,26 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxpro__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxpro__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , openedx_user_id - , useractivity_event_source - , useractivity_event_type - , useractivity_path - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_discussion_post_id - , json_query(useractivity_event_object, 'lax $.title' omit quotes) as useractivity_discussion_post_title - , json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as useractivity_discussion_block_id - , json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as useractivity_discussion_block_name - , json_query(useractivity_event_object, 'lax $.url' omit quotes) as useractivity_discussion_page_url - , json_query(useractivity_event_object, 'lax $.query' omit quotes) as useractivity_discussion_search_query - , json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as useractivity_discussion_roles + user_username, + courserun_readable_id, + openedx_user_id, + useractivity_event_source, + useractivity_event_type, + useractivity_path, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_discussion_post_id, + json_query(useractivity_event_object, 'lax $.title' omit quotes) as useractivity_discussion_post_title, + json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as useractivity_discussion_block_id, + json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as useractivity_discussion_block_name, + json_query(useractivity_event_object, 'lax $.url' omit quotes) as useractivity_discussion_page_url, + json_query(useractivity_event_object, 'lax $.query' omit quotes) as useractivity_discussion_search_query, + json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as useractivity_discussion_roles from course_activities where useractivity_event_type like 'edx.forum.%' diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_problemcheck.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_problemcheck.sql index e0151b729..797bdac22 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_problemcheck.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_problemcheck.sql @@ -1,25 +1,28 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxpro__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxpro__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , openedx_user_id - , useractivity_event_type - , useractivity_timestamp - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id - , json_query(useractivity_event_object, 'lax $.answers' omit quotes) as useractivity_problem_student_answers - , json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as useractivity_problem_attempts - , json_query(useractivity_event_object, 'lax $.success' omit quotes) as useractivity_problem_success - , json_query(useractivity_event_object, 'lax $.grade' omit quotes) as useractivity_problem_current_grade - , json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as useractivity_problem_max_grade + user_username, + courserun_readable_id, + openedx_user_id, + useractivity_event_type, + useractivity_timestamp, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id, + json_query(useractivity_event_object, 'lax $.answers' omit quotes) as useractivity_problem_student_answers, + json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as useractivity_problem_attempts, + json_query(useractivity_event_object, 'lax $.success' omit quotes) as useractivity_problem_success, + json_query(useractivity_event_object, 'lax $.grade' omit quotes) as useractivity_problem_current_grade, + json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as useractivity_problem_max_grade from course_activities -where useractivity_event_type = 'problem_check' ---- This event emitted by the browser contain all of the GET parameters, --- only events emitted by the server are useful -and useractivity_event_source = 'server' +where + useractivity_event_type = 'problem_check' + -- - This event emitted by the browser contain all of the GET parameters, + -- only events emitted by the server are useful + and useractivity_event_source = 'server' diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_problemsubmitted.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_problemsubmitted.sql index 23a90ebca..b00ca6420 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_problemsubmitted.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_problemsubmitted.sql @@ -1,23 +1,25 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxpro__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxpro__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , openedx_user_id - , useractivity_event_source - , useractivity_event_type - , useractivity_path - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.event_transaction_id' omit quotes) as useractivity_event_id - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id - , json_query(useractivity_event_object, 'lax $.weight' omit quotes) as useractivity_problem_weight - , json_query(useractivity_event_object, 'lax $.weighted_earned' omit quotes) as useractivity_problem_earned_score - , json_query(useractivity_event_object, 'lax $.weighted_possible' omit quotes) as useractivity_problem_max_score + user_username, + courserun_readable_id, + openedx_user_id, + useractivity_event_source, + useractivity_event_type, + useractivity_path, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.event_transaction_id' omit quotes) as useractivity_event_id, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id, + json_query(useractivity_event_object, 'lax $.weight' omit quotes) as useractivity_problem_weight, + json_query(useractivity_event_object, 'lax $.weighted_earned' omit quotes) as useractivity_problem_earned_score, + json_query(useractivity_event_object, 'lax $.weighted_possible' omit quotes) as useractivity_problem_max_score from course_activities where useractivity_event_type = 'edx.grades.problem.submitted' diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_showanswer.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_showanswer.sql index 957cdf9de..1884b6e66 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_showanswer.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_showanswer.sql @@ -1,16 +1,18 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxpro__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxpro__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , openedx_user_id - , useractivity_path - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id + user_username, + courserun_readable_id, + openedx_user_id, + useractivity_path, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id from course_activities where useractivity_event_type = 'showanswer' diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_video.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_video.sql index 66b4d3b1d..81092ccd6 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_video.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__user_courseactivity_video.sql @@ -1,28 +1,31 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxpro__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxpro__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , openedx_user_id - , useractivity_event_source - , useractivity_event_type - , useractivity_page_url - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_video_id - , case - when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' then null + user_username, + courserun_readable_id, + openedx_user_id, + useractivity_event_source, + useractivity_event_type, + useractivity_page_url, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_video_id, + case + when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' + then null else cast(json_query(useractivity_event_object, 'lax $.duration' omit quotes) as decimal(38, 4)) - end as useractivity_video_duration - , json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as useractivity_video_currenttime - , json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as useractivity_video_old_time - , json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as useractivity_video_new_time - , json_query(useractivity_event_object, 'lax $.new_speed' omit quotes) as useractivity_video_new_speed - , json_query(useractivity_event_object, 'lax $.old_speed' omit quotes) as useractivity_video_old_speed + end as useractivity_video_duration, + json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as useractivity_video_currenttime, + json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as useractivity_video_old_time, + json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as useractivity_video_new_time, + json_query(useractivity_event_object, 'lax $.new_speed' omit quotes) as useractivity_video_new_speed, + json_query(useractivity_event_object, 'lax $.old_speed' omit quotes) as useractivity_video_old_speed from course_activities where regexp_like(useractivity_event_type, '(^[\w]+)_video') = true diff --git a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__users.sql b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__users.sql index 0f2269102..c17a24e8d 100644 --- a/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__users.sql +++ b/src/ol_dbt/models/intermediate/mitxpro/int__mitxpro__users.sql @@ -1,75 +1,51 @@ -with users as ( - select * - from {{ ref('stg__mitxpro__app__postgres__users_user') }} -) - -, users_legaladdress as ( - select * - from {{ ref('stg__mitxpro__app__postgres__users_legaladdress') }} -) - -, users_profile as ( - select * - from {{ ref('stg__mitxpro__app__postgres__users_profile') }} -) - -, openedx_users as ( - select * from {{ ref('stg__mitxpro__openedx__mysql__auth_user') }} -) - -, company as ( - select * from {{ ref('stg__mitxpro__app__postgres__ecommerce_company') }} -) - -, user_company_from_enrollments as ( - select - user_id - , ecommerce_company_id - , row_number() over ( - partition by user_id - order by courserunenrollment_created_on desc - ) as enrollment_rank - from {{ ref('stg__mitxpro__app__postgres__courses_courserunenrollment') }} - where ecommerce_company_id is not null -) - -, user_company as ( - select - user_company_from_enrollments.user_id - , company.company_name - from user_company_from_enrollments - inner join company on user_company_from_enrollments.ecommerce_company_id = company.company_id - where user_company_from_enrollments.enrollment_rank = 1 -) +with + users as (select * from {{ ref("stg__mitxpro__app__postgres__users_user") }}), + users_legaladdress as (select * from {{ ref("stg__mitxpro__app__postgres__users_legaladdress") }}), + users_profile as (select * from {{ ref("stg__mitxpro__app__postgres__users_profile") }}), + openedx_users as (select * from {{ ref("stg__mitxpro__openedx__mysql__auth_user") }}), + company as (select * from {{ ref("stg__mitxpro__app__postgres__ecommerce_company") }}), + user_company_from_enrollments as ( + select + user_id, + ecommerce_company_id, + row_number() over (partition by user_id order by courserunenrollment_created_on desc) as enrollment_rank + from {{ ref("stg__mitxpro__app__postgres__courses_courserunenrollment") }} + where ecommerce_company_id is not null + ), + user_company as ( + select user_company_from_enrollments.user_id, company.company_name + from user_company_from_enrollments + inner join company on user_company_from_enrollments.ecommerce_company_id = company.company_id + where user_company_from_enrollments.enrollment_rank = 1 + ) select - users.user_id - , openedx_users.openedx_user_id - , users.user_username - , users.user_full_name - , users.user_email - , users.user_joined_on - , users.user_last_login - , users.user_is_active - , users_legaladdress.user_address_country - , users_legaladdress.user_address_state_or_territory - , users_legaladdress.user_address_city - , users_legaladdress.user_street_address - , users_legaladdress.user_address_postal_code - , users_legaladdress.user_vat_id - , users_profile.user_birth_year - , users_profile.user_job_title - , users_profile.user_industry - , users_profile.user_job_function - , users_profile.user_leadership_level - , users_profile.user_highest_education - , users_profile.user_gender - , users_profile.user_company_size - , users_profile.user_years_experience - , coalesce(user_company.company_name, users_profile.user_company) as user_company + users.user_id, + openedx_users.openedx_user_id, + users.user_username, + users.user_full_name, + users.user_email, + users.user_joined_on, + users.user_last_login, + users.user_is_active, + users_legaladdress.user_address_country, + users_legaladdress.user_address_state_or_territory, + users_legaladdress.user_address_city, + users_legaladdress.user_street_address, + users_legaladdress.user_address_postal_code, + users_legaladdress.user_vat_id, + users_profile.user_birth_year, + users_profile.user_job_title, + users_profile.user_industry, + users_profile.user_job_function, + users_profile.user_leadership_level, + users_profile.user_highest_education, + users_profile.user_gender, + users_profile.user_company_size, + users_profile.user_years_experience, + coalesce(user_company.company_name, users_profile.user_company) as user_company from users left join users_legaladdress on users.user_id = users_legaladdress.user_id left join users_profile on users.user_id = users_profile.user_id -left join openedx_users - on users.user_username = openedx_users.user_username +left join openedx_users on users.user_username = openedx_users.user_username left join user_company on users.user_id = user_company.user_id diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__course_structure.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__course_structure.sql index 0031b4303..f4b3141ea 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__course_structure.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__course_structure.sql @@ -1,66 +1,60 @@ -with course_structure as ( - select * from {{ ref('stg__mitxresidential__openedx__api__course_structure') }} - order by courserun_readable_id, coursestructure_retrieved_at, coursestructure_block_index +with + course_structure as ( + select * + from {{ ref("stg__mitxresidential__openedx__api__course_structure") }} + order by courserun_readable_id, coursestructure_retrieved_at, coursestructure_block_index -) - -, latest_course_structure_date as ( - select - courserun_readable_id - , max(coursestructure_retrieved_at) as max_retrieved_date - from course_structure - group by courserun_readable_id -) - -, chapters as ( - select * from course_structure - where coursestructure_block_category = 'chapter' -) - -, course_structure_with_chapters as ( - select - course_structure.* - , chapters.coursestructure_block_id as coursestructure_chapter_id - , chapters.coursestructure_block_title as coursestructure_chapter_title - , row_number() over ( - partition by - course_structure.courserun_readable_id - , course_structure.coursestructure_block_index - , course_structure.coursestructure_retrieved_at - order by chapters.coursestructure_block_index desc - ) as row_num - from course_structure - inner join chapters - on - course_structure.courserun_readable_id = chapters.courserun_readable_id + ), + latest_course_structure_date as ( + select courserun_readable_id, max(coursestructure_retrieved_at) as max_retrieved_date + from course_structure + group by courserun_readable_id + ), + chapters as (select * from course_structure where coursestructure_block_category = 'chapter'), + course_structure_with_chapters as ( + select + course_structure.*, + chapters.coursestructure_block_id as coursestructure_chapter_id, + chapters.coursestructure_block_title as coursestructure_chapter_title, + row_number() over ( + partition by + course_structure.courserun_readable_id, + course_structure.coursestructure_block_index, + course_structure.coursestructure_retrieved_at + order by chapters.coursestructure_block_index desc + ) as row_num + from course_structure + inner join + chapters + on course_structure.courserun_readable_id = chapters.courserun_readable_id and course_structure.coursestructure_retrieved_at = chapters.coursestructure_retrieved_at and course_structure.coursestructure_block_index >= chapters.coursestructure_block_index -) + ) select - course_structure.courserun_readable_id - , course_structure.courserun_title - , course_structure.coursestructure_block_index - , course_structure.coursestructure_block_id - , course_structure.coursestructure_parent_block_id - , course_structure.coursestructure_block_category - , course_structure.coursestructure_block_title - , course_structure.coursestructure_content_hash - , course_structure.coursestructure_block_content_hash - , course_structure.coursestructure_block_metadata - , course_structure.courserun_start_on - , course_structure.coursestructure_retrieved_at - , course_structure_with_chapters.coursestructure_chapter_id - , course_structure_with_chapters.coursestructure_chapter_title - , if(latest_course_structure_date.max_retrieved_date is not null, true, false) as coursestructure_is_latest + course_structure.courserun_readable_id, + course_structure.courserun_title, + course_structure.coursestructure_block_index, + course_structure.coursestructure_block_id, + course_structure.coursestructure_parent_block_id, + course_structure.coursestructure_block_category, + course_structure.coursestructure_block_title, + course_structure.coursestructure_content_hash, + course_structure.coursestructure_block_content_hash, + course_structure.coursestructure_block_metadata, + course_structure.courserun_start_on, + course_structure.coursestructure_retrieved_at, + course_structure_with_chapters.coursestructure_chapter_id, + course_structure_with_chapters.coursestructure_chapter_title, + if(latest_course_structure_date.max_retrieved_date is not null, true, false) as coursestructure_is_latest from course_structure -left join latest_course_structure_date - on - course_structure.courserun_readable_id = latest_course_structure_date.courserun_readable_id - and course_structure.coursestructure_retrieved_at = latest_course_structure_date.max_retrieved_date -left join course_structure_with_chapters - on - course_structure.courserun_readable_id = course_structure_with_chapters.courserun_readable_id - and course_structure.coursestructure_block_id = course_structure_with_chapters.coursestructure_block_id - and course_structure.coursestructure_retrieved_at = course_structure_with_chapters.coursestructure_retrieved_at - and course_structure_with_chapters.row_num = 1 +left join + latest_course_structure_date + on course_structure.courserun_readable_id = latest_course_structure_date.courserun_readable_id + and course_structure.coursestructure_retrieved_at = latest_course_structure_date.max_retrieved_date +left join + course_structure_with_chapters + on course_structure.courserun_readable_id = course_structure_with_chapters.courserun_readable_id + and course_structure.coursestructure_block_id = course_structure_with_chapters.coursestructure_block_id + and course_structure.coursestructure_retrieved_at = course_structure_with_chapters.coursestructure_retrieved_at + and course_structure_with_chapters.row_num = 1 diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courserun_enrollments.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courserun_enrollments.sql index a101d9579..4a94c3c8c 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courserun_enrollments.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courserun_enrollments.sql @@ -1,30 +1,23 @@ -with enrollments as ( - select * from {{ ref('stg__mitxresidential__openedx__courserun_enrollment') }} -) +with + enrollments as (select * from {{ ref("stg__mitxresidential__openedx__courserun_enrollment") }}), + runs as (select * from {{ ref("stg__mitxresidential__openedx__courserun") }}), + users as (select * from {{ ref("int__mitxresidential__users") }}), + courserun_enrollments as ( + select + enrollments.courserunenrollment_id, + enrollments.user_id, + enrollments.courserun_readable_id, + enrollments.courserunenrollment_enrollment_mode, + enrollments.courserunenrollment_is_active, + enrollments.courserunenrollment_created_on, + runs.courserun_title, + users.user_username, + users.user_email, + users.user_full_name + from enrollments + left join runs on enrollments.courserun_readable_id = runs.courserun_readable_id + left join users on enrollments.user_id = users.user_id + ) -, runs as ( - select * from {{ ref('stg__mitxresidential__openedx__courserun') }} -) - -, users as ( - select * from {{ ref('int__mitxresidential__users') }} -) - -, courserun_enrollments as ( - select - enrollments.courserunenrollment_id - , enrollments.user_id - , enrollments.courserun_readable_id - , enrollments.courserunenrollment_enrollment_mode - , enrollments.courserunenrollment_is_active - , enrollments.courserunenrollment_created_on - , runs.courserun_title - , users.user_username - , users.user_email - , users.user_full_name - from enrollments - left join runs on enrollments.courserun_readable_id = runs.courserun_readable_id - left join users on enrollments.user_id = users.user_id -) - -select * from courserun_enrollments +select * +from courserun_enrollments diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courserun_grades.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courserun_grades.sql index e75b1324a..55c5b7b8f 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courserun_grades.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courserun_grades.sql @@ -1,30 +1,23 @@ -with grades as ( - select * from {{ ref('stg__mitxresidential__openedx__courserun_grade') }} -) +with + grades as (select * from {{ ref("stg__mitxresidential__openedx__courserun_grade") }}), + runs as (select * from {{ ref("stg__mitxresidential__openedx__courserun") }}), + users as (select * from {{ ref("stg__mitxresidential__openedx__auth_user") }}), + courserun_grades as ( + select + grades.courserungrade_id, + grades.courserun_readable_id, + runs.courserun_title, + grades.courserungrade_grade, + grades.courserungrade_letter_grade, + grades.courserungrade_created_on, + grades.courserungrade_updated_on, + grades.user_id, + users.user_username, + users.user_email + from grades + left join runs on grades.courserun_readable_id = runs.courserun_readable_id + inner join users on grades.user_id = users.user_id + ) -, runs as ( - select * from {{ ref('stg__mitxresidential__openedx__courserun') }} -) - -, users as ( - select * from {{ ref('stg__mitxresidential__openedx__auth_user') }} -) - -, courserun_grades as ( - select - grades.courserungrade_id - , grades.courserun_readable_id - , runs.courserun_title - , grades.courserungrade_grade - , grades.courserungrade_letter_grade - , grades.courserungrade_created_on - , grades.courserungrade_updated_on - , grades.user_id - , users.user_username - , users.user_email - from grades - left join runs on grades.courserun_readable_id = runs.courserun_readable_id - inner join users on grades.user_id = users.user_id -) - -select * from courserun_grades +select * +from courserun_grades diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courserun_videos.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courserun_videos.sql index 7ba8d75be..fefdb37a3 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courserun_videos.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courserun_videos.sql @@ -1,17 +1,13 @@ -with coursevideos as ( - select * from {{ ref('stg__mitxresidential__openedx__edxval_coursevideo') }} -) - -, videos as ( - select * from {{ ref('stg__mitxresidential__openedx__edxval_video') }} -) +with + coursevideos as (select * from {{ ref("stg__mitxresidential__openedx__edxval_coursevideo") }}), + videos as (select * from {{ ref("stg__mitxresidential__openedx__edxval_video") }}) select - coursevideos.courserun_readable_id - , coursevideos.coursevideo_is_hidden - , videos.video_edx_uuid - , videos.video_client_id - , videos.video_status - , videos.video_duration + coursevideos.courserun_readable_id, + coursevideos.coursevideo_is_hidden, + videos.video_edx_uuid, + videos.video_client_id, + videos.video_status, + videos.video_duration from coursevideos inner join videos on coursevideos.video_id = videos.video_id diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courseruns.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courseruns.sql index d0c6c90da..3835d9cca 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courseruns.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__courseruns.sql @@ -1,16 +1,14 @@ -with runs as ( - select * from {{ ref('stg__mitxresidential__openedx__courserun') }} -) +with runs as (select * from {{ ref("stg__mitxresidential__openedx__courserun") }}) select - courserun_readable_id - , courserun_title - , courserun_org - , courserun_course_number - , courserun_start_on - , courserun_end_on - , courserun_enrollment_start_on - , courserun_enrollment_end_on - , courserun_is_self_paced - , courserun_created_on + courserun_readable_id, + courserun_title, + courserun_org, + courserun_course_number, + courserun_start_on, + courserun_end_on, + courserun_enrollment_start_on, + courserun_enrollment_end_on, + courserun_is_self_paced, + courserun_created_on from runs diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivities.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivities.sql index d3e053772..b93053da5 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivities.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivities.sql @@ -1,69 +1,61 @@ -with course_activities as ( - select * from {{ ref('stg__mitxresidential__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) - -, course_activities_video as ( - select * from {{ ref('int__mitxresidential__user_courseactivity_video') }} -) - -, problem_check as ( - select * from {{ ref('int__mitxresidential__user_courseactivity_problemcheck') }} -) - -, problem_check_stats as ( - select - user_username - , courserun_readable_id - , max(useractivity_timestamp) as courseactivity_last_problem_check_timestamp - from problem_check - group by user_username, courserun_readable_id -) - -, play_video_stats as ( - select - user_username - , courserun_readable_id - , count(distinct useractivity_video_id) as courseactivity_num_unique_play_video - , count(*) as courseactivity_num_play_video - , max(useractivity_timestamp) as courseactivity_last_play_video_timestamp - from course_activities_video - where useractivity_event_type = 'play_video' - group by user_username, courserun_readable_id -) - -, all_course_activities_stats as ( - select - course_activities.user_username - , course_activities.courserun_readable_id - , count(distinct date(from_iso8601_timestamp(course_activities.useractivity_timestamp))) - as courseactivity_num_days_activity - , count(*) as courseactivity_num_events - , min(course_activities.useractivity_timestamp) as courseactivity_first_event_timestamp - , max(course_activities.useractivity_timestamp) as courseactivity_last_event_timestamp - from course_activities - group by - course_activities.user_username - , course_activities.courserun_readable_id -) +with + course_activities as ( + select * + from {{ ref("stg__mitxresidential__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ), + course_activities_video as (select * from {{ ref("int__mitxresidential__user_courseactivity_video") }}), + problem_check as (select * from {{ ref("int__mitxresidential__user_courseactivity_problemcheck") }}), + problem_check_stats as ( + select + user_username, + courserun_readable_id, + max(useractivity_timestamp) as courseactivity_last_problem_check_timestamp + from problem_check + group by user_username, courserun_readable_id + ), + play_video_stats as ( + select + user_username, + courserun_readable_id, + count(distinct useractivity_video_id) as courseactivity_num_unique_play_video, + count(*) as courseactivity_num_play_video, + max(useractivity_timestamp) as courseactivity_last_play_video_timestamp + from course_activities_video + where useractivity_event_type = 'play_video' + group by user_username, courserun_readable_id + ), + all_course_activities_stats as ( + select + course_activities.user_username, + course_activities.courserun_readable_id, + count( + distinct date(from_iso8601_timestamp(course_activities.useractivity_timestamp)) + ) as courseactivity_num_days_activity, + count(*) as courseactivity_num_events, + min(course_activities.useractivity_timestamp) as courseactivity_first_event_timestamp, + max(course_activities.useractivity_timestamp) as courseactivity_last_event_timestamp + from course_activities + group by course_activities.user_username, course_activities.courserun_readable_id + ) select - all_course_activities_stats.user_username - , all_course_activities_stats.courserun_readable_id - , all_course_activities_stats.courseactivity_num_days_activity - , all_course_activities_stats.courseactivity_num_events - , play_video_stats.courseactivity_num_unique_play_video - , play_video_stats.courseactivity_num_play_video - , play_video_stats.courseactivity_last_play_video_timestamp - , problem_check_stats.courseactivity_last_problem_check_timestamp - , all_course_activities_stats.courseactivity_first_event_timestamp - , all_course_activities_stats.courseactivity_last_event_timestamp + all_course_activities_stats.user_username, + all_course_activities_stats.courserun_readable_id, + all_course_activities_stats.courseactivity_num_days_activity, + all_course_activities_stats.courseactivity_num_events, + play_video_stats.courseactivity_num_unique_play_video, + play_video_stats.courseactivity_num_play_video, + play_video_stats.courseactivity_last_play_video_timestamp, + problem_check_stats.courseactivity_last_problem_check_timestamp, + all_course_activities_stats.courseactivity_first_event_timestamp, + all_course_activities_stats.courseactivity_last_event_timestamp from all_course_activities_stats -left join play_video_stats - on - all_course_activities_stats.user_username = play_video_stats.user_username - and all_course_activities_stats.courserun_readable_id = play_video_stats.courserun_readable_id -left join problem_check_stats - on - all_course_activities_stats.user_username = problem_check_stats.user_username - and all_course_activities_stats.courserun_readable_id = problem_check_stats.courserun_readable_id +left join + play_video_stats + on all_course_activities_stats.user_username = play_video_stats.user_username + and all_course_activities_stats.courserun_readable_id = play_video_stats.courserun_readable_id +left join + problem_check_stats + on all_course_activities_stats.user_username = problem_check_stats.user_username + and all_course_activities_stats.courserun_readable_id = problem_check_stats.courserun_readable_id diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivities_daily.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivities_daily.sql index 01cfedb90..1477e5f82 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivities_daily.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivities_daily.sql @@ -1,19 +1,18 @@ -with course_activities as ( - select * from {{ ref('stg__mitxresidential__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxresidential__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ), + daily_activities_stats as ( + select + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date, + count(*) as courseactivity_num_events + from course_activities + group by user_username, courserun_readable_id, date(from_iso8601_timestamp(useractivity_timestamp)) + ) -, daily_activities_stats as ( - select - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - , count(*) as courseactivity_num_events - from course_activities - group by - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) -) - -select * from daily_activities_stats +select * +from daily_activities_stats diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_discussion.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_discussion.sql index aecd441cd..f5f40c316 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_discussion.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_discussion.sql @@ -1,24 +1,26 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxresidential__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxresidential__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , user_id - , useractivity_event_source - , useractivity_event_type - , useractivity_path - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_discussion_post_id - , json_query(useractivity_event_object, 'lax $.title' omit quotes) as useractivity_discussion_post_title - , json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as useractivity_discussion_block_id - , json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as useractivity_discussion_block_name - , json_query(useractivity_event_object, 'lax $.url' omit quotes) as useractivity_discussion_page_url - , json_query(useractivity_event_object, 'lax $.query' omit quotes) as useractivity_discussion_search_query - , json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as useractivity_discussion_roles + user_username, + courserun_readable_id, + user_id, + useractivity_event_source, + useractivity_event_type, + useractivity_path, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_discussion_post_id, + json_query(useractivity_event_object, 'lax $.title' omit quotes) as useractivity_discussion_post_title, + json_query(useractivity_event_object, 'lax $.category_id' omit quotes) as useractivity_discussion_block_id, + json_query(useractivity_event_object, 'lax $.category_name' omit quotes) as useractivity_discussion_block_name, + json_query(useractivity_event_object, 'lax $.url' omit quotes) as useractivity_discussion_page_url, + json_query(useractivity_event_object, 'lax $.query' omit quotes) as useractivity_discussion_search_query, + json_query(useractivity_event_object, 'lax $.user_forums_roles' omit quotes) as useractivity_discussion_roles from course_activities where useractivity_event_type like 'edx.forum.%' diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_problemcheck.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_problemcheck.sql index f03a40517..afe6fb86c 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_problemcheck.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_problemcheck.sql @@ -1,25 +1,28 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxresidential__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxresidential__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select distinct - user_username - , courserun_readable_id - , user_id - , useractivity_event_type - , useractivity_timestamp - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id - , json_query(useractivity_event_object, 'lax $.answers' omit quotes) as useractivity_problem_student_answers - , json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as useractivity_problem_attempts - , json_query(useractivity_event_object, 'lax $.success' omit quotes) as useractivity_problem_success - , json_query(useractivity_event_object, 'lax $.grade' omit quotes) as useractivity_problem_current_grade - , json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as useractivity_problem_max_grade + user_username, + courserun_readable_id, + user_id, + useractivity_event_type, + useractivity_timestamp, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id, + json_query(useractivity_event_object, 'lax $.answers' omit quotes) as useractivity_problem_student_answers, + json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as useractivity_problem_attempts, + json_query(useractivity_event_object, 'lax $.success' omit quotes) as useractivity_problem_success, + json_query(useractivity_event_object, 'lax $.grade' omit quotes) as useractivity_problem_current_grade, + json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as useractivity_problem_max_grade from course_activities -where useractivity_event_type = 'problem_check' ---- This event emitted by the browser contain all of the GET parameters, --- only events emitted by the server are useful -and useractivity_event_source = 'server' +where + useractivity_event_type = 'problem_check' + -- - This event emitted by the browser contain all of the GET parameters, + -- only events emitted by the server are useful + and useractivity_event_source = 'server' diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_problemsubmitted.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_problemsubmitted.sql index 802a0c08d..f13c5177e 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_problemsubmitted.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_problemsubmitted.sql @@ -1,23 +1,25 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxresidential__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxresidential__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , user_id - , useractivity_event_source - , useractivity_event_type - , useractivity_path - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.event_transaction_id' omit quotes) as useractivity_event_id - , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id - , json_query(useractivity_event_object, 'lax $.weight' omit quotes) as useractivity_problem_weight - , json_query(useractivity_event_object, 'lax $.weighted_earned' omit quotes) as useractivity_problem_earned_score - , json_query(useractivity_event_object, 'lax $.weighted_possible' omit quotes) as useractivity_problem_max_score + user_username, + courserun_readable_id, + user_id, + useractivity_event_source, + useractivity_event_type, + useractivity_path, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.event_transaction_id' omit quotes) as useractivity_event_id, + json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id, + json_query(useractivity_event_object, 'lax $.weight' omit quotes) as useractivity_problem_weight, + json_query(useractivity_event_object, 'lax $.weighted_earned' omit quotes) as useractivity_problem_earned_score, + json_query(useractivity_event_object, 'lax $.weighted_possible' omit quotes) as useractivity_problem_max_score from course_activities where useractivity_event_type = 'edx.grades.problem.submitted' diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_showanswer.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_showanswer.sql index afb5dacef..7a9e843bc 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_showanswer.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_showanswer.sql @@ -1,16 +1,18 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxresidential__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxresidential__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , user_id - , useractivity_path - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id + user_username, + courserun_readable_id, + user_id, + useractivity_path, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id from course_activities where useractivity_event_type = 'showanswer' diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_video.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_video.sql index 85cc5e4f5..3c1eadfd6 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_video.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__user_courseactivity_video.sql @@ -1,29 +1,32 @@ -{{ config(materialized='view') }} +{{ config(materialized="view") }} -with course_activities as ( - select * from {{ ref('stg__mitxresidential__openedx__tracking_logs__user_activity') }} - where courserun_readable_id is not null -) +with + course_activities as ( + select * + from {{ ref("stg__mitxresidential__openedx__tracking_logs__user_activity") }} + where courserun_readable_id is not null + ) select - user_username - , courserun_readable_id - , user_id - , useractivity_event_source - , useractivity_event_type - , useractivity_event_object - , useractivity_page_url - , useractivity_timestamp - , json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_video_id - , case - when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' then null + user_username, + courserun_readable_id, + user_id, + useractivity_event_source, + useractivity_event_type, + useractivity_event_object, + useractivity_page_url, + useractivity_timestamp, + json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_video_id, + case + when lower(json_query(useractivity_event_object, 'lax $.duration' omit quotes)) = 'null' + then null else cast(json_query(useractivity_event_object, 'lax $.duration' omit quotes) as decimal(38, 4)) - end as useractivity_video_duration - , json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as useractivity_video_currenttime - , json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as useractivity_video_old_time - , json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as useractivity_video_new_time - , json_query(useractivity_event_object, 'lax $.new_speed' omit quotes) as useractivity_video_new_speed - , json_query(useractivity_event_object, 'lax $.old_speed' omit quotes) as useractivity_video_old_speed + end as useractivity_video_duration, + json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as useractivity_video_currenttime, + json_query(useractivity_event_object, 'lax $.old_time' omit quotes) as useractivity_video_old_time, + json_query(useractivity_event_object, 'lax $.new_time' omit quotes) as useractivity_video_new_time, + json_query(useractivity_event_object, 'lax $.new_speed' omit quotes) as useractivity_video_new_speed, + json_query(useractivity_event_object, 'lax $.old_speed' omit quotes) as useractivity_video_old_speed from course_activities where regexp_like(useractivity_event_type, '(^[\w]+)_video') = true diff --git a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__users.sql b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__users.sql index 344b4240f..bb77128f1 100644 --- a/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__users.sql +++ b/src/ol_dbt/models/intermediate/mitxresidential/int__mitxresidential__users.sql @@ -1,25 +1,21 @@ -with users as ( - select * from {{ ref('stg__mitxresidential__openedx__auth_user') }} -) - -, profiles as ( - select * from {{ ref('stg__mitxresidential__openedx__auth_userprofile') }} -) +with + users as (select * from {{ ref("stg__mitxresidential__openedx__auth_user") }}), + profiles as (select * from {{ ref("stg__mitxresidential__openedx__auth_userprofile") }}) select - users.user_id - , users.user_username - , users.user_first_name - , users.user_last_name - , users.user_email - , users.user_joined_on - , users.user_last_login - , users.user_is_active - , profiles.user_address_city - , profiles.user_address_country - , profiles.user_birth_year - , profiles.user_gender - , profiles.user_highest_education - , coalesce(users.user_full_name, profiles.user_full_name) as user_full_name + users.user_id, + users.user_username, + users.user_first_name, + users.user_last_name, + users.user_email, + users.user_joined_on, + users.user_last_login, + users.user_is_active, + profiles.user_address_city, + profiles.user_address_country, + profiles.user_birth_year, + profiles.user_gender, + profiles.user_highest_education, + coalesce(users.user_full_name, profiles.user_full_name) as user_full_name from users left join profiles on users.user_id = profiles.user_id diff --git a/src/ol_dbt/models/intermediate/ocw/int__ocw__course_departments.sql b/src/ol_dbt/models/intermediate/ocw/int__ocw__course_departments.sql index 396852140..5d96e991e 100644 --- a/src/ol_dbt/models/intermediate/ocw/int__ocw__course_departments.sql +++ b/src/ol_dbt/models/intermediate/ocw/int__ocw__course_departments.sql @@ -1,11 +1,9 @@ -with websitecontents as ( - select * from {{ ref('stg__ocw__studio__postgres__websites_websitecontent') }} -) +with websitecontents as (select * from {{ ref("stg__ocw__studio__postgres__websites_websitecontent") }}) select - websitecontents.website_uuid as course_uuid - , course_department_number -- noqa - , {{ transform_ocw_department_number('course_department_number') }} as course_department_name + websitecontents.website_uuid as course_uuid, + course_department_number, -- noqa + {{ transform_ocw_department_number("course_department_number") }} as course_department_name from websitecontents -cross join unnest(cast(json_parse(course_department_numbers_json) as array(varchar))) as t(course_department_number) -- noqa +cross join unnest(cast(json_parse(course_department_numbers_json) as array(varchar))) as t(course_department_number) -- noqa where websitecontents.websitecontent_type = 'sitemetadata' diff --git a/src/ol_dbt/models/intermediate/ocw/int__ocw__course_instructors.sql b/src/ol_dbt/models/intermediate/ocw/int__ocw__course_instructors.sql index 8076f0bd2..6a24ed4cf 100644 --- a/src/ol_dbt/models/intermediate/ocw/int__ocw__course_instructors.sql +++ b/src/ol_dbt/models/intermediate/ocw/int__ocw__course_instructors.sql @@ -1,37 +1,32 @@ -with websitecontents as ( - select * from {{ ref('stg__ocw__studio__postgres__websites_websitecontent') }} -) - -, instructor_contents as ( - select - websitecontent_text_id as course_instructor_uuid - , websitecontent_title as course_instructor_title - , json_query(websitecontent_metadata, 'lax $.first_name' omit quotes) as course_instructor_first_name - , json_query(websitecontent_metadata, 'lax $.last_name' omit quotes) as course_instructor_last_name - , json_query(websitecontent_metadata, 'lax $.middle_initial' omit quotes) as course_instructor_middle_initial - , json_query(websitecontent_metadata, 'lax $.salutation' omit quotes) as course_instructor_salutation - from websitecontents - where websitecontent_type = 'instructor' -) - -, unnested_instructor_contents as ( - select - websitecontents.website_uuid as course_uuid - , course_instructor_uuid -- noqa - from websitecontents - cross join - unnest(cast(json_parse(course_instructor_uuids) as array (varchar))) as t(course_instructor_uuid) -- noqa - where websitecontents.websitecontent_type = 'sitemetadata' -) +with + websitecontents as (select * from {{ ref("stg__ocw__studio__postgres__websites_websitecontent") }}), + instructor_contents as ( + select + websitecontent_text_id as course_instructor_uuid, + websitecontent_title as course_instructor_title, + json_query(websitecontent_metadata, 'lax $.first_name' omit quotes) as course_instructor_first_name, + json_query(websitecontent_metadata, 'lax $.last_name' omit quotes) as course_instructor_last_name, + json_query(websitecontent_metadata, 'lax $.middle_initial' omit quotes) as course_instructor_middle_initial, + json_query(websitecontent_metadata, 'lax $.salutation' omit quotes) as course_instructor_salutation + from websitecontents + where websitecontent_type = 'instructor' + ), + unnested_instructor_contents as ( + select websitecontents.website_uuid as course_uuid, course_instructor_uuid -- noqa + from websitecontents + cross join unnest(cast(json_parse(course_instructor_uuids) as array(varchar))) as t(course_instructor_uuid) -- noqa + where websitecontents.websitecontent_type = 'sitemetadata' + ) select - unnested_instructor_contents.course_uuid - , unnested_instructor_contents.course_instructor_uuid - , instructor_contents.course_instructor_title - , instructor_contents.course_instructor_first_name - , instructor_contents.course_instructor_last_name - , instructor_contents.course_instructor_salutation - , instructor_contents.course_instructor_middle_initial + unnested_instructor_contents.course_uuid, + unnested_instructor_contents.course_instructor_uuid, + instructor_contents.course_instructor_title, + instructor_contents.course_instructor_first_name, + instructor_contents.course_instructor_last_name, + instructor_contents.course_instructor_salutation, + instructor_contents.course_instructor_middle_initial from unnested_instructor_contents -inner join instructor_contents +inner join + instructor_contents on unnested_instructor_contents.course_instructor_uuid = instructor_contents.course_instructor_uuid diff --git a/src/ol_dbt/models/intermediate/ocw/int__ocw__course_topics.sql b/src/ol_dbt/models/intermediate/ocw/int__ocw__course_topics.sql index 6197d660d..04f322987 100644 --- a/src/ol_dbt/models/intermediate/ocw/int__ocw__course_topics.sql +++ b/src/ol_dbt/models/intermediate/ocw/int__ocw__course_topics.sql @@ -1,12 +1,10 @@ -with websitecontents as ( - select * from {{ ref('stg__ocw__studio__postgres__websites_websitecontent') }} -) +with websitecontents as (select * from {{ ref("stg__ocw__studio__postgres__websites_websitecontent") }}) -select distinct -- a few courses have duplicated topic, subtopic and speciality combination - websitecontents.website_uuid as course_uuid - , element_at(course_topic, 1) as course_topic -- noqa - , element_at(course_topic, 2) as course_subtopic -- noqa - , element_at(course_topic, 3) as course_speciality -- noqa +select distinct -- a few courses have duplicated topic, subtopic and speciality combination + websitecontents.website_uuid as course_uuid, + element_at(course_topic, 1) as course_topic, -- noqa + element_at(course_topic, 2) as course_subtopic, -- noqa + element_at(course_topic, 3) as course_speciality -- noqa from websitecontents -cross join unnest(cast(json_parse(course_topics) as array(array(varchar)))) as t(course_topic) -- noqa +cross join unnest(cast(json_parse(course_topics) as array(array(varchar)))) as t(course_topic) -- noqa where websitecontents.websitecontent_type = 'sitemetadata' diff --git a/src/ol_dbt/models/intermediate/ocw/int__ocw__courses.sql b/src/ol_dbt/models/intermediate/ocw/int__ocw__courses.sql index e50b39a73..2cbadb4c2 100644 --- a/src/ol_dbt/models/intermediate/ocw/int__ocw__courses.sql +++ b/src/ol_dbt/models/intermediate/ocw/int__ocw__courses.sql @@ -1,45 +1,34 @@ -with websites as ( - select * from {{ ref('stg__ocw__studio__postgres__websites_website') }} -) - -, websitecontents as ( - select * from {{ ref('stg__ocw__studio__postgres__websites_websitecontent') }} -) - -, websitestarters as ( - select * from {{ ref('stg__ocw__studio__postgres__websites_websitestarter') }} -) +with + websites as (select * from {{ ref("stg__ocw__studio__postgres__websites_website") }}), + websitecontents as (select * from {{ ref("stg__ocw__studio__postgres__websites_websitecontent") }}), + websitestarters as (select * from {{ ref("stg__ocw__studio__postgres__websites_websitestarter") }}) select - websites.website_uuid as course_uuid - , websites.website_name as course_name - , websites.website_title as course_title - , websites.website_short_id as course_readable_id - , websites.website_source as course_source - , websites.website_is_live as course_is_live - , websites.website_is_unpublished as course_is_unpublished - , websites.website_has_never_published as course_has_never_published - , websites.website_url_path as course_url_path - , websites.website_live_url as course_live_url - , websites.website_first_published_on as course_first_published_on - , websites.website_publish_date_updated_on as course_publish_date_updated_on - , websites.website_created_on as course_created_on - , websites.website_updated_on as course_updated_on - , websitecontents.course_description - , websitecontents.course_term - , websitecontents.course_year - , websitecontents.course_level - , websitecontents.course_primary_course_number - , websitecontents.course_extra_course_numbers - , websitecontents.course_topics - , websitecontents.course_department_numbers - , websitecontents.learning_resource_types as course_learning_resource_types + websites.website_uuid as course_uuid, + websites.website_name as course_name, + websites.website_title as course_title, + websites.website_short_id as course_readable_id, + websites.website_source as course_source, + websites.website_is_live as course_is_live, + websites.website_is_unpublished as course_is_unpublished, + websites.website_has_never_published as course_has_never_published, + websites.website_url_path as course_url_path, + websites.website_live_url as course_live_url, + websites.website_first_published_on as course_first_published_on, + websites.website_publish_date_updated_on as course_publish_date_updated_on, + websites.website_created_on as course_created_on, + websites.website_updated_on as course_updated_on, + websitecontents.course_description, + websitecontents.course_term, + websitecontents.course_year, + websitecontents.course_level, + websitecontents.course_primary_course_number, + websitecontents.course_extra_course_numbers, + websitecontents.course_topics, + websitecontents.course_department_numbers, + websitecontents.learning_resource_types as course_learning_resource_types from websites -inner join websitecontents - on websites.website_uuid = websitecontents.website_uuid -inner join websitestarters - on websites.websitestarter_id = websitestarters.websitestarter_id ---- the where clause ensure the records are OCW courses -where - websitecontents.websitecontent_type = 'sitemetadata' - and websitestarters.websitestarter_name = 'ocw-course' +inner join websitecontents on websites.website_uuid = websitecontents.website_uuid +inner join websitestarters on websites.websitestarter_id = websitestarters.websitestarter_id +-- - the where clause ensure the records are OCW courses +where websitecontents.websitecontent_type = 'sitemetadata' and websitestarters.websitestarter_name = 'ocw-course' diff --git a/src/ol_dbt/models/intermediate/ocw/int__ocw__resources.sql b/src/ol_dbt/models/intermediate/ocw/int__ocw__resources.sql index 5f5b98d4b..8444043ea 100644 --- a/src/ol_dbt/models/intermediate/ocw/int__ocw__resources.sql +++ b/src/ol_dbt/models/intermediate/ocw/int__ocw__resources.sql @@ -1,135 +1,149 @@ -with websites as ( - select * from {{ ref('stg__ocw__studio__postgres__websites_website') }} -) - -, websitecontents as ( - select * from {{ ref('stg__ocw__studio__postgres__websites_websitecontent') }} -) - -, websitestarters as ( - select * from {{ ref('stg__ocw__studio__postgres__websites_websitestarter') }} -) - -, sitemetadata as ( - select - website_uuid - , course_primary_course_number as sitemetadata_primary_course_number - , course_term as sitemetadata_course_term - , course_title as sitemetadata_course_title - , course_year as sitemetadata_course_year - from websitecontents - where websitecontent_type = 'sitemetadata' -) +with + websites as (select * from {{ ref("stg__ocw__studio__postgres__websites_website") }}), + websitecontents as (select * from {{ ref("stg__ocw__studio__postgres__websites_websitecontent") }}), + websitestarters as (select * from {{ ref("stg__ocw__studio__postgres__websites_websitestarter") }}), + sitemetadata as ( + select + website_uuid, + course_primary_course_number as sitemetadata_primary_course_number, + course_term as sitemetadata_course_term, + course_title as sitemetadata_course_title, + course_year as sitemetadata_course_year + from websitecontents + where websitecontent_type = 'sitemetadata' + ) select - websites.website_name as course_name - , websites.website_live_url as course_live_url - , websites.website_uuid as course_uuid - , websitecontents.websitecontent_type as content_type - , websitecontents.learning_resource_types - , websitecontents.websitecontent_metadata as metadata --noqa: disable=RF04 - , websitecontents.metadata_draft as resource_draft - , websitecontents.websitecontent_filename as resource_filename - , websitecontents.websitecontent_title as resource_title - , websitecontents.metadata_resource_type as resource_type - , websitecontents.websitecontent_text_id as resource_uuid - , websites.website_title + websites.website_name as course_name, + websites.website_live_url as course_live_url, + websites.website_uuid as course_uuid, + websitecontents.websitecontent_type as content_type, + websitecontents.learning_resource_types, + websitecontents.websitecontent_metadata as metadata, -- noqa: disable=RF04 + websitecontents.metadata_draft as resource_draft, + websitecontents.websitecontent_filename as resource_filename, + websitecontents.websitecontent_title as resource_title, + websitecontents.metadata_resource_type as resource_type, + websitecontents.websitecontent_text_id as resource_uuid, + websites.website_title, -- noqa: disable=RF02 -- external resources - , cast( + cast( nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.is_broken' omit quotes), '') as boolean - ) as external_resource_is_broken - , cast( + ) as external_resource_is_broken, + cast( nullif( json_query(websitecontents.websitecontent_metadata, 'lax $.has_external_license_warning' omit quotes), '' ) as boolean - ) as external_resource_license_warning - , nullif(nullif( - json_query(websitecontents.websitecontent_metadata, 'lax $.url_status_code' omit quotes), '' - ), 'null') as external_resource_url_status_code - , nullif(nullif( - json_query(websitecontents.websitecontent_metadata, 'lax $.backup_url_status_code' omit quotes), '' - ), 'null') as external_resource_backup_url_status_code - , coalesce(sitemetadata.sitemetadata_primary_course_number, websites.primary_course_number) as course_number - , coalesce(sitemetadata.sitemetadata_course_term, websites.metadata_course_term) as course_term - , coalesce(sitemetadata.sitemetadata_course_title, websites.metadata_course_title) as course_title - , coalesce(sitemetadata.sitemetadata_course_year, websites.metadata_course_year) as course_year - , websites.website_live_url || '/resources/' || websitecontents.websitecontent_filename as resource_live_url - , 'https://ocw-studio.odl.mit.edu/sites/' + ) as external_resource_license_warning, + nullif( + nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.url_status_code' omit quotes), ''), 'null' + ) as external_resource_url_status_code, + nullif( + nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.backup_url_status_code' omit quotes), ''), + 'null' + ) as external_resource_backup_url_status_code, + coalesce(sitemetadata.sitemetadata_primary_course_number, websites.primary_course_number) as course_number, + coalesce(sitemetadata.sitemetadata_course_term, websites.metadata_course_term) as course_term, + coalesce(sitemetadata.sitemetadata_course_title, websites.metadata_course_title) as course_title, + coalesce(sitemetadata.sitemetadata_course_year, websites.metadata_course_year) as course_year, + websites.website_live_url || '/resources/' || websitecontents.websitecontent_filename as resource_live_url, + 'https://ocw-studio.odl.mit.edu/sites/' || websites.website_name || '/type/' || websitecontents.websitecontent_type || '/edit/' || websitecontents.websitecontent_text_id - || '/' as studio_url - , nullif(nullif(json_query( - websitecontents.websitecontent_metadata, 'lax $.backup_url' omit quotes - ), ''), 'null') as external_resource_backup_url - , nullif( + || '/' as studio_url, + nullif( + nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.backup_url' omit quotes), ''), 'null' + ) as external_resource_backup_url, + nullif( nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.external_url' omit quotes), ''), 'null' - ) as external_resource_url + ) as external_resource_url, -- image_metadata for image resources; could be in metadata or image_metadata - , coalesce( + coalesce( nullif( - nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.metadata.image_alt' omit quotes), '') - , 'null' - ) - , nullif(nullif( - json_query(websitecontents.websitecontent_metadata, 'lax $.image_metadata."image-alt"' omit quotes), '' - ), 'null') - ) as image_alt_text - , coalesce( + nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.metadata.image_alt' omit quotes), ''), + 'null' + ), nullif( - nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.metadata.caption' omit quotes), '') - , 'null' + nullif( + json_query(websitecontents.websitecontent_metadata, 'lax $.image_metadata."image-alt"' omit quotes), '' + ), + 'null' ) - , nullif( - nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.image_metadata.caption' omit quotes), '') - , 'null' + ) as image_alt_text, + coalesce( + nullif( + nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.metadata.caption' omit quotes), ''), + 'null' + ), + nullif( + nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.image_metadata.caption' omit quotes), ''), + 'null' ) - ) as image_caption - , coalesce( + ) as image_caption, + coalesce( nullif( nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.metadata.credit' omit quotes), ''), 'null' + ), + nullif( + nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.image_metadata.credit' omit quotes), ''), + 'null' ) - , nullif( - nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.image_metadata.credit' omit quotes), '') - , 'null' - ) - ) as image_credit + ) as image_credit, -- video_metadata for video resources - , nullif(nullif(json_query( - websitecontents.websitecontent_metadata, 'lax $.video_metadata.youtube_description' omit quotes - ), ''), 'null') as video_youtube_description - , nullif(nullif(json_query( - websitecontents.websitecontent_metadata, 'lax $.video_metadata.youtube_id' omit quotes - ), ''), 'null') as video_youtube_id - , nullif(nullif(json_query( - websitecontents.websitecontent_metadata, 'lax $.video_metadata.video_speakers' omit quotes - ), ''), 'null') as video_youtube_speakers - , nullif(nullif(json_query( - websitecontents.websitecontent_metadata, 'lax $.video_metadata.video_tags' omit quotes - ), ''), 'null') as video_youtube_tags + nullif( + nullif( + json_query(websitecontents.websitecontent_metadata, 'lax $.video_metadata.youtube_description' omit quotes), + '' + ), + 'null' + ) as video_youtube_description, + nullif( + nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.video_metadata.youtube_id' omit quotes), ''), + 'null' + ) as video_youtube_id, + nullif( + nullif( + json_query(websitecontents.websitecontent_metadata, 'lax $.video_metadata.video_speakers' omit quotes), '' + ), + 'null' + ) as video_youtube_speakers, + nullif( + nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.video_metadata.video_tags' omit quotes), ''), + 'null' + ) as video_youtube_tags, -- video_files for video resources - , nullif(nullif(json_query( - websitecontents.websitecontent_metadata, 'lax $.video_files.archive_url' omit quotes - ), ''), 'null') as video_archive_url - , nullif(nullif(json_query( - websitecontents.websitecontent_metadata, 'lax $.video_files.video_captions_file' omit quotes - ), ''), 'null') as video_captions_file - , nullif(nullif(json_query( - websitecontents.websitecontent_metadata, 'lax $.video_files.video_thumbnail_file' omit quotes - ), ''), 'null') as video_thumbnail_file - , nullif(nullif(json_query( - websitecontents.websitecontent_metadata, 'lax $.video_files.video_transcript_file' omit quotes - ), ''), 'null') as video_transcript_file + nullif( + nullif(json_query(websitecontents.websitecontent_metadata, 'lax $.video_files.archive_url' omit quotes), ''), + 'null' + ) as video_archive_url, + nullif( + nullif( + json_query(websitecontents.websitecontent_metadata, 'lax $.video_files.video_captions_file' omit quotes), '' + ), + 'null' + ) as video_captions_file, + nullif( + nullif( + json_query(websitecontents.websitecontent_metadata, 'lax $.video_files.video_thumbnail_file' omit quotes), + '' + ), + 'null' + ) as video_thumbnail_file, + nullif( + nullif( + json_query(websitecontents.websitecontent_metadata, 'lax $.video_files.video_transcript_file' omit quotes), + '' + ), + 'null' + ) as video_transcript_file from websites -inner join websitecontents - on websites.website_uuid = websitecontents.website_uuid -left join sitemetadata - on websites.website_uuid = sitemetadata.website_uuid -inner join websitestarters - on websites.websitestarter_id = websitestarters.websitestarter_id +inner join websitecontents on websites.website_uuid = websitecontents.website_uuid +left join sitemetadata on websites.website_uuid = sitemetadata.website_uuid +inner join websitestarters on websites.websitestarter_id = websitestarters.websitestarter_id where (websitecontents.websitecontent_type = 'resource' or websitecontents.websitecontent_type = 'external-resource') and websitestarters.websitestarter_name = 'ocw-course' diff --git a/src/ol_dbt/models/intermediate/ovs/int__ovs__videos.sql b/src/ol_dbt/models/intermediate/ovs/int__ovs__videos.sql index 1efc9ee96..0d18be83b 100644 --- a/src/ol_dbt/models/intermediate/ovs/int__ovs__videos.sql +++ b/src/ol_dbt/models/intermediate/ovs/int__ovs__videos.sql @@ -1,47 +1,33 @@ -with video_encodejobs as ( - select - * - --- there may be multiple encodejobs for the same video, but we only need the last one for video duration - , row_number() over ( - partition by video_id - order by encodejob_created_on desc - ) as encodejob_rank - from {{ ref('stg__ovs__studio__postgres__ui_encodejob') }} -) - -, videos as ( - select * from {{ ref('stg__ovs__studio__postgres__ui_video') }} -) - -, collections as ( - select * from {{ ref('stg__ovs__studio__postgres__ui_collection') }} -) - -, collection_edxendpoints as ( - select * from {{ ref('stg__ovs__studio__postgres__ui_collectionedxendpoint') }} -) - -, edxendpoints as ( - select * from {{ ref('stg__ovs__studio__postgres__ui_edxendpoint') }} -) +with + video_encodejobs as ( + select + *, + -- - there may be multiple encodejobs for the same video, but we only need the last one for video duration + row_number() over (partition by video_id order by encodejob_created_on desc) as encodejob_rank + from {{ ref("stg__ovs__studio__postgres__ui_encodejob") }} + ), + videos as (select * from {{ ref("stg__ovs__studio__postgres__ui_video") }}), + collections as (select * from {{ ref("stg__ovs__studio__postgres__ui_collection") }}), + collection_edxendpoints as (select * from {{ ref("stg__ovs__studio__postgres__ui_collectionedxendpoint") }}), + edxendpoints as (select * from {{ ref("stg__ovs__studio__postgres__ui_edxendpoint") }}) select - collections.collection_id - , collections.collection_uuid - , collections.collection_title - , collections.courserun_readable_id - , edxendpoints.edxendpoint_base_url - , videos.video_id - , videos.video_uuid - , videos.video_title - , videos.video_status - , videos.video_created_on - , video_encodejobs.video_duration - , case + collections.collection_id, + collections.collection_uuid, + collections.collection_title, + collections.courserun_readable_id, + edxendpoints.edxendpoint_base_url, + videos.video_id, + videos.video_uuid, + videos.video_title, + videos.video_status, + videos.video_created_on, + video_encodejobs.video_duration, + case when edxendpoints.edxendpoint_base_url = '{{ var("mitxonline_openedx_url") }}' - then '{{ var("mitxonline") }}' + then '{{ var("mitxonline") }}' when edxendpoints.edxendpoint_base_url = '{{ var("mitxpro_openedx_url") }}' - then '{{ var("mitxpro") }}' + then '{{ var("mitxpro") }}' end as platform from videos inner join collections on videos.collection_id = collections.collection_id diff --git a/src/ol_dbt/models/intermediate/salesforce/int__salesforce__opportunity.sql b/src/ol_dbt/models/intermediate/salesforce/int__salesforce__opportunity.sql index 84d5f6fab..ac970574a 100644 --- a/src/ol_dbt/models/intermediate/salesforce/int__salesforce__opportunity.sql +++ b/src/ol_dbt/models/intermediate/salesforce/int__salesforce__opportunity.sql @@ -1,30 +1,28 @@ -with stg_opportunity as ( - select * from {{ ref('stg__salesforce__opportunity') }} -) +with stg_opportunity as (select * from {{ ref("stg__salesforce__opportunity") }}) select - opportunity_id - , opportunity_name - , opportunity_type - , opportunity_business_type - , opportunity_agreement_type - , opportunity_mit_is_sales_lead - , opportunity_revenue_share_or_commission_contract - , opportunity_revenue_share_or_commission_amount - , opportunity_stage - , opportunity_amount - , opportunity_probability - , opportunity_is_won - , opportunity_is_closed - , opportunity_num_enrollment_codes_redeemed - , opportunity_num_of_seats_purchased - , opportunity_price_per_seat - , opportunity_leadsource - , opportunity_nextstep - , opportunity_has_lineitem - , opportunity_data_quality_score - , opportunity_data_quality_description - , opportunity_close_date - , opportunity_created_on - , opportunity_modified_on + opportunity_id, + opportunity_name, + opportunity_type, + opportunity_business_type, + opportunity_agreement_type, + opportunity_mit_is_sales_lead, + opportunity_revenue_share_or_commission_contract, + opportunity_revenue_share_or_commission_amount, + opportunity_stage, + opportunity_amount, + opportunity_probability, + opportunity_is_won, + opportunity_is_closed, + opportunity_num_enrollment_codes_redeemed, + opportunity_num_of_seats_purchased, + opportunity_price_per_seat, + opportunity_leadsource, + opportunity_nextstep, + opportunity_has_lineitem, + opportunity_data_quality_score, + opportunity_data_quality_description, + opportunity_close_date, + opportunity_created_on, + opportunity_modified_on from stg_opportunity diff --git a/src/ol_dbt/models/intermediate/salesforce/int__salesforce__opportunitylineitem.sql b/src/ol_dbt/models/intermediate/salesforce/int__salesforce__opportunitylineitem.sql index 244b01bed..0673326dc 100644 --- a/src/ol_dbt/models/intermediate/salesforce/int__salesforce__opportunitylineitem.sql +++ b/src/ol_dbt/models/intermediate/salesforce/int__salesforce__opportunitylineitem.sql @@ -1,19 +1,17 @@ -with stg_opportunitylineitem as ( - select * from {{ ref('stg__salesforce__opportunitylineitem') }} -) +with stg_opportunitylineitem as (select * from {{ ref("stg__salesforce__opportunitylineitem") }}) select - opportunitylineitem_id - , opportunity_id - , opportunitylineitem_product_name - , opportunitylineitem_description - , opportunitylineitem_product_code - , opportunitylineitem_list_price - , opportunitylineitem_sales_price - , opportunitylineitem_discount_percent - , opportunitylineitem_quantity - , opportunitylineitem_total_price - , opportunitylineitem_service_date - , opportunitylineitem_created_on - , opportunitylineitem_modified_on + opportunitylineitem_id, + opportunity_id, + opportunitylineitem_product_name, + opportunitylineitem_description, + opportunitylineitem_product_code, + opportunitylineitem_list_price, + opportunitylineitem_sales_price, + opportunitylineitem_discount_percent, + opportunitylineitem_quantity, + opportunitylineitem_total_price, + opportunitylineitem_service_date, + opportunitylineitem_created_on, + opportunitylineitem_modified_on from stg_opportunitylineitem diff --git a/src/ol_dbt/models/intermediate/zendesk/int__zendesk__ticket.sql b/src/ol_dbt/models/intermediate/zendesk/int__zendesk__ticket.sql index c1347fd0c..5b2f33a2a 100644 --- a/src/ol_dbt/models/intermediate/zendesk/int__zendesk__ticket.sql +++ b/src/ol_dbt/models/intermediate/zendesk/int__zendesk__ticket.sql @@ -1,126 +1,77 @@ -with ticket as ( -- noqa: PRS - select * from {{ ref('stg__zendesk__ticket') }} -) - -, user as ( - select * from {{ ref('stg__zendesk__user') }} -) - -, organization as ( - select * from {{ ref('stg__zendesk__organization') }} -) - -, brand as ( - select * from {{ ref('stg__zendesk__brand') }} -) - -, field as ( - select * from {{ ref('stg__zendesk__ticket_field') }} -) - -, groups as ( - select * from {{ ref('stg__zendesk__group') }} -) - -, followers as ( - select - ticket.ticket_id, - json_format( - cast(array_agg(user.user_name) AS json) - ) as followers - from ticket - cross join unnest(ticket.ticket_follower_user_ids) AS t(user_id) - join user on t.user_id = user.user_id - group by ticket.ticket_id -) - -, collaborators as ( - select - ticket.ticket_id, - json_format( - cast(array_agg(user.user_name) AS json) - ) as collaborators - from ticket - cross join unnest(ticket.ticket_collaborator_user_ids) AS t(user_id) - join user on t.user_id = user.user_id - group by ticket.ticket_id -) - -, named_custom_fields as ( - select - ticket.ticket_id - , json_format( - cast( - array_agg( - json_parse( - json_object( - field.field_title : value - ) - ) - ) as json - ) - ) as custom_fields - from ticket - cross join unnest(ticket.ticket_custom_fields) AS t(json_str) - cross join unnest( - array[ - cast(json_parse(json_str) AS row(id bigint, value varchar)) - ] - ) as u(id, value) - join field on id = field.field_id - where value is not null - group by ticket.ticket_id -) - +with + ticket as ( -- noqa: PRS + select * from {{ ref("stg__zendesk__ticket") }} + ), + user as (select * from {{ ref("stg__zendesk__user") }}), + organization as (select * from {{ ref("stg__zendesk__organization") }}), + brand as (select * from {{ ref("stg__zendesk__brand") }}), + field as (select * from {{ ref("stg__zendesk__ticket_field") }}), + groups as (select * from {{ ref("stg__zendesk__group") }}), + followers as ( + select ticket.ticket_id, json_format(cast(array_agg(user.user_name) as json)) as followers + from ticket + cross join unnest(ticket.ticket_follower_user_ids) as t(user_id) + join user on t.user_id = user.user_id + group by ticket.ticket_id + ), + collaborators as ( + select ticket.ticket_id, json_format(cast(array_agg(user.user_name) as json)) as collaborators + from ticket + cross join unnest(ticket.ticket_collaborator_user_ids) as t(user_id) + join user on t.user_id = user.user_id + group by ticket.ticket_id + ), + named_custom_fields as ( + select + ticket.ticket_id, + json_format(cast(array_agg(json_parse(json_object(field.field_title:value))) as json)) as custom_fields + from ticket + cross join unnest(ticket.ticket_custom_fields) as t(json_str) + cross join unnest(array[cast(json_parse(json_str) as row(id bigint, value varchar))]) as u(id, value) + join field on id = field.field_id + where value is not null + group by ticket.ticket_id + ) select - ticket.ticket_id - , ticket.ticket_api_url - , ticket.ticket_type - , ticket.ticket_tags - , ticket.ticket_subject - , ticket.ticket_description - , ticket.ticket_priority - , ticket.ticket_status - , ticket.ticket_source_channel - , ticket.ticket_source_email - , ticket.ticket_source_rel - , requester.user_name as ticket_requester - , assignee.user_name as ticket_assignee - , submitter.user_name as ticket_submitter - , ticket.ticket_recipient_email - , followers.followers as ticket_followers - , collaborators.collaborators as ticket_collaborators - , brand.brand_name - , groups.group_name - , organization.organization_name - , named_custom_fields.custom_fields - , ticket.ticket_due_at - , ticket.ticket_has_incidents - , ticket.ticket_is_public - , ticket.ticket_satisfaction_rating_score - , ticket.ticket_satisfaction_rating_comment - , ticket.ticket_satisfaction_rating_reason - , ticket.ticket_via_object - , ticket.ticket_unix_timestamp - , ticket.ticket_created_at - , ticket.ticket_updated_at + ticket.ticket_id, + ticket.ticket_api_url, + ticket.ticket_type, + ticket.ticket_tags, + ticket.ticket_subject, + ticket.ticket_description, + ticket.ticket_priority, + ticket.ticket_status, + ticket.ticket_source_channel, + ticket.ticket_source_email, + ticket.ticket_source_rel, + requester.user_name as ticket_requester, + assignee.user_name as ticket_assignee, + submitter.user_name as ticket_submitter, + ticket.ticket_recipient_email, + followers.followers as ticket_followers, + collaborators.collaborators as ticket_collaborators, + brand.brand_name, + groups.group_name, + organization.organization_name, + named_custom_fields.custom_fields, + ticket.ticket_due_at, + ticket.ticket_has_incidents, + ticket.ticket_is_public, + ticket.ticket_satisfaction_rating_score, + ticket.ticket_satisfaction_rating_comment, + ticket.ticket_satisfaction_rating_reason, + ticket.ticket_via_object, + ticket.ticket_unix_timestamp, + ticket.ticket_created_at, + ticket.ticket_updated_at from ticket -left join organization - on ticket.organization_id = organization.organization_id -left join brand - on ticket.brand_id = brand.brand_id -left join groups - on ticket.group_id = groups.group_id -left join user as submitter - on ticket.ticket_submitter_user_id = submitter.user_id -left join user as requester - on ticket.ticket_requester_user_id = requester.user_id -left join user as assignee - on ticket.ticket_assignee_user_id = assignee.user_id -left join named_custom_fields - on ticket.ticket_id = named_custom_fields.ticket_id -left join followers - on ticket.ticket_id = followers.ticket_id -left join collaborators - on ticket.ticket_id = collaborators.ticket_id +left join organization on ticket.organization_id = organization.organization_id +left join brand on ticket.brand_id = brand.brand_id +left join groups on ticket.group_id = groups.group_id +left join user as submitter on ticket.ticket_submitter_user_id = submitter.user_id +left join user as requester on ticket.ticket_requester_user_id = requester.user_id +left join user as assignee on ticket.ticket_assignee_user_id = assignee.user_id +left join named_custom_fields on ticket.ticket_id = named_custom_fields.ticket_id +left join followers on ticket.ticket_id = followers.ticket_id +left join collaborators on ticket.ticket_id = collaborators.ticket_id diff --git a/src/ol_dbt/models/intermediate/zendesk/int__zendesk__ticket_comment.sql b/src/ol_dbt/models/intermediate/zendesk/int__zendesk__ticket_comment.sql index 4b3bf6e12..0abadf59c 100644 --- a/src/ol_dbt/models/intermediate/zendesk/int__zendesk__ticket_comment.sql +++ b/src/ol_dbt/models/intermediate/zendesk/int__zendesk__ticket_comment.sql @@ -1,27 +1,25 @@ -with ticket_comment as ( -- noqa: PRS - select * from {{ ref('stg__zendesk__ticket_comment') }} -) - -, user as ( - select * from {{ ref('stg__zendesk__user') }} -) +with + ticket_comment as ( -- noqa: PRS + select * from {{ ref("stg__zendesk__ticket_comment") }} + ), + user as (select * from {{ ref("stg__zendesk__user") }}) select - ticket_comment.comment_id - , ticket_comment.ticket_id - , ticket_comment.comment_type - , ticket_comment.comment_is_public - , ticket_comment.comment_uploads - , user.user_name as comment_author - , ticket_comment.audit_id - , ticket_comment.comment_html_body - , ticket_comment.comment_plain_body - , ticket_comment.comment_attachments - , ticket_comment.comment_source_channel - , ticket_comment.comment_source_email - , ticket_comment.comment_source_rel - , ticket_comment.comment_via_object - , ticket_comment.comment_unix_timestamp - , ticket_comment.comment_created_at + ticket_comment.comment_id, + ticket_comment.ticket_id, + ticket_comment.comment_type, + ticket_comment.comment_is_public, + ticket_comment.comment_uploads, + user.user_name as comment_author, + ticket_comment.audit_id, + ticket_comment.comment_html_body, + ticket_comment.comment_plain_body, + ticket_comment.comment_attachments, + ticket_comment.comment_source_channel, + ticket_comment.comment_source_email, + ticket_comment.comment_source_rel, + ticket_comment.comment_via_object, + ticket_comment.comment_unix_timestamp, + ticket_comment.comment_created_at from ticket_comment left join user on ticket_comment.comment_author_user_id = user.user_id diff --git a/src/ol_dbt/models/marts/combined/marts__combined__orders.sql b/src/ol_dbt/models/marts/combined/marts__combined__orders.sql index b7d146f34..0d56e1eb8 100644 --- a/src/ol_dbt/models/marts/combined/marts__combined__orders.sql +++ b/src/ol_dbt/models/marts/combined/marts__combined__orders.sql @@ -1,470 +1,424 @@ ---- This model combines intermediate orders from different platforms - -with bootcamps__ecommerce_order as ( - select * from {{ ref('int__bootcamps__ecommerce_order') }} -) - -, bootcamps__receipt as ( - select * from {{ ref('int__bootcamps__ecommerce_receipt') }} -) - -, mitxpro__ecommerce_allorders as ( - select * from {{ ref('int__mitxpro__ecommerce_allorders') }} -) - -, mitxonline__ecommerce_order as ( - select * from {{ ref('int__mitxonline__ecommerce_order') }} -) - -, mitxonline__transaction as ( - select * from {{ ref('int__mitxonline__ecommerce_transaction') }} -) - -, mitxpro__lines as ( - select * from {{ ref('int__mitxpro__ecommerce_line') }} -) - -, mitxpro__programruns as ( - select * from {{ ref('int__mitxpro__program_runs') }} -) - -, mitxpro__receipts as ( - select * - from {{ ref('int__mitxpro__ecommerce_receipt') }} - where receipt_transaction_status != 'ERROR' -) - -, mitxpro__ecommerce_order as ( - select * from {{ ref('int__mitxpro__ecommerce_order') }} -) - -, mitxpro__ecommerce_allcoupons as ( - select * from {{ ref('int__mitxpro__ecommerce_allcoupons') }} -) - -, micromasters_orders as ( - select * from {{ ref('int__micromasters__orders') }} -) - -, mitxonline_orders as ( - select - mitxonline__ecommerce_order.order_id - , mitxonline__ecommerce_order.line_id - , mitxonline__ecommerce_order.courserun_id - , mitxonline__ecommerce_order.courserun_readable_id - , mitxonline__ecommerce_order.product_id - , mitxonline__ecommerce_order.product_type - , mitxonline__ecommerce_order.product_price - , mitxonline__ecommerce_order.user_email - , mitxonline__ecommerce_order.user_id - , mitxonline__ecommerce_order.discount_code - , mitxonline__ecommerce_order.discount_redemption_type - , mitxonline__ecommerce_order.discountredemption_timestamp - , mitxonline__transaction.transaction_bill_to_address_state - , mitxonline__transaction.transaction_bill_to_address_country - , mitxonline__transaction.transaction_uuid - , mitxonline__transaction.transaction_req_type - , mitxonline__transaction.transaction_payment_amount - , mitxonline__transaction.transaction_payment_currency - , mitxonline__transaction.transaction_payment_card_number - , mitxonline__transaction.transaction_payment_card_type - , mitxonline__transaction.transaction_payer_name - , mitxonline__transaction.transaction_payer_email - , mitxonline__transaction.transaction_payer_ip_address - , mitxonline__transaction.transaction_payment_method - , mitxonline__transaction.transaction_timestamp as transaction_payment_timestamp - , mitxonline__transaction.transaction_reference_number - , mitxonline__ecommerce_order.order_created_on - , mitxonline__ecommerce_order.order_reference_number - , mitxonline__ecommerce_order.order_state - , mitxonline__ecommerce_order.order_total_price_paid - , mitxonline__ecommerce_order.discount_amount_text - ----In MITx Online, there are two transactions for an refunded order. - ----Here we picked the refund transaction ID and auth code until we change the unique key - , coalesce( - mitxonline__refund.transaction_readable_identifier - , mitxonline__transaction.transaction_readable_identifier - ) as transaction_readable_identifier - , coalesce( - mitxonline__refund.transaction_authorization_code - , mitxonline__transaction.transaction_authorization_code - ) as transaction_authorization_code - from mitxonline__ecommerce_order - left join mitxonline__transaction - on mitxonline__ecommerce_order.transaction_id = mitxonline__transaction.transaction_id - left join mitxonline__transaction as mitxonline__refund - on - mitxonline__ecommerce_order.order_id = mitxonline__refund.order_id +-- - This model combines intermediate orders from different platforms +with + bootcamps__ecommerce_order as (select * from {{ ref("int__bootcamps__ecommerce_order") }}), + bootcamps__receipt as (select * from {{ ref("int__bootcamps__ecommerce_receipt") }}), + mitxpro__ecommerce_allorders as (select * from {{ ref("int__mitxpro__ecommerce_allorders") }}), + mitxonline__ecommerce_order as (select * from {{ ref("int__mitxonline__ecommerce_order") }}), + mitxonline__transaction as (select * from {{ ref("int__mitxonline__ecommerce_transaction") }}), + mitxpro__lines as (select * from {{ ref("int__mitxpro__ecommerce_line") }}), + mitxpro__programruns as (select * from {{ ref("int__mitxpro__program_runs") }}), + mitxpro__receipts as ( + select * from {{ ref("int__mitxpro__ecommerce_receipt") }} where receipt_transaction_status != 'ERROR' + ), + mitxpro__ecommerce_order as (select * from {{ ref("int__mitxpro__ecommerce_order") }}), + mitxpro__ecommerce_allcoupons as (select * from {{ ref("int__mitxpro__ecommerce_allcoupons") }}), + micromasters_orders as (select * from {{ ref("int__micromasters__orders") }}), + mitxonline_orders as ( + select + mitxonline__ecommerce_order.order_id, + mitxonline__ecommerce_order.line_id, + mitxonline__ecommerce_order.courserun_id, + mitxonline__ecommerce_order.courserun_readable_id, + mitxonline__ecommerce_order.product_id, + mitxonline__ecommerce_order.product_type, + mitxonline__ecommerce_order.product_price, + mitxonline__ecommerce_order.user_email, + mitxonline__ecommerce_order.user_id, + mitxonline__ecommerce_order.discount_code, + mitxonline__ecommerce_order.discount_redemption_type, + mitxonline__ecommerce_order.discountredemption_timestamp, + mitxonline__transaction.transaction_bill_to_address_state, + mitxonline__transaction.transaction_bill_to_address_country, + mitxonline__transaction.transaction_uuid, + mitxonline__transaction.transaction_req_type, + mitxonline__transaction.transaction_payment_amount, + mitxonline__transaction.transaction_payment_currency, + mitxonline__transaction.transaction_payment_card_number, + mitxonline__transaction.transaction_payment_card_type, + mitxonline__transaction.transaction_payer_name, + mitxonline__transaction.transaction_payer_email, + mitxonline__transaction.transaction_payer_ip_address, + mitxonline__transaction.transaction_payment_method, + mitxonline__transaction.transaction_timestamp as transaction_payment_timestamp, + mitxonline__transaction.transaction_reference_number, + mitxonline__ecommerce_order.order_created_on, + mitxonline__ecommerce_order.order_reference_number, + mitxonline__ecommerce_order.order_state, + mitxonline__ecommerce_order.order_total_price_paid, + mitxonline__ecommerce_order.discount_amount_text, + -- --In MITx Online, there are two transactions for an refunded order. + -- --Here we picked the refund transaction ID and auth code until we change the unique key + coalesce( + mitxonline__refund.transaction_readable_identifier, + mitxonline__transaction.transaction_readable_identifier + ) as transaction_readable_identifier, + coalesce( + mitxonline__refund.transaction_authorization_code, + mitxonline__transaction.transaction_authorization_code + ) as transaction_authorization_code + from mitxonline__ecommerce_order + left join + mitxonline__transaction + on mitxonline__ecommerce_order.transaction_id = mitxonline__transaction.transaction_id + left join + mitxonline__transaction as mitxonline__refund + on mitxonline__ecommerce_order.order_id = mitxonline__refund.order_id and mitxonline__refund.transaction_type = 'refund' -) - -, bootcamps_orders as ( - select - bootcamps__ecommerce_order.order_id - , bootcamps__ecommerce_order.line_id - , bootcamps__ecommerce_order.courserun_id - , bootcamps__ecommerce_order.courserun_readable_id - , bootcamps__ecommerce_order.line_price - , bootcamps__ecommerce_order.user_email - , bootcamps__ecommerce_order.order_purchaser_user_id - , bootcamps__receipt.receipt_authorization_code - , bootcamps__receipt.receipt_bill_to_address_state - , bootcamps__receipt.receipt_bill_to_address_country - , bootcamps__receipt.receipt_transaction_uuid - , bootcamps__receipt.receipt_transaction_type - , bootcamps__receipt.receipt_payment_amount - , bootcamps__receipt.receipt_payment_currency - , bootcamps__receipt.receipt_payment_card_number - , bootcamps__receipt.receipt_payment_card_type - , bootcamps__receipt.receipt_payer_name - , bootcamps__receipt.receipt_payer_email - , bootcamps__receipt.receipt_payer_ip_address - , bootcamps__receipt.receipt_payment_method - , bootcamps__receipt.receipt_transaction_id - , bootcamps__receipt.receipt_reference_number - , bootcamps__receipt.receipt_payment_timestamp - , bootcamps__ecommerce_order.order_created_on - , bootcamps__ecommerce_order.order_reference_number - , bootcamps__ecommerce_order.order_state - , bootcamps__ecommerce_order.order_total_price_paid - from bootcamps__ecommerce_order - left join bootcamps__receipt - on bootcamps__ecommerce_order.order_id = bootcamps__receipt.order_id -) - -, mitxpro_orders as ( - select - mitxpro__ecommerce_allorders.line_id - , mitxpro__ecommerce_allcoupons.coupon_code - , mitxpro__ecommerce_allcoupons.coupon_name - , mitxpro__ecommerce_allcoupons.coupon_type - , mitxpro__ecommerce_allorders.coupon_redeemed_on - , mitxpro__ecommerce_allorders.order_created_on - , mitxpro__ecommerce_allorders.order_state - , mitxpro__ecommerce_allorders.product_id - , mitxpro__ecommerce_allorders.product_type - , mitxpro__lines.product_price - , mitxpro__ecommerce_allorders.user_email - , mitxpro__ecommerce_allorders.courserun_id - , mitxpro__ecommerce_allorders.courserun_readable_id - , mitxpro__ecommerce_order.order_purchaser_user_id - , mitxpro__receipts.receipt_authorization_code - , mitxpro__receipts.receipt_bill_to_address_state - , mitxpro__receipts.receipt_bill_to_address_country - , mitxpro__receipts.receipt_transaction_uuid - , mitxpro__receipts.receipt_transaction_type - , mitxpro__receipts.receipt_payment_amount - , mitxpro__receipts.receipt_payment_currency - , mitxpro__receipts.receipt_payer_email - , mitxpro__receipts.receipt_payment_card_number - , mitxpro__receipts.receipt_payer_ip_address - , mitxpro__receipts.receipt_payment_card_type - , mitxpro__receipts.receipt_payer_name - , mitxpro__receipts.receipt_payment_method - , mitxpro__receipts.receipt_transaction_id - , mitxpro__receipts.receipt_payment_timestamp - , mitxpro__ecommerce_order.receipt_reference_number - , mitxpro__ecommerce_order.order_tax_country_code - , mitxpro__ecommerce_order.order_tax_rate - , mitxpro__ecommerce_order.order_tax_rate_name - , mitxpro__ecommerce_order.order_tax_amount - , mitxpro__ecommerce_order.order_total_price_paid_plus_tax - , mitxpro__ecommerce_allorders.coupon_id - , mitxpro__ecommerce_allorders.b2bcoupon_id - , mitxpro__ecommerce_allorders.b2border_contract_number - , mitxpro__ecommerce_allorders.order_id - , mitxpro__ecommerce_order.couponpaymentversion_payment_transaction - , mitxpro__ecommerce_order.order_total_price_paid - , mitxpro__ecommerce_order.couponpaymentversion_discount_amount_text as discount - , concat('xpro-b2c-production-', cast(mitxpro__ecommerce_allorders.order_id as varchar)) - as order_reference_number - , coalesce( - mitxpro__ecommerce_allorders.courserun_readable_id - , mitxpro__programruns.programrun_readable_id - , mitxpro__ecommerce_allorders.program_readable_id - ) as product_readable_id - from mitxpro__ecommerce_allorders - left join mitxpro__ecommerce_order - on mitxpro__ecommerce_allorders.order_id = mitxpro__ecommerce_order.order_id - left join mitxpro__receipts - on mitxpro__ecommerce_order.order_id = mitxpro__receipts.order_id - left join mitxpro__lines - on mitxpro__ecommerce_allorders.line_id = mitxpro__lines.line_id - left join mitxpro__ecommerce_allcoupons - on mitxpro__ecommerce_allorders.coupon_id = mitxpro__ecommerce_allcoupons.coupon_id - left join mitxpro__programruns - on mitxpro__lines.programrun_id = mitxpro__programruns.programrun_id - where mitxpro__ecommerce_allorders.order_id is not null -) - -, combined_orders as ( - select - '{{ var("mitxonline") }}' as platform - , order_id - , line_id - , courserun_id - , courserun_readable_id - , product_id - , courserun_readable_id as product_readable_id - , product_type - , product_price as unit_price - , user_email - , user_id - , discount_code as coupon_code - , null as coupon_id - , null as b2bcoupon_id - , null as b2border_contract_number - , null as coupon_name - , discount_redemption_type as coupon_type - , discountredemption_timestamp as coupon_redeemed_on - , transaction_authorization_code as receipt_authorization_code - , transaction_bill_to_address_state as receipt_bill_to_address_state - , transaction_bill_to_address_country as receipt_bill_to_address_country - , transaction_uuid as receipt_payment_transaction_uuid - , transaction_req_type as receipt_payment_transaction_type - , transaction_payment_amount as receipt_payment_amount - , transaction_payment_currency as receipt_payment_currency - , transaction_payment_card_number as receipt_payment_card_number - , transaction_payment_card_type as receipt_payment_card_type - , transaction_payer_name as receipt_payer_name - , transaction_payer_email as receipt_payer_email - , transaction_payer_ip_address as receipt_payer_ip_address - , transaction_payment_method as receipt_payment_method - , transaction_payment_timestamp as receipt_payment_timestamp - , transaction_readable_identifier as receipt_transaction_id - , transaction_reference_number as req_reference_number - , order_created_on - , order_reference_number - , order_state - , null as order_tax_country_code - , null as order_tax_rate - , null as order_tax_rate_name - , null as order_tax_amount - , null as couponpaymentversion_payment_transaction - , order_total_price_paid as order_total_price_paid_plus_tax - , order_total_price_paid - , case - when discount_amount_text like '%Fixed%' - then cast( - product_price - cast( - substring(discount_amount_text, 14) - as decimal(38, 2) - ) as varchar - ) - else discount_amount_text - end as discount - from mitxonline_orders + ), + bootcamps_orders as ( + select + bootcamps__ecommerce_order.order_id, + bootcamps__ecommerce_order.line_id, + bootcamps__ecommerce_order.courserun_id, + bootcamps__ecommerce_order.courserun_readable_id, + bootcamps__ecommerce_order.line_price, + bootcamps__ecommerce_order.user_email, + bootcamps__ecommerce_order.order_purchaser_user_id, + bootcamps__receipt.receipt_authorization_code, + bootcamps__receipt.receipt_bill_to_address_state, + bootcamps__receipt.receipt_bill_to_address_country, + bootcamps__receipt.receipt_transaction_uuid, + bootcamps__receipt.receipt_transaction_type, + bootcamps__receipt.receipt_payment_amount, + bootcamps__receipt.receipt_payment_currency, + bootcamps__receipt.receipt_payment_card_number, + bootcamps__receipt.receipt_payment_card_type, + bootcamps__receipt.receipt_payer_name, + bootcamps__receipt.receipt_payer_email, + bootcamps__receipt.receipt_payer_ip_address, + bootcamps__receipt.receipt_payment_method, + bootcamps__receipt.receipt_transaction_id, + bootcamps__receipt.receipt_reference_number, + bootcamps__receipt.receipt_payment_timestamp, + bootcamps__ecommerce_order.order_created_on, + bootcamps__ecommerce_order.order_reference_number, + bootcamps__ecommerce_order.order_state, + bootcamps__ecommerce_order.order_total_price_paid + from bootcamps__ecommerce_order + left join bootcamps__receipt on bootcamps__ecommerce_order.order_id = bootcamps__receipt.order_id + ), + mitxpro_orders as ( + select + mitxpro__ecommerce_allorders.line_id, + mitxpro__ecommerce_allcoupons.coupon_code, + mitxpro__ecommerce_allcoupons.coupon_name, + mitxpro__ecommerce_allcoupons.coupon_type, + mitxpro__ecommerce_allorders.coupon_redeemed_on, + mitxpro__ecommerce_allorders.order_created_on, + mitxpro__ecommerce_allorders.order_state, + mitxpro__ecommerce_allorders.product_id, + mitxpro__ecommerce_allorders.product_type, + mitxpro__lines.product_price, + mitxpro__ecommerce_allorders.user_email, + mitxpro__ecommerce_allorders.courserun_id, + mitxpro__ecommerce_allorders.courserun_readable_id, + mitxpro__ecommerce_order.order_purchaser_user_id, + mitxpro__receipts.receipt_authorization_code, + mitxpro__receipts.receipt_bill_to_address_state, + mitxpro__receipts.receipt_bill_to_address_country, + mitxpro__receipts.receipt_transaction_uuid, + mitxpro__receipts.receipt_transaction_type, + mitxpro__receipts.receipt_payment_amount, + mitxpro__receipts.receipt_payment_currency, + mitxpro__receipts.receipt_payer_email, + mitxpro__receipts.receipt_payment_card_number, + mitxpro__receipts.receipt_payer_ip_address, + mitxpro__receipts.receipt_payment_card_type, + mitxpro__receipts.receipt_payer_name, + mitxpro__receipts.receipt_payment_method, + mitxpro__receipts.receipt_transaction_id, + mitxpro__receipts.receipt_payment_timestamp, + mitxpro__ecommerce_order.receipt_reference_number, + mitxpro__ecommerce_order.order_tax_country_code, + mitxpro__ecommerce_order.order_tax_rate, + mitxpro__ecommerce_order.order_tax_rate_name, + mitxpro__ecommerce_order.order_tax_amount, + mitxpro__ecommerce_order.order_total_price_paid_plus_tax, + mitxpro__ecommerce_allorders.coupon_id, + mitxpro__ecommerce_allorders.b2bcoupon_id, + mitxpro__ecommerce_allorders.b2border_contract_number, + mitxpro__ecommerce_allorders.order_id, + mitxpro__ecommerce_order.couponpaymentversion_payment_transaction, + mitxpro__ecommerce_order.order_total_price_paid, + mitxpro__ecommerce_order.couponpaymentversion_discount_amount_text as discount, + concat( + 'xpro-b2c-production-', cast(mitxpro__ecommerce_allorders.order_id as varchar) + ) as order_reference_number, + coalesce( + mitxpro__ecommerce_allorders.courserun_readable_id, + mitxpro__programruns.programrun_readable_id, + mitxpro__ecommerce_allorders.program_readable_id + ) as product_readable_id + from mitxpro__ecommerce_allorders + left join mitxpro__ecommerce_order on mitxpro__ecommerce_allorders.order_id = mitxpro__ecommerce_order.order_id + left join mitxpro__receipts on mitxpro__ecommerce_order.order_id = mitxpro__receipts.order_id + left join mitxpro__lines on mitxpro__ecommerce_allorders.line_id = mitxpro__lines.line_id + left join + mitxpro__ecommerce_allcoupons + on mitxpro__ecommerce_allorders.coupon_id = mitxpro__ecommerce_allcoupons.coupon_id + left join mitxpro__programruns on mitxpro__lines.programrun_id = mitxpro__programruns.programrun_id + where mitxpro__ecommerce_allorders.order_id is not null + ), + combined_orders as ( + select + '{{ var("mitxonline") }}' as platform, + order_id, + line_id, + courserun_id, + courserun_readable_id, + product_id, + courserun_readable_id as product_readable_id, + product_type, + product_price as unit_price, + user_email, + user_id, + discount_code as coupon_code, + null as coupon_id, + null as b2bcoupon_id, + null as b2border_contract_number, + null as coupon_name, + discount_redemption_type as coupon_type, + discountredemption_timestamp as coupon_redeemed_on, + transaction_authorization_code as receipt_authorization_code, + transaction_bill_to_address_state as receipt_bill_to_address_state, + transaction_bill_to_address_country as receipt_bill_to_address_country, + transaction_uuid as receipt_payment_transaction_uuid, + transaction_req_type as receipt_payment_transaction_type, + transaction_payment_amount as receipt_payment_amount, + transaction_payment_currency as receipt_payment_currency, + transaction_payment_card_number as receipt_payment_card_number, + transaction_payment_card_type as receipt_payment_card_type, + transaction_payer_name as receipt_payer_name, + transaction_payer_email as receipt_payer_email, + transaction_payer_ip_address as receipt_payer_ip_address, + transaction_payment_method as receipt_payment_method, + transaction_payment_timestamp as receipt_payment_timestamp, + transaction_readable_identifier as receipt_transaction_id, + transaction_reference_number as req_reference_number, + order_created_on, + order_reference_number, + order_state, + null as order_tax_country_code, + null as order_tax_rate, + null as order_tax_rate_name, + null as order_tax_amount, + null as couponpaymentversion_payment_transaction, + order_total_price_paid as order_total_price_paid_plus_tax, + order_total_price_paid, + case + when discount_amount_text like '%Fixed%' + then cast(product_price - cast(substring(discount_amount_text, 14) as decimal(38, 2)) as varchar) + else discount_amount_text + end as discount + from mitxonline_orders - union all + union all - select - '{{ var("mitxpro") }}' as platform - , order_id - , line_id - , courserun_id - , courserun_readable_id - , product_id - , product_readable_id - , product_type - , product_price as unit_price - , user_email - , order_purchaser_user_id as user_id - , coupon_code - , coupon_id - , b2bcoupon_id - , b2border_contract_number - , coupon_name - , coupon_type - , coupon_redeemed_on - , receipt_authorization_code - , receipt_bill_to_address_state - , receipt_bill_to_address_country - , receipt_transaction_uuid as receipt_payment_transaction_uuid - , receipt_transaction_type as receipt_payment_transaction_type - , receipt_payment_amount - , receipt_payment_currency - , receipt_payment_card_number - , receipt_payment_card_type - , receipt_payer_name - , receipt_payer_email - , receipt_payer_ip_address - , receipt_payment_method - , receipt_payment_timestamp - , receipt_transaction_id - , receipt_reference_number as req_reference_number - , order_created_on - , order_reference_number - , order_state - , order_tax_country_code - , order_tax_rate - , order_tax_rate_name - , order_tax_amount - , couponpaymentversion_payment_transaction - , order_total_price_paid_plus_tax - , order_total_price_paid - , discount - from mitxpro_orders + select + '{{ var("mitxpro") }}' as platform, + order_id, + line_id, + courserun_id, + courserun_readable_id, + product_id, + product_readable_id, + product_type, + product_price as unit_price, + user_email, + order_purchaser_user_id as user_id, + coupon_code, + coupon_id, + b2bcoupon_id, + b2border_contract_number, + coupon_name, + coupon_type, + coupon_redeemed_on, + receipt_authorization_code, + receipt_bill_to_address_state, + receipt_bill_to_address_country, + receipt_transaction_uuid as receipt_payment_transaction_uuid, + receipt_transaction_type as receipt_payment_transaction_type, + receipt_payment_amount, + receipt_payment_currency, + receipt_payment_card_number, + receipt_payment_card_type, + receipt_payer_name, + receipt_payer_email, + receipt_payer_ip_address, + receipt_payment_method, + receipt_payment_timestamp, + receipt_transaction_id, + receipt_reference_number as req_reference_number, + order_created_on, + order_reference_number, + order_state, + order_tax_country_code, + order_tax_rate, + order_tax_rate_name, + order_tax_amount, + couponpaymentversion_payment_transaction, + order_total_price_paid_plus_tax, + order_total_price_paid, + discount + from mitxpro_orders - union all + union all - select - '{{ var("bootcamps") }}' as platform - , order_id - , line_id - , courserun_id - , courserun_readable_id - , null as product_id - , null as product_readable_id - , null as product_type - , line_price as unit_price - , user_email - , order_purchaser_user_id as user_id - , null as coupon_code - , null as coupon_id - , null as b2bcoupon_id - , null as b2border_contract_number - , null as coupon_name - , null as coupon_type - , null as coupon_redeemed_on - , receipt_authorization_code - , receipt_bill_to_address_state - , receipt_bill_to_address_country - , receipt_transaction_uuid as receipt_payment_transaction_uuid - , receipt_transaction_type as receipt_payment_transaction_type - , receipt_payment_amount - , receipt_payment_currency - , receipt_payment_card_number - , receipt_payment_card_type - , receipt_payer_name - , receipt_payer_email - , receipt_payer_ip_address - , receipt_payment_method - , receipt_payment_timestamp - , receipt_transaction_id - , receipt_reference_number as req_reference_number - , order_created_on - , order_reference_number - , order_state - , null as order_tax_country_code - , null as order_tax_rate - , null as order_tax_rate_name - , null as order_tax_amount - , null as couponpaymentversion_payment_transaction - , order_total_price_paid as order_total_price_paid_plus_tax - , order_total_price_paid - , null as discount - from bootcamps_orders + select + '{{ var("bootcamps") }}' as platform, + order_id, + line_id, + courserun_id, + courserun_readable_id, + null as product_id, + null as product_readable_id, + null as product_type, + line_price as unit_price, + user_email, + order_purchaser_user_id as user_id, + null as coupon_code, + null as coupon_id, + null as b2bcoupon_id, + null as b2border_contract_number, + null as coupon_name, + null as coupon_type, + null as coupon_redeemed_on, + receipt_authorization_code, + receipt_bill_to_address_state, + receipt_bill_to_address_country, + receipt_transaction_uuid as receipt_payment_transaction_uuid, + receipt_transaction_type as receipt_payment_transaction_type, + receipt_payment_amount, + receipt_payment_currency, + receipt_payment_card_number, + receipt_payment_card_type, + receipt_payer_name, + receipt_payer_email, + receipt_payer_ip_address, + receipt_payment_method, + receipt_payment_timestamp, + receipt_transaction_id, + receipt_reference_number as req_reference_number, + order_created_on, + order_reference_number, + order_state, + null as order_tax_country_code, + null as order_tax_rate, + null as order_tax_rate_name, + null as order_tax_amount, + null as couponpaymentversion_payment_transaction, + order_total_price_paid as order_total_price_paid_plus_tax, + order_total_price_paid, + null as discount + from bootcamps_orders - union all + union all - select - '{{ var("edxorg") }}' as platform - , order_id - , line_id - , null as courserun_id - , courserun_readable_id - , null as product_id - , null as product_readable_id - , null as product_type - , line_price as unit_price - , user_edxorg_email as user_email - , user_edxorg_id as user_id - , coupon_code - , coupon_id - , null as b2bcoupon_id - , null as b2border_contract_number - , null as coupon_name - , coupon_type - , redeemedcoupon_created_on as coupon_redeemed_on - , receipt_authorization_code - , receipt_bill_to_address_state - , receipt_bill_to_address_country - , receipt_transaction_uuid as receipt_payment_transaction_uuid - , receipt_transaction_type as receipt_payment_transaction_type - , receipt_payment_amount - , receipt_payment_currency - , receipt_payment_card_number - , receipt_payment_card_type - , receipt_payer_name - , receipt_payer_email - , receipt_payer_ip_address - , receipt_payment_method - , receipt_payment_timestamp - , receipt_transaction_id - , receipt_reference_number as req_reference_number - , order_created_on - , order_reference_number - , order_state - , null as order_tax_country_code - , null as order_tax_rate - , null as order_tax_rate_name - , null as order_tax_amount - , null as couponpaymentversion_payment_transaction - , order_total_price_paid as order_total_price_paid_plus_tax - , order_total_price_paid - , case - when coupon_discount_amount_text like '%Fixed%' - then cast( - line_price - cast( - substring(coupon_discount_amount_text, 14) - as decimal(38, 2) - ) as varchar - ) - else coupon_discount_amount_text - end as discount - from micromasters_orders - where courserun_platform = '{{ var("edxorg") }}' + select + '{{ var("edxorg") }}' as platform, + order_id, + line_id, + null as courserun_id, + courserun_readable_id, + null as product_id, + null as product_readable_id, + null as product_type, + line_price as unit_price, + user_edxorg_email as user_email, + user_edxorg_id as user_id, + coupon_code, + coupon_id, + null as b2bcoupon_id, + null as b2border_contract_number, + null as coupon_name, + coupon_type, + redeemedcoupon_created_on as coupon_redeemed_on, + receipt_authorization_code, + receipt_bill_to_address_state, + receipt_bill_to_address_country, + receipt_transaction_uuid as receipt_payment_transaction_uuid, + receipt_transaction_type as receipt_payment_transaction_type, + receipt_payment_amount, + receipt_payment_currency, + receipt_payment_card_number, + receipt_payment_card_type, + receipt_payer_name, + receipt_payer_email, + receipt_payer_ip_address, + receipt_payment_method, + receipt_payment_timestamp, + receipt_transaction_id, + receipt_reference_number as req_reference_number, + order_created_on, + order_reference_number, + order_state, + null as order_tax_country_code, + null as order_tax_rate, + null as order_tax_rate_name, + null as order_tax_amount, + null as couponpaymentversion_payment_transaction, + order_total_price_paid as order_total_price_paid_plus_tax, + order_total_price_paid, + case + when coupon_discount_amount_text like '%Fixed%' + then cast(line_price - cast(substring(coupon_discount_amount_text, 14) as decimal(38, 2)) as varchar) + else coupon_discount_amount_text + end as discount + from micromasters_orders + where courserun_platform = '{{ var("edxorg") }}' -) + ) select - {{ generate_hash_id('cast(order_id as varchar) - || cast(coalesce(line_id, 9) as varchar) - || platform') }} as combined_orders_hash_id - , platform - , order_id - , line_id - , coupon_code - , coupon_id - , b2bcoupon_id - , b2border_contract_number - , coupon_name - , coupon_redeemed_on - , coupon_type - , couponpaymentversion_payment_transaction - , courserun_id - , courserun_readable_id - , discount - , order_created_on - , order_reference_number - , order_state - , order_tax_amount - , order_tax_country_code - , order_tax_rate - , order_tax_rate_name - , order_total_price_paid_plus_tax - , order_total_price_paid - , product_id - , product_readable_id - , product_type - , receipt_authorization_code - , receipt_bill_to_address_state - , receipt_bill_to_address_country - , receipt_payment_amount - , receipt_payment_currency - , receipt_payment_card_number - , receipt_payment_card_type - , receipt_payment_method - , receipt_payment_timestamp - , receipt_payment_transaction_type - , receipt_payment_transaction_uuid - , receipt_payer_name - , receipt_payer_email - , receipt_payer_ip_address - , receipt_transaction_id - , req_reference_number - , unit_price - , user_email - , user_id - , {{ generate_hash_id('cast(user_id as varchar) || platform') }} as user_hashed_id + {{ + generate_hash_id( + "cast(order_id as varchar) || cast(coalesce(line_id, 9) as varchar) || platform" + ) + }} as combined_orders_hash_id, + platform, + order_id, + line_id, + coupon_code, + coupon_id, + b2bcoupon_id, + b2border_contract_number, + coupon_name, + coupon_redeemed_on, + coupon_type, + couponpaymentversion_payment_transaction, + courserun_id, + courserun_readable_id, + discount, + order_created_on, + order_reference_number, + order_state, + order_tax_amount, + order_tax_country_code, + order_tax_rate, + order_tax_rate_name, + order_total_price_paid_plus_tax, + order_total_price_paid, + product_id, + product_readable_id, + product_type, + receipt_authorization_code, + receipt_bill_to_address_state, + receipt_bill_to_address_country, + receipt_payment_amount, + receipt_payment_currency, + receipt_payment_card_number, + receipt_payment_card_type, + receipt_payment_method, + receipt_payment_timestamp, + receipt_payment_transaction_type, + receipt_payment_transaction_uuid, + receipt_payer_name, + receipt_payer_email, + receipt_payer_ip_address, + receipt_transaction_id, + req_reference_number, + unit_price, + user_email, + user_id, + {{ generate_hash_id("cast(user_id as varchar) || platform") }} as user_hashed_id from combined_orders diff --git a/src/ol_dbt/models/marts/combined/marts__combined__products.sql b/src/ol_dbt/models/marts/combined/marts__combined__products.sql index b71a018e6..3a658418c 100644 --- a/src/ol_dbt/models/marts/combined/marts__combined__products.sql +++ b/src/ol_dbt/models/marts/combined/marts__combined__products.sql @@ -1,281 +1,234 @@ -with mitxonline_product as ( - select * from {{ ref('int__mitxonline__ecommerce_product') }} -) - -, mitxonline_course_runs as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, mitxonline_courses as ( - select * from {{ ref('int__mitxonline__courses') }} -) - -, mitxonline_programs as ( - select * from {{ ref('int__mitxonline__programs') }} -) - -, mitxpro_product as ( - select * from {{ ref('int__mitxpro__ecommerce_product') }} -) - -, mitxpro_course_runs as ( - select * from {{ ref('int__mitxpro__course_runs') }} -) - -, mitxpro_courses as ( - select * from {{ ref('int__mitxpro__courses') }} -) - -, mitxpro_programs as ( - select * from {{ ref('int__mitxpro__programs') }} -) - -, mitxpro_program_runs as ( - select * from {{ ref('int__mitxpro__program_runs') }} -) - -, edxorg_product as ( - select * from {{ ref('int__edxorg__mitx_product') }} -) - -, edxorg_runs as ( - select * from {{ ref('int__edxorg__mitx_courseruns') }} -) - -, mitxonline_product_view as ( - select - mitxonline_product.product_id - , coalesce(mitxonline_product.product_type, 'course run') as product_type - , coalesce( - mitxonline_product.courserun_readable_id, mitxonline_course_runs.courserun_readable_id - ) as product_readable_id - , mitxonline_product.product_price as list_price - , mitxonline_product.product_description - , mitxonline_product.product_is_active - , mitxonline_product.product_created_on - , mitxonline_course_runs.courserun_title as product_name - , mitxonline_course_runs.courserun_start_on as start_on - , mitxonline_course_runs.courserun_end_on as end_on - , mitxonline_course_runs.courserun_enrollment_start_on as enrollment_start_on - , mitxonline_course_runs.courserun_enrollment_end_on as enrollment_end_on - , mitxonline_course_runs.courserun_upgrade_deadline as upgrade_deadline - , mitxonline_courses.course_length as duration - , mitxonline_courses.course_effort as time_commitment - , mitxonline_courses.course_certification_type as certification_type - , mitxonline_courses.course_topics as topics - , mitxonline_courses.course_instructors as instructors - , if(mitxonline_course_runs.courserun_is_self_paced = true, 'Self-paced', 'Instructor-paced') - as pace - , if( - mitxonline_courses.course_page_is_live = true and mitxonline_courses.course_is_live = true - , true - , false - ) as is_live - from mitxonline_course_runs - inner join mitxonline_courses - on mitxonline_course_runs.course_id = mitxonline_courses.course_id - left join mitxonline_product - on mitxonline_course_runs.courserun_id = mitxonline_product.courserun_id - - union all - - select - null as product_id - , 'program' as product_type - , mitxonline_programs.program_readable_id as product_readable_id - , null as list_price - , mitxonline_programs.program_description as product_description - , mitxonline_product.product_is_active - , null as product_created_on - , mitxonline_programs.program_title as product_name - , null as start_on - , null as end_on - , null as enrollment_start_on - , null as enrollment_end_on - , null as upgrade_deadline - , mitxonline_programs.program_length as duration - , mitxonline_programs.program_effort as time_commitment - , mitxonline_programs.program_certification_type as certification_type - , mitxonline_programs.program_topics as topics - , mitxonline_programs.program_instructors as instructors - , null as pace - , if( - mitxonline_programs.program_is_live = true and mitxonline_programs.program_page_is_live = true - , true - , false - ) as is_live - from mitxonline_programs - left join mitxonline_product - on mitxonline_programs.program_readable_id = mitxonline_product.program_readable_id - where mitxonline_product.program_readable_id is null -) - -, mitxpro_product_view as ( - select - mitxpro_product.product_id - , mitxpro_product.product_list_price as list_price - , mitxpro_product.product_description - , mitxpro_product.product_is_private - , mitxpro_product.product_is_active - , mitxpro_product.product_created_on - , mitxpro_course_runs.courserun_enrollment_start_on as enrollment_start_on - , mitxpro_course_runs.courserun_enrollment_end_on as enrollment_end_on - , mitxpro_courses.course_topics as topics - , coalesce(mitxpro_courses.cms_coursepage_duration, mitxpro_programs.cms_programpage_duration) as duration - , coalesce( - mitxpro_courses.cms_coursepage_time_commitment, mitxpro_programs.cms_programpage_time_commitment - ) as time_commitment - , coalesce(mitxpro_courses.cms_coursepage_format, mitxpro_programs.cms_programpage_format) as delivery - , coalesce(mitxpro_courses.cms_certificate_ceus, mitxpro_programs.cms_certificate_ceus) - as continuing_education_credits - , coalesce(mitxpro_courses.course_instructors, mitxpro_programs.program_instructors) as instructors - , if(mitxpro_product.product_type = 'program', 'program run', mitxpro_product.product_type) as product_type - , coalesce(mitxpro_courses.platform_name, mitxpro_programs.platform_name) as product_platform - , coalesce( - mitxpro_course_runs.courserun_readable_id, mitxpro_program_runs.programrun_readable_id - ) as product_readable_id - , coalesce(mitxpro_course_runs.courserun_title, mitxpro_program_runs.program_title) as product_name - , coalesce(mitxpro_course_runs.courserun_start_on, mitxpro_program_runs.programrun_start_on) as start_on - , coalesce(mitxpro_course_runs.courserun_end_on, mitxpro_program_runs.programrun_end_on) as end_on - , case - when mitxpro_courses.course_is_live = true and mitxpro_courses.cms_coursepage_is_live = true +with + mitxonline_product as (select * from {{ ref("int__mitxonline__ecommerce_product") }}), + mitxonline_course_runs as (select * from {{ ref("int__mitxonline__course_runs") }}), + mitxonline_courses as (select * from {{ ref("int__mitxonline__courses") }}), + mitxonline_programs as (select * from {{ ref("int__mitxonline__programs") }}), + mitxpro_product as (select * from {{ ref("int__mitxpro__ecommerce_product") }}), + mitxpro_course_runs as (select * from {{ ref("int__mitxpro__course_runs") }}), + mitxpro_courses as (select * from {{ ref("int__mitxpro__courses") }}), + mitxpro_programs as (select * from {{ ref("int__mitxpro__programs") }}), + mitxpro_program_runs as (select * from {{ ref("int__mitxpro__program_runs") }}), + edxorg_product as (select * from {{ ref("int__edxorg__mitx_product") }}), + edxorg_runs as (select * from {{ ref("int__edxorg__mitx_courseruns") }}), + mitxonline_product_view as ( + select + mitxonline_product.product_id, + coalesce(mitxonline_product.product_type, 'course run') as product_type, + coalesce( + mitxonline_product.courserun_readable_id, mitxonline_course_runs.courserun_readable_id + ) as product_readable_id, + mitxonline_product.product_price as list_price, + mitxonline_product.product_description, + mitxonline_product.product_is_active, + mitxonline_product.product_created_on, + mitxonline_course_runs.courserun_title as product_name, + mitxonline_course_runs.courserun_start_on as start_on, + mitxonline_course_runs.courserun_end_on as end_on, + mitxonline_course_runs.courserun_enrollment_start_on as enrollment_start_on, + mitxonline_course_runs.courserun_enrollment_end_on as enrollment_end_on, + mitxonline_course_runs.courserun_upgrade_deadline as upgrade_deadline, + mitxonline_courses.course_length as duration, + mitxonline_courses.course_effort as time_commitment, + mitxonline_courses.course_certification_type as certification_type, + mitxonline_courses.course_topics as topics, + mitxonline_courses.course_instructors as instructors, + if(mitxonline_course_runs.courserun_is_self_paced = true, 'Self-paced', 'Instructor-paced') as pace, + if( + mitxonline_courses.course_page_is_live = true and mitxonline_courses.course_is_live = true, true, false + ) as is_live + from mitxonline_course_runs + inner join mitxonline_courses on mitxonline_course_runs.course_id = mitxonline_courses.course_id + left join mitxonline_product on mitxonline_course_runs.courserun_id = mitxonline_product.courserun_id + + union all + + select + null as product_id, + 'program' as product_type, + mitxonline_programs.program_readable_id as product_readable_id, + null as list_price, + mitxonline_programs.program_description as product_description, + mitxonline_product.product_is_active, + null as product_created_on, + mitxonline_programs.program_title as product_name, + null as start_on, + null as end_on, + null as enrollment_start_on, + null as enrollment_end_on, + null as upgrade_deadline, + mitxonline_programs.program_length as duration, + mitxonline_programs.program_effort as time_commitment, + mitxonline_programs.program_certification_type as certification_type, + mitxonline_programs.program_topics as topics, + mitxonline_programs.program_instructors as instructors, + null as pace, + if( + mitxonline_programs.program_is_live = true and mitxonline_programs.program_page_is_live = true, + true, + false + ) as is_live + from mitxonline_programs + left join mitxonline_product on mitxonline_programs.program_readable_id = mitxonline_product.program_readable_id + where mitxonline_product.program_readable_id is null + ), + mitxpro_product_view as ( + select + mitxpro_product.product_id, + mitxpro_product.product_list_price as list_price, + mitxpro_product.product_description, + mitxpro_product.product_is_private, + mitxpro_product.product_is_active, + mitxpro_product.product_created_on, + mitxpro_course_runs.courserun_enrollment_start_on as enrollment_start_on, + mitxpro_course_runs.courserun_enrollment_end_on as enrollment_end_on, + mitxpro_courses.course_topics as topics, + coalesce(mitxpro_courses.cms_coursepage_duration, mitxpro_programs.cms_programpage_duration) as duration, + coalesce( + mitxpro_courses.cms_coursepage_time_commitment, mitxpro_programs.cms_programpage_time_commitment + ) as time_commitment, + coalesce(mitxpro_courses.cms_coursepage_format, mitxpro_programs.cms_programpage_format) as delivery, + coalesce( + mitxpro_courses.cms_certificate_ceus, mitxpro_programs.cms_certificate_ceus + ) as continuing_education_credits, + coalesce(mitxpro_courses.course_instructors, mitxpro_programs.program_instructors) as instructors, + if(mitxpro_product.product_type = 'program', 'program run', mitxpro_product.product_type) as product_type, + coalesce(mitxpro_courses.platform_name, mitxpro_programs.platform_name) as product_platform, + coalesce( + mitxpro_course_runs.courserun_readable_id, mitxpro_program_runs.programrun_readable_id + ) as product_readable_id, + coalesce(mitxpro_course_runs.courserun_title, mitxpro_program_runs.program_title) as product_name, + coalesce(mitxpro_course_runs.courserun_start_on, mitxpro_program_runs.programrun_start_on) as start_on, + coalesce(mitxpro_course_runs.courserun_end_on, mitxpro_program_runs.programrun_end_on) as end_on, + case + when mitxpro_courses.course_is_live = true and mitxpro_courses.cms_coursepage_is_live = true then true - when mitxpro_programs.program_is_live = true and mitxpro_programs.cms_programpage_is_live = true + when mitxpro_programs.program_is_live = true and mitxpro_programs.cms_programpage_is_live = true then true - else false - end as is_live - from mitxpro_product - left join mitxpro_course_runs - on mitxpro_product.courserun_id = mitxpro_course_runs.courserun_id - left join mitxpro_courses - on mitxpro_course_runs.course_id = mitxpro_courses.course_id - left join mitxpro_program_runs - on mitxpro_product.program_id = mitxpro_program_runs.program_id - left join mitxpro_programs - on mitxpro_program_runs.program_id = mitxpro_programs.program_id - where mitxpro_product.product_type in ('program', 'course run') -) - -, edxorg_product_view as ( - select - edxorg_runs.courserun_readable_id as product_readable_id - , edxorg_runs.courserun_title as product_name - , edxorg_runs.courserun_description as product_description - , edxorg_product.price as list_price - , edxorg_runs.courserun_start_date as start_on - , edxorg_runs.courserun_end_date as end_on - , edxorg_runs.courserun_enrollment_start_date as enrollment_start_on - , edxorg_runs.courserun_enrollment_end_date as enrollment_end_on - , edxorg_product.upgrade_deadline - , edxorg_runs.courserun_pace as pace - , edxorg_runs.courserun_duration as duration - , edxorg_runs.courserun_time_commitment as time_commitment - , edxorg_runs.course_topics as topics - , edxorg_runs.courserun_instructors as instructors - , if(edxorg_runs.courserun_is_published, true, false) as is_live - , if( - edxorg_runs.micromasters_program_id is not null - , 'MicroMasters Credential' - , 'Certificate of Completion' - ) as certification_type - from edxorg_product - inner join edxorg_runs - on edxorg_product.courserun_readable_id = edxorg_runs.courserun_readable_id - left join mitxonline_product_view - on edxorg_product.courserun_readable_id = mitxonline_product_view.product_readable_id - where - edxorg_product.courserun_mode = 'verified' - and mitxonline_product_view.product_readable_id is null - -) + else false + end as is_live + from mitxpro_product + left join mitxpro_course_runs on mitxpro_product.courserun_id = mitxpro_course_runs.courserun_id + left join mitxpro_courses on mitxpro_course_runs.course_id = mitxpro_courses.course_id + left join mitxpro_program_runs on mitxpro_product.program_id = mitxpro_program_runs.program_id + left join mitxpro_programs on mitxpro_program_runs.program_id = mitxpro_programs.program_id + where mitxpro_product.product_type in ('program', 'course run') + ), + edxorg_product_view as ( + select + edxorg_runs.courserun_readable_id as product_readable_id, + edxorg_runs.courserun_title as product_name, + edxorg_runs.courserun_description as product_description, + edxorg_product.price as list_price, + edxorg_runs.courserun_start_date as start_on, + edxorg_runs.courserun_end_date as end_on, + edxorg_runs.courserun_enrollment_start_date as enrollment_start_on, + edxorg_runs.courserun_enrollment_end_date as enrollment_end_on, + edxorg_product.upgrade_deadline, + edxorg_runs.courserun_pace as pace, + edxorg_runs.courserun_duration as duration, + edxorg_runs.courserun_time_commitment as time_commitment, + edxorg_runs.course_topics as topics, + edxorg_runs.courserun_instructors as instructors, + if(edxorg_runs.courserun_is_published, true, false) as is_live, + if( + edxorg_runs.micromasters_program_id is not null, 'MicroMasters Credential', 'Certificate of Completion' + ) as certification_type + from edxorg_product + inner join edxorg_runs on edxorg_product.courserun_readable_id = edxorg_runs.courserun_readable_id + left join + mitxonline_product_view + on edxorg_product.courserun_readable_id = mitxonline_product_view.product_readable_id + where edxorg_product.courserun_mode = 'verified' and mitxonline_product_view.product_readable_id is null + + ) select - '{{ var("mitxonline") }}' as platform - , '{{ var("mitxonline") }}' as product_platform - , product_readable_id - , product_name - , product_id - , product_type - , product_description - , list_price - , product_is_active - , false as product_is_private - , product_created_on - , start_on - , end_on - , enrollment_start_on - , enrollment_end_on - , upgrade_deadline - , pace - , duration - , time_commitment - , certification_type - , 'Online' as delivery - , null as continuing_education_credits - , topics - , instructors - , 'MITx' as offered_by - , is_live + '{{ var("mitxonline") }}' as platform, + '{{ var("mitxonline") }}' as product_platform, + product_readable_id, + product_name, + product_id, + product_type, + product_description, + list_price, + product_is_active, + false as product_is_private, + product_created_on, + start_on, + end_on, + enrollment_start_on, + enrollment_end_on, + upgrade_deadline, + pace, + duration, + time_commitment, + certification_type, + 'Online' as delivery, + null as continuing_education_credits, + topics, + instructors, + 'MITx' as offered_by, + is_live from mitxonline_product_view union all select - '{{ var("mitxpro") }}' as platform - , product_platform - , product_readable_id - , product_name - , product_id - , product_type - , product_description - , list_price - , product_is_active - , product_is_private - , product_created_on - , start_on - , end_on - , enrollment_start_on - , enrollment_end_on - , null as upgrade_deadline - , null as pace - , duration - , time_commitment - , 'Professional Certificate' as certification_type - , delivery - , continuing_education_credits - , topics - , instructors - , 'xPro' as offered_by - , is_live + '{{ var("mitxpro") }}' as platform, + product_platform, + product_readable_id, + product_name, + product_id, + product_type, + product_description, + list_price, + product_is_active, + product_is_private, + product_created_on, + start_on, + end_on, + enrollment_start_on, + enrollment_end_on, + null as upgrade_deadline, + null as pace, + duration, + time_commitment, + 'Professional Certificate' as certification_type, + delivery, + continuing_education_credits, + topics, + instructors, + 'xPro' as offered_by, + is_live from mitxpro_product_view union all select - '{{ var("edxorg") }}' as platform - , '{{ var("edxorg") }}' as product_platform - , product_readable_id - , product_name - , null as product_id - , 'course run' as product_type - , product_description - , list_price - , null as product_is_active - , false as product_is_private - , null as product_created_on - , start_on - , end_on - , enrollment_start_on - , enrollment_end_on - , upgrade_deadline - , pace - , duration - , time_commitment - , certification_type - , 'Online' as delivery - , null as continuing_education_credits - , topics - , instructors - , 'MITx' as offered_by - , is_live + '{{ var("edxorg") }}' as platform, + '{{ var("edxorg") }}' as product_platform, + product_readable_id, + product_name, + null as product_id, + 'course run' as product_type, + product_description, + list_price, + null as product_is_active, + false as product_is_private, + null as product_created_on, + start_on, + end_on, + enrollment_start_on, + enrollment_end_on, + upgrade_deadline, + pace, + duration, + time_commitment, + certification_type, + 'Online' as delivery, + null as continuing_education_credits, + topics, + instructors, + 'MITx' as offered_by, + is_live from edxorg_product_view diff --git a/src/ol_dbt/models/marts/combined/marts__combined__users.sql b/src/ol_dbt/models/marts/combined/marts__combined__users.sql index dc59b1ddb..c6fcddae0 100644 --- a/src/ol_dbt/models/marts/combined/marts__combined__users.sql +++ b/src/ol_dbt/models/marts/combined/marts__combined__users.sql @@ -1,269 +1,240 @@ ---- This model combines intermediate users from different platforms - -with mitx__users as ( - select * from {{ ref('int__mitx__users') }} -) - -, non_mitx_users as ( - select * from {{ ref('int__combined__users') }} -) - -, combined_enrollments as ( - select * from {{ ref('int__combined__courserun_enrollments') }} -) - -, combined_programs as ( - select * from {{ ref('marts__combined_program_enrollment_detail') }} -) - -, orders as ( - select * from {{ ref('marts__combined__orders') }} -) - -, income as ( - select * from {{ ref('marts__mitxonline_user_profiles') }} -) - -, combined_courseruns as ( - select * from {{ ref('int__combined__course_runs') }} -) - -, program_stats as ( - select - user_email - , count(distinct programcertificate_uuid) as cert_count - , sum(case - when - program_title in ( - 'Data, Economics, and Design of Policy' - , 'Data, Economics, and Design of Policy: International Development' - , 'Data, Economics, and Design of Policy: Public Policy' - ) - and - programcertificate_uuid is not null - then 1 - else 0 - end) as dedp_program_cred_count - from combined_programs - group by user_email -) - -, orders_stats as ( - select - user_email - , sum(order_total_price_paid) as total_amount_paid_orders - from orders - group by user_email -) - -, course_stats as ( - select - combined_enrollments.user_email - , count(distinct combined_enrollments.course_title) as num_of_course_enrolled - , count( - distinct +-- - This model combines intermediate users from different platforms +with + mitx__users as (select * from {{ ref("int__mitx__users") }}), + non_mitx_users as (select * from {{ ref("int__combined__users") }}), + combined_enrollments as (select * from {{ ref("int__combined__courserun_enrollments") }}), + combined_programs as (select * from {{ ref("marts__combined_program_enrollment_detail") }}), + orders as (select * from {{ ref("marts__combined__orders") }}), + income as (select * from {{ ref("marts__mitxonline_user_profiles") }}), + combined_courseruns as (select * from {{ ref("int__combined__course_runs") }}), + program_stats as ( + select + user_email, + count(distinct programcertificate_uuid) as cert_count, + sum( + case + when + program_title in ( + 'Data, Economics, and Design of Policy', + 'Data, Economics, and Design of Policy: International Development', + 'Data, Economics, and Design of Policy: Public Policy' + ) + and programcertificate_uuid is not null + then 1 + else 0 + end + ) as dedp_program_cred_count + from combined_programs + group by user_email + ), + orders_stats as ( + select user_email, sum(order_total_price_paid) as total_amount_paid_orders from orders group by user_email + ), + course_stats as ( + select + combined_enrollments.user_email, + count(distinct combined_enrollments.course_title) as num_of_course_enrolled, + count( + distinct + case + when combined_enrollments.courserungrade_is_passing = true then combined_enrollments.course_title + end + ) as num_of_course_passed, + min(combined_courseruns.courserun_start_on) as first_course_start_datetime, + max(combined_courseruns.courserun_start_on) as last_course_start_datetime, + count(distinct combined_enrollments.courseruncertificate_uuid) as number_of_courserun_certificates + from combined_enrollments + left join + combined_courseruns + on combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id + group by combined_enrollments.user_email + ), + combined_users as ( + select + user_hashed_id, + user_mitxonline_id, + user_edxorg_id, + null as user_mitxpro_id, + null as user_bootcamps_id, + user_mitxonline_username, + user_edxorg_username, + null as user_mitxpro_username, + null as user_bootcamps_username, case - when combined_enrollments.courserungrade_is_passing = true - then combined_enrollments.course_title - end - ) as num_of_course_passed - , min(combined_courseruns.courserun_start_on) as first_course_start_datetime - , max(combined_courseruns.courserun_start_on) as last_course_start_datetime - , count(distinct combined_enrollments.courseruncertificate_uuid) as number_of_courserun_certificates - from combined_enrollments - left join combined_courseruns - on combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id - group by combined_enrollments.user_email -) - -, combined_users as ( - select - user_hashed_id - , user_mitxonline_id - , user_edxorg_id - , null as user_mitxpro_id - , null as user_bootcamps_id - , user_mitxonline_username - , user_edxorg_username - , null as user_mitxpro_username - , null as user_bootcamps_username - , case - when user_is_active_on_mitxonline and user_joined_on_mitxonline > user_joined_on_edxorg + when user_is_active_on_mitxonline and user_joined_on_mitxonline > user_joined_on_edxorg then user_mitxonline_email - else coalesce(user_edxorg_email, user_mitxonline_email, user_micromasters_email) - end as user_email - , case - when user_is_active_on_mitxonline and user_joined_on_mitxonline > user_joined_on_edxorg + else coalesce(user_edxorg_email, user_mitxonline_email, user_micromasters_email) + end as user_email, + case + when user_is_active_on_mitxonline and user_joined_on_mitxonline > user_joined_on_edxorg then user_joined_on_mitxonline - else coalesce(user_joined_on_edxorg, user_joined_on_mitxonline) - end as user_joined_on - , case - when user_is_active_on_mitxonline and user_last_login_on_mitxonline > user_last_login_on_edxorg + else coalesce(user_joined_on_edxorg, user_joined_on_mitxonline) + end as user_joined_on, + case + when user_is_active_on_mitxonline and user_last_login_on_mitxonline > user_last_login_on_edxorg then user_last_login_on_mitxonline - else coalesce(user_last_login_on_edxorg, user_last_login_on_mitxonline) - end as user_last_login - , case - when user_is_active_on_mitxonline - then user_is_active_on_mitxonline - else user_is_active_on_edxorg - end as user_is_active - , case - when is_mitxonline_user = true and is_edxorg_user = true + else coalesce(user_last_login_on_edxorg, user_last_login_on_mitxonline) + end as user_last_login, + case + when user_is_active_on_mitxonline then user_is_active_on_mitxonline else user_is_active_on_edxorg + end as user_is_active, + case + when is_mitxonline_user = true and is_edxorg_user = true then concat('{{ var("mitxonline") }}', ' and ', '{{ var("edxorg") }}') - when is_mitxonline_user = true + when is_mitxonline_user = true then '{{ var("mitxonline") }}' - when is_edxorg_user = true + when is_edxorg_user = true then '{{ var("edxorg") }}' - end as platforms - , user_full_name - , user_address_country - , user_highest_education - , user_gender - , user_birth_year - , user_company - , user_job_title - , user_industry - , user_street_address - , user_address_city - , user_address_state as user_address_state_or_territory - , user_address_postal_code - from mitx__users - where is_mitxonline_user = true or is_edxorg_user = true - - union all - - select - user_hashed_id - , null as user_mitxonline_id - , null as user_edxorg_id - , user_id as user_mitxpro_id - , null as user_bootcamps_id - , null as user_mitxonline_username - , null as user_edxorg_username - , user_username as user_mitxpro_username - , null as user_bootcamps_username - , user_email - , user_joined_on - , user_last_login - , user_is_active - , '{{ var("mitxpro") }}' as platforms - , user_full_name - , user_address_country - , user_highest_education - , user_gender - , user_birth_year - , user_company - , user_job_title - , user_industry - , user_street_address - , user_address_city - , user_address_state_or_territory - , user_address_postal_code - from non_mitx_users - where platform = '{{ var("mitxpro") }}' - - union all - - select - user_hashed_id - , null as user_mitxonline_id - , null as user_edxorg_id - , null as user_mitxpro_id - , user_id as user_bootcamps_id - , null as user_mitxonline_username - , null as user_edxorg_username - , null as user_mitxpro_username - , user_username as user_bootcamps_username - , user_email - , user_joined_on - , user_last_login - , user_is_active - , '{{ var("bootcamps") }}' as platforms - , user_full_name - , user_address_country - , user_highest_education - , user_gender - , user_birth_year - , user_company - , user_job_title - , user_industry - , user_street_address - , user_address_city - , user_address_state_or_territory - , user_address_postal_code - from non_mitx_users - where platform = '{{ var("bootcamps") }}' - - union all - - select - user_hashed_id - , null as user_mitxonline_id - , null as user_edxorg_id - , null as user_mitxpro_id - , null as user_bootcamps_id - , null as user_mitxonline_username - , null as user_edxorg_username - , null as user_mitxpro_username - , null as user_bootcamps_username - , user_email - , user_joined_on - , user_last_login - , user_is_active - , platform as platforms - , user_full_name - , user_address_country - , user_highest_education - , user_gender - , user_birth_year - , user_company - , user_job_title - , user_industry - , user_street_address - , user_address_city - , user_address_state_or_territory - , user_address_postal_code - from non_mitx_users - where platform in ('{{ var("emeritus") }}', '{{ var("global_alumni") }}') - -) + end as platforms, + user_full_name, + user_address_country, + user_highest_education, + user_gender, + user_birth_year, + user_company, + user_job_title, + user_industry, + user_street_address, + user_address_city, + user_address_state as user_address_state_or_territory, + user_address_postal_code + from mitx__users + where is_mitxonline_user = true or is_edxorg_user = true + + union all + + select + user_hashed_id, + null as user_mitxonline_id, + null as user_edxorg_id, + user_id as user_mitxpro_id, + null as user_bootcamps_id, + null as user_mitxonline_username, + null as user_edxorg_username, + user_username as user_mitxpro_username, + null as user_bootcamps_username, + user_email, + user_joined_on, + user_last_login, + user_is_active, + '{{ var("mitxpro") }}' as platforms, + user_full_name, + user_address_country, + user_highest_education, + user_gender, + user_birth_year, + user_company, + user_job_title, + user_industry, + user_street_address, + user_address_city, + user_address_state_or_territory, + user_address_postal_code + from non_mitx_users + where platform = '{{ var("mitxpro") }}' + + union all + + select + user_hashed_id, + null as user_mitxonline_id, + null as user_edxorg_id, + null as user_mitxpro_id, + user_id as user_bootcamps_id, + null as user_mitxonline_username, + null as user_edxorg_username, + null as user_mitxpro_username, + user_username as user_bootcamps_username, + user_email, + user_joined_on, + user_last_login, + user_is_active, + '{{ var("bootcamps") }}' as platforms, + user_full_name, + user_address_country, + user_highest_education, + user_gender, + user_birth_year, + user_company, + user_job_title, + user_industry, + user_street_address, + user_address_city, + user_address_state_or_territory, + user_address_postal_code + from non_mitx_users + where platform = '{{ var("bootcamps") }}' + + union all + + select + user_hashed_id, + null as user_mitxonline_id, + null as user_edxorg_id, + null as user_mitxpro_id, + null as user_bootcamps_id, + null as user_mitxonline_username, + null as user_edxorg_username, + null as user_mitxpro_username, + null as user_bootcamps_username, + user_email, + user_joined_on, + user_last_login, + user_is_active, + platform as platforms, + user_full_name, + user_address_country, + user_highest_education, + user_gender, + user_birth_year, + user_company, + user_job_title, + user_industry, + user_street_address, + user_address_city, + user_address_state_or_territory, + user_address_postal_code + from non_mitx_users + where platform in ('{{ var("emeritus") }}', '{{ var("global_alumni") }}') + + ) select - combined_users.user_hashed_id - , combined_users.platforms - , combined_users.user_email - , combined_users.user_joined_on - , combined_users.user_last_login - , combined_users.user_is_active - , combined_users.user_full_name - , combined_users.user_address_country - , combined_users.user_highest_education - , combined_users.user_gender - , combined_users.user_birth_year - , combined_users.user_company - , combined_users.user_job_title - , combined_users.user_industry - , combined_users.user_street_address - , combined_users.user_address_city - , combined_users.user_address_state_or_territory - , combined_users.user_address_postal_code - , combined_users.user_mitxonline_id - , combined_users.user_edxorg_id - , combined_users.user_mitxpro_id - , combined_users.user_bootcamps_id - , combined_users.user_mitxonline_username - , combined_users.user_edxorg_username - , combined_users.user_mitxpro_username - , combined_users.user_bootcamps_username - , course_stats.num_of_course_enrolled - , course_stats.num_of_course_passed - , course_stats.first_course_start_datetime - , course_stats.last_course_start_datetime - , course_stats.number_of_courserun_certificates - , orders_stats.total_amount_paid_orders - , income.latest_income_usd - , case when program_stats.cert_count > 0 then true end as has_program_certificate - , case when program_stats.dedp_program_cred_count > 0 then true end as has_dedp_program_certificate + combined_users.user_hashed_id, + combined_users.platforms, + combined_users.user_email, + combined_users.user_joined_on, + combined_users.user_last_login, + combined_users.user_is_active, + combined_users.user_full_name, + combined_users.user_address_country, + combined_users.user_highest_education, + combined_users.user_gender, + combined_users.user_birth_year, + combined_users.user_company, + combined_users.user_job_title, + combined_users.user_industry, + combined_users.user_street_address, + combined_users.user_address_city, + combined_users.user_address_state_or_territory, + combined_users.user_address_postal_code, + combined_users.user_mitxonline_id, + combined_users.user_edxorg_id, + combined_users.user_mitxpro_id, + combined_users.user_bootcamps_id, + combined_users.user_mitxonline_username, + combined_users.user_edxorg_username, + combined_users.user_mitxpro_username, + combined_users.user_bootcamps_username, + course_stats.num_of_course_enrolled, + course_stats.num_of_course_passed, + course_stats.first_course_start_datetime, + course_stats.last_course_start_datetime, + course_stats.number_of_courserun_certificates, + orders_stats.total_amount_paid_orders, + income.latest_income_usd, + case when program_stats.cert_count > 0 then true end as has_program_certificate, + case when program_stats.dedp_program_cred_count > 0 then true end as has_dedp_program_certificate from combined_users left join course_stats on combined_users.user_email = course_stats.user_email left join program_stats on combined_users.user_email = program_stats.user_email diff --git a/src/ol_dbt/models/marts/combined/marts__combined_course_engagements.sql b/src/ol_dbt/models/marts/combined/marts__combined_course_engagements.sql index 91621ccd2..4f8031f2f 100644 --- a/src/ol_dbt/models/marts/combined/marts__combined_course_engagements.sql +++ b/src/ol_dbt/models/marts/combined/marts__combined_course_engagements.sql @@ -1,272 +1,237 @@ -with combined_course_activities_daily as ( - select - '{{ var("mitxonline") }}' as platform - , user_username - , courserun_readable_id - , courseactivity_date - , courseactivity_num_events - from {{ ref('int__mitxonline__user_courseactivities_daily') }} - - union all - - select - '{{ var("edxorg") }}' as platform - , user_username - , courserun_readable_id - , courseactivity_date - , courseactivity_num_events - from {{ ref('int__edxorg__mitx_user_courseactivities_daily') }} - - union all - - select - '{{ var("mitxpro") }}' as platform - , user_username - , courserun_readable_id - , courseactivity_date - , courseactivity_num_events - from {{ ref('int__mitxpro__user_courseactivities_daily') }} +with + combined_course_activities_daily as ( + select + '{{ var("mitxonline") }}' as platform, + user_username, + courserun_readable_id, + courseactivity_date, + courseactivity_num_events + from {{ ref("int__mitxonline__user_courseactivities_daily") }} - union all + union all - select - '{{ var("residential") }}' as platform - , user_username - , courserun_readable_id - , courseactivity_date - , courseactivity_num_events - from {{ ref('int__mitxresidential__user_courseactivities_daily') }} + select + '{{ var("edxorg") }}' as platform, + user_username, + courserun_readable_id, + courseactivity_date, + courseactivity_num_events + from {{ ref("int__edxorg__mitx_user_courseactivities_daily") }} -) + union all -, combined_play_video as ( + select + '{{ var("mitxpro") }}' as platform, + user_username, + courserun_readable_id, + courseactivity_date, + courseactivity_num_events + from {{ ref("int__mitxpro__user_courseactivities_daily") }} - select - '{{ var("mitxonline") }}' as platform - , user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - from {{ ref('int__mitxonline__user_courseactivity_video') }} - where useractivity_event_type = 'play_video' + union all - union all + select + '{{ var("residential") }}' as platform, + user_username, + courserun_readable_id, + courseactivity_date, + courseactivity_num_events + from {{ ref("int__mitxresidential__user_courseactivities_daily") }} - select - '{{ var("edxorg") }}' as platform - , user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - from {{ ref('int__edxorg__mitx_user_courseactivity_video') }} - where useractivity_event_type = 'play_video' + ), + combined_play_video as ( - union all + select + '{{ var("mitxonline") }}' as platform, + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date + from {{ ref("int__mitxonline__user_courseactivity_video") }} + where useractivity_event_type = 'play_video' - select - '{{ var("mitxpro") }}' as platform - , user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - from {{ ref('int__mitxpro__user_courseactivity_video') }} - where useractivity_event_type = 'play_video' + union all - union all + select + '{{ var("edxorg") }}' as platform, + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date + from {{ ref("int__edxorg__mitx_user_courseactivity_video") }} + where useractivity_event_type = 'play_video' - select - '{{ var("residential") }}' as platform - , user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - from {{ ref('int__mitxresidential__user_courseactivity_video') }} - where useractivity_event_type = 'play_video' -) + union all -, combined_problem_submitted as ( + select + '{{ var("mitxpro") }}' as platform, + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date + from {{ ref("int__mitxpro__user_courseactivity_video") }} + where useractivity_event_type = 'play_video' - select - '{{ var("mitxonline") }}' as platform - , user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - from {{ ref('int__mitxonline__user_courseactivity_problemsubmitted') }} + union all - union all + select + '{{ var("residential") }}' as platform, + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date + from {{ ref("int__mitxresidential__user_courseactivity_video") }} + where useractivity_event_type = 'play_video' + ), + combined_problem_submitted as ( - select - '{{ var("edxorg") }}' as platform - , user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - from {{ ref('int__edxorg__mitx_user_courseactivity_problemsubmitted') }} + select + '{{ var("mitxonline") }}' as platform, + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date + from {{ ref("int__mitxonline__user_courseactivity_problemsubmitted") }} - union all + union all - select - '{{ var("mitxpro") }}' as platform - , user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - from {{ ref('int__mitxpro__user_courseactivity_problemsubmitted') }} + select + '{{ var("edxorg") }}' as platform, + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date + from {{ ref("int__edxorg__mitx_user_courseactivity_problemsubmitted") }} - union all + union all - select - '{{ var("residential") }}' as platform - , user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - from {{ ref('int__mitxresidential__user_courseactivity_problemsubmitted') }} -) + select + '{{ var("mitxpro") }}' as platform, + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date + from {{ ref("int__mitxpro__user_courseactivity_problemsubmitted") }} -, combined_discussion as ( + union all - select - '{{ var("mitxonline") }}' as platform - , user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - from {{ ref('int__mitxonline__user_courseactivity_discussion') }} - -- edx.forum.comment.created, edx.forum.response.created, edx.forum.thread.created - where useractivity_event_type like 'edx.forum.%.created' + select + '{{ var("residential") }}' as platform, + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date + from {{ ref("int__mitxresidential__user_courseactivity_problemsubmitted") }} + ), + combined_discussion as ( - union all + select + '{{ var("mitxonline") }}' as platform, + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date + from {{ ref("int__mitxonline__user_courseactivity_discussion") }} + -- edx.forum.comment.created, edx.forum.response.created, edx.forum.thread.created + where useractivity_event_type like 'edx.forum.%.created' - select - '{{ var("edxorg") }}' as platform - , user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - from {{ ref('int__edxorg__mitx_user_courseactivity_discussion') }} - -- edx.forum.comment.created, edx.forum.response.created, edx.forum.thread.created - where useractivity_event_type like 'edx.forum.%.created' + union all - union all + select + '{{ var("edxorg") }}' as platform, + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date + from {{ ref("int__edxorg__mitx_user_courseactivity_discussion") }} + -- edx.forum.comment.created, edx.forum.response.created, edx.forum.thread.created + where useractivity_event_type like 'edx.forum.%.created' - select - '{{ var("mitxpro") }}' as platform - , user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - from {{ ref('int__mitxpro__user_courseactivity_discussion') }} - -- edx.forum.comment.created, edx.forum.response.created, edx.forum.thread.created - where useractivity_event_type like 'edx.forum.%.created' + union all - union all + select + '{{ var("mitxpro") }}' as platform, + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date + from {{ ref("int__mitxpro__user_courseactivity_discussion") }} + -- edx.forum.comment.created, edx.forum.response.created, edx.forum.thread.created + where useractivity_event_type like 'edx.forum.%.created' - select - '{{ var("residential") }}' as platform - , user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - from {{ ref('int__mitxresidential__user_courseactivity_discussion') }} - -- edx.forum.comment.created, edx.forum.response.created, edx.forum.thread.created - where useractivity_event_type like 'edx.forum.%.created' -) + union all -, combined_users as ( - select * from ( select - * - , row_number() over ( - partition by user_username, platform - order by openedx_user_id asc nulls last - ) as row_num - from {{ ref('int__combined__users') }} + '{{ var("residential") }}' as platform, + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date + from {{ ref("int__mitxresidential__user_courseactivity_discussion") }} + -- edx.forum.comment.created, edx.forum.response.created, edx.forum.thread.created + where useractivity_event_type like 'edx.forum.%.created' + ), + combined_users as ( + select * + from + ( + select + *, + row_number() over ( + partition by user_username, platform order by openedx_user_id asc nulls last + ) as row_num + from {{ ref("int__combined__users") }} + ) + where row_num = 1 + ), + combined_runs as (select * from {{ ref("int__combined__course_runs") }}), + combined_play_video_daily as ( + select platform, user_username, courserun_readable_id, courseactivity_date, count(*) as num_video_played + from combined_play_video + group by platform, user_username, courserun_readable_id, courseactivity_date + ), + combined_problem_submitted_daily as ( + select platform, user_username, courserun_readable_id, courseactivity_date, count(*) as num_problem_submitted + from combined_problem_submitted + group by platform, user_username, courserun_readable_id, courseactivity_date + ), + combined_discussion_daily as ( + select + platform, user_username, courserun_readable_id, courseactivity_date, count(*) as num_discussion_participated + from combined_discussion + group by platform, user_username, courserun_readable_id, courseactivity_date ) - where row_num = 1 -) - -, combined_runs as ( - select * from {{ ref('int__combined__course_runs') }} -) - -, combined_play_video_daily as ( - select - platform - , user_username - , courserun_readable_id - , courseactivity_date - , count(*) as num_video_played - from combined_play_video - group by - platform - , user_username - , courserun_readable_id - , courseactivity_date -) - -, combined_problem_submitted_daily as ( - select - platform - , user_username - , courserun_readable_id - , courseactivity_date - , count(*) as num_problem_submitted - from combined_problem_submitted - group by - platform - , user_username - , courserun_readable_id - , courseactivity_date -) - -, combined_discussion_daily as ( - select - platform - , user_username - , courserun_readable_id - , courseactivity_date - , count(*) as num_discussion_participated - from combined_discussion - group by - platform - , user_username - , courserun_readable_id - , courseactivity_date -) select - combined_course_activities_daily.platform - , combined_users.user_hashed_id - , combined_course_activities_daily.user_username - , combined_course_activities_daily.courserun_readable_id - , combined_course_activities_daily.courseactivity_date - , combined_course_activities_daily.courseactivity_num_events as num_events - , combined_problem_submitted_daily.num_problem_submitted - , combined_play_video_daily.num_video_played - , combined_discussion_daily.num_discussion_participated - , combined_users.user_full_name - , combined_users.user_email - , combined_users.user_address_country as user_country_code - , combined_users.user_highest_education - , combined_users.user_gender - , combined_runs.courserun_title - , combined_runs.course_readable_id - , combined_runs.course_number - , combined_runs.courserun_is_current - , combined_runs.courserun_start_on - , combined_runs.courserun_end_on + combined_course_activities_daily.platform, + combined_users.user_hashed_id, + combined_course_activities_daily.user_username, + combined_course_activities_daily.courserun_readable_id, + combined_course_activities_daily.courseactivity_date, + combined_course_activities_daily.courseactivity_num_events as num_events, + combined_problem_submitted_daily.num_problem_submitted, + combined_play_video_daily.num_video_played, + combined_discussion_daily.num_discussion_participated, + combined_users.user_full_name, + combined_users.user_email, + combined_users.user_address_country as user_country_code, + combined_users.user_highest_education, + combined_users.user_gender, + combined_runs.courserun_title, + combined_runs.course_readable_id, + combined_runs.course_number, + combined_runs.courserun_is_current, + combined_runs.courserun_start_on, + combined_runs.courserun_end_on from combined_course_activities_daily -inner join combined_runs - on - combined_course_activities_daily.courserun_readable_id = combined_runs.courserun_readable_id - and combined_course_activities_daily.platform = combined_runs.platform -left join combined_users - on - combined_course_activities_daily.user_username = combined_users.user_username - and combined_course_activities_daily.platform = combined_users.platform -left join combined_problem_submitted_daily - on - combined_course_activities_daily.courseactivity_date = combined_problem_submitted_daily.courseactivity_date - and combined_course_activities_daily.courserun_readable_id - = combined_problem_submitted_daily.courserun_readable_id - and combined_course_activities_daily.user_username = combined_problem_submitted_daily.user_username -left join combined_play_video_daily - on - combined_course_activities_daily.courseactivity_date = combined_play_video_daily.courseactivity_date - and combined_course_activities_daily.courserun_readable_id = combined_play_video_daily.courserun_readable_id - and combined_course_activities_daily.user_username = combined_play_video_daily.user_username -left join combined_discussion_daily - on - combined_course_activities_daily.courseactivity_date = combined_discussion_daily.courseactivity_date - and combined_course_activities_daily.courserun_readable_id = combined_discussion_daily.courserun_readable_id - and combined_course_activities_daily.user_username = combined_discussion_daily.user_username +inner join + combined_runs + on combined_course_activities_daily.courserun_readable_id = combined_runs.courserun_readable_id + and combined_course_activities_daily.platform = combined_runs.platform +left join + combined_users + on combined_course_activities_daily.user_username = combined_users.user_username + and combined_course_activities_daily.platform = combined_users.platform +left join + combined_problem_submitted_daily + on combined_course_activities_daily.courseactivity_date = combined_problem_submitted_daily.courseactivity_date + and combined_course_activities_daily.courserun_readable_id = combined_problem_submitted_daily.courserun_readable_id + and combined_course_activities_daily.user_username = combined_problem_submitted_daily.user_username +left join + combined_play_video_daily + on combined_course_activities_daily.courseactivity_date = combined_play_video_daily.courseactivity_date + and combined_course_activities_daily.courserun_readable_id = combined_play_video_daily.courserun_readable_id + and combined_course_activities_daily.user_username = combined_play_video_daily.user_username +left join + combined_discussion_daily + on combined_course_activities_daily.courseactivity_date = combined_discussion_daily.courseactivity_date + and combined_course_activities_daily.courserun_readable_id = combined_discussion_daily.courserun_readable_id + and combined_course_activities_daily.user_username = combined_discussion_daily.user_username diff --git a/src/ol_dbt/models/marts/combined/marts__combined_course_enrollment_detail.sql b/src/ol_dbt/models/marts/combined/marts__combined_course_enrollment_detail.sql index 4b65ff28d..d7e15cd39 100644 --- a/src/ol_dbt/models/marts/combined/marts__combined_course_enrollment_detail.sql +++ b/src/ol_dbt/models/marts/combined/marts__combined_course_enrollment_detail.sql @@ -1,485 +1,453 @@ -with combined_enrollments as ( - select * from {{ ref('int__combined__courserun_enrollments') }} -) - -, mitxpro_enrollments as ( - select * from {{ ref('int__mitxpro__courserunenrollments') }} -) - -, combined_courseruns as ( - select * from {{ ref('int__combined__course_runs') }} -) - -, combined_users as ( - select * from {{ ref('int__combined__users') }} -) - -, mitxonline_transactions as ( - select - order_id - , max(transaction_timestamp) as payment_timestamp - from {{ ref('int__mitxonline__ecommerce_transaction') }} - where transaction_type= 'payment' - group by order_id -) - -, mitxonline_completed_orders as ( - select - * - , row_number() over ( - partition by user_id, courserun_id - order by order_created_on desc - ) as row_num - from {{ ref('int__mitxonline__ecommerce_order') }} - where order_state in ('fulfilled', 'refunded') -) - -, micromasters_completed_orders as ( - select - * - , row_number() over ( - partition by user_id, courserun_readable_id - order by order_created_on desc, order_id desc - ) as row_num - from {{ ref('int__micromasters__orders') }} - where order_state in ('fulfilled', 'refunded', 'partially_refunded') -) - -, micromasters_users as ( - select * from {{ ref('int__micromasters__users') }} -) - -, mitxpro_completed_orders as ( - select * from {{ ref('int__mitxpro__ecommerce_order') }} - where order_state in ('fulfilled', 'refunded') -) - -, mitxpro_receipts as ( - select - order_id - , max(receipt_payment_timestamp) as payment_timestamp - from {{ ref('int__mitxpro__ecommerce_receipt') }} - where receipt_transaction_status != 'ERROR' - group by order_id -) - -, bootcamps_completed_orders as ( - select * from {{ ref('int__bootcamps__ecommerce_order') }} - where order_state in ('fulfilled', 'refunded') -) - -, bootcamps_receipts as ( - select - order_id - , max(receipt_payment_timestamp) as receipt_payment_timestamp - from {{ ref('int__bootcamps__ecommerce_receipt') }} - group by order_id -) - -, mitxpro__ecommerce_line as ( - select * from {{ ref('int__mitxpro__ecommerce_line') }} -) - -, mitxonline_certificates as ( - select * from {{ ref('int__mitxonline__courserun_certificates') }} -) - -, combined_enrollment_detail as ( - select - '{{ var("mitxonline") }}' as platform - , combined_enrollments.courserunenrollment_id - , combined_enrollments.courserunenrollment_is_active - , combined_enrollments.courserunenrollment_created_on - , combined_enrollments.courserunenrollment_enrollment_mode - , combined_enrollments.courserunenrollment_enrollment_status - , combined_enrollments.courserunenrollment_is_edx_enrolled - , combined_enrollments.user_id - , combined_enrollments.courserun_id - , combined_enrollments.courserun_title - , combined_enrollments.courserun_readable_id - , combined_courseruns.courserun_start_on - , combined_courseruns.courserun_end_on - , combined_courseruns.courserun_is_current - , combined_enrollments.user_username - , combined_enrollments.user_email - , combined_enrollments.user_full_name - , combined_users.user_address_country as user_country_code - , combined_users.user_highest_education - , combined_users.user_company - , combined_users.user_gender - , combined_enrollments.courseruncertificate_is_earned - , combined_enrollments.courseruncertificate_created_on - , combined_enrollments.courseruncertificate_url - , combined_enrollments.courseruncertificate_uuid - , mitxonline_completed_orders.order_id - , mitxonline_completed_orders.line_id - , mitxonline_completed_orders.order_reference_number - , mitxonline_completed_orders.discount_code as coupon_code - , combined_enrollments.courserungrade_grade - , combined_enrollments.courserungrade_is_passing - , combined_enrollments.course_title - , combined_enrollments.course_readable_id - , combined_enrollments.courserun_upgrade_deadline - , if(mitxonline_completed_orders.order_id is not null - , coalesce(mitxonline_transactions.payment_timestamp, mitxonline_completed_orders.order_created_on), null) - as courserunenrollment_upgraded_on - from combined_enrollments - left join combined_users - on - combined_enrollments.user_id = combined_users.user_id +with + combined_enrollments as (select * from {{ ref("int__combined__courserun_enrollments") }}), + mitxpro_enrollments as (select * from {{ ref("int__mitxpro__courserunenrollments") }}), + combined_courseruns as (select * from {{ ref("int__combined__course_runs") }}), + combined_users as (select * from {{ ref("int__combined__users") }}), + mitxonline_transactions as ( + select order_id, max(transaction_timestamp) as payment_timestamp + from {{ ref("int__mitxonline__ecommerce_transaction") }} + where transaction_type = 'payment' + group by order_id + ), + mitxonline_completed_orders as ( + select *, row_number() over (partition by user_id, courserun_id order by order_created_on desc) as row_num + from {{ ref("int__mitxonline__ecommerce_order") }} + where order_state in ('fulfilled', 'refunded') + ), + micromasters_completed_orders as ( + select + *, + row_number() over ( + partition by user_id, courserun_readable_id order by order_created_on desc, order_id desc + ) as row_num + from {{ ref("int__micromasters__orders") }} + where order_state in ('fulfilled', 'refunded', 'partially_refunded') + ), + micromasters_users as (select * from {{ ref("int__micromasters__users") }}), + mitxpro_completed_orders as ( + select * from {{ ref("int__mitxpro__ecommerce_order") }} where order_state in ('fulfilled', 'refunded') + ), + mitxpro_receipts as ( + select order_id, max(receipt_payment_timestamp) as payment_timestamp + from {{ ref("int__mitxpro__ecommerce_receipt") }} + where receipt_transaction_status != 'ERROR' + group by order_id + ), + bootcamps_completed_orders as ( + select * from {{ ref("int__bootcamps__ecommerce_order") }} where order_state in ('fulfilled', 'refunded') + ), + bootcamps_receipts as ( + select order_id, max(receipt_payment_timestamp) as receipt_payment_timestamp + from {{ ref("int__bootcamps__ecommerce_receipt") }} + group by order_id + ), + mitxpro__ecommerce_line as (select * from {{ ref("int__mitxpro__ecommerce_line") }}), + mitxonline_certificates as (select * from {{ ref("int__mitxonline__courserun_certificates") }}), + combined_enrollment_detail as ( + select + '{{ var("mitxonline") }}' as platform, + combined_enrollments.courserunenrollment_id, + combined_enrollments.courserunenrollment_is_active, + combined_enrollments.courserunenrollment_created_on, + combined_enrollments.courserunenrollment_enrollment_mode, + combined_enrollments.courserunenrollment_enrollment_status, + combined_enrollments.courserunenrollment_is_edx_enrolled, + combined_enrollments.user_id, + combined_enrollments.courserun_id, + combined_enrollments.courserun_title, + combined_enrollments.courserun_readable_id, + combined_courseruns.courserun_start_on, + combined_courseruns.courserun_end_on, + combined_courseruns.courserun_is_current, + combined_enrollments.user_username, + combined_enrollments.user_email, + combined_enrollments.user_full_name, + combined_users.user_address_country as user_country_code, + combined_users.user_highest_education, + combined_users.user_company, + combined_users.user_gender, + combined_enrollments.courseruncertificate_is_earned, + combined_enrollments.courseruncertificate_created_on, + combined_enrollments.courseruncertificate_url, + combined_enrollments.courseruncertificate_uuid, + mitxonline_completed_orders.order_id, + mitxonline_completed_orders.line_id, + mitxonline_completed_orders.order_reference_number, + mitxonline_completed_orders.discount_code as coupon_code, + combined_enrollments.courserungrade_grade, + combined_enrollments.courserungrade_is_passing, + combined_enrollments.course_title, + combined_enrollments.course_readable_id, + combined_enrollments.courserun_upgrade_deadline, + if( + mitxonline_completed_orders.order_id is not null, + coalesce(mitxonline_transactions.payment_timestamp, mitxonline_completed_orders.order_created_on), + null + ) as courserunenrollment_upgraded_on + from combined_enrollments + left join + combined_users + on combined_enrollments.user_id = combined_users.user_id and combined_enrollments.platform = combined_users.platform - left join mitxonline_completed_orders - on - combined_enrollments.user_id = cast(mitxonline_completed_orders.user_id as varchar) + left join + mitxonline_completed_orders + on combined_enrollments.user_id = cast(mitxonline_completed_orders.user_id as varchar) and combined_enrollments.courserun_id = mitxonline_completed_orders.courserun_id and mitxonline_completed_orders.row_num = 1 - left join combined_courseruns - on - combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id + left join + combined_courseruns + on combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id and combined_enrollments.platform = combined_courseruns.platform - left join mitxonline_transactions - on mitxonline_completed_orders.order_id = mitxonline_transactions.order_id - where combined_enrollments.platform = '{{ var("mitxonline") }}' + left join mitxonline_transactions on mitxonline_completed_orders.order_id = mitxonline_transactions.order_id + where combined_enrollments.platform = '{{ var("mitxonline") }}' - union all + union all - select - '{{ var("edxorg") }}' as platform - , combined_enrollments.courserunenrollment_id - , combined_enrollments.courserunenrollment_is_active - , combined_enrollments.courserunenrollment_created_on - , combined_enrollments.courserunenrollment_enrollment_mode - , combined_enrollments.courserunenrollment_enrollment_status - , combined_enrollments.courserunenrollment_is_edx_enrolled - , combined_enrollments.user_id - , combined_enrollments.courserun_id - , combined_enrollments.courserun_title - , combined_enrollments.courserun_readable_id - , combined_courseruns.courserun_start_on - , combined_courseruns.courserun_end_on - , if(combined_courseruns.courserun_is_current is null, false, combined_courseruns.courserun_is_current) - as courserun_is_current - , combined_enrollments.user_username - , combined_enrollments.user_email - , combined_enrollments.user_full_name - , combined_users.user_address_country as user_country_code - , combined_users.user_highest_education - , combined_users.user_company - , combined_users.user_gender - , case - when mitxonline_certificates.courseruncertificate_is_revoked = false then true - when combined_enrollments.courseruncertificate_created_on is not null then true - else false - end as courseruncertificate_is_earned - , coalesce( - mitxonline_certificates.courseruncertificate_created_on - , combined_enrollments.courseruncertificate_created_on - ) as courseruncertificate_created_on - , coalesce( - mitxonline_certificates.courseruncertificate_url - , combined_enrollments.courseruncertificate_url - ) as courseruncertificate_url - , coalesce( - mitxonline_certificates.courseruncertificate_uuid - , combined_enrollments.courseruncertificate_uuid - ) as courseruncertificate_uuid - , micromasters_completed_orders.order_id - , micromasters_completed_orders.line_id - , micromasters_completed_orders.order_reference_number - , micromasters_completed_orders.coupon_code - , combined_enrollments.courserungrade_grade - , combined_enrollments.courserungrade_is_passing - , combined_enrollments.course_title - , combined_enrollments.course_readable_id - , combined_enrollments.courserun_upgrade_deadline - , if(micromasters_completed_orders.order_id is not null - , coalesce(micromasters_completed_orders.receipt_payment_timestamp, micromasters_completed_orders.order_created_on) - , null - ) as courserunenrollment_upgraded_on - from combined_enrollments - left join combined_users - on - combined_enrollments.user_username = combined_users.user_username + select + '{{ var("edxorg") }}' as platform, + combined_enrollments.courserunenrollment_id, + combined_enrollments.courserunenrollment_is_active, + combined_enrollments.courserunenrollment_created_on, + combined_enrollments.courserunenrollment_enrollment_mode, + combined_enrollments.courserunenrollment_enrollment_status, + combined_enrollments.courserunenrollment_is_edx_enrolled, + combined_enrollments.user_id, + combined_enrollments.courserun_id, + combined_enrollments.courserun_title, + combined_enrollments.courserun_readable_id, + combined_courseruns.courserun_start_on, + combined_courseruns.courserun_end_on, + if( + combined_courseruns.courserun_is_current is null, false, combined_courseruns.courserun_is_current + ) as courserun_is_current, + combined_enrollments.user_username, + combined_enrollments.user_email, + combined_enrollments.user_full_name, + combined_users.user_address_country as user_country_code, + combined_users.user_highest_education, + combined_users.user_company, + combined_users.user_gender, + case + when mitxonline_certificates.courseruncertificate_is_revoked = false + then true + when combined_enrollments.courseruncertificate_created_on is not null + then true + else false + end as courseruncertificate_is_earned, + coalesce( + mitxonline_certificates.courseruncertificate_created_on, + combined_enrollments.courseruncertificate_created_on + ) as courseruncertificate_created_on, + coalesce( + mitxonline_certificates.courseruncertificate_url, combined_enrollments.courseruncertificate_url + ) as courseruncertificate_url, + coalesce( + mitxonline_certificates.courseruncertificate_uuid, combined_enrollments.courseruncertificate_uuid + ) as courseruncertificate_uuid, + micromasters_completed_orders.order_id, + micromasters_completed_orders.line_id, + micromasters_completed_orders.order_reference_number, + micromasters_completed_orders.coupon_code, + combined_enrollments.courserungrade_grade, + combined_enrollments.courserungrade_is_passing, + combined_enrollments.course_title, + combined_enrollments.course_readable_id, + combined_enrollments.courserun_upgrade_deadline, + if( + micromasters_completed_orders.order_id is not null, + coalesce( + micromasters_completed_orders.receipt_payment_timestamp, + micromasters_completed_orders.order_created_on + ), + null + ) as courserunenrollment_upgraded_on + from combined_enrollments + left join + combined_users + on combined_enrollments.user_username = combined_users.user_username and combined_enrollments.platform = combined_users.platform - left join micromasters_users on combined_enrollments.user_username = micromasters_users.user_edxorg_username - left join micromasters_completed_orders - on - micromasters_users.user_id = micromasters_completed_orders.user_id + left join micromasters_users on combined_enrollments.user_username = micromasters_users.user_edxorg_username + left join + micromasters_completed_orders + on micromasters_users.user_id = micromasters_completed_orders.user_id and combined_enrollments.courserun_readable_id = micromasters_completed_orders.courserun_edxorg_readable_id and micromasters_completed_orders.row_num = 1 - left join combined_courseruns - on - combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id + left join + combined_courseruns + on combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id and combined_enrollments.platform = combined_courseruns.platform - left join mitxonline_certificates - on ( - micromasters_users.user_mitxonline_username = mitxonline_certificates.user_username - or micromasters_users.user_email = mitxonline_certificates.user_email - ) - and combined_enrollments.courserun_readable_id - = replace(replace(mitxonline_certificates.courserun_readable_id, 'course-v1:', ''), '+', '/') - where combined_enrollments.platform = '{{ var("edxorg") }}' + left join + mitxonline_certificates + on ( + micromasters_users.user_mitxonline_username = mitxonline_certificates.user_username + or micromasters_users.user_email = mitxonline_certificates.user_email + ) + and combined_enrollments.courserun_readable_id + = replace(replace(mitxonline_certificates.courserun_readable_id, 'course-v1:', ''), '+', '/') + where combined_enrollments.platform = '{{ var("edxorg") }}' - union all + union all - select - '{{ var("mitxpro") }}' as platform - , combined_enrollments.courserunenrollment_id - , combined_enrollments.courserunenrollment_is_active - , combined_enrollments.courserunenrollment_created_on - , combined_enrollments.courserunenrollment_enrollment_mode - , combined_enrollments.courserunenrollment_enrollment_status - , combined_enrollments.courserunenrollment_is_edx_enrolled - , combined_enrollments.user_id - , combined_enrollments.courserun_id - , combined_enrollments.courserun_title - , combined_enrollments.courserun_readable_id - , combined_courseruns.courserun_start_on - , combined_courseruns.courserun_end_on - , combined_courseruns.courserun_is_current - , combined_enrollments.user_username - , combined_enrollments.user_email - , combined_enrollments.user_full_name - , combined_users.user_address_country as user_country_code - , combined_users.user_highest_education - , combined_users.user_company - , combined_users.user_gender - , combined_enrollments.courseruncertificate_is_earned - , combined_enrollments.courseruncertificate_created_on - , combined_enrollments.courseruncertificate_url - , combined_enrollments.courseruncertificate_uuid - , mitxpro_completed_orders.order_id - , mitxpro__ecommerce_line.line_id - , mitxpro_completed_orders.receipt_reference_number as order_reference_number - , mitxpro_completed_orders.coupon_code - , combined_enrollments.courserungrade_grade - , combined_enrollments.courserungrade_is_passing - , combined_enrollments.course_title - , combined_enrollments.course_readable_id - , combined_enrollments.courserun_upgrade_deadline - , if(mitxpro_completed_orders.order_id is not null - , coalesce(mitxpro_receipts.payment_timestamp, mitxpro_completed_orders.order_created_on) - , null - ) as courserunenrollment_upgraded_on - from mitxpro_enrollments - inner join combined_enrollments - on mitxpro_enrollments.courserunenrollment_id = combined_enrollments.courserunenrollment_id - left join combined_users - on - mitxpro_enrollments.user_username = combined_users.user_username + select + '{{ var("mitxpro") }}' as platform, + combined_enrollments.courserunenrollment_id, + combined_enrollments.courserunenrollment_is_active, + combined_enrollments.courserunenrollment_created_on, + combined_enrollments.courserunenrollment_enrollment_mode, + combined_enrollments.courserunenrollment_enrollment_status, + combined_enrollments.courserunenrollment_is_edx_enrolled, + combined_enrollments.user_id, + combined_enrollments.courserun_id, + combined_enrollments.courserun_title, + combined_enrollments.courserun_readable_id, + combined_courseruns.courserun_start_on, + combined_courseruns.courserun_end_on, + combined_courseruns.courserun_is_current, + combined_enrollments.user_username, + combined_enrollments.user_email, + combined_enrollments.user_full_name, + combined_users.user_address_country as user_country_code, + combined_users.user_highest_education, + combined_users.user_company, + combined_users.user_gender, + combined_enrollments.courseruncertificate_is_earned, + combined_enrollments.courseruncertificate_created_on, + combined_enrollments.courseruncertificate_url, + combined_enrollments.courseruncertificate_uuid, + mitxpro_completed_orders.order_id, + mitxpro__ecommerce_line.line_id, + mitxpro_completed_orders.receipt_reference_number as order_reference_number, + mitxpro_completed_orders.coupon_code, + combined_enrollments.courserungrade_grade, + combined_enrollments.courserungrade_is_passing, + combined_enrollments.course_title, + combined_enrollments.course_readable_id, + combined_enrollments.courserun_upgrade_deadline, + if( + mitxpro_completed_orders.order_id is not null, + coalesce(mitxpro_receipts.payment_timestamp, mitxpro_completed_orders.order_created_on), + null + ) as courserunenrollment_upgraded_on + from mitxpro_enrollments + inner join + combined_enrollments + on mitxpro_enrollments.courserunenrollment_id = combined_enrollments.courserunenrollment_id + left join + combined_users + on mitxpro_enrollments.user_username = combined_users.user_username and combined_enrollments.platform = combined_users.platform - left join mitxpro_completed_orders - on mitxpro_enrollments.ecommerce_order_id = mitxpro_completed_orders.order_id - left join combined_courseruns - on mitxpro_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id - left join mitxpro__ecommerce_line - on mitxpro_completed_orders.order_id = mitxpro__ecommerce_line.order_id - left join mitxpro_receipts - on mitxpro_completed_orders.order_id = mitxpro_receipts.order_id - where combined_enrollments.platform = '{{ var("mitxpro") }}' + left join mitxpro_completed_orders on mitxpro_enrollments.ecommerce_order_id = mitxpro_completed_orders.order_id + left join + combined_courseruns on mitxpro_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id + left join mitxpro__ecommerce_line on mitxpro_completed_orders.order_id = mitxpro__ecommerce_line.order_id + left join mitxpro_receipts on mitxpro_completed_orders.order_id = mitxpro_receipts.order_id + where combined_enrollments.platform = '{{ var("mitxpro") }}' - union all + union all - select - combined_enrollments.platform - , combined_enrollments.courserunenrollment_id - , combined_enrollments.courserunenrollment_is_active - , combined_enrollments.courserunenrollment_created_on - , combined_enrollments.courserunenrollment_enrollment_mode - , combined_enrollments.courserunenrollment_enrollment_status - , combined_enrollments.courserunenrollment_is_edx_enrolled - , combined_enrollments.user_id - , combined_enrollments.courserun_id - , combined_enrollments.courserun_title - , combined_enrollments.courserun_readable_id - , combined_courseruns.courserun_start_on - , combined_courseruns.courserun_end_on - , combined_courseruns.courserun_is_current - , combined_enrollments.user_username - , combined_enrollments.user_email - , combined_enrollments.user_full_name - , combined_users.user_address_country as user_country_code - , combined_users.user_highest_education - , combined_users.user_company - , combined_users.user_gender - , combined_enrollments.courseruncertificate_is_earned - , combined_enrollments.courseruncertificate_created_on - , combined_enrollments.courseruncertificate_url - , combined_enrollments.courseruncertificate_uuid - , null as order_id - , null as line_id - , null as order_reference_number - , null as coupon_code - , combined_enrollments.courserungrade_grade - , combined_enrollments.courserungrade_is_passing - , combined_enrollments.course_title - , combined_enrollments.course_readable_id - , combined_enrollments.courserun_upgrade_deadline - , null as courserunenrollment_upgraded_on - from combined_enrollments - left join combined_users - on - combined_enrollments.user_id = combined_users.user_id + select + combined_enrollments.platform, + combined_enrollments.courserunenrollment_id, + combined_enrollments.courserunenrollment_is_active, + combined_enrollments.courserunenrollment_created_on, + combined_enrollments.courserunenrollment_enrollment_mode, + combined_enrollments.courserunenrollment_enrollment_status, + combined_enrollments.courserunenrollment_is_edx_enrolled, + combined_enrollments.user_id, + combined_enrollments.courserun_id, + combined_enrollments.courserun_title, + combined_enrollments.courserun_readable_id, + combined_courseruns.courserun_start_on, + combined_courseruns.courserun_end_on, + combined_courseruns.courserun_is_current, + combined_enrollments.user_username, + combined_enrollments.user_email, + combined_enrollments.user_full_name, + combined_users.user_address_country as user_country_code, + combined_users.user_highest_education, + combined_users.user_company, + combined_users.user_gender, + combined_enrollments.courseruncertificate_is_earned, + combined_enrollments.courseruncertificate_created_on, + combined_enrollments.courseruncertificate_url, + combined_enrollments.courseruncertificate_uuid, + null as order_id, + null as line_id, + null as order_reference_number, + null as coupon_code, + combined_enrollments.courserungrade_grade, + combined_enrollments.courserungrade_is_passing, + combined_enrollments.course_title, + combined_enrollments.course_readable_id, + combined_enrollments.courserun_upgrade_deadline, + null as courserunenrollment_upgraded_on + from combined_enrollments + left join + combined_users + on combined_enrollments.user_id = combined_users.user_id and combined_enrollments.user_email = combined_users.user_email and combined_enrollments.platform = combined_users.platform - left join combined_courseruns - on combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id - where combined_enrollments.platform in ('{{ var("emeritus") }}', '{{ var("global_alumni") }}') + left join + combined_courseruns + on combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id + where combined_enrollments.platform in ('{{ var("emeritus") }}', '{{ var("global_alumni") }}') - union all + union all - select - '{{ var("bootcamps") }}' as platform - , combined_enrollments.courserunenrollment_id - , combined_enrollments.courserunenrollment_is_active - , combined_enrollments.courserunenrollment_created_on - , combined_enrollments.courserunenrollment_enrollment_mode - , combined_enrollments.courserunenrollment_enrollment_status - , combined_enrollments.courserunenrollment_is_edx_enrolled - , combined_enrollments.user_id - , combined_enrollments.courserun_id - , combined_enrollments.courserun_title - , combined_enrollments.courserun_readable_id - , combined_courseruns.courserun_start_on - , combined_courseruns.courserun_end_on - , combined_courseruns.courserun_is_current - , combined_enrollments.user_username - , combined_enrollments.user_email - , combined_enrollments.user_full_name - , combined_users.user_address_country as user_country_code - , combined_users.user_highest_education - , combined_users.user_company - , combined_users.user_gender - , combined_enrollments.courseruncertificate_is_earned - , combined_enrollments.courseruncertificate_created_on - , combined_enrollments.courseruncertificate_url - , combined_enrollments.courseruncertificate_uuid - , bootcamps_completed_orders.order_id - , bootcamps_completed_orders.line_id - , bootcamps_completed_orders.order_reference_number - , null as coupon_code - , combined_enrollments.courserungrade_grade - , combined_enrollments.courserungrade_is_passing - , combined_enrollments.course_title - , combined_enrollments.course_readable_id - , combined_enrollments.courserun_upgrade_deadline - , if(bootcamps_completed_orders.order_id is not null - , coalesce(bootcamps_receipts.receipt_payment_timestamp, bootcamps_completed_orders.order_created_on) - , null - ) as courserunenrollment_upgraded_on - from combined_enrollments - left join combined_users - on - combined_enrollments.user_username = combined_users.user_username + select + '{{ var("bootcamps") }}' as platform, + combined_enrollments.courserunenrollment_id, + combined_enrollments.courserunenrollment_is_active, + combined_enrollments.courserunenrollment_created_on, + combined_enrollments.courserunenrollment_enrollment_mode, + combined_enrollments.courserunenrollment_enrollment_status, + combined_enrollments.courserunenrollment_is_edx_enrolled, + combined_enrollments.user_id, + combined_enrollments.courserun_id, + combined_enrollments.courserun_title, + combined_enrollments.courserun_readable_id, + combined_courseruns.courserun_start_on, + combined_courseruns.courserun_end_on, + combined_courseruns.courserun_is_current, + combined_enrollments.user_username, + combined_enrollments.user_email, + combined_enrollments.user_full_name, + combined_users.user_address_country as user_country_code, + combined_users.user_highest_education, + combined_users.user_company, + combined_users.user_gender, + combined_enrollments.courseruncertificate_is_earned, + combined_enrollments.courseruncertificate_created_on, + combined_enrollments.courseruncertificate_url, + combined_enrollments.courseruncertificate_uuid, + bootcamps_completed_orders.order_id, + bootcamps_completed_orders.line_id, + bootcamps_completed_orders.order_reference_number, + null as coupon_code, + combined_enrollments.courserungrade_grade, + combined_enrollments.courserungrade_is_passing, + combined_enrollments.course_title, + combined_enrollments.course_readable_id, + combined_enrollments.courserun_upgrade_deadline, + if( + bootcamps_completed_orders.order_id is not null, + coalesce(bootcamps_receipts.receipt_payment_timestamp, bootcamps_completed_orders.order_created_on), + null + ) as courserunenrollment_upgraded_on + from combined_enrollments + left join + combined_users + on combined_enrollments.user_username = combined_users.user_username and combined_enrollments.platform = combined_users.platform - left join bootcamps_completed_orders - on - combined_enrollments.user_username = bootcamps_completed_orders.user_username + left join + bootcamps_completed_orders + on combined_enrollments.user_username = bootcamps_completed_orders.user_username and combined_enrollments.courserun_id = bootcamps_completed_orders.courserun_id - left join combined_courseruns - on - combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id + left join + combined_courseruns + on combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id and combined_enrollments.platform = combined_courseruns.platform - left join bootcamps_receipts - on bootcamps_completed_orders.order_id = bootcamps_receipts.order_id - where combined_enrollments.platform = '{{ var("bootcamps") }}' + left join bootcamps_receipts on bootcamps_completed_orders.order_id = bootcamps_receipts.order_id + where combined_enrollments.platform = '{{ var("bootcamps") }}' - union all + union all - select - '{{ var("residential") }}' as platform - , combined_enrollments.courserunenrollment_id - , combined_enrollments.courserunenrollment_is_active - , combined_enrollments.courserunenrollment_created_on - , combined_enrollments.courserunenrollment_enrollment_mode - , combined_enrollments.courserunenrollment_enrollment_status - , combined_enrollments.courserunenrollment_is_edx_enrolled - , combined_enrollments.user_id - , combined_enrollments.courserun_id - , combined_enrollments.courserun_title - , combined_enrollments.courserun_readable_id - , combined_courseruns.courserun_start_on - , combined_courseruns.courserun_end_on - , if(combined_courseruns.courserun_is_current is null, false, combined_courseruns.courserun_is_current) - as courserun_is_current - , combined_enrollments.user_username - , combined_enrollments.user_email - , combined_enrollments.user_full_name - , combined_users.user_address_country as user_country_code - , combined_users.user_highest_education - , combined_users.user_company - , combined_users.user_gender - , combined_enrollments.courseruncertificate_is_earned - , combined_enrollments.courseruncertificate_created_on - , combined_enrollments.courseruncertificate_url - , combined_enrollments.courseruncertificate_uuid - , null as order_id - , null as line_id - , null as order_reference_number - , null as coupon_code - , combined_enrollments.courserungrade_grade - , combined_enrollments.courserungrade_is_passing - , combined_enrollments.course_title - , combined_enrollments.course_readable_id - , combined_enrollments.courserun_upgrade_deadline - , null as courserunenrollment_upgraded_on - from combined_enrollments - left join combined_users - on - combined_enrollments.user_username = combined_users.user_username + select + '{{ var("residential") }}' as platform, + combined_enrollments.courserunenrollment_id, + combined_enrollments.courserunenrollment_is_active, + combined_enrollments.courserunenrollment_created_on, + combined_enrollments.courserunenrollment_enrollment_mode, + combined_enrollments.courserunenrollment_enrollment_status, + combined_enrollments.courserunenrollment_is_edx_enrolled, + combined_enrollments.user_id, + combined_enrollments.courserun_id, + combined_enrollments.courserun_title, + combined_enrollments.courserun_readable_id, + combined_courseruns.courserun_start_on, + combined_courseruns.courserun_end_on, + if( + combined_courseruns.courserun_is_current is null, false, combined_courseruns.courserun_is_current + ) as courserun_is_current, + combined_enrollments.user_username, + combined_enrollments.user_email, + combined_enrollments.user_full_name, + combined_users.user_address_country as user_country_code, + combined_users.user_highest_education, + combined_users.user_company, + combined_users.user_gender, + combined_enrollments.courseruncertificate_is_earned, + combined_enrollments.courseruncertificate_created_on, + combined_enrollments.courseruncertificate_url, + combined_enrollments.courseruncertificate_uuid, + null as order_id, + null as line_id, + null as order_reference_number, + null as coupon_code, + combined_enrollments.courserungrade_grade, + combined_enrollments.courserungrade_is_passing, + combined_enrollments.course_title, + combined_enrollments.course_readable_id, + combined_enrollments.courserun_upgrade_deadline, + null as courserunenrollment_upgraded_on + from combined_enrollments + left join + combined_users + on combined_enrollments.user_username = combined_users.user_username and combined_enrollments.platform = combined_users.platform - left join combined_courseruns - on - combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id + left join + combined_courseruns + on combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id and combined_enrollments.platform = combined_courseruns.platform - where combined_enrollments.platform = '{{ var("residential") }}' -) + where combined_enrollments.platform = '{{ var("residential") }}' + ) select - platform - , courserunenrollment_id - , {{ generate_hash_id('cast(order_id as varchar) - || cast(coalesce(line_id, 9) as varchar) - || platform') }} as combined_orders_hash_id - , course_readable_id - , course_title - , courserun_id - , courserun_is_current - , courserun_readable_id - , courserun_start_on - , courserun_end_on - , courserun_title - , courserun_upgrade_deadline - , courseruncertificate_created_on - , courseruncertificate_is_earned - , courseruncertificate_url - , courseruncertificate_uuid - , courserunenrollment_created_on - , courserunenrollment_enrollment_mode - , courserunenrollment_enrollment_status - , courserunenrollment_is_active - , courserunenrollment_is_edx_enrolled - , courserunenrollment_upgraded_on - , courserungrade_grade - , courserungrade_is_passing - , line_id - , order_id - , order_reference_number - , coupon_code - , user_company - , user_country_code - , user_email - , user_full_name - , user_highest_education - , user_gender - , case + platform, + courserunenrollment_id, + {{ + generate_hash_id( + "cast(order_id as varchar) || cast(coalesce(line_id, 9) as varchar) || platform" + ) + }} as combined_orders_hash_id, + course_readable_id, + course_title, + courserun_id, + courserun_is_current, + courserun_readable_id, + courserun_start_on, + courserun_end_on, + courserun_title, + courserun_upgrade_deadline, + courseruncertificate_created_on, + courseruncertificate_is_earned, + courseruncertificate_url, + courseruncertificate_uuid, + courserunenrollment_created_on, + courserunenrollment_enrollment_mode, + courserunenrollment_enrollment_status, + courserunenrollment_is_active, + courserunenrollment_is_edx_enrolled, + courserunenrollment_upgraded_on, + courserungrade_grade, + courserungrade_is_passing, + line_id, + order_id, + order_reference_number, + coupon_code, + user_company, + user_country_code, + user_email, + user_full_name, + user_highest_education, + user_gender, + case when user_id is not null - then {{ generate_hash_id('user_id || platform') }} + then {{ generate_hash_id("user_id || platform") }} when user_email is not null - then {{ generate_hash_id('user_email || platform') }} - else - {{ generate_hash_id('user_full_name || platform') }} - end as user_hashed_id - , user_id - , user_username + then {{ generate_hash_id("user_email || platform") }} + else {{ generate_hash_id("user_full_name || platform") }} + end as user_hashed_id, + user_id, + user_username from combined_enrollment_detail diff --git a/src/ol_dbt/models/marts/combined/marts__combined_coursesinprogram.sql b/src/ol_dbt/models/marts/combined/marts__combined_coursesinprogram.sql index fd7fa502c..87f9a212e 100644 --- a/src/ol_dbt/models/marts/combined/marts__combined_coursesinprogram.sql +++ b/src/ol_dbt/models/marts/combined/marts__combined_coursesinprogram.sql @@ -1,86 +1,65 @@ -with mitx_programs as ( - select * from {{ ref('int__mitx__program_requirements') }} -) - -, edxorg_mitx_program_courses as ( - select * from {{ ref('int__edxorg__mitx_program_courses') }} -) - -, mitx__courses as ( - select * from {{ ref('int__mitx__courses') }} -) - -, mitxpro__courses as ( - select * from {{ ref('int__mitxpro__courses') }} -) - -, mitxpro__programs as ( - select * from {{ ref('int__mitxpro__programs') }} -) - -, mitxonline__programs as ( - select * from {{ ref('int__mitxonline__programs') }} -) +with + mitx_programs as (select * from {{ ref("int__mitx__program_requirements") }}), + edxorg_mitx_program_courses as (select * from {{ ref("int__edxorg__mitx_program_courses") }}), + mitx__courses as (select * from {{ ref("int__mitx__courses") }}), + mitxpro__courses as (select * from {{ ref("int__mitxpro__courses") }}), + mitxpro__programs as (select * from {{ ref("int__mitxpro__programs") }}), + mitxonline__programs as (select * from {{ ref("int__mitxonline__programs") }}) select - '{{ var("mitxonline") }}' as platform - , mitx__courses.course_title - , mitx_programs.program_title - , mitxonline__programs.program_name - , mitx_programs.mitxonline_program_id as program_id - , mitx__courses.course_readable_id - , mitxonline__programs.program_readable_id + '{{ var("mitxonline") }}' as platform, + mitx__courses.course_title, + mitx_programs.program_title, + mitxonline__programs.program_name, + mitx_programs.mitxonline_program_id as program_id, + mitx__courses.course_readable_id, + mitxonline__programs.program_readable_id from mitx_programs -inner join mitx__courses - on mitx_programs.course_number = mitx__courses.course_number -left join mitxonline__programs - on mitx_programs.mitxonline_program_id = mitxonline__programs.program_id +inner join mitx__courses on mitx_programs.course_number = mitx__courses.course_number +left join mitxonline__programs on mitx_programs.mitxonline_program_id = mitxonline__programs.program_id where mitx__courses.is_on_mitxonline = true union all select - '{{ var("edxorg") }}' as platform - , mitx__courses.course_title - , mitx_programs.program_title - , mitx_programs.program_title as program_name - , mitx_programs.micromasters_program_id as program_id - , mitx__courses.course_readable_id - , null as program_readable_id + '{{ var("edxorg") }}' as platform, + mitx__courses.course_title, + mitx_programs.program_title, + mitx_programs.program_title as program_name, + mitx_programs.micromasters_program_id as program_id, + mitx__courses.course_readable_id, + null as program_readable_id from mitx_programs -inner join mitx__courses - on mitx_programs.course_number = mitx__courses.course_number +inner join mitx__courses on mitx_programs.course_number = mitx__courses.course_number where mitx__courses.is_on_mitxonline = false union all select - '{{ var("edxorg") }}' as platform - , edxorg_mitx_program_courses.course_title - , edxorg_mitx_program_courses.program_title - , edxorg_mitx_program_courses.program_name - , null as program_id - , coalesce(mitx__courses.course_readable_id, edxorg_mitx_program_courses.course_readable_id) as course_readable_id - , null as program_readable_id + '{{ var("edxorg") }}' as platform, + edxorg_mitx_program_courses.course_title, + edxorg_mitx_program_courses.program_title, + edxorg_mitx_program_courses.program_name, + null as program_id, + coalesce(mitx__courses.course_readable_id, edxorg_mitx_program_courses.course_readable_id) as course_readable_id, + null as program_readable_id from edxorg_mitx_program_courses -left join mitx_programs - on edxorg_mitx_program_courses.program_name = mitx_programs.program_title -left join mitx__courses - on - replace(edxorg_mitx_program_courses.course_readable_id, 'MITx/', '') - = replace(mitx__courses.course_readable_id, 'course-v1:MITxT+', '') +left join mitx_programs on edxorg_mitx_program_courses.program_name = mitx_programs.program_title +left join + mitx__courses + on replace(edxorg_mitx_program_courses.course_readable_id, 'MITx/', '') + = replace(mitx__courses.course_readable_id, 'course-v1:MITxT+', '') where mitx_programs.program_title is null union all select - '{{ var("mitxpro") }}' as platform - , mitxpro__courses.course_title - , mitxpro__programs.program_title - , mitxpro__programs.program_title as program_name - , mitxpro__courses.program_id - , mitxpro__courses.course_readable_id - , mitxpro__programs.program_readable_id + '{{ var("mitxpro") }}' as platform, + mitxpro__courses.course_title, + mitxpro__programs.program_title, + mitxpro__programs.program_title as program_name, + mitxpro__courses.program_id, + mitxpro__courses.course_readable_id, + mitxpro__programs.program_readable_id from mitxpro__courses -inner join mitxpro__programs - on mitxpro__courses.program_id = mitxpro__programs.program_id +inner join mitxpro__programs on mitxpro__courses.program_id = mitxpro__programs.program_id diff --git a/src/ol_dbt/models/marts/combined/marts__combined_discounts.sql b/src/ol_dbt/models/marts/combined/marts__combined_discounts.sql index ca83c0529..d7d5611f1 100644 --- a/src/ol_dbt/models/marts/combined/marts__combined_discounts.sql +++ b/src/ol_dbt/models/marts/combined/marts__combined_discounts.sql @@ -1,108 +1,85 @@ -with mitxonline_discount as ( - select * from {{ ref('int__mitxonline__ecommerce_discount') }} -) - -, mitxonline_discountproduct as ( - select * from {{ ref('int__mitxonline__ecommerce_discountproduct') }} -) - -, mitxonline_product as ( - select * from {{ ref('int__mitxonline__ecommerce_product') }} -) - -, mitxonline_order as ( - select * from {{ ref('int__mitxonline__ecommerce_order') }} -) - -, mitxpro_all_coupons as ( - select * from {{ ref('marts__mitxpro_all_coupons') }} -) - -, mitxonline_order_sum as ( - select - discount_code - , count(distinct case when order_state = 'fulfilled' then order_id end) as coupons_used_count - from mitxonline_order - group by discount_code -) - -, mitxonline_product_joins as ( - select - mitxonline_discountproduct.discount_id - , coalesce( - mitxonline_product.courserun_readable_id - , mitxonline_product.program_readable_id - ) as product_readable_id - from mitxonline_discountproduct - left join mitxonline_product - on mitxonline_discountproduct.product_id = mitxonline_product.product_id - group by - mitxonline_discountproduct.discount_id - , coalesce( - mitxonline_product.courserun_readable_id - , mitxonline_product.program_readable_id - ) -) +with + mitxonline_discount as (select * from {{ ref("int__mitxonline__ecommerce_discount") }}), + mitxonline_discountproduct as (select * from {{ ref("int__mitxonline__ecommerce_discountproduct") }}), + mitxonline_product as (select * from {{ ref("int__mitxonline__ecommerce_product") }}), + mitxonline_order as (select * from {{ ref("int__mitxonline__ecommerce_order") }}), + mitxpro_all_coupons as (select * from {{ ref("marts__mitxpro_all_coupons") }}), + mitxonline_order_sum as ( + select + discount_code, count(distinct case when order_state = 'fulfilled' then order_id end) as coupons_used_count + from mitxonline_order + group by discount_code + ), + mitxonline_product_joins as ( + select + mitxonline_discountproduct.discount_id, + coalesce( + mitxonline_product.courserun_readable_id, mitxonline_product.program_readable_id + ) as product_readable_id + from mitxonline_discountproduct + left join mitxonline_product on mitxonline_discountproduct.product_id = mitxonline_product.product_id + group by + mitxonline_discountproduct.discount_id, + coalesce(mitxonline_product.courserun_readable_id, mitxonline_product.program_readable_id) + ) select - 'MITx Online' as platform - , mitxonline_discount.discount_code - , null as discount_name - , mitxonline_discount.discount_created_on - , null as payment_transaction - , mitxonline_discount.discount_source - , mitxonline_discount.discount_activated_on - , mitxonline_discount.discount_expires_on - , null as num_discount_codes - , mitxonline_discount.discount_max_redemptions - , null as couponpayment_name - , null as b2b_contract_number - , null as b2breceipt_reference_number - , coalesce(case when mitxonline_order_sum.discount_code is not null then true end, false) as redeemed - , mitxonline_order_sum.coupons_used_count as discounts_used_count - , mitxonline_product_joins.product_readable_id - , case + 'MITx Online' as platform, + mitxonline_discount.discount_code, + null as discount_name, + mitxonline_discount.discount_created_on, + null as payment_transaction, + mitxonline_discount.discount_source, + mitxonline_discount.discount_activated_on, + mitxonline_discount.discount_expires_on, + null as num_discount_codes, + mitxonline_discount.discount_max_redemptions, + null as couponpayment_name, + null as b2b_contract_number, + null as b2breceipt_reference_number, + coalesce(case when mitxonline_order_sum.discount_code is not null then true end, false) as redeemed, + mitxonline_order_sum.coupons_used_count as discounts_used_count, + mitxonline_product_joins.product_readable_id, + case when discount_type = 'percent-off' - then concat(format('%.2f', mitxonline_discount.discount_amount), '%') + then concat(format('%.2f', mitxonline_discount.discount_amount), '%') when discount_type = 'dollars-off' - then concat('$', format('%.2f', mitxonline_discount.discount_amount)) + then concat('$', format('%.2f', mitxonline_discount.discount_amount)) when discount_type = 'fixed-price' - then concat('Fixed Price: ', format('%.2f', mitxonline_discount.discount_amount)) - end as discount_amount_text - , null as company_name - , null as discount_type - , null as b2bcoupon_id - , mitxonline_discount.discount_id - , null as couponpaymentversion_id + then concat('Fixed Price: ', format('%.2f', mitxonline_discount.discount_amount)) + end as discount_amount_text, + null as company_name, + null as discount_type, + null as b2bcoupon_id, + mitxonline_discount.discount_id, + null as couponpaymentversion_id from mitxonline_discount -left join mitxonline_product_joins - on mitxonline_discount.discount_id = mitxonline_product_joins.discount_id -left join mitxonline_order_sum - on mitxonline_discount.discount_code = mitxonline_order_sum.discount_code +left join mitxonline_product_joins on mitxonline_discount.discount_id = mitxonline_product_joins.discount_id +left join mitxonline_order_sum on mitxonline_discount.discount_code = mitxonline_order_sum.discount_code union all select - 'xPro' as platform - , coupon_code as discount_code - , coupon_name as discount_name - , coupon_created_on as discount_created_on - , payment_transaction - , case when coupon_id is null then 'b2b' else discount_source end as discount_source - , activated_on as discount_activated_on - , expires_on as discount_expires_on - , couponpaymentversion_num_coupon_codes as num_discount_codes - , couponpaymentversion_max_redemptions as discount_max_redemptions - , couponpayment_name - , b2border_contract_number as b2b_contract_number - , b2breceipt_reference_number - , redeemed - , coupons_used_count as discounts_used_count - , product_readable_id - , discount_amount as discount_amount_text - , company_name - , coupon_type as discount_type - , b2bcoupon_id - , coupon_id as discount_id - , couponpaymentversion_id + 'xPro' as platform, + coupon_code as discount_code, + coupon_name as discount_name, + coupon_created_on as discount_created_on, + payment_transaction, + case when coupon_id is null then 'b2b' else discount_source end as discount_source, + activated_on as discount_activated_on, + expires_on as discount_expires_on, + couponpaymentversion_num_coupon_codes as num_discount_codes, + couponpaymentversion_max_redemptions as discount_max_redemptions, + couponpayment_name, + b2border_contract_number as b2b_contract_number, + b2breceipt_reference_number, + redeemed, + coupons_used_count as discounts_used_count, + product_readable_id, + discount_amount as discount_amount_text, + company_name, + coupon_type as discount_type, + b2bcoupon_id, + coupon_id as discount_id, + couponpaymentversion_id from mitxpro_all_coupons diff --git a/src/ol_dbt/models/marts/combined/marts__combined_problem_submissions.sql b/src/ol_dbt/models/marts/combined/marts__combined_problem_submissions.sql index 18cd259da..2d603e580 100644 --- a/src/ol_dbt/models/marts/combined/marts__combined_problem_submissions.sql +++ b/src/ol_dbt/models/marts/combined/marts__combined_problem_submissions.sql @@ -1,105 +1,99 @@ -with combined_problem_checks as ( - select - '{{ var("mitxonline") }}' as platform - , user_username - , courserun_readable_id - , useractivity_problem_id - , useractivity_problem_name - , useractivity_problem_student_answers - , useractivity_problem_attempts - , useractivity_problem_success - , useractivity_problem_current_grade - , useractivity_problem_max_grade - , useractivity_timestamp - from {{ ref('int__mitxonline__user_courseactivity_problemcheck') }} +with + combined_problem_checks as ( + select + '{{ var("mitxonline") }}' as platform, + user_username, + courserun_readable_id, + useractivity_problem_id, + useractivity_problem_name, + useractivity_problem_student_answers, + useractivity_problem_attempts, + useractivity_problem_success, + useractivity_problem_current_grade, + useractivity_problem_max_grade, + useractivity_timestamp + from {{ ref("int__mitxonline__user_courseactivity_problemcheck") }} - union all + union all - select - '{{ var("edxorg") }}' as platform - , user_username - , courserun_readable_id - , useractivity_problem_id - , useractivity_problem_name - , useractivity_problem_student_answers - , useractivity_problem_attempts - , useractivity_problem_success - , useractivity_problem_current_grade - , useractivity_problem_max_grade - , useractivity_timestamp - from {{ ref('int__edxorg__mitx_user_courseactivity_problemcheck') }} + select + '{{ var("edxorg") }}' as platform, + user_username, + courserun_readable_id, + useractivity_problem_id, + useractivity_problem_name, + useractivity_problem_student_answers, + useractivity_problem_attempts, + useractivity_problem_success, + useractivity_problem_current_grade, + useractivity_problem_max_grade, + useractivity_timestamp + from {{ ref("int__edxorg__mitx_user_courseactivity_problemcheck") }} - union all + union all - select - '{{ var("mitxpro") }}' as platform - , user_username - , courserun_readable_id - , useractivity_problem_id - , useractivity_problem_name - , useractivity_problem_student_answers - , useractivity_problem_attempts - , useractivity_problem_success - , useractivity_problem_current_grade - , useractivity_problem_max_grade - , useractivity_timestamp - from {{ ref('int__mitxpro__user_courseactivity_problemcheck') }} + select + '{{ var("mitxpro") }}' as platform, + user_username, + courserun_readable_id, + useractivity_problem_id, + useractivity_problem_name, + useractivity_problem_student_answers, + useractivity_problem_attempts, + useractivity_problem_success, + useractivity_problem_current_grade, + useractivity_problem_max_grade, + useractivity_timestamp + from {{ ref("int__mitxpro__user_courseactivity_problemcheck") }} - union all + union all - select - '{{ var("residential") }}' as platform - , user_username - , courserun_readable_id - , useractivity_problem_id - , useractivity_problem_name - , useractivity_problem_student_answers - , useractivity_problem_attempts - , useractivity_problem_success - , useractivity_problem_current_grade - , useractivity_problem_max_grade - , useractivity_timestamp - from {{ ref('int__mitxresidential__user_courseactivity_problemcheck') }} -) - -, combined_runs as ( - select * from {{ ref('int__combined__course_runs') }} -) - -, combined_users as ( - select * from {{ ref('int__combined__users') }} - -) + select + '{{ var("residential") }}' as platform, + user_username, + courserun_readable_id, + useractivity_problem_id, + useractivity_problem_name, + useractivity_problem_student_answers, + useractivity_problem_attempts, + useractivity_problem_success, + useractivity_problem_current_grade, + useractivity_problem_max_grade, + useractivity_timestamp + from {{ ref("int__mitxresidential__user_courseactivity_problemcheck") }} + ), + combined_runs as (select * from {{ ref("int__combined__course_runs") }}), + combined_users as (select * from {{ ref("int__combined__users") }}) select - combined_problem_checks.platform - , combined_problem_checks.user_username - , combined_problem_checks.courserun_readable_id - , combined_problem_checks.useractivity_problem_id as problem_id - , combined_problem_checks.useractivity_problem_name as problem_name - , combined_problem_checks.useractivity_problem_attempts as num_attempts - , combined_problem_checks.useractivity_problem_student_answers as student_answers - , combined_problem_checks.useractivity_problem_success as problem_success - , combined_problem_checks.useractivity_problem_current_grade as problem_grade - , combined_problem_checks.useractivity_problem_max_grade as problem_max_grade - , combined_problem_checks.useractivity_timestamp as problem_submission_timestamp - , combined_users.user_hashed_id - , combined_users.user_full_name - , combined_users.user_email - , combined_users.user_address_country as user_country_code - , combined_users.user_highest_education - , combined_users.user_gender - , combined_runs.courserun_title - , combined_runs.course_readable_id - , combined_runs.courserun_is_current - , combined_runs.courserun_start_on - , combined_runs.courserun_end_on + combined_problem_checks.platform, + combined_problem_checks.user_username, + combined_problem_checks.courserun_readable_id, + combined_problem_checks.useractivity_problem_id as problem_id, + combined_problem_checks.useractivity_problem_name as problem_name, + combined_problem_checks.useractivity_problem_attempts as num_attempts, + combined_problem_checks.useractivity_problem_student_answers as student_answers, + combined_problem_checks.useractivity_problem_success as problem_success, + combined_problem_checks.useractivity_problem_current_grade as problem_grade, + combined_problem_checks.useractivity_problem_max_grade as problem_max_grade, + combined_problem_checks.useractivity_timestamp as problem_submission_timestamp, + combined_users.user_hashed_id, + combined_users.user_full_name, + combined_users.user_email, + combined_users.user_address_country as user_country_code, + combined_users.user_highest_education, + combined_users.user_gender, + combined_runs.courserun_title, + combined_runs.course_readable_id, + combined_runs.courserun_is_current, + combined_runs.courserun_start_on, + combined_runs.courserun_end_on from combined_problem_checks -inner join combined_runs - on - combined_problem_checks.courserun_readable_id = combined_runs.courserun_readable_id - and combined_problem_checks.platform = combined_runs.platform -left join combined_users - on - combined_problem_checks.user_username = combined_users.user_username - and combined_problem_checks.platform = combined_users.platform +inner join + combined_runs + on combined_problem_checks.courserun_readable_id = combined_runs.courserun_readable_id + and combined_problem_checks.platform = combined_runs.platform +left join + combined_users + on combined_problem_checks.user_username = combined_users.user_username + and combined_problem_checks.platform = combined_users.platform diff --git a/src/ol_dbt/models/marts/combined/marts__combined_program_enrollment_detail.sql b/src/ol_dbt/models/marts/combined/marts__combined_program_enrollment_detail.sql index 3344d6a97..694f1c329 100644 --- a/src/ol_dbt/models/marts/combined/marts__combined_program_enrollment_detail.sql +++ b/src/ol_dbt/models/marts/combined/marts__combined_program_enrollment_detail.sql @@ -1,360 +1,292 @@ -with mitxpro__programenrollments as ( - select * from {{ ref('int__mitxpro__programenrollments') }} -) - -, mitx_programs as ( - select * from {{ ref('int__mitx__program_requirements') }} -) - -, mitxpro__programs as ( - select * from {{ ref('int__mitxpro__programs') }} -) - -, mitxpro__program_certificates as ( - select * from {{ ref('int__mitxpro__program_certificates') }} -) - -, mitxpro__users as ( - select * from {{ ref('int__mitxpro__users') }} -) - -, mitxonline__programenrollments as ( - select * from {{ ref('int__mitxonline__programenrollments') }} -) - -, mitxonline__programs as ( - select * from {{ ref('int__mitxonline__programs') }} -) - -, mitxonline__users as ( - select * from {{ ref('int__mitxonline__users') }} -) - -, mitx__programs as ( - select * from {{ ref('int__mitx__programs') }} -) - -, mitxonline__program_certificates as ( - select * from {{ ref('int__mitxonline__program_certificates') }} -) - -, edx_program_certificates as ( - select * from {{ ref('int__edxorg__mitx_program_certificates') }} -) - -, edx_program_enrollments as ( - select * from {{ ref('int__edxorg__mitx_program_enrollments') }} -) - -, edxorg__users as ( - select * from {{ ref('int__edxorg__mitx_users') }} -) - -, micromasters__program_enrollments as ( - select * from {{ ref('int__micromasters__program_enrollments') }} -) - -, edxorg_mitx_program_courses as ( - select * from {{ ref('int__edxorg__mitx_program_courses') }} -) - -, mitx__courses as ( - select * from {{ ref('int__mitx__courses') }} -) - -, mitxpro__courses as ( - select * from {{ ref('int__mitxpro__courses') }} -) - -, combined_enrollments as ( - select * from {{ ref('int__combined__courserun_enrollments') }} -) - -, combined_courseruns as ( - select * from {{ ref('int__combined__course_runs') }} -) - -, combined_programs as ( - select - mitxpro__programs.platform_name - , mitxpro__programs.program_id - , mitxpro__programs.program_title - , mitxpro__programs.program_title as program_name - , null as program_track - , 'Professional Certificate' as program_type - , mitxpro__programs.program_is_live - , mitxpro__programs.program_readable_id - , mitxpro__programenrollments.user_id - , mitxpro__programenrollments.user_email - , mitxpro__programenrollments.user_username - , mitxpro__users.user_full_name - , mitxpro__programenrollments.programenrollment_is_active - , mitxpro__programenrollments.programenrollment_created_on - , mitxpro__programenrollments.programenrollment_enrollment_status - , mitxpro__program_certificates.programcertificate_created_on - , mitxpro__program_certificates.programcertificate_is_revoked - , mitxpro__program_certificates.programcertificate_uuid - , mitxpro__program_certificates.programcertificate_url - , if(mitxpro__program_certificates.programcertificate_url is not null, true, false) - as user_has_completed_program - from mitxpro__programenrollments - inner join mitxpro__programs - on mitxpro__programenrollments.program_id = mitxpro__programs.program_id - left join mitxpro__users - on mitxpro__programenrollments.user_id = mitxpro__users.user_id - left join mitxpro__program_certificates - on - mitxpro__programenrollments.program_id = mitxpro__program_certificates.program_id +with + mitxpro__programenrollments as (select * from {{ ref("int__mitxpro__programenrollments") }}), + mitx_programs as (select * from {{ ref("int__mitx__program_requirements") }}), + mitxpro__programs as (select * from {{ ref("int__mitxpro__programs") }}), + mitxpro__program_certificates as (select * from {{ ref("int__mitxpro__program_certificates") }}), + mitxpro__users as (select * from {{ ref("int__mitxpro__users") }}), + mitxonline__programenrollments as (select * from {{ ref("int__mitxonline__programenrollments") }}), + mitxonline__programs as (select * from {{ ref("int__mitxonline__programs") }}), + mitxonline__users as (select * from {{ ref("int__mitxonline__users") }}), + mitx__programs as (select * from {{ ref("int__mitx__programs") }}), + mitxonline__program_certificates as (select * from {{ ref("int__mitxonline__program_certificates") }}), + edx_program_certificates as (select * from {{ ref("int__edxorg__mitx_program_certificates") }}), + edx_program_enrollments as (select * from {{ ref("int__edxorg__mitx_program_enrollments") }}), + edxorg__users as (select * from {{ ref("int__edxorg__mitx_users") }}), + micromasters__program_enrollments as (select * from {{ ref("int__micromasters__program_enrollments") }}), + edxorg_mitx_program_courses as (select * from {{ ref("int__edxorg__mitx_program_courses") }}), + mitx__courses as (select * from {{ ref("int__mitx__courses") }}), + mitxpro__courses as (select * from {{ ref("int__mitxpro__courses") }}), + combined_enrollments as (select * from {{ ref("int__combined__courserun_enrollments") }}), + combined_courseruns as (select * from {{ ref("int__combined__course_runs") }}), + combined_programs as ( + select + mitxpro__programs.platform_name, + mitxpro__programs.program_id, + mitxpro__programs.program_title, + mitxpro__programs.program_title as program_name, + null as program_track, + 'Professional Certificate' as program_type, + mitxpro__programs.program_is_live, + mitxpro__programs.program_readable_id, + mitxpro__programenrollments.user_id, + mitxpro__programenrollments.user_email, + mitxpro__programenrollments.user_username, + mitxpro__users.user_full_name, + mitxpro__programenrollments.programenrollment_is_active, + mitxpro__programenrollments.programenrollment_created_on, + mitxpro__programenrollments.programenrollment_enrollment_status, + mitxpro__program_certificates.programcertificate_created_on, + mitxpro__program_certificates.programcertificate_is_revoked, + mitxpro__program_certificates.programcertificate_uuid, + mitxpro__program_certificates.programcertificate_url, + if( + mitxpro__program_certificates.programcertificate_url is not null, true, false + ) as user_has_completed_program + from mitxpro__programenrollments + inner join mitxpro__programs on mitxpro__programenrollments.program_id = mitxpro__programs.program_id + left join mitxpro__users on mitxpro__programenrollments.user_id = mitxpro__users.user_id + left join + mitxpro__program_certificates + on mitxpro__programenrollments.program_id = mitxpro__program_certificates.program_id and mitxpro__programenrollments.user_id = mitxpro__program_certificates.user_id - union all - - select - '{{ var("mitxonline") }}' as platform_name - , mitxonline__programs.program_id - , mitxonline__programs.program_title - , mitxonline__programs.program_name - , mitxonline__programs.program_track - , mitxonline__programs.program_type - , mitxonline__programs.program_is_live - , mitxonline__programs.program_readable_id - , mitxonline__programenrollments.user_id - , mitxonline__programenrollments.user_email - , mitxonline__programenrollments.user_username - , mitxonline__users.user_full_name - , mitxonline__programenrollments.programenrollment_is_active - , mitxonline__programenrollments.programenrollment_created_on - , mitxonline__programenrollments.programenrollment_enrollment_status - , mitxonline__program_certificates.programcertificate_created_on - , mitxonline__program_certificates.programcertificate_is_revoked - , mitxonline__program_certificates.programcertificate_uuid - , mitxonline__program_certificates.programcertificate_url - , if(mitxonline__program_certificates.programcertificate_url is not null, true, false) - as user_has_completed_program - from mitxonline__programenrollments - inner join mitxonline__programs - on mitxonline__programenrollments.program_id = mitxonline__programs.program_id - left join mitxonline__users - on mitxonline__programenrollments.user_id = mitxonline__users.user_id - left join mitxonline__program_certificates - on - mitxonline__programenrollments.user_id = mitxonline__program_certificates.user_id + union all + + select + '{{ var("mitxonline") }}' as platform_name, + mitxonline__programs.program_id, + mitxonline__programs.program_title, + mitxonline__programs.program_name, + mitxonline__programs.program_track, + mitxonline__programs.program_type, + mitxonline__programs.program_is_live, + mitxonline__programs.program_readable_id, + mitxonline__programenrollments.user_id, + mitxonline__programenrollments.user_email, + mitxonline__programenrollments.user_username, + mitxonline__users.user_full_name, + mitxonline__programenrollments.programenrollment_is_active, + mitxonline__programenrollments.programenrollment_created_on, + mitxonline__programenrollments.programenrollment_enrollment_status, + mitxonline__program_certificates.programcertificate_created_on, + mitxonline__program_certificates.programcertificate_is_revoked, + mitxonline__program_certificates.programcertificate_uuid, + mitxonline__program_certificates.programcertificate_url, + if( + mitxonline__program_certificates.programcertificate_url is not null, true, false + ) as user_has_completed_program + from mitxonline__programenrollments + inner join mitxonline__programs on mitxonline__programenrollments.program_id = mitxonline__programs.program_id + left join mitxonline__users on mitxonline__programenrollments.user_id = mitxonline__users.user_id + left join + mitxonline__program_certificates + on mitxonline__programenrollments.user_id = mitxonline__program_certificates.user_id and mitxonline__programenrollments.program_id = mitxonline__program_certificates.program_id - union all - - select - '{{ var("edxorg") }}' as platform_name - , edx_program_enrollments.micromasters_program_id as program_id - , edx_program_enrollments.program_title - , edx_program_enrollments.program_name - , edx_program_enrollments.program_track - , edx_program_enrollments.program_type - , null as program_is_live - , {{ generate_micromasters_program_readable_id( - 'edx_program_enrollments.micromasters_program_id' - ,'edx_program_enrollments.program_title') - }} as program_readable_id - , edx_program_enrollments.user_id - , edxorg__users.user_email - , edx_program_enrollments.user_username - , edxorg__users.user_full_name - , null as programenrollment_is_active - , null as programenrollment_created_on - , null as programenrollment_enrollment_status - , edx_program_certificates.program_certificate_awarded_on as programcertificate_created_on - , null as programcertificate_is_revoked - , edx_program_certificates.program_certificate_hashed_id as programcertificate_uuid - , null as programcertificate_url - , if(edx_program_certificates.user_has_completed_program = true, true, false) as user_has_completed_program - from edx_program_enrollments - left join edxorg__users - on edx_program_enrollments.user_id = edxorg__users.user_id - left join edx_program_certificates - on - edx_program_enrollments.program_uuid = edx_program_certificates.program_uuid + union all + + select + '{{ var("edxorg") }}' as platform_name, + edx_program_enrollments.micromasters_program_id as program_id, + edx_program_enrollments.program_title, + edx_program_enrollments.program_name, + edx_program_enrollments.program_track, + edx_program_enrollments.program_type, + null as program_is_live, + {{ + generate_micromasters_program_readable_id( + "edx_program_enrollments.micromasters_program_id", "edx_program_enrollments.program_title" + ) + }} as program_readable_id, + edx_program_enrollments.user_id, + edxorg__users.user_email, + edx_program_enrollments.user_username, + edxorg__users.user_full_name, + null as programenrollment_is_active, + null as programenrollment_created_on, + null as programenrollment_enrollment_status, + edx_program_certificates.program_certificate_awarded_on as programcertificate_created_on, + null as programcertificate_is_revoked, + edx_program_certificates.program_certificate_hashed_id as programcertificate_uuid, + null as programcertificate_url, + if(edx_program_certificates.user_has_completed_program = true, true, false) as user_has_completed_program + from edx_program_enrollments + left join edxorg__users on edx_program_enrollments.user_id = edxorg__users.user_id + left join + edx_program_certificates + on edx_program_enrollments.program_uuid = edx_program_certificates.program_uuid and edx_program_enrollments.user_id = edx_program_certificates.user_id - union all - - select - micromasters__program_enrollments.platform_name - , micromasters__program_enrollments.micromasters_program_id as program_id - , micromasters__program_enrollments.program_title - , mitxonline__programs.program_name - , mitxonline__programs.program_track - , mitxonline__programs.program_type - , mitxonline__programs.program_is_live - , mitxonline__programs.program_readable_id - , micromasters__program_enrollments.user_edxorg_id as user_id - , micromasters__program_enrollments.user_email - , micromasters__program_enrollments.user_edxorg_username as user_username - , micromasters__program_enrollments.user_full_name - , null as programenrollment_is_active - , null as programenrollment_created_on - , null as programenrollment_enrollment_status - , edx_program_certificates.program_certificate_awarded_on as programcertificate_created_on - , null as programcertificate_is_revoked - , edx_program_certificates.program_certificate_hashed_id as programcertificate_uuid - , null as programcertificate_url - , if(edx_program_certificates.user_has_completed_program = true, true, false) as user_has_completed_program - from micromasters__program_enrollments - left join mitx__programs - on micromasters__program_enrollments.micromasters_program_id = mitx__programs.micromasters_program_id - left join mitxonline__programs - on mitx__programs.mitxonline_program_id = mitxonline__programs.program_id - left join edx_program_certificates - on - micromasters__program_enrollments.micromasters_program_id = edx_program_certificates.micromasters_program_id + union all + + select + micromasters__program_enrollments.platform_name, + micromasters__program_enrollments.micromasters_program_id as program_id, + micromasters__program_enrollments.program_title, + mitxonline__programs.program_name, + mitxonline__programs.program_track, + mitxonline__programs.program_type, + mitxonline__programs.program_is_live, + mitxonline__programs.program_readable_id, + micromasters__program_enrollments.user_edxorg_id as user_id, + micromasters__program_enrollments.user_email, + micromasters__program_enrollments.user_edxorg_username as user_username, + micromasters__program_enrollments.user_full_name, + null as programenrollment_is_active, + null as programenrollment_created_on, + null as programenrollment_enrollment_status, + edx_program_certificates.program_certificate_awarded_on as programcertificate_created_on, + null as programcertificate_is_revoked, + edx_program_certificates.program_certificate_hashed_id as programcertificate_uuid, + null as programcertificate_url, + if(edx_program_certificates.user_has_completed_program = true, true, false) as user_has_completed_program + from micromasters__program_enrollments + left join + mitx__programs + on micromasters__program_enrollments.micromasters_program_id = mitx__programs.micromasters_program_id + left join mitxonline__programs on mitx__programs.mitxonline_program_id = mitxonline__programs.program_id + left join + edx_program_certificates + on micromasters__program_enrollments.micromasters_program_id + = edx_program_certificates.micromasters_program_id and micromasters__program_enrollments.user_edxorg_id = edx_program_certificates.user_id - left join edx_program_enrollments - on - micromasters__program_enrollments.micromasters_program_id = edx_program_enrollments.micromasters_program_id + left join + edx_program_enrollments + on micromasters__program_enrollments.micromasters_program_id + = edx_program_enrollments.micromasters_program_id and micromasters__program_enrollments.user_edxorg_id = edx_program_enrollments.user_id - where - micromasters__program_enrollments.platform_name = '{{ var("edxorg") }}' - and edx_program_enrollments.user_id is null - and edx_program_enrollments.micromasters_program_id is null -) - -, courses_in_programs as ( - select - mitx__courses.course_readable_id - , mitxonline__programs.program_readable_id - from mitx_programs - inner join mitx__courses - on mitx_programs.course_number = mitx__courses.course_number - left join mitxonline__programs - on mitx_programs.mitxonline_program_id = mitxonline__programs.program_id - where mitx__courses.is_on_mitxonline = true - - union all - - select - mitx__courses.course_readable_id - , null as program_readable_id - from mitx_programs - inner join mitx__courses - on mitx_programs.course_number = mitx__courses.course_number - where mitx__courses.is_on_mitxonline = false - - union all - - select - coalesce(mitx__courses.course_readable_id, edxorg_mitx_program_courses.course_readable_id) as course_readable_id - , null as program_readable_id - from edxorg_mitx_program_courses - left join mitx_programs - on edxorg_mitx_program_courses.program_name = mitx_programs.program_title - left join mitx__courses - on - replace(edxorg_mitx_program_courses.course_readable_id, 'MITx/', '') + where + micromasters__program_enrollments.platform_name = '{{ var("edxorg") }}' + and edx_program_enrollments.user_id is null + and edx_program_enrollments.micromasters_program_id is null + ), + courses_in_programs as ( + select mitx__courses.course_readable_id, mitxonline__programs.program_readable_id + from mitx_programs + inner join mitx__courses on mitx_programs.course_number = mitx__courses.course_number + left join mitxonline__programs on mitx_programs.mitxonline_program_id = mitxonline__programs.program_id + where mitx__courses.is_on_mitxonline = true + + union all + + select mitx__courses.course_readable_id, null as program_readable_id + from mitx_programs + inner join mitx__courses on mitx_programs.course_number = mitx__courses.course_number + where mitx__courses.is_on_mitxonline = false + + union all + + select + coalesce( + mitx__courses.course_readable_id, edxorg_mitx_program_courses.course_readable_id + ) as course_readable_id, + null as program_readable_id + from edxorg_mitx_program_courses + left join mitx_programs on edxorg_mitx_program_courses.program_name = mitx_programs.program_title + left join + mitx__courses + on replace(edxorg_mitx_program_courses.course_readable_id, 'MITx/', '') = replace(mitx__courses.course_readable_id, 'course-v1:MITxT+', '') - where mitx_programs.program_title is null - - union all - - select - mitxpro__courses.course_readable_id - , mitxpro__programs.program_readable_id - from mitxpro__courses - inner join mitxpro__programs - on mitxpro__courses.program_id = mitxpro__programs.program_id -) - -, final_combined_programs as ( - select - combined_programs.platform_name - , combined_programs.program_id - , combined_programs.program_title - , combined_programs.program_name - , combined_programs.program_track - , combined_programs.program_type - , combined_programs.program_is_live - , combined_programs.program_readable_id - , {{ generate_hash_id('cast(combined_programs.user_id as varchar) || combined_programs.platform_name') }} - as user_hashed_id - , combined_programs.user_id - , combined_programs.user_email - , combined_programs.user_username - , combined_programs.user_full_name - , combined_programs.user_has_completed_program - , combined_programs.programenrollment_is_active - , combined_programs.programenrollment_created_on - , combined_programs.programenrollment_enrollment_status - , combined_programs.programcertificate_created_on - , combined_programs.programcertificate_is_revoked - , combined_programs.programcertificate_uuid - , combined_programs.programcertificate_url - , cast(substring(combined_programs.programcertificate_created_on, 1, 10) as date) - as programcertificate_created_on_date - , sum(case when upper(combined_enrollments.course_title) like '%CAPSTONE%' then 1 else 0 end) - as capstone_sum - , min(cast(substring(combined_courseruns.courserun_start_on, 1, 10) as date)) - as first_courserun_start_on_date - , count(distinct combined_enrollments.course_readable_id) as unique_courses_taken_in_program - from combined_programs - left join courses_in_programs - on combined_programs.program_readable_id = courses_in_programs.program_readable_id - left join combined_enrollments - on - cast(combined_programs.user_id as varchar) = combined_enrollments.user_id + where mitx_programs.program_title is null + + union all + + select mitxpro__courses.course_readable_id, mitxpro__programs.program_readable_id + from mitxpro__courses + inner join mitxpro__programs on mitxpro__courses.program_id = mitxpro__programs.program_id + ), + final_combined_programs as ( + select + combined_programs.platform_name, + combined_programs.program_id, + combined_programs.program_title, + combined_programs.program_name, + combined_programs.program_track, + combined_programs.program_type, + combined_programs.program_is_live, + combined_programs.program_readable_id, + {{ generate_hash_id("cast(combined_programs.user_id as varchar) || combined_programs.platform_name") }} + as user_hashed_id, + combined_programs.user_id, + combined_programs.user_email, + combined_programs.user_username, + combined_programs.user_full_name, + combined_programs.user_has_completed_program, + combined_programs.programenrollment_is_active, + combined_programs.programenrollment_created_on, + combined_programs.programenrollment_enrollment_status, + combined_programs.programcertificate_created_on, + combined_programs.programcertificate_is_revoked, + combined_programs.programcertificate_uuid, + combined_programs.programcertificate_url, + cast( + substring(combined_programs.programcertificate_created_on, 1, 10) as date + ) as programcertificate_created_on_date, + sum(case when upper(combined_enrollments.course_title) like '%CAPSTONE%' then 1 else 0 end) as capstone_sum, + min(cast(substring(combined_courseruns.courserun_start_on, 1, 10) as date)) as first_courserun_start_on_date + , + count(distinct combined_enrollments.course_readable_id) as unique_courses_taken_in_program + from combined_programs + left join courses_in_programs on combined_programs.program_readable_id = courses_in_programs.program_readable_id + left join + combined_enrollments + on cast(combined_programs.user_id as varchar) = combined_enrollments.user_id and courses_in_programs.course_readable_id = combined_enrollments.course_readable_id - left join combined_courseruns - on combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id - group by - combined_programs.platform_name - , combined_programs.program_id - , combined_programs.program_title - , combined_programs.program_name - , combined_programs.program_track - , combined_programs.program_type - , combined_programs.program_is_live - , combined_programs.program_readable_id - , {{ generate_hash_id('cast(combined_programs.user_id as varchar) || combined_programs.platform_name') }} - , combined_programs.user_id - , combined_programs.user_email - , combined_programs.user_username - , combined_programs.user_full_name - , combined_programs.user_has_completed_program - , combined_programs.programenrollment_is_active - , combined_programs.programenrollment_created_on - , combined_programs.programenrollment_enrollment_status - , combined_programs.programcertificate_created_on - , combined_programs.programcertificate_is_revoked - , combined_programs.programcertificate_uuid - , combined_programs.programcertificate_url - , cast(substring(combined_programs.programcertificate_created_on, 1, 10) as date) -) + left join + combined_courseruns + on combined_enrollments.courserun_readable_id = combined_courseruns.courserun_readable_id + group by + combined_programs.platform_name, + combined_programs.program_id, + combined_programs.program_title, + combined_programs.program_name, + combined_programs.program_track, + combined_programs.program_type, + combined_programs.program_is_live, + combined_programs.program_readable_id, + {{ generate_hash_id("cast(combined_programs.user_id as varchar) || combined_programs.platform_name") }}, + combined_programs.user_id, + combined_programs.user_email, + combined_programs.user_username, + combined_programs.user_full_name, + combined_programs.user_has_completed_program, + combined_programs.programenrollment_is_active, + combined_programs.programenrollment_created_on, + combined_programs.programenrollment_enrollment_status, + combined_programs.programcertificate_created_on, + combined_programs.programcertificate_is_revoked, + combined_programs.programcertificate_uuid, + combined_programs.programcertificate_url, + cast(substring(combined_programs.programcertificate_created_on, 1, 10) as date) + ) select - final_combined_programs.platform_name - , final_combined_programs.program_id - , final_combined_programs.program_title - , final_combined_programs.program_name - , final_combined_programs.program_track - , final_combined_programs.program_type - , final_combined_programs.program_is_live - , final_combined_programs.program_readable_id - , final_combined_programs.user_hashed_id - , final_combined_programs.user_id - , final_combined_programs.user_email - , final_combined_programs.user_username - , final_combined_programs.user_full_name - , final_combined_programs.user_has_completed_program - , final_combined_programs.programenrollment_is_active - , final_combined_programs.programenrollment_created_on - , final_combined_programs.programenrollment_enrollment_status - , final_combined_programs.programcertificate_created_on - , final_combined_programs.programcertificate_is_revoked - , final_combined_programs.programcertificate_uuid - , final_combined_programs.programcertificate_url - , final_combined_programs.unique_courses_taken_in_program - , case when final_combined_programs.capstone_sum > 0 then 'Y' else 'N' end - as capstone_indicator - , date_diff( - 'day' - , final_combined_programs.programcertificate_created_on_date - , final_combined_programs.first_courserun_start_on_date + final_combined_programs.platform_name, + final_combined_programs.program_id, + final_combined_programs.program_title, + final_combined_programs.program_name, + final_combined_programs.program_track, + final_combined_programs.program_type, + final_combined_programs.program_is_live, + final_combined_programs.program_readable_id, + final_combined_programs.user_hashed_id, + final_combined_programs.user_id, + final_combined_programs.user_email, + final_combined_programs.user_username, + final_combined_programs.user_full_name, + final_combined_programs.user_has_completed_program, + final_combined_programs.programenrollment_is_active, + final_combined_programs.programenrollment_created_on, + final_combined_programs.programenrollment_enrollment_status, + final_combined_programs.programcertificate_created_on, + final_combined_programs.programcertificate_is_revoked, + final_combined_programs.programcertificate_uuid, + final_combined_programs.programcertificate_url, + final_combined_programs.unique_courses_taken_in_program, + case when final_combined_programs.capstone_sum > 0 then 'Y' else 'N' end as capstone_indicator, + date_diff( + 'day', + final_combined_programs.programcertificate_created_on_date, + final_combined_programs.first_courserun_start_on_date ) as program_completion_days from final_combined_programs diff --git a/src/ol_dbt/models/marts/combined/marts__combined_total_course_engagements.sql b/src/ol_dbt/models/marts/combined/marts__combined_total_course_engagements.sql index 108029409..6005a5e10 100644 --- a/src/ol_dbt/models/marts/combined/marts__combined_total_course_engagements.sql +++ b/src/ol_dbt/models/marts/combined/marts__combined_total_course_engagements.sql @@ -1,315 +1,232 @@ -with combined_engagements as ( - select - platform - , courserun_readable_id - , count( - distinct - case - when coursestructure_block_category = 'discussion' - then coursestructure_block_id - end - ) as total_courserun_discussions - , count( - distinct - case - when coursestructure_block_category = 'video' - then coursestructure_block_id - end - ) as total_courserun_videos - , count( - distinct - case - when coursestructure_block_category = 'problem' - then coursestructure_block_id - end - ) as total_courserun_problems - from {{ ref('int__combined__course_structure') }} - group by - platform - , courserun_readable_id -) - -, combined_runs as ( - select * - from {{ ref('int__combined__course_runs') }} - where courserun_readable_id is not null -) - -, combined_enrollments as ( - select - platform - , courserun_readable_id - , user_username - from {{ ref('int__combined__courserun_enrollments') }} - where courserun_readable_id is not null - group by - platform - , courserun_readable_id - , user_username -) - -, combined_user_video as ( - - select - '{{ var("mitxonline") }}' as platform - , user_username - , courserun_readable_id - , count(distinct useractivity_video_id) as videos_user_watched - from {{ ref('int__mitxonline__user_courseactivity_video') }} - where useractivity_event_type = 'play_video' - group by - '{{ var("mitxonline") }}' - , user_username - , courserun_readable_id - - union all - - select - '{{ var("edxorg") }}' as platform - , user_username - , courserun_readable_id - , count(distinct useractivity_video_id) as videos_user_watched - from {{ ref('int__edxorg__mitx_user_courseactivity_video') }} - where useractivity_event_type = 'play_video' - group by - '{{ var("edxorg") }}' - , user_username - , courserun_readable_id - - union all - - select - '{{ var("mitxpro") }}' as platform - , user_username - , courserun_readable_id - , count(distinct useractivity_video_id) as videos_user_watched - from {{ ref('int__mitxpro__user_courseactivity_video') }} - where useractivity_event_type = 'play_video' - group by - '{{ var("mitxpro") }}' - , user_username - , courserun_readable_id - - union all - - select - '{{ var("residential") }}' as platform - , user_username - , courserun_readable_id - , count(distinct useractivity_video_id) as videos_user_watched - from {{ ref('int__mitxresidential__user_courseactivity_video') }} - where useractivity_event_type = 'play_video' - group by - '{{ var("residential") }}' - , user_username - , courserun_readable_id -) - -, combined_user_problem as ( - - select - '{{ var("mitxonline") }}' as platform - , user_username - , courserun_readable_id - , count(distinct useractivity_problem_id) as problems_user_submitted - from {{ ref('int__mitxonline__user_courseactivity_problemsubmitted') }} - group by - '{{ var("mitxonline") }}' - , user_username - , courserun_readable_id - - union all - - select - '{{ var("edxorg") }}' as platform - , user_username - , courserun_readable_id - , count(distinct useractivity_problem_id) as problems_user_submitted - from {{ ref('int__edxorg__mitx_user_courseactivity_problemsubmitted') }} - group by - '{{ var("edxorg") }}' - , user_username - , courserun_readable_id - - union all - - select - '{{ var("mitxpro") }}' as platform - , user_username - , courserun_readable_id - , count(distinct useractivity_problem_id) as problems_user_submitted - from {{ ref('int__mitxpro__user_courseactivity_problemsubmitted') }} - group by - '{{ var("mitxpro") }}' - , user_username - , courserun_readable_id - - union all - - select - '{{ var("residential") }}' as platform - , user_username - , courserun_readable_id - , count(distinct useractivity_problem_id) as problems_user_submitted - from {{ ref('int__mitxresidential__user_courseactivity_problemsubmitted') }} - group by - '{{ var("residential") }}' - , user_username - , courserun_readable_id -) - -, combined_user_discussion as ( - - select - '{{ var("mitxonline") }}' as platform - , user_username - , courserun_readable_id - , count(distinct useractivity_discussion_post_id) as user_discussion_count - from {{ ref('int__mitxonline__user_courseactivity_discussion') }} - where useractivity_event_type like 'edx.forum.%.created' - group by - '{{ var("mitxonline") }}' - , user_username - , courserun_readable_id - - union all - - select - '{{ var("edxorg") }}' as platform - , user_username - , courserun_readable_id - , count(distinct useractivity_discussion_post_id) as user_discussion_count - from {{ ref('int__edxorg__mitx_user_courseactivity_discussion') }} - where useractivity_event_type like 'edx.forum.%.created' - group by - '{{ var("edxorg") }}' - , user_username - , courserun_readable_id - - union all - - select - '{{ var("mitxpro") }}' as platform - , user_username - , courserun_readable_id - , count(distinct useractivity_discussion_post_id) as user_discussion_count - from {{ ref('int__mitxpro__user_courseactivity_discussion') }} - where useractivity_event_type like 'edx.forum.%.created' - group by - '{{ var("mitxpro") }}' - , user_username - , courserun_readable_id - - union all - - select - '{{ var("residential") }}' as platform - , user_username - , courserun_readable_id - , count(distinct useractivity_discussion_post_id) as user_discussion_count - from {{ ref('int__mitxresidential__user_courseactivity_discussion') }} - where useractivity_event_type like 'edx.forum.%.created' - group by - '{{ var("residential") }}' - , user_username - , courserun_readable_id -) - -, user_video_correction as ( - select - platform - , courserun_readable_id - , max(videos_user_watched) as max_videos_per_courserun - from combined_user_video - group by - platform - , courserun_readable_id -) - -, user_problem_correction as ( - select - platform - , courserun_readable_id - , max(problems_user_submitted) as max_problems_per_courserun - from combined_user_problem - group by - platform - , courserun_readable_id -) - -, user_discussion_correction as ( - select - platform - , courserun_readable_id - , max(user_discussion_count) as max_discussions_per_courserun - from combined_user_discussion - group by - platform - , courserun_readable_id -) +with + combined_engagements as ( + select + platform, + courserun_readable_id, + count( + distinct case when coursestructure_block_category = 'discussion' then coursestructure_block_id end + ) as total_courserun_discussions, + count( + distinct case when coursestructure_block_category = 'video' then coursestructure_block_id end + ) as total_courserun_videos, + count( + distinct case when coursestructure_block_category = 'problem' then coursestructure_block_id end + ) as total_courserun_problems + from {{ ref("int__combined__course_structure") }} + group by platform, courserun_readable_id + ), + combined_runs as (select * from {{ ref("int__combined__course_runs") }} where courserun_readable_id is not null), + combined_enrollments as ( + select platform, courserun_readable_id, user_username + from {{ ref("int__combined__courserun_enrollments") }} + where courserun_readable_id is not null + group by platform, courserun_readable_id, user_username + ), + combined_user_video as ( + + select + '{{ var("mitxonline") }}' as platform, + user_username, + courserun_readable_id, + count(distinct useractivity_video_id) as videos_user_watched + from {{ ref("int__mitxonline__user_courseactivity_video") }} + where useractivity_event_type = 'play_video' + group by '{{ var("mitxonline") }}', user_username, courserun_readable_id + + union all + + select + '{{ var("edxorg") }}' as platform, + user_username, + courserun_readable_id, + count(distinct useractivity_video_id) as videos_user_watched + from {{ ref("int__edxorg__mitx_user_courseactivity_video") }} + where useractivity_event_type = 'play_video' + group by '{{ var("edxorg") }}', user_username, courserun_readable_id + + union all + + select + '{{ var("mitxpro") }}' as platform, + user_username, + courserun_readable_id, + count(distinct useractivity_video_id) as videos_user_watched + from {{ ref("int__mitxpro__user_courseactivity_video") }} + where useractivity_event_type = 'play_video' + group by '{{ var("mitxpro") }}', user_username, courserun_readable_id + + union all + + select + '{{ var("residential") }}' as platform, + user_username, + courserun_readable_id, + count(distinct useractivity_video_id) as videos_user_watched + from {{ ref("int__mitxresidential__user_courseactivity_video") }} + where useractivity_event_type = 'play_video' + group by '{{ var("residential") }}', user_username, courserun_readable_id + ), + combined_user_problem as ( + + select + '{{ var("mitxonline") }}' as platform, + user_username, + courserun_readable_id, + count(distinct useractivity_problem_id) as problems_user_submitted + from {{ ref("int__mitxonline__user_courseactivity_problemsubmitted") }} + group by '{{ var("mitxonline") }}', user_username, courserun_readable_id + + union all + + select + '{{ var("edxorg") }}' as platform, + user_username, + courserun_readable_id, + count(distinct useractivity_problem_id) as problems_user_submitted + from {{ ref("int__edxorg__mitx_user_courseactivity_problemsubmitted") }} + group by '{{ var("edxorg") }}', user_username, courserun_readable_id + + union all + + select + '{{ var("mitxpro") }}' as platform, + user_username, + courserun_readable_id, + count(distinct useractivity_problem_id) as problems_user_submitted + from {{ ref("int__mitxpro__user_courseactivity_problemsubmitted") }} + group by '{{ var("mitxpro") }}', user_username, courserun_readable_id + + union all + + select + '{{ var("residential") }}' as platform, + user_username, + courserun_readable_id, + count(distinct useractivity_problem_id) as problems_user_submitted + from {{ ref("int__mitxresidential__user_courseactivity_problemsubmitted") }} + group by '{{ var("residential") }}', user_username, courserun_readable_id + ), + combined_user_discussion as ( + + select + '{{ var("mitxonline") }}' as platform, + user_username, + courserun_readable_id, + count(distinct useractivity_discussion_post_id) as user_discussion_count + from {{ ref("int__mitxonline__user_courseactivity_discussion") }} + where useractivity_event_type like 'edx.forum.%.created' + group by '{{ var("mitxonline") }}', user_username, courserun_readable_id + + union all + + select + '{{ var("edxorg") }}' as platform, + user_username, + courserun_readable_id, + count(distinct useractivity_discussion_post_id) as user_discussion_count + from {{ ref("int__edxorg__mitx_user_courseactivity_discussion") }} + where useractivity_event_type like 'edx.forum.%.created' + group by '{{ var("edxorg") }}', user_username, courserun_readable_id + + union all + + select + '{{ var("mitxpro") }}' as platform, + user_username, + courserun_readable_id, + count(distinct useractivity_discussion_post_id) as user_discussion_count + from {{ ref("int__mitxpro__user_courseactivity_discussion") }} + where useractivity_event_type like 'edx.forum.%.created' + group by '{{ var("mitxpro") }}', user_username, courserun_readable_id + + union all + + select + '{{ var("residential") }}' as platform, + user_username, + courserun_readable_id, + count(distinct useractivity_discussion_post_id) as user_discussion_count + from {{ ref("int__mitxresidential__user_courseactivity_discussion") }} + where useractivity_event_type like 'edx.forum.%.created' + group by '{{ var("residential") }}', user_username, courserun_readable_id + ), + user_video_correction as ( + select platform, courserun_readable_id, max(videos_user_watched) as max_videos_per_courserun + from combined_user_video + group by platform, courserun_readable_id + ), + user_problem_correction as ( + select platform, courserun_readable_id, max(problems_user_submitted) as max_problems_per_courserun + from combined_user_problem + group by platform, courserun_readable_id + ), + user_discussion_correction as ( + select platform, courserun_readable_id, max(user_discussion_count) as max_discussions_per_courserun + from combined_user_discussion + group by platform, courserun_readable_id + ) select - combined_runs.platform - , combined_runs.courserun_readable_id - , combined_runs.courserun_title - , combined_runs.course_readable_id - , combined_runs.courserun_is_current - , combined_runs.courserun_start_on - , combined_runs.courserun_end_on - , combined_enrollments.user_username - , combined_user_video.videos_user_watched - , combined_user_problem.problems_user_submitted - , combined_user_discussion.user_discussion_count - , case + combined_runs.platform, + combined_runs.courserun_readable_id, + combined_runs.courserun_title, + combined_runs.course_readable_id, + combined_runs.courserun_is_current, + combined_runs.courserun_start_on, + combined_runs.courserun_end_on, + combined_enrollments.user_username, + combined_user_video.videos_user_watched, + combined_user_problem.problems_user_submitted, + combined_user_discussion.user_discussion_count, + case when combined_engagements.total_courserun_problems >= user_problem_correction.max_problems_per_courserun - then combined_engagements.total_courserun_problems + then combined_engagements.total_courserun_problems else user_problem_correction.max_problems_per_courserun - end as total_courserun_problems - , case + end as total_courserun_problems, + case when combined_engagements.total_courserun_videos >= user_video_correction.max_videos_per_courserun - then combined_engagements.total_courserun_videos + then combined_engagements.total_courserun_videos else user_video_correction.max_videos_per_courserun - end as total_courserun_videos - , case + end as total_courserun_videos, + case when - combined_engagements.total_courserun_discussions - >= user_discussion_correction.max_discussions_per_courserun - then combined_engagements.total_courserun_discussions + combined_engagements.total_courserun_discussions >= user_discussion_correction.max_discussions_per_courserun + then combined_engagements.total_courserun_discussions else user_discussion_correction.max_discussions_per_courserun end as total_courserun_discussions from combined_runs -inner join combined_engagements - on - combined_runs.courserun_readable_id = combined_engagements.courserun_readable_id - and combined_runs.platform = combined_engagements.platform -inner join combined_enrollments - on - combined_runs.courserun_readable_id = combined_enrollments.courserun_readable_id - and combined_runs.platform = combined_enrollments.platform -left join combined_user_video - on - combined_enrollments.courserun_readable_id = combined_user_video.courserun_readable_id - and combined_enrollments.platform = combined_user_video.platform - and combined_enrollments.user_username = combined_user_video.user_username -left join combined_user_problem - on - combined_enrollments.courserun_readable_id = combined_user_problem.courserun_readable_id - and combined_enrollments.platform = combined_user_problem.platform - and combined_enrollments.user_username = combined_user_problem.user_username -left join combined_user_discussion - on - combined_enrollments.courserun_readable_id = combined_user_discussion.courserun_readable_id - and combined_enrollments.platform = combined_user_discussion.platform - and combined_enrollments.user_username = combined_user_discussion.user_username -left join user_video_correction - on - combined_runs.courserun_readable_id = user_video_correction.courserun_readable_id - and combined_runs.platform = user_video_correction.platform -left join user_problem_correction - on - combined_runs.courserun_readable_id = user_problem_correction.courserun_readable_id - and combined_runs.platform = user_problem_correction.platform -left join user_discussion_correction - on - combined_runs.courserun_readable_id = user_discussion_correction.courserun_readable_id - and combined_runs.platform = user_discussion_correction.platform +inner join + combined_engagements + on combined_runs.courserun_readable_id = combined_engagements.courserun_readable_id + and combined_runs.platform = combined_engagements.platform +inner join + combined_enrollments + on combined_runs.courserun_readable_id = combined_enrollments.courserun_readable_id + and combined_runs.platform = combined_enrollments.platform +left join + combined_user_video + on combined_enrollments.courserun_readable_id = combined_user_video.courserun_readable_id + and combined_enrollments.platform = combined_user_video.platform + and combined_enrollments.user_username = combined_user_video.user_username +left join + combined_user_problem + on combined_enrollments.courserun_readable_id = combined_user_problem.courserun_readable_id + and combined_enrollments.platform = combined_user_problem.platform + and combined_enrollments.user_username = combined_user_problem.user_username +left join + combined_user_discussion + on combined_enrollments.courserun_readable_id = combined_user_discussion.courserun_readable_id + and combined_enrollments.platform = combined_user_discussion.platform + and combined_enrollments.user_username = combined_user_discussion.user_username +left join + user_video_correction + on combined_runs.courserun_readable_id = user_video_correction.courserun_readable_id + and combined_runs.platform = user_video_correction.platform +left join + user_problem_correction + on combined_runs.courserun_readable_id = user_problem_correction.courserun_readable_id + and combined_runs.platform = user_problem_correction.platform +left join + user_discussion_correction + on combined_runs.courserun_readable_id = user_discussion_correction.courserun_readable_id + and combined_runs.platform = user_discussion_correction.platform diff --git a/src/ol_dbt/models/marts/combined/marts__combined_video_engagements.sql b/src/ol_dbt/models/marts/combined/marts__combined_video_engagements.sql index 39177f87b..5df154e3c 100644 --- a/src/ol_dbt/models/marts/combined/marts__combined_video_engagements.sql +++ b/src/ol_dbt/models/marts/combined/marts__combined_video_engagements.sql @@ -1,126 +1,116 @@ -with combined_video_engagements as ( - select - '{{ var("mitxonline") }}' as platform - , user_username - , courserun_readable_id - , useractivity_video_id - , useractivity_page_url - , useractivity_event_type - , useractivity_timestamp - , useractivity_video_duration - , useractivity_video_currenttime - , useractivity_video_old_time - , useractivity_video_new_time - from {{ ref('int__mitxonline__user_courseactivity_video') }} +with + combined_video_engagements as ( + select + '{{ var("mitxonline") }}' as platform, + user_username, + courserun_readable_id, + useractivity_video_id, + useractivity_page_url, + useractivity_event_type, + useractivity_timestamp, + useractivity_video_duration, + useractivity_video_currenttime, + useractivity_video_old_time, + useractivity_video_new_time + from {{ ref("int__mitxonline__user_courseactivity_video") }} - union all + union all - select - '{{ var("edxorg") }}' as platform - , user_username - , courserun_readable_id - , useractivity_video_id - , useractivity_page_url - , useractivity_event_type - , useractivity_timestamp - , useractivity_video_duration - , useractivity_video_currenttime - , useractivity_video_old_time - , useractivity_video_new_time - from {{ ref('int__edxorg__mitx_user_courseactivity_video') }} + select + '{{ var("edxorg") }}' as platform, + user_username, + courserun_readable_id, + useractivity_video_id, + useractivity_page_url, + useractivity_event_type, + useractivity_timestamp, + useractivity_video_duration, + useractivity_video_currenttime, + useractivity_video_old_time, + useractivity_video_new_time + from {{ ref("int__edxorg__mitx_user_courseactivity_video") }} - union all + union all - select - '{{ var("mitxpro") }}' as platform - , user_username - , courserun_readable_id - , useractivity_video_id - , useractivity_page_url - , useractivity_event_type - , useractivity_timestamp - , useractivity_video_duration - , useractivity_video_currenttime - , useractivity_video_old_time - , useractivity_video_new_time - from {{ ref('int__mitxpro__user_courseactivity_video') }} + select + '{{ var("mitxpro") }}' as platform, + user_username, + courserun_readable_id, + useractivity_video_id, + useractivity_page_url, + useractivity_event_type, + useractivity_timestamp, + useractivity_video_duration, + useractivity_video_currenttime, + useractivity_video_old_time, + useractivity_video_new_time + from {{ ref("int__mitxpro__user_courseactivity_video") }} - union all + union all - select - '{{ var("residential") }}' as platform - , user_username - , courserun_readable_id - , useractivity_video_id - , useractivity_page_url - , useractivity_event_type - , useractivity_timestamp - , useractivity_video_duration - , useractivity_video_currenttime - , useractivity_video_old_time - , useractivity_video_new_time - from {{ ref('int__mitxresidential__user_courseactivity_video') }} -) - -, combined_video_structure as ( - select * from {{ ref('int__combined__course_videos') }} -) - -, combined_runs as ( - select * from {{ ref('int__combined__course_runs') }} -) - -, combined_users as ( - select * from {{ ref('int__combined__users') }} - -) + select + '{{ var("residential") }}' as platform, + user_username, + courserun_readable_id, + useractivity_video_id, + useractivity_page_url, + useractivity_event_type, + useractivity_timestamp, + useractivity_video_duration, + useractivity_video_currenttime, + useractivity_video_old_time, + useractivity_video_new_time + from {{ ref("int__mitxresidential__user_courseactivity_video") }} + ), + combined_video_structure as (select * from {{ ref("int__combined__course_videos") }}), + combined_runs as (select * from {{ ref("int__combined__course_runs") }}), + combined_users as (select * from {{ ref("int__combined__users") }}) select - combined_video_engagements.platform - , combined_video_engagements.user_username - , combined_video_engagements.courserun_readable_id - , combined_video_engagements.useractivity_video_id as video_id - , combined_video_structure.video_edx_id - , combined_video_structure.video_title - , combined_video_structure.video_index - , combined_video_structure.chapter_title - , combined_video_structure.chapter_id - , combined_video_engagements.useractivity_page_url as page_url - , combined_video_engagements.useractivity_event_type as video_event_type - , combined_video_engagements.useractivity_timestamp as video_event_timestamp - , combined_video_engagements.useractivity_video_currenttime as video_currenttime - , combined_video_engagements.useractivity_video_old_time as video_old_time - , combined_video_engagements.useractivity_video_new_time as video_new_time - , combined_users.user_hashed_id - , combined_users.user_full_name - , combined_users.user_email - , combined_users.user_address_country as user_country_code - , combined_users.user_highest_education - , combined_users.user_gender - , combined_runs.courserun_title - , combined_runs.course_readable_id - , combined_runs.course_number - , combined_runs.courserun_is_current - , combined_runs.courserun_start_on - , combined_runs.courserun_end_on - , coalesce( - combined_video_structure.video_duration - , combined_video_engagements.useractivity_video_duration + combined_video_engagements.platform, + combined_video_engagements.user_username, + combined_video_engagements.courserun_readable_id, + combined_video_engagements.useractivity_video_id as video_id, + combined_video_structure.video_edx_id, + combined_video_structure.video_title, + combined_video_structure.video_index, + combined_video_structure.chapter_title, + combined_video_structure.chapter_id, + combined_video_engagements.useractivity_page_url as page_url, + combined_video_engagements.useractivity_event_type as video_event_type, + combined_video_engagements.useractivity_timestamp as video_event_timestamp, + combined_video_engagements.useractivity_video_currenttime as video_currenttime, + combined_video_engagements.useractivity_video_old_time as video_old_time, + combined_video_engagements.useractivity_video_new_time as video_new_time, + combined_users.user_hashed_id, + combined_users.user_full_name, + combined_users.user_email, + combined_users.user_address_country as user_country_code, + combined_users.user_highest_education, + combined_users.user_gender, + combined_runs.courserun_title, + combined_runs.course_readable_id, + combined_runs.course_number, + combined_runs.courserun_is_current, + combined_runs.courserun_start_on, + combined_runs.courserun_end_on, + coalesce( + combined_video_structure.video_duration, combined_video_engagements.useractivity_video_duration ) as video_duration from combined_video_engagements -inner join combined_runs - on - combined_video_engagements.courserun_readable_id = combined_runs.courserun_readable_id - and combined_video_engagements.platform = combined_runs.platform -left join combined_video_structure - on - combined_video_engagements.courserun_readable_id = combined_video_structure.courserun_readable_id - and combined_video_engagements.useractivity_video_id = combined_video_structure.video_id - and combined_video_engagements.platform = combined_video_structure.platform -left join combined_users - on - combined_video_engagements.user_username = combined_users.user_username - and combined_video_engagements.platform = combined_users.platform +inner join + combined_runs + on combined_video_engagements.courserun_readable_id = combined_runs.courserun_readable_id + and combined_video_engagements.platform = combined_runs.platform +left join + combined_video_structure + on combined_video_engagements.courserun_readable_id = combined_video_structure.courserun_readable_id + and combined_video_engagements.useractivity_video_id = combined_video_structure.video_id + and combined_video_engagements.platform = combined_video_structure.platform +left join + combined_users + on combined_video_engagements.user_username = combined_users.user_username + and combined_video_engagements.platform = combined_users.platform where combined_video_engagements.useractivity_event_type in ('play_video', 'seek_video', 'complete_video', 'pause_video', 'stop_video') diff --git a/src/ol_dbt/models/marts/micromasters/marts__micromasters_course_certificates.sql b/src/ol_dbt/models/marts/micromasters/marts__micromasters_course_certificates.sql index c2bd2f629..f1c195665 100644 --- a/src/ol_dbt/models/marts/micromasters/marts__micromasters_course_certificates.sql +++ b/src/ol_dbt/models/marts/micromasters/marts__micromasters_course_certificates.sql @@ -1,36 +1,30 @@ -with course_certificates as ( - select * from {{ ref('int__micromasters__course_certificates') }} -) - -, grades as ( - select * - from {{ ref('int__micromasters__course_grades') }} -) +with + course_certificates as (select * from {{ ref("int__micromasters__course_certificates") }}), + grades as (select * from {{ ref("int__micromasters__course_grades") }}) select - course_certificates.program_title - , course_certificates.mitxonline_program_id - , course_certificates.micromasters_program_id - , course_certificates.courserun_title - , course_certificates.courserun_readable_id - , course_certificates.courserun_platform - , course_certificates.course_number - , course_certificates.user_edxorg_username - , course_certificates.user_mitxonline_username - , course_certificates.user_full_name - , course_certificates.user_country - , course_certificates.user_email - , course_certificates.courseruncertificate_url - , course_certificates.courseruncertificate_created_on - , grades.grade - , grades.is_passing + course_certificates.program_title, + course_certificates.mitxonline_program_id, + course_certificates.micromasters_program_id, + course_certificates.courserun_title, + course_certificates.courserun_readable_id, + course_certificates.courserun_platform, + course_certificates.course_number, + course_certificates.user_edxorg_username, + course_certificates.user_mitxonline_username, + course_certificates.user_full_name, + course_certificates.user_country, + course_certificates.user_email, + course_certificates.courseruncertificate_url, + course_certificates.courseruncertificate_created_on, + grades.grade, + grades.is_passing from course_certificates -left join grades - on - course_certificates.user_email = grades.user_email - and course_certificates.courserun_readable_id = grades.courserun_readable_id - and - ( - course_certificates.mitxonline_program_id = grades.mitxonline_program_id - or course_certificates.micromasters_program_id = grades.micromasters_program_id - ) +left join + grades + on course_certificates.user_email = grades.user_email + and course_certificates.courserun_readable_id = grades.courserun_readable_id + and ( + course_certificates.mitxonline_program_id = grades.mitxonline_program_id + or course_certificates.micromasters_program_id = grades.micromasters_program_id + ) diff --git a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql index 2a1190c53..0626ddbc7 100644 --- a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql +++ b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql @@ -1,49 +1,42 @@ -with micromasters_exam_grades as ( - select * from {{ ref('int__micromasters__dedp_proctored_exam_grades') }} -) - -, mitxonline_exam_grades as ( - select * from {{ ref('int__mitxonline__proctored_exam_grades') }} -) - -, micromasters_users as ( - select * from {{ ref('int__micromasters__users') }} -) +with + micromasters_exam_grades as (select * from {{ ref("int__micromasters__dedp_proctored_exam_grades") }}), + mitxonline_exam_grades as (select * from {{ ref("int__mitxonline__proctored_exam_grades") }}), + micromasters_users as (select * from {{ ref("int__micromasters__users") }}) select - course_number - , course_title - , examrun_readable_id as examrun_courserun_readable_id - , user_edxorg_username - , user_mitxonline_username - , user_full_name - , user_micromasters_email - , user_mitxonline_email - , examrun_passing_grade as proctoredexamgrade_passing_grade - , proctoredexamgrade_percentage_grade - , proctoredexamgrade_created_on - , examrun_semester as semester + course_number, + course_title, + examrun_readable_id as examrun_courserun_readable_id, + user_edxorg_username, + user_mitxonline_username, + user_full_name, + user_micromasters_email, + user_mitxonline_email, + examrun_passing_grade as proctoredexamgrade_passing_grade, + proctoredexamgrade_percentage_grade, + proctoredexamgrade_created_on, + examrun_semester as semester from micromasters_exam_grades union all select - mitxonline_exam_grades.course_number - , mitxonline_exam_grades.course_title - , mitxonline_exam_grades.courserun_readable_id as examrun_courserun_readable_id - , mitxonline_exam_grades.user_edxorg_username - , mitxonline_exam_grades.user_username as user_mitxonline_username - , mitxonline_exam_grades.user_full_name - , micromasters_users.user_email as user_micromasters_email - , mitxonline_exam_grades.user_email as user_mitxonline_email - , mitxonline_exam_grades.proctoredexamgrade_passing_grade - , mitxonline_exam_grades.proctoredexamgrade_grade as proctoredexamgrade_percentage_grade - , mitxonline_exam_grades.proctoredexamgrade_created_on - , mitxonline_exam_grades.semester + mitxonline_exam_grades.course_number, + mitxonline_exam_grades.course_title, + mitxonline_exam_grades.courserun_readable_id as examrun_courserun_readable_id, + mitxonline_exam_grades.user_edxorg_username, + mitxonline_exam_grades.user_username as user_mitxonline_username, + mitxonline_exam_grades.user_full_name, + micromasters_users.user_email as user_micromasters_email, + mitxonline_exam_grades.user_email as user_mitxonline_email, + mitxonline_exam_grades.proctoredexamgrade_passing_grade, + mitxonline_exam_grades.proctoredexamgrade_grade as proctoredexamgrade_percentage_grade, + mitxonline_exam_grades.proctoredexamgrade_created_on, + mitxonline_exam_grades.semester from mitxonline_exam_grades left join micromasters_users on mitxonline_exam_grades.user_username = micromasters_users.user_mitxonline_username -left join micromasters_exam_grades - on - mitxonline_exam_grades.courserun_readable_id = micromasters_exam_grades.examrun_readable_id - and mitxonline_exam_grades.user_username = micromasters_exam_grades.user_mitxonline_username +left join + micromasters_exam_grades + on mitxonline_exam_grades.courserun_readable_id = micromasters_exam_grades.examrun_readable_id + and mitxonline_exam_grades.user_username = micromasters_exam_grades.user_mitxonline_username where micromasters_exam_grades.user_mitxonline_username is null and micromasters_exam_grades.examrun_readable_id is null diff --git a/src/ol_dbt/models/marts/micromasters/marts__micromasters_program_certificates.sql b/src/ol_dbt/models/marts/micromasters/marts__micromasters_program_certificates.sql index ce6cb2ff3..340dbd166 100644 --- a/src/ol_dbt/models/marts/micromasters/marts__micromasters_program_certificates.sql +++ b/src/ol_dbt/models/marts/micromasters/marts__micromasters_program_certificates.sql @@ -1,7 +1,4 @@ -with program_certificates as ( - select * - from {{ ref('int__micromasters__program_certificates') }} -) +with program_certificates as (select * from {{ ref("int__micromasters__program_certificates") }}) select * from program_certificates diff --git a/src/ol_dbt/models/marts/micromasters/marts__micromasters_summary.sql b/src/ol_dbt/models/marts/micromasters/marts__micromasters_summary.sql index ec7d931a6..553bfb5e4 100644 --- a/src/ol_dbt/models/marts/micromasters/marts__micromasters_summary.sql +++ b/src/ol_dbt/models/marts/micromasters/marts__micromasters_summary.sql @@ -1,95 +1,84 @@ -with micromasters_enrollments as ( - select * - from {{ ref('int__micromasters__course_enrollments') }} -) - -, micromasters_course_certificates as ( - select * - from {{ ref('int__micromasters__course_certificates') }} -) - -, micromasters_program_certificates as ( - select * - from {{ ref('int__micromasters__program_certificates') }} -) - -, mitx_programs as ( - select * - from {{ ref('int__mitx__programs') }} -) - -, enrollments as ( - select - case when - mitx_programs.is_dedp_program = true then 'Data, Economics, and Design of Policy' - else micromasters_enrollments.program_title end as program_title - , count( - distinct micromasters_enrollments.courserun_readable_id - || micromasters_enrollments.user_email - ) as total_enrollments - , count(distinct micromasters_enrollments.user_email) as unique_users - , count(distinct micromasters_enrollments.user_address_country) as unique_countries - , count(distinct case - when micromasters_enrollments.courserunenrollment_enrollment_mode = 'verified' - then (micromasters_enrollments.courserun_readable_id || micromasters_enrollments.user_email) - end) as verified_enrollments - , count(distinct case - when micromasters_enrollments.courserunenrollment_enrollment_mode = 'verified' - then micromasters_enrollments.user_email - end) as unique_verified_users - from micromasters_enrollments - inner join mitx_programs - on - micromasters_enrollments.micromasters_program_id = mitx_programs.micromasters_program_id +with + micromasters_enrollments as (select * from {{ ref("int__micromasters__course_enrollments") }}), + micromasters_course_certificates as (select * from {{ ref("int__micromasters__course_certificates") }}), + micromasters_program_certificates as (select * from {{ ref("int__micromasters__program_certificates") }}), + mitx_programs as (select * from {{ ref("int__mitx__programs") }}), + enrollments as ( + select + case + when mitx_programs.is_dedp_program = true + then 'Data, Economics, and Design of Policy' + else micromasters_enrollments.program_title + end as program_title, + count( + distinct micromasters_enrollments.courserun_readable_id || micromasters_enrollments.user_email + ) as total_enrollments, + count(distinct micromasters_enrollments.user_email) as unique_users, + count(distinct micromasters_enrollments.user_address_country) as unique_countries, + count( + distinct case + when micromasters_enrollments.courserunenrollment_enrollment_mode = 'verified' + then (micromasters_enrollments.courserun_readable_id || micromasters_enrollments.user_email) + end + ) as verified_enrollments, + count( + distinct case + when micromasters_enrollments.courserunenrollment_enrollment_mode = 'verified' + then micromasters_enrollments.user_email + end + ) as unique_verified_users + from micromasters_enrollments + inner join + mitx_programs + on micromasters_enrollments.micromasters_program_id = mitx_programs.micromasters_program_id or micromasters_enrollments.mitxonline_program_id = mitx_programs.mitxonline_program_id - group by 1 -) - -, course_certs as ( - select - case when - mitx_programs.is_dedp_program = true then 'Data, Economics, and Design of Policy' - else micromasters_course_certificates.program_title end as program_title - , count( - distinct - micromasters_course_certificates.courserun_readable_id - || micromasters_course_certificates.user_email - ) as course_certificates - , count(distinct micromasters_course_certificates.user_email) as unique_course_certificate_earners - from micromasters_course_certificates - inner join mitx_programs - on - micromasters_course_certificates.micromasters_program_id = mitx_programs.micromasters_program_id + group by 1 + ), + course_certs as ( + select + case + when mitx_programs.is_dedp_program = true + then 'Data, Economics, and Design of Policy' + else micromasters_course_certificates.program_title + end as program_title, + count( + distinct + micromasters_course_certificates.courserun_readable_id || micromasters_course_certificates.user_email + ) as course_certificates, + count(distinct micromasters_course_certificates.user_email) as unique_course_certificate_earners + from micromasters_course_certificates + inner join + mitx_programs + on micromasters_course_certificates.micromasters_program_id = mitx_programs.micromasters_program_id or micromasters_course_certificates.mitxonline_program_id = mitx_programs.mitxonline_program_id - group by 1 -) - -, program_certs as ( - select - case when - mitx_programs.is_dedp_program = true then 'Data, Economics, and Design of Policy' - else micromasters_program_certificates.program_title end as program_title - , count(distinct micromasters_program_certificates.user_email) as program_certificates - from micromasters_program_certificates - inner join mitx_programs - on - micromasters_program_certificates.micromasters_program_id = mitx_programs.micromasters_program_id + group by 1 + ), + program_certs as ( + select + case + when mitx_programs.is_dedp_program = true + then 'Data, Economics, and Design of Policy' + else micromasters_program_certificates.program_title + end as program_title, + count(distinct micromasters_program_certificates.user_email) as program_certificates + from micromasters_program_certificates + inner join + mitx_programs + on micromasters_program_certificates.micromasters_program_id = mitx_programs.micromasters_program_id or micromasters_program_certificates.mitxonline_program_id = mitx_programs.mitxonline_program_id - group by 1 -) + group by 1 + ) select - enrollments.program_title - , enrollments.total_enrollments - , enrollments.unique_users - , enrollments.unique_countries - , enrollments.verified_enrollments - , enrollments.unique_verified_users - , course_certs.course_certificates - , course_certs.unique_course_certificate_earners - , program_certs.program_certificates + enrollments.program_title, + enrollments.total_enrollments, + enrollments.unique_users, + enrollments.unique_countries, + enrollments.verified_enrollments, + enrollments.unique_verified_users, + course_certs.course_certificates, + course_certs.unique_course_certificate_earners, + program_certs.program_certificates from enrollments -left join course_certs - on enrollments.program_title = course_certs.program_title -left join program_certs - on enrollments.program_title = program_certs.program_title +left join course_certs on enrollments.program_title = course_certs.program_title +left join program_certs on enrollments.program_title = program_certs.program_title diff --git a/src/ol_dbt/models/marts/micromasters/marts__micromasters_summary_timeseries.sql b/src/ol_dbt/models/marts/micromasters/marts__micromasters_summary_timeseries.sql index c3ec7bd6a..31bea7fb1 100644 --- a/src/ol_dbt/models/marts/micromasters/marts__micromasters_summary_timeseries.sql +++ b/src/ol_dbt/models/marts/micromasters/marts__micromasters_summary_timeseries.sql @@ -1,222 +1,193 @@ -with enrollments as ( - select * - from {{ ref('int__micromasters__course_enrollments') }} -) - -, course_certificates as ( - select * - from {{ ref('int__micromasters__course_certificates') }} -) - -, program_certificates as ( - select * - from {{ ref('int__micromasters__program_certificates') }} -) - -, timeseries as ( - select end_date - from unnest( - sequence(date '2014-01-01', current_date, interval '1' month) || current_date - ) as t(end_date) -- noqa -) - -, enrollments_by_program as ( - select - timeseries.end_date - , enrollments.micromasters_program_id - , enrollments.mitxonline_program_id - , arbitrary(enrollments.program_title) as program_title - , count(*) as total_enrollments - , count(distinct enrollments.user_email) as unique_users - , count(distinct enrollments.user_address_country) as unique_countries - , count_if(enrollments.courserunenrollment_enrollment_mode = 'verified') as verified_enrollments - , count( - distinct case - when enrollments.courserunenrollment_enrollment_mode = 'verified' then enrollments.user_email - end - ) - as unique_verified_users - from enrollments - inner join timeseries on from_iso8601_timestamp(enrollments.courserunenrollment_created_on) < timeseries.end_date - group by enrollments.micromasters_program_id, enrollments.mitxonline_program_id, timeseries.end_date -) - -, enrollments_total as ( - select - timeseries.end_date - , 'total' as program_title - , 0 as micromasters_program_id - , 0 as mitxonline_program_id - , count( - distinct concat_ws( - ',', cast(enrollments.user_id as varchar), enrollments.platform, enrollments.courserun_readable_id - ) - ) - as total_enrollments - , count(distinct enrollments.user_email) as unique_users - , count(distinct enrollments.user_address_country) as unique_countries - , count_if(enrollments.courserunenrollment_enrollment_mode = 'verified') as verified_enrollments - , count( - distinct case - when enrollments.courserunenrollment_enrollment_mode = 'verified' then enrollments.user_email - end - ) - as unique_verified_users - from enrollments - inner join timeseries on from_iso8601_timestamp(enrollments.courserunenrollment_created_on) < timeseries.end_date - group by timeseries.end_date -) - -, enrollments_combined as ( - select - end_date - , program_title - , micromasters_program_id - , mitxonline_program_id - , total_enrollments - , unique_users - , unique_countries - , verified_enrollments - , unique_verified_users - from enrollments_by_program - union all - select - end_date - , program_title - , micromasters_program_id - , mitxonline_program_id - , total_enrollments - , unique_users - , unique_countries - , verified_enrollments - , unique_verified_users - from enrollments_total -) - -, course_certificates_by_program as ( - select - timeseries.end_date - , course_certificates.micromasters_program_id - , course_certificates.mitxonline_program_id - , count(*) as course_certificates - , count(distinct course_certificates.user_email) as unique_course_certificate_earners - from course_certificates - inner join - timeseries - on from_iso8601_timestamp(course_certificates.courseruncertificate_created_on) < timeseries.end_date - group by course_certificates.micromasters_program_id, course_certificates.mitxonline_program_id, timeseries.end_date -) - -, course_certificates_total as ( - select - timeseries.end_date - , 0 as micromasters_program_id - , 0 as mitxonline_program_id - , count( - distinct concat_ws( - ',' - , course_certificates.courserun_readable_id - , course_certificates.user_edxorg_username - , course_certificates.user_mitxonline_username - ) - ) - as course_certificates - , count(distinct course_certificates.user_email) as unique_course_certificate_earners - from course_certificates - inner join - timeseries - on from_iso8601_timestamp(course_certificates.courseruncertificate_created_on) < timeseries.end_date - group by timeseries.end_date -) - -, course_certificates_combined as ( - select - end_date - , micromasters_program_id - , mitxonline_program_id - , course_certificates - , unique_course_certificate_earners - from course_certificates_by_program - union all - select - end_date - , micromasters_program_id - , mitxonline_program_id - , course_certificates - , unique_course_certificate_earners - from course_certificates_total -) - -, program_certificates_by_program as ( - select - timeseries.end_date - , program_certificates.micromasters_program_id - , program_certificates.mitxonline_program_id - , count(*) as program_certificates - from program_certificates - inner join - timeseries - on from_iso8601_timestamp(program_certificates.program_completion_timestamp) < timeseries.end_date - group by - program_certificates.micromasters_program_id, program_certificates.mitxonline_program_id, timeseries.end_date -) - -, program_certificates_total as ( - select - timeseries.end_date - , 0 as micromasters_program_id - , 0 as mitxonline_program_id - , count(*) as program_certificates - from program_certificates - inner join - timeseries - on from_iso8601_timestamp(program_certificates.program_completion_timestamp) < timeseries.end_date - group by timeseries.end_date -) - -, program_certificates_combined as ( - select - end_date - , micromasters_program_id - , mitxonline_program_id - , program_certificates - from program_certificates_by_program - union all - select - end_date - , micromasters_program_id - , mitxonline_program_id - , program_certificates - from program_certificates_total -) +with + enrollments as (select * from {{ ref("int__micromasters__course_enrollments") }}), + course_certificates as (select * from {{ ref("int__micromasters__course_certificates") }}), + program_certificates as (select * from {{ ref("int__micromasters__program_certificates") }}), + timeseries as ( + select end_date + from unnest(sequence(date '2014-01-01', current_date, interval '1' month) || current_date) as t(end_date) -- noqa + ), + enrollments_by_program as ( + select + timeseries.end_date, + enrollments.micromasters_program_id, + enrollments.mitxonline_program_id, + arbitrary(enrollments.program_title) as program_title, + count(*) as total_enrollments, + count(distinct enrollments.user_email) as unique_users, + count(distinct enrollments.user_address_country) as unique_countries, + count_if(enrollments.courserunenrollment_enrollment_mode = 'verified') as verified_enrollments, + count( + distinct case + when enrollments.courserunenrollment_enrollment_mode = 'verified' then enrollments.user_email + end + ) as unique_verified_users + from enrollments + inner join + timeseries on from_iso8601_timestamp(enrollments.courserunenrollment_created_on) < timeseries.end_date + group by enrollments.micromasters_program_id, enrollments.mitxonline_program_id, timeseries.end_date + ), + enrollments_total as ( + select + timeseries.end_date, + 'total' as program_title, + 0 as micromasters_program_id, + 0 as mitxonline_program_id, + count( + distinct concat_ws( + ',', cast(enrollments.user_id as varchar), enrollments.platform, enrollments.courserun_readable_id + ) + ) as total_enrollments, + count(distinct enrollments.user_email) as unique_users, + count(distinct enrollments.user_address_country) as unique_countries, + count_if(enrollments.courserunenrollment_enrollment_mode = 'verified') as verified_enrollments, + count( + distinct case + when enrollments.courserunenrollment_enrollment_mode = 'verified' then enrollments.user_email + end + ) as unique_verified_users + from enrollments + inner join + timeseries on from_iso8601_timestamp(enrollments.courserunenrollment_created_on) < timeseries.end_date + group by timeseries.end_date + ), + enrollments_combined as ( + select + end_date, + program_title, + micromasters_program_id, + mitxonline_program_id, + total_enrollments, + unique_users, + unique_countries, + verified_enrollments, + unique_verified_users + from enrollments_by_program + union all + select + end_date, + program_title, + micromasters_program_id, + mitxonline_program_id, + total_enrollments, + unique_users, + unique_countries, + verified_enrollments, + unique_verified_users + from enrollments_total + ), + course_certificates_by_program as ( + select + timeseries.end_date, + course_certificates.micromasters_program_id, + course_certificates.mitxonline_program_id, + count(*) as course_certificates, + count(distinct course_certificates.user_email) as unique_course_certificate_earners + from course_certificates + inner join + timeseries + on from_iso8601_timestamp(course_certificates.courseruncertificate_created_on) < timeseries.end_date + group by + course_certificates.micromasters_program_id, course_certificates.mitxonline_program_id, timeseries.end_date + ), + course_certificates_total as ( + select + timeseries.end_date, + 0 as micromasters_program_id, + 0 as mitxonline_program_id, + count( + distinct concat_ws( + ',', + course_certificates.courserun_readable_id, + course_certificates.user_edxorg_username, + course_certificates.user_mitxonline_username + ) + ) as course_certificates, + count(distinct course_certificates.user_email) as unique_course_certificate_earners + from course_certificates + inner join + timeseries + on from_iso8601_timestamp(course_certificates.courseruncertificate_created_on) < timeseries.end_date + group by timeseries.end_date + ), + course_certificates_combined as ( + select + end_date, + micromasters_program_id, + mitxonline_program_id, + course_certificates, + unique_course_certificate_earners + from course_certificates_by_program + union all + select + end_date, + micromasters_program_id, + mitxonline_program_id, + course_certificates, + unique_course_certificate_earners + from course_certificates_total + ), + program_certificates_by_program as ( + select + timeseries.end_date, + program_certificates.micromasters_program_id, + program_certificates.mitxonline_program_id, + count(*) as program_certificates + from program_certificates + inner join + timeseries + on from_iso8601_timestamp(program_certificates.program_completion_timestamp) < timeseries.end_date + group by + program_certificates.micromasters_program_id, + program_certificates.mitxonline_program_id, + timeseries.end_date + ), + program_certificates_total as ( + select + timeseries.end_date, + 0 as micromasters_program_id, + 0 as mitxonline_program_id, + count(*) as program_certificates + from program_certificates + inner join + timeseries + on from_iso8601_timestamp(program_certificates.program_completion_timestamp) < timeseries.end_date + group by timeseries.end_date + ), + program_certificates_combined as ( + select end_date, micromasters_program_id, mitxonline_program_id, program_certificates + from program_certificates_by_program + union all + select end_date, micromasters_program_id, mitxonline_program_id, program_certificates + from program_certificates_total + ) select - enrollments_combined.program_title - , enrollments_combined.total_enrollments - , enrollments_combined.unique_users - , enrollments_combined.unique_countries - , enrollments_combined.verified_enrollments - , enrollments_combined.unique_verified_users - , course_certificates_combined.course_certificates - , course_certificates_combined.unique_course_certificate_earners - , program_certificates_combined.program_certificates - , to_iso8601(enrollments_combined.end_date) as end_date + enrollments_combined.program_title, + enrollments_combined.total_enrollments, + enrollments_combined.unique_users, + enrollments_combined.unique_countries, + enrollments_combined.verified_enrollments, + enrollments_combined.unique_verified_users, + course_certificates_combined.course_certificates, + course_certificates_combined.unique_course_certificate_earners, + program_certificates_combined.program_certificates, + to_iso8601(enrollments_combined.end_date) as end_date from enrollments_combined -left join course_certificates_combined - on - ( - enrollments_combined.micromasters_program_id = course_certificates_combined.micromasters_program_id - or enrollments_combined.mitxonline_program_id = course_certificates_combined.mitxonline_program_id - ) - and enrollments_combined.end_date = course_certificates_combined.end_date -left join program_certificates_combined - on - ( - enrollments_combined.micromasters_program_id = program_certificates_combined.micromasters_program_id - or enrollments_combined.mitxonline_program_id = program_certificates_combined.mitxonline_program_id - ) - and enrollments_combined.end_date = program_certificates_combined.end_date +left join + course_certificates_combined + on ( + enrollments_combined.micromasters_program_id = course_certificates_combined.micromasters_program_id + or enrollments_combined.mitxonline_program_id = course_certificates_combined.mitxonline_program_id + ) + and enrollments_combined.end_date = course_certificates_combined.end_date +left join + program_certificates_combined + on ( + enrollments_combined.micromasters_program_id = program_certificates_combined.micromasters_program_id + or enrollments_combined.mitxonline_program_id = program_certificates_combined.mitxonline_program_id + ) + and enrollments_combined.end_date = program_certificates_combined.end_date order by - enrollments_combined.micromasters_program_id - , enrollments_combined.mitxonline_program_id - , to_iso8601(enrollments_combined.end_date) + enrollments_combined.micromasters_program_id, + enrollments_combined.mitxonline_program_id, + to_iso8601(enrollments_combined.end_date) diff --git a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_course_certificates.sql b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_course_certificates.sql index 425f49e55..524a24d24 100644 --- a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_course_certificates.sql +++ b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_course_certificates.sql @@ -1,22 +1,18 @@ -with course_certificates as ( - select * from {{ ref('int__mitx__courserun_certificates') }} - where platform = '{{ var("mitxonline") }}' -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} -) +with + course_certificates as ( + select * from {{ ref("int__mitx__courserun_certificates") }} where platform = '{{ var("mitxonline") }}' + ), + users as (select * from {{ ref("int__mitxonline__users") }}) select - course_certificates.course_number - , course_certificates.courserun_title - , course_certificates.courserun_readable_id - , course_certificates.courseruncertificate_url - , course_certificates.courseruncertificate_created_on - , course_certificates.user_mitxonline_username as user_username - , course_certificates.user_email - , course_certificates.user_full_name - , users.openedx_user_id + course_certificates.course_number, + course_certificates.courserun_title, + course_certificates.courserun_readable_id, + course_certificates.courseruncertificate_url, + course_certificates.courseruncertificate_created_on, + course_certificates.user_mitxonline_username as user_username, + course_certificates.user_email, + course_certificates.user_full_name, + users.openedx_user_id from course_certificates -left join users - on course_certificates.user_email = users.user_email +left join users on course_certificates.user_email = users.user_email diff --git a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_course_engagements_daily.sql b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_course_engagements_daily.sql index 9ea26b1ed..98a3f6266 100644 --- a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_course_engagements_daily.sql +++ b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_course_engagements_daily.sql @@ -1,103 +1,70 @@ -with course_activities_daily as ( - select * from {{ ref('int__mitxonline__user_courseactivities_daily') }} -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} - where openedx_user_id is not null -) - -, video as ( - select * from {{ ref('int__mitxonline__user_courseactivity_video') }} -) - -, problem_submitted as ( - select * from {{ ref('int__mitxonline__user_courseactivity_problemsubmitted') }} -) - -, discussion as ( - select * from {{ ref('int__mitxonline__user_courseactivity_discussion') }} -) - -, course_runs as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, problem_submitted_users_daily as ( - select - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - , count(*) as num_problem_submitted - from problem_submitted - group by - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) -) - -, played_video_users_daily as ( - select - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - , count(*) as num_video_played - from video - where useractivity_event_type = 'play_video' - group by - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) -) - -, participated_discussion_users_daily as ( - select - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date - , count(*) as num_discussion_participated - from discussion - -- edx.forum.comment.created, edx.forum.response.created, edx.forum.thread.created - where useractivity_event_type like 'edx.forum.%.created' - group by - user_username - , courserun_readable_id - , date(from_iso8601_timestamp(useractivity_timestamp)) -) +with + course_activities_daily as (select * from {{ ref("int__mitxonline__user_courseactivities_daily") }}), + users as (select * from {{ ref("int__mitxonline__users") }} where openedx_user_id is not null), + video as (select * from {{ ref("int__mitxonline__user_courseactivity_video") }}), + problem_submitted as (select * from {{ ref("int__mitxonline__user_courseactivity_problemsubmitted") }}), + discussion as (select * from {{ ref("int__mitxonline__user_courseactivity_discussion") }}), + course_runs as (select * from {{ ref("int__mitxonline__course_runs") }}), + problem_submitted_users_daily as ( + select + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date, + count(*) as num_problem_submitted + from problem_submitted + group by user_username, courserun_readable_id, date(from_iso8601_timestamp(useractivity_timestamp)) + ), + played_video_users_daily as ( + select + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date, + count(*) as num_video_played + from video + where useractivity_event_type = 'play_video' + group by user_username, courserun_readable_id, date(from_iso8601_timestamp(useractivity_timestamp)) + ), + participated_discussion_users_daily as ( + select + user_username, + courserun_readable_id, + date(from_iso8601_timestamp(useractivity_timestamp)) as courseactivity_date, + count(*) as num_discussion_participated + from discussion + -- edx.forum.comment.created, edx.forum.response.created, edx.forum.thread.created + where useractivity_event_type like 'edx.forum.%.created' + group by user_username, courserun_readable_id, date(from_iso8601_timestamp(useractivity_timestamp)) + ) select - course_activities_daily.courseactivity_date - , course_activities_daily.user_username - , users.user_full_name - , users.user_email - , course_activities_daily.courserun_readable_id - , course_runs.courserun_title - , course_runs.course_number - , course_runs.courserun_start_on - , course_runs.courserun_end_on - , course_activities_daily.courseactivity_num_events as num_events - , problem_submitted_users_daily.num_problem_submitted - , played_video_users_daily.num_video_played - , participated_discussion_users_daily.num_discussion_participated + course_activities_daily.courseactivity_date, + course_activities_daily.user_username, + users.user_full_name, + users.user_email, + course_activities_daily.courserun_readable_id, + course_runs.courserun_title, + course_runs.course_number, + course_runs.courserun_start_on, + course_runs.courserun_end_on, + course_activities_daily.courseactivity_num_events as num_events, + problem_submitted_users_daily.num_problem_submitted, + played_video_users_daily.num_video_played, + participated_discussion_users_daily.num_discussion_participated from course_activities_daily -inner join course_runs - on - course_activities_daily.courserun_readable_id = course_runs.courserun_readable_id -left join users - on - course_activities_daily.user_username = users.user_username -left join problem_submitted_users_daily - on - course_activities_daily.courseactivity_date = problem_submitted_users_daily.courseactivity_date - and course_activities_daily.courserun_readable_id = problem_submitted_users_daily.courserun_readable_id - and course_activities_daily.user_username = problem_submitted_users_daily.user_username -left join played_video_users_daily - on - course_activities_daily.courseactivity_date = played_video_users_daily.courseactivity_date - and course_activities_daily.courserun_readable_id = played_video_users_daily.courserun_readable_id - and course_activities_daily.user_username = played_video_users_daily.user_username -left join participated_discussion_users_daily - on - course_activities_daily.courseactivity_date = participated_discussion_users_daily.courseactivity_date - and course_activities_daily.courserun_readable_id = participated_discussion_users_daily.courserun_readable_id - and course_activities_daily.user_username = participated_discussion_users_daily.user_username +inner join course_runs on course_activities_daily.courserun_readable_id = course_runs.courserun_readable_id +left join users on course_activities_daily.user_username = users.user_username +left join + problem_submitted_users_daily + on course_activities_daily.courseactivity_date = problem_submitted_users_daily.courseactivity_date + and course_activities_daily.courserun_readable_id = problem_submitted_users_daily.courserun_readable_id + and course_activities_daily.user_username = problem_submitted_users_daily.user_username +left join + played_video_users_daily + on course_activities_daily.courseactivity_date = played_video_users_daily.courseactivity_date + and course_activities_daily.courserun_readable_id = played_video_users_daily.courserun_readable_id + and course_activities_daily.user_username = played_video_users_daily.user_username +left join + participated_discussion_users_daily + on course_activities_daily.courseactivity_date = participated_discussion_users_daily.courseactivity_date + and course_activities_daily.courserun_readable_id = participated_discussion_users_daily.courserun_readable_id + and course_activities_daily.user_username = participated_discussion_users_daily.user_username diff --git a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_course_enrollments.sql b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_course_enrollments.sql index 0ec0800fb..b13ba7360 100644 --- a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_course_enrollments.sql +++ b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_course_enrollments.sql @@ -1,29 +1,26 @@ -with enrollments as ( - select * from {{ ref('int__mitx__courserun_enrollments') }} - where platform = '{{ var("mitxonline") }}' -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} -) +with + enrollments as ( + select * from {{ ref("int__mitx__courserun_enrollments") }} where platform = '{{ var("mitxonline") }}' + ), + users as (select * from {{ ref("int__mitxonline__users") }}) select - enrollments.course_number - , enrollments.courserun_title - , enrollments.courserun_readable_id - , enrollments.courserunenrollment_is_active - , enrollments.courserunenrollment_enrollment_mode - , enrollments.courserunenrollment_enrollment_status - , enrollments.courserunenrollment_created_on - , enrollments.user_mitxonline_username as user_username - , enrollments.user_email - , enrollments.user_full_name - , users.user_address_country - , users.user_highest_education - , users.user_gender - , users.user_birth_year - , users.user_company - , users.user_job_title - , users.user_industry + enrollments.course_number, + enrollments.courserun_title, + enrollments.courserun_readable_id, + enrollments.courserunenrollment_is_active, + enrollments.courserunenrollment_enrollment_mode, + enrollments.courserunenrollment_enrollment_status, + enrollments.courserunenrollment_created_on, + enrollments.user_mitxonline_username as user_username, + enrollments.user_email, + enrollments.user_full_name, + users.user_address_country, + users.user_highest_education, + users.user_gender, + users.user_birth_year, + users.user_company, + users.user_job_title, + users.user_industry from enrollments left join users on enrollments.user_email = users.user_email diff --git a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_discussions.sql b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_discussions.sql index 469af8136..4fcb15386 100644 --- a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_discussions.sql +++ b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_discussions.sql @@ -1,34 +1,26 @@ -with discussions as ( - select * from {{ ref('int__mitxonline__user_courseactivity_discussion') }} -) - -, course_runs as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} -) +with + discussions as (select * from {{ ref("int__mitxonline__user_courseactivity_discussion") }}), + course_runs as (select * from {{ ref("int__mitxonline__course_runs") }}), + users as (select * from {{ ref("int__mitxonline__users") }}) select - discussions.user_username - , discussions.courserun_readable_id - , discussions.useractivity_discussion_page_url as page_url - , discussions.useractivity_path as discussion_event_path - , discussions.useractivity_event_type as discussion_event_type - , discussions.useractivity_timestamp as discussion_event_timestamp - , discussions.useractivity_discussion_block_id as discussion_id - , discussions.useractivity_discussion_block_name as discussion_name - , discussions.useractivity_discussion_post_id as post_id - , discussions.useractivity_discussion_post_title as post_title - , users.user_full_name - , users.user_email - , course_runs.courserun_title - , course_runs.course_number - , course_runs.courserun_start_on - , course_runs.courserun_end_on + discussions.user_username, + discussions.courserun_readable_id, + discussions.useractivity_discussion_page_url as page_url, + discussions.useractivity_path as discussion_event_path, + discussions.useractivity_event_type as discussion_event_type, + discussions.useractivity_timestamp as discussion_event_timestamp, + discussions.useractivity_discussion_block_id as discussion_id, + discussions.useractivity_discussion_block_name as discussion_name, + discussions.useractivity_discussion_post_id as post_id, + discussions.useractivity_discussion_post_title as post_title, + users.user_full_name, + users.user_email, + course_runs.courserun_title, + course_runs.course_number, + course_runs.courserun_start_on, + course_runs.courserun_end_on from discussions inner join course_runs on discussions.courserun_readable_id = course_runs.courserun_readable_id -left join users on - discussions.user_username = users.user_username - and discussions.openedx_user_id = users.openedx_user_id +left join + users on discussions.user_username = users.user_username and discussions.openedx_user_id = users.openedx_user_id diff --git a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_problem_submissions.sql b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_problem_submissions.sql index e94b70505..c15a80b7e 100644 --- a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_problem_submissions.sql +++ b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_problem_submissions.sql @@ -1,42 +1,37 @@ -with problem_response as ( - select - * - , row_number() over ( - partition by courserun_readable_id, user_username, useractivity_problem_id - order by useractivity_problem_attempts desc - ) as most_recent_num - from {{ ref('int__mitxonline__user_courseactivity_problemcheck') }} -) - -, course_runs as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} -) +with + problem_response as ( + select + *, + row_number() over ( + partition by courserun_readable_id, user_username, useractivity_problem_id + order by useractivity_problem_attempts desc + ) as most_recent_num + from {{ ref("int__mitxonline__user_courseactivity_problemcheck") }} + ), + course_runs as (select * from {{ ref("int__mitxonline__course_runs") }}), + users as (select * from {{ ref("int__mitxonline__users") }}) select - problem_response.user_username - , problem_response.courserun_readable_id - , problem_response.useractivity_problem_id as problem_id - , problem_response.useractivity_problem_name as problem_name - , problem_response.useractivity_problem_attempts as num_attempts - , problem_response.useractivity_problem_student_answers as student_answers - , problem_response.useractivity_problem_success as problem_success - , problem_response.useractivity_problem_current_grade as problem_grade - , problem_response.useractivity_problem_max_grade as problem_max_grade - , problem_response.useractivity_timestamp as problem_submission_timestamp - , users.user_full_name - , users.user_email - , course_runs.courserun_title - , course_runs.course_number - , course_runs.courserun_start_on - , course_runs.courserun_end_on - , if(problem_response.most_recent_num = 1, true, false) as is_most_recent_attempt + problem_response.user_username, + problem_response.courserun_readable_id, + problem_response.useractivity_problem_id as problem_id, + problem_response.useractivity_problem_name as problem_name, + problem_response.useractivity_problem_attempts as num_attempts, + problem_response.useractivity_problem_student_answers as student_answers, + problem_response.useractivity_problem_success as problem_success, + problem_response.useractivity_problem_current_grade as problem_grade, + problem_response.useractivity_problem_max_grade as problem_max_grade, + problem_response.useractivity_timestamp as problem_submission_timestamp, + users.user_full_name, + users.user_email, + course_runs.courserun_title, + course_runs.course_number, + course_runs.courserun_start_on, + course_runs.courserun_end_on, + if(problem_response.most_recent_num = 1, true, false) as is_most_recent_attempt from problem_response inner join course_runs on problem_response.courserun_readable_id = course_runs.courserun_readable_id -left join users - on - problem_response.user_username = users.user_username - and problem_response.openedx_user_id = users.openedx_user_id +left join + users + on problem_response.user_username = users.user_username + and problem_response.openedx_user_id = users.openedx_user_id diff --git a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_problem_summary.sql b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_problem_summary.sql index 37afc93bc..417a92261 100644 --- a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_problem_summary.sql +++ b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_problem_summary.sql @@ -1,76 +1,55 @@ -with showanswers as ( - select * from {{ ref('int__mitxonline__user_courseactivity_showanswer') }} -) - -, showanswers_stats as ( - select - user_username - , courserun_readable_id - , useractivity_problem_id - , count(*) as num_showanswer - from showanswers - group by - user_username - , courserun_readable_id - , useractivity_problem_id -) - -, problem_attempts as ( - select - * - , row_number() over ( - partition by courserun_readable_id, user_username, useractivity_problem_id - order by useractivity_problem_attempts desc - ) as row_num - from {{ ref('int__mitxonline__user_courseactivity_problemcheck') }} -) - -, most_recent_attempts as ( - select * - from problem_attempts - where row_num = 1 -) - -, course_runs as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} - where openedx_user_id is not null -) - -, combined as ( - select - showanswers_stats.num_showanswer - , most_recent_attempts.useractivity_problem_attempts as num_attempts - , most_recent_attempts.useractivity_problem_success as problem_success - , coalesce(showanswers_stats.user_username, most_recent_attempts.user_username) as user_username - , coalesce(showanswers_stats.courserun_readable_id, most_recent_attempts.courserun_readable_id) - as courserun_readable_id - , coalesce(showanswers_stats.useractivity_problem_id, most_recent_attempts.useractivity_problem_id) - as useractivity_problem_id - from showanswers_stats - full outer join most_recent_attempts - on - showanswers_stats.user_username = most_recent_attempts.user_username +with + showanswers as (select * from {{ ref("int__mitxonline__user_courseactivity_showanswer") }}), + showanswers_stats as ( + select user_username, courserun_readable_id, useractivity_problem_id, count(*) as num_showanswer + from showanswers + group by user_username, courserun_readable_id, useractivity_problem_id + ), + problem_attempts as ( + select + *, + row_number() over ( + partition by courserun_readable_id, user_username, useractivity_problem_id + order by useractivity_problem_attempts desc + ) as row_num + from {{ ref("int__mitxonline__user_courseactivity_problemcheck") }} + ), + most_recent_attempts as (select * from problem_attempts where row_num = 1), + course_runs as (select * from {{ ref("int__mitxonline__course_runs") }}), + users as (select * from {{ ref("int__mitxonline__users") }} where openedx_user_id is not null), + combined as ( + select + showanswers_stats.num_showanswer, + most_recent_attempts.useractivity_problem_attempts as num_attempts, + most_recent_attempts.useractivity_problem_success as problem_success, + coalesce(showanswers_stats.user_username, most_recent_attempts.user_username) as user_username, + coalesce( + showanswers_stats.courserun_readable_id, most_recent_attempts.courserun_readable_id + ) as courserun_readable_id, + coalesce( + showanswers_stats.useractivity_problem_id, most_recent_attempts.useractivity_problem_id + ) as useractivity_problem_id + from showanswers_stats + full outer join + most_recent_attempts + on showanswers_stats.user_username = most_recent_attempts.user_username and showanswers_stats.courserun_readable_id = most_recent_attempts.courserun_readable_id and showanswers_stats.useractivity_problem_id = most_recent_attempts.useractivity_problem_id -) + ) select - combined.user_username - , combined.courserun_readable_id - , combined.useractivity_problem_id as problem_id - , combined.num_showanswer - , combined.num_attempts - , combined.problem_success - , users.user_full_name - , users.user_email - , course_runs.courserun_title - , course_runs.course_number - , course_runs.courserun_start_on - , course_runs.courserun_end_on + combined.user_username, + combined.courserun_readable_id, + combined.useractivity_problem_id as problem_id, + combined.num_showanswer, + combined.num_attempts, + combined.problem_success, + users.user_full_name, + users.user_email, + course_runs.courserun_title, + course_runs.course_number, + course_runs.courserun_start_on, + course_runs.courserun_end_on from combined inner join course_runs on combined.courserun_readable_id = course_runs.courserun_readable_id left join users on combined.user_username = users.user_username diff --git a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_user_profiles.sql b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_user_profiles.sql index f9858340f..b17702647 100644 --- a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_user_profiles.sql +++ b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_user_profiles.sql @@ -1,38 +1,27 @@ -with users as ( - select * from {{ ref('int__mitxonline__users') }} -) - -, income as ( - select * from {{ ref('int__mitxonline__flexiblepricing_flexiblepriceapplication') }} -) - -, user_income as ( - select - user_id - , flexiblepriceapplication_income_usd - , flexiblepriceapplication_original_income - , flexiblepriceapplication_original_currency - , rank() over ( - partition by user_id - order by flexiblepriceapplication_updated_on desc - ) as rnk - from income -) +with + users as (select * from {{ ref("int__mitxonline__users") }}), + income as (select * from {{ ref("int__mitxonline__flexiblepricing_flexiblepriceapplication") }}), + user_income as ( + select + user_id, + flexiblepriceapplication_income_usd, + flexiblepriceapplication_original_income, + flexiblepriceapplication_original_currency, + rank() over (partition by user_id order by flexiblepriceapplication_updated_on desc) as rnk + from income + ) select - users.user_username - , users.user_full_name - , users.user_email - , users.user_address_country - , users.user_address_state - , users.user_birth_year - , users.user_gender - , users.user_highest_education - , user_income.flexiblepriceapplication_income_usd as latest_income_usd - , user_income.flexiblepriceapplication_original_income as latest_original_income - , user_income.flexiblepriceapplication_original_currency as latest_original_currency + users.user_username, + users.user_full_name, + users.user_email, + users.user_address_country, + users.user_address_state, + users.user_birth_year, + users.user_gender, + users.user_highest_education, + user_income.flexiblepriceapplication_income_usd as latest_income_usd, + user_income.flexiblepriceapplication_original_income as latest_original_income, + user_income.flexiblepriceapplication_original_currency as latest_original_currency from users -left join user_income - on - users.user_id = user_income.user_id - and user_income.rnk = 1 +left join user_income on users.user_id = user_income.user_id and user_income.rnk = 1 diff --git a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_video_engagements.sql b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_video_engagements.sql index 4823511be..11137f681 100644 --- a/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_video_engagements.sql +++ b/src/ol_dbt/models/marts/mitxonline/marts__mitxonline_video_engagements.sql @@ -1,61 +1,47 @@ -with video as ( - select * from {{ ref('int__mitxonline__user_courseactivity_video') }} -) - -, video_structure as ( - select - * - , json_query(coursestructure_block_metadata, 'lax $.edx_video_id' omit quotes) as video_edx_uuid - , element_at(split(coursestructure_block_id, '@'), -1) as video_id - from {{ ref('int__mitxonline__course_structure') }} - where coursestructure_block_category = 'video' and coursestructure_is_latest = true -) - -, mitxonline_videos as ( - select * from {{ ref('int__mitxonline__courserun_videos') }} -) - -, course_runs as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, users as ( - select * from {{ ref('int__mitxonline__users') }} -) +with + video as (select * from {{ ref("int__mitxonline__user_courseactivity_video") }}), + video_structure as ( + select + *, + json_query(coursestructure_block_metadata, 'lax $.edx_video_id' omit quotes) as video_edx_uuid, + element_at(split(coursestructure_block_id, '@'), -1) as video_id + from {{ ref("int__mitxonline__course_structure") }} + where coursestructure_block_category = 'video' and coursestructure_is_latest = true + ), + mitxonline_videos as (select * from {{ ref("int__mitxonline__courserun_videos") }}), + course_runs as (select * from {{ ref("int__mitxonline__course_runs") }}), + users as (select * from {{ ref("int__mitxonline__users") }}) select - video.user_username - , video.courserun_readable_id - , video.useractivity_video_id as video_id - , mitxonline_videos.video_edx_uuid - , video_structure.coursestructure_block_title as video_title - , video_structure.coursestructure_chapter_title as section_title - , video_structure.coursestructure_block_index - , video.useractivity_page_url as page_url - , video.useractivity_event_type as video_event_type - , video.useractivity_timestamp as video_event_timestamp - , video.useractivity_video_currenttime as video_currenttime - , video.useractivity_video_old_time as video_old_time - , video.useractivity_video_new_time as video_new_time - , users.user_full_name - , users.user_email - , course_runs.courserun_title - , course_runs.course_number - , course_runs.courserun_start_on - , course_runs.courserun_end_on - , coalesce(mitxonline_videos.video_duration, video.useractivity_video_duration) as video_duration + video.user_username, + video.courserun_readable_id, + video.useractivity_video_id as video_id, + mitxonline_videos.video_edx_uuid, + video_structure.coursestructure_block_title as video_title, + video_structure.coursestructure_chapter_title as section_title, + video_structure.coursestructure_block_index, + video.useractivity_page_url as page_url, + video.useractivity_event_type as video_event_type, + video.useractivity_timestamp as video_event_timestamp, + video.useractivity_video_currenttime as video_currenttime, + video.useractivity_video_old_time as video_old_time, + video.useractivity_video_new_time as video_new_time, + users.user_full_name, + users.user_email, + course_runs.courserun_title, + course_runs.course_number, + course_runs.courserun_start_on, + course_runs.courserun_end_on, + coalesce(mitxonline_videos.video_duration, video.useractivity_video_duration) as video_duration from video inner join course_runs on video.courserun_readable_id = course_runs.courserun_readable_id -left join video_structure - on - video.courserun_readable_id = video_structure.courserun_readable_id - and video.useractivity_video_id = video_structure.video_id -left join mitxonline_videos - on - video_structure.courserun_readable_id = mitxonline_videos.courserun_readable_id - and video_structure.video_edx_uuid = mitxonline_videos.video_edx_uuid -left join users - on - video.user_username = users.user_username - and video.openedx_user_id = users.openedx_user_id +left join + video_structure + on video.courserun_readable_id = video_structure.courserun_readable_id + and video.useractivity_video_id = video_structure.video_id +left join + mitxonline_videos + on video_structure.courserun_readable_id = mitxonline_videos.courserun_readable_id + and video_structure.video_edx_uuid = mitxonline_videos.video_edx_uuid +left join users on video.user_username = users.user_username and video.openedx_user_id = users.openedx_user_id where video.useractivity_event_type in ('play_video', 'seek_video', 'complete_video', 'pause_video', 'stop_video') diff --git a/src/ol_dbt/models/marts/mitxpro/marts__mitxpro_all_coupons.sql b/src/ol_dbt/models/marts/mitxpro/marts__mitxpro_all_coupons.sql index 005cc0083..874a4e985 100644 --- a/src/ol_dbt/models/marts/mitxpro/marts__mitxpro_all_coupons.sql +++ b/src/ol_dbt/models/marts/mitxpro/marts__mitxpro_all_coupons.sql @@ -1,249 +1,151 @@ -with allcoupons as ( - select * - from {{ ref('int__mitxpro__ecommerce_allcoupons') }} -) - -, allorders as ( - select * - from {{ ref('int__mitxpro__ecommerce_allorders') }} -) - -, ecommerce_coupon as ( - select * - from {{ ref('int__mitxpro__ecommerce_coupon') }} -) - -, ecommerce_couponpaymentversion as ( - select * - from {{ ref('int__mitxpro__ecommerce_couponpaymentversion') }} -) - -, ecommerce_company as ( - select * - from {{ ref('int__mitxpro__ecommerce_company') }} -) - -, ecommerce_line as ( - select * - from {{ ref('int__mitxpro__ecommerce_line') }} -) - -, mitxpro__programruns as ( - select * - from {{ ref('int__mitxpro__program_runs') }} -) - -, ecommerce_order as ( - select * - from {{ ref('int__mitxpro__ecommerce_order') }} -) - -, b2b_order as ( - select * - from {{ ref('int__mitxpro__b2becommerce_b2border') }} -) - -, b2b_receipt as ( - select * - from {{ ref('int__mitxpro__b2becommerce_b2breceipt') }} -) - -, product as ( - select * - from {{ ref('int__mitxpro__ecommerce_product') }} -) - -, couponproduct as ( - select * - from {{ ref('int__mitxpro__ecommerce_couponproduct') }} -) - -, pull_req_ref_num as ( - select - ecommerce_coupon.coupon_id - , b2b_receipt.b2breceipt_reference_number - from b2b_order - inner join b2b_receipt - on b2b_order.b2border_id = b2b_receipt.b2border_id - inner join ecommerce_couponpaymentversion - on b2b_order.couponpaymentversion_id = ecommerce_couponpaymentversion.couponpaymentversion_id - inner join ecommerce_coupon - on ecommerce_couponpaymentversion.couponpayment_name = ecommerce_coupon.couponpayment_name - group by - ecommerce_coupon.coupon_id - , b2b_receipt.b2breceipt_reference_number -) - -, redeemed_coupons as ( - select coupon_id - from allorders - where redeemed = true - group by coupon_id -) - -, redeemed_b2b_coupons as ( - select - b2bcoupon_id - , b2border_contract_number - from allorders - where - redeemed = true - and coupon_id is null - group by - b2bcoupon_id - , b2border_contract_number -) - -, coupons_used_by_name as ( - select - ecommerce_coupon.couponpayment_name - , count(distinct ecommerce_order.order_id) as coupons_used_count - from ecommerce_order - inner join ecommerce_coupon - on ecommerce_order.coupon_id = ecommerce_coupon.coupon_id - where ecommerce_order.order_state = 'fulfilled' - group by ecommerce_coupon.couponpayment_name -) - -, pull_product_regular as ( - select - allorders.coupon_id - , ecommerce_couponpaymentversion.couponpaymentversion_id - , coalesce( - allorders.courserun_readable_id - , mitxpro__programruns.programrun_readable_id - , allorders.program_readable_id - ) as product_readable_id - from allorders - left join ecommerce_line - on allorders.line_id = ecommerce_line.line_id - left join mitxpro__programruns - on ecommerce_line.programrun_id = mitxpro__programruns.programrun_id - left join ecommerce_couponpaymentversion - on - allorders.couponpaymentversion_payment_transaction +with + allcoupons as (select * from {{ ref("int__mitxpro__ecommerce_allcoupons") }}), + allorders as (select * from {{ ref("int__mitxpro__ecommerce_allorders") }}), + ecommerce_coupon as (select * from {{ ref("int__mitxpro__ecommerce_coupon") }}), + ecommerce_couponpaymentversion as (select * from {{ ref("int__mitxpro__ecommerce_couponpaymentversion") }}), + ecommerce_company as (select * from {{ ref("int__mitxpro__ecommerce_company") }}), + ecommerce_line as (select * from {{ ref("int__mitxpro__ecommerce_line") }}), + mitxpro__programruns as (select * from {{ ref("int__mitxpro__program_runs") }}), + ecommerce_order as (select * from {{ ref("int__mitxpro__ecommerce_order") }}), + b2b_order as (select * from {{ ref("int__mitxpro__b2becommerce_b2border") }}), + b2b_receipt as (select * from {{ ref("int__mitxpro__b2becommerce_b2breceipt") }}), + product as (select * from {{ ref("int__mitxpro__ecommerce_product") }}), + couponproduct as (select * from {{ ref("int__mitxpro__ecommerce_couponproduct") }}), + pull_req_ref_num as ( + select ecommerce_coupon.coupon_id, b2b_receipt.b2breceipt_reference_number + from b2b_order + inner join b2b_receipt on b2b_order.b2border_id = b2b_receipt.b2border_id + inner join + ecommerce_couponpaymentversion + on b2b_order.couponpaymentversion_id = ecommerce_couponpaymentversion.couponpaymentversion_id + inner join + ecommerce_coupon on ecommerce_couponpaymentversion.couponpayment_name = ecommerce_coupon.couponpayment_name + group by ecommerce_coupon.coupon_id, b2b_receipt.b2breceipt_reference_number + ), + redeemed_coupons as (select coupon_id from allorders where redeemed = true group by coupon_id), + redeemed_b2b_coupons as ( + select b2bcoupon_id, b2border_contract_number + from allorders + where redeemed = true and coupon_id is null + group by b2bcoupon_id, b2border_contract_number + ), + coupons_used_by_name as ( + select ecommerce_coupon.couponpayment_name, count(distinct ecommerce_order.order_id) as coupons_used_count + from ecommerce_order + inner join ecommerce_coupon on ecommerce_order.coupon_id = ecommerce_coupon.coupon_id + where ecommerce_order.order_state = 'fulfilled' + group by ecommerce_coupon.couponpayment_name + ), + pull_product_regular as ( + select + allorders.coupon_id, + ecommerce_couponpaymentversion.couponpaymentversion_id, + coalesce( + allorders.courserun_readable_id, + mitxpro__programruns.programrun_readable_id, + allorders.program_readable_id + ) as product_readable_id + from allorders + left join ecommerce_line on allorders.line_id = ecommerce_line.line_id + left join mitxpro__programruns on ecommerce_line.programrun_id = mitxpro__programruns.programrun_id + left join + ecommerce_couponpaymentversion + on allorders.couponpaymentversion_payment_transaction = ecommerce_couponpaymentversion.couponpaymentversion_payment_transaction - where - allorders.order_id is not null - and allorders.coupon_id is not null - group by - allorders.coupon_id - , ecommerce_couponpaymentversion.couponpaymentversion_id - , coalesce( - allorders.courserun_readable_id - , mitxpro__programruns.programrun_readable_id - , allorders.program_readable_id - ) -) - -, pull_product_b2b as ( - select - allorders.b2bcoupon_id - , allorders.b2border_contract_number - , coalesce( - allorders.courserun_readable_id - , mitxpro__programruns.programrun_readable_id - , allorders.program_readable_id - ) as product_readable_id - from allorders - left join ecommerce_line - on allorders.line_id = ecommerce_line.line_id - left join mitxpro__programruns - on ecommerce_line.programrun_id = mitxpro__programruns.programrun_id - where - allorders.order_id is not null - and allorders.b2bcoupon_id is not null - group by - allorders.b2bcoupon_id - , allorders.b2border_contract_number - , coalesce( - allorders.courserun_readable_id - , mitxpro__programruns.programrun_readable_id - , allorders.program_readable_id - ) -) - -, pull_product_nonorders as ( - select - cp.coupon_id - , coalesce( - p.courserun_readable_id - , p.program_readable_id - ) as product_readable_id - from couponproduct as cp - inner join product as p - on cp.product_id = p.product_id - group by - cp.coupon_id - , coalesce( - p.courserun_readable_id - , p.program_readable_id - ) -) + where allorders.order_id is not null and allorders.coupon_id is not null + group by + allorders.coupon_id, + ecommerce_couponpaymentversion.couponpaymentversion_id, + coalesce( + allorders.courserun_readable_id, + mitxpro__programruns.programrun_readable_id, + allorders.program_readable_id + ) + ), + pull_product_b2b as ( + select + allorders.b2bcoupon_id, + allorders.b2border_contract_number, + coalesce( + allorders.courserun_readable_id, + mitxpro__programruns.programrun_readable_id, + allorders.program_readable_id + ) as product_readable_id + from allorders + left join ecommerce_line on allorders.line_id = ecommerce_line.line_id + left join mitxpro__programruns on ecommerce_line.programrun_id = mitxpro__programruns.programrun_id + where allorders.order_id is not null and allorders.b2bcoupon_id is not null + group by + allorders.b2bcoupon_id, + allorders.b2border_contract_number, + coalesce( + allorders.courserun_readable_id, + mitxpro__programruns.programrun_readable_id, + allorders.program_readable_id + ) + ), + pull_product_nonorders as ( + select cp.coupon_id, coalesce(p.courserun_readable_id, p.program_readable_id) as product_readable_id + from couponproduct as cp + inner join product as p on cp.product_id = p.product_id + group by cp.coupon_id, coalesce(p.courserun_readable_id, p.program_readable_id) + ) select - allcoupons.coupon_code - , allcoupons.coupon_name - , allcoupons.coupon_created_on - , allcoupons.payment_transaction - , allcoupons.discount_amount - , allcoupons.coupon_type - , allcoupons.discount_source - , ecommerce_couponpaymentversion.couponpaymentversion_activated_on as activated_on - , ecommerce_couponpaymentversion.couponpaymentversion_expires_on as expires_on - , allcoupons.coupon_source_table - , allcoupons.b2bcoupon_id - , allcoupons.coupon_id - , ecommerce_couponpaymentversion.couponpaymentversion_num_coupon_codes - , ecommerce_couponpaymentversion.couponpaymentversion_max_redemptions - , ecommerce_couponpaymentversion.couponpayment_name - , ecommerce_couponpaymentversion.couponpaymentversion_id - , ecommerce_couponpaymentversion.couponpaymentversion_created_on - , ecommerce_couponpaymentversion.couponpaymentversion_discount_amount_text - , ecommerce_company.company_name - , redeemed_b2b_coupons.b2border_contract_number - , pull_req_ref_num.b2breceipt_reference_number - , coupons_used_by_name.coupons_used_count - , coalesce( - pull_product_regular.product_readable_id - , pull_product_b2b.product_readable_id - , pull_product_nonorders.product_readable_id - ) as product_readable_id - , case - when - redeemed_coupons.coupon_id is not null - or redeemed_b2b_coupons.b2bcoupon_id is not null - then true - when - redeemed_coupons.coupon_id is null - and redeemed_b2b_coupons.b2bcoupon_id is null - then false + allcoupons.coupon_code, + allcoupons.coupon_name, + allcoupons.coupon_created_on, + allcoupons.payment_transaction, + allcoupons.discount_amount, + allcoupons.coupon_type, + allcoupons.discount_source, + ecommerce_couponpaymentversion.couponpaymentversion_activated_on as activated_on, + ecommerce_couponpaymentversion.couponpaymentversion_expires_on as expires_on, + allcoupons.coupon_source_table, + allcoupons.b2bcoupon_id, + allcoupons.coupon_id, + ecommerce_couponpaymentversion.couponpaymentversion_num_coupon_codes, + ecommerce_couponpaymentversion.couponpaymentversion_max_redemptions, + ecommerce_couponpaymentversion.couponpayment_name, + ecommerce_couponpaymentversion.couponpaymentversion_id, + ecommerce_couponpaymentversion.couponpaymentversion_created_on, + ecommerce_couponpaymentversion.couponpaymentversion_discount_amount_text, + ecommerce_company.company_name, + redeemed_b2b_coupons.b2border_contract_number, + pull_req_ref_num.b2breceipt_reference_number, + coupons_used_by_name.coupons_used_count, + coalesce( + pull_product_regular.product_readable_id, + pull_product_b2b.product_readable_id, + pull_product_nonorders.product_readable_id + ) as product_readable_id, + case + when redeemed_coupons.coupon_id is not null or redeemed_b2b_coupons.b2bcoupon_id is not null + then true + when redeemed_coupons.coupon_id is null and redeemed_b2b_coupons.b2bcoupon_id is null + then false end as redeemed from allcoupons -left join redeemed_coupons - on allcoupons.coupon_id = redeemed_coupons.coupon_id -left join redeemed_b2b_coupons - on allcoupons.b2bcoupon_id = redeemed_b2b_coupons.b2bcoupon_id -left join ecommerce_coupon - on allcoupons.coupon_id = ecommerce_coupon.coupon_id -left join ecommerce_couponpaymentversion +left join redeemed_coupons on allcoupons.coupon_id = redeemed_coupons.coupon_id +left join redeemed_b2b_coupons on allcoupons.b2bcoupon_id = redeemed_b2b_coupons.b2bcoupon_id +left join ecommerce_coupon on allcoupons.coupon_id = ecommerce_coupon.coupon_id +left join + ecommerce_couponpaymentversion on ecommerce_coupon.couponpayment_name = ecommerce_couponpaymentversion.couponpayment_name -left join ecommerce_company - on ecommerce_couponpaymentversion.company_id = ecommerce_company.company_id -left join coupons_used_by_name - on ecommerce_couponpaymentversion.couponpayment_name = coupons_used_by_name.couponpayment_name -left join pull_product_regular - on - allcoupons.coupon_id = pull_product_regular.coupon_id - and ecommerce_couponpaymentversion.couponpaymentversion_id = pull_product_regular.couponpaymentversion_id -left join pull_product_b2b - on - allcoupons.b2bcoupon_id = pull_product_b2b.b2bcoupon_id - and redeemed_b2b_coupons.b2border_contract_number = pull_product_b2b.b2border_contract_number -left join pull_req_ref_num - on allcoupons.coupon_id = pull_req_ref_num.coupon_id -left join pull_product_nonorders - on - allcoupons.coupon_id = pull_product_nonorders.coupon_id - and pull_product_regular.product_readable_id is null - and pull_product_b2b.product_readable_id is null +left join ecommerce_company on ecommerce_couponpaymentversion.company_id = ecommerce_company.company_id +left join + coupons_used_by_name on ecommerce_couponpaymentversion.couponpayment_name = coupons_used_by_name.couponpayment_name +left join + pull_product_regular + on allcoupons.coupon_id = pull_product_regular.coupon_id + and ecommerce_couponpaymentversion.couponpaymentversion_id = pull_product_regular.couponpaymentversion_id +left join + pull_product_b2b + on allcoupons.b2bcoupon_id = pull_product_b2b.b2bcoupon_id + and redeemed_b2b_coupons.b2border_contract_number = pull_product_b2b.b2border_contract_number +left join pull_req_ref_num on allcoupons.coupon_id = pull_req_ref_num.coupon_id +left join + pull_product_nonorders + on allcoupons.coupon_id = pull_product_nonorders.coupon_id + and pull_product_regular.product_readable_id is null + and pull_product_b2b.product_readable_id is null diff --git a/src/ol_dbt/models/marts/mitxpro/marts__mitxpro_ecommerce_productlist.sql b/src/ol_dbt/models/marts/mitxpro/marts__mitxpro_ecommerce_productlist.sql index c18a5830e..28c8dc735 100644 --- a/src/ol_dbt/models/marts/mitxpro/marts__mitxpro_ecommerce_productlist.sql +++ b/src/ol_dbt/models/marts/mitxpro/marts__mitxpro_ecommerce_productlist.sql @@ -1,141 +1,102 @@ -with ecommerce_product as ( - select * - from {{ ref('int__mitxpro__ecommerce_product') }} -) - -, ecommerce_productversion as ( - select * - from {{ ref('int__mitxpro__ecommerce_productversion') }} -) - -, course_runs as ( - select * - from {{ ref('int__mitxpro__course_runs') }} -) - -, programs as ( - select * - from {{ ref('int__mitxpro__programs') }} -) - -, program_runs as ( - select * - from {{ ref('int__mitxpro__program_runs') }} -) - -, courses as ( - select * - from {{ ref('int__mitxpro__courses') }} -) - -, course_to_topics as ( - select * - from {{ ref('int__mitxpro__courses_to_topics') }} -) - -, coursetopic as ( - select * - from {{ ref('int__mitxpro__coursetopic') }} -) - -, ecommerce_productversion_latest as ( - select * - from ( - select - ecommerce_productversion.* - , rank() over ( - partition by ecommerce_productversion.product_id - order by ecommerce_productversion.productversion_updated_on desc - ) as myrank - from ecommerce_productversion - ) as a - where myrank = 1 - order by product_id -) - -, ecommerce_course_to_topics as ( - select - course_to_topics.course_id - , array_join(array_agg(coursetopic.coursetopic_name), ', ') as coursetopic_name - from course_to_topics - inner join coursetopic - on course_to_topics.coursetopic_id = coursetopic.coursetopic_id - group by course_to_topics.course_id -) ---there can occationally be multiple topics per course - +with + ecommerce_product as (select * from {{ ref("int__mitxpro__ecommerce_product") }}), + ecommerce_productversion as (select * from {{ ref("int__mitxpro__ecommerce_productversion") }}), + course_runs as (select * from {{ ref("int__mitxpro__course_runs") }}), + programs as (select * from {{ ref("int__mitxpro__programs") }}), + program_runs as (select * from {{ ref("int__mitxpro__program_runs") }}), + courses as (select * from {{ ref("int__mitxpro__courses") }}), + course_to_topics as (select * from {{ ref("int__mitxpro__courses_to_topics") }}), + coursetopic as (select * from {{ ref("int__mitxpro__coursetopic") }}), + ecommerce_productversion_latest as ( + select * + from + ( + select + ecommerce_productversion.*, + rank() over ( + partition by ecommerce_productversion.product_id + order by ecommerce_productversion.productversion_updated_on desc + ) as myrank + from ecommerce_productversion + ) as a + where myrank = 1 + order by product_id + ), + ecommerce_course_to_topics as ( + select course_to_topics.course_id, array_join(array_agg(coursetopic.coursetopic_name), ', ') as coursetopic_name + from course_to_topics + inner join coursetopic on course_to_topics.coursetopic_id = coursetopic.coursetopic_id + group by course_to_topics.course_id + ) +-- there can occationally be multiple topics per course select - 'xPRO' as product_platform - , ecommerce_product.product_id as productid - , course_runs.courserun_title as product_name - , course_runs.courserun_readable_id as product_readable_id - , ecommerce_productversion_latest.productversion_readable_id - , ecommerce_product.product_type - , courses.short_program_code - , cast(ecommerce_productversion_latest.productversion_price as decimal(38, 2)) as list_price - , ecommerce_productversion_latest.productversion_description as product_description - , substring(course_runs.courserun_start_on, 1, 10) as start_date - , substring(course_runs.courserun_end_on, 1, 10) as end_date - , substring(course_runs.courserun_enrollment_start_on, 1, 10) as enrollment_start - , substring(course_runs.courserun_enrollment_end_on, 1, 10) as enrollment_end - , concat( - '', course_runs.courserun_readable_id, '' - ) as link - , concat(programs.program_readable_id, '+', course_runs.courserun_tag) as product_parent_run_id - , courses.cms_coursepage_duration as duration - , courses.cms_coursepage_format as courseware_format - , courses.cms_coursepage_time_commitment as time_commitment - , ecommerce_course_to_topics.coursetopic_name as coursetopic_names - , ecommerce_product.product_is_private - , courses.platform_name + 'xPRO' as product_platform, + ecommerce_product.product_id as productid, + course_runs.courserun_title as product_name, + course_runs.courserun_readable_id as product_readable_id, + ecommerce_productversion_latest.productversion_readable_id, + ecommerce_product.product_type, + courses.short_program_code, + cast(ecommerce_productversion_latest.productversion_price as decimal(38, 2)) as list_price, + ecommerce_productversion_latest.productversion_description as product_description, + substring(course_runs.courserun_start_on, 1, 10) as start_date, + substring(course_runs.courserun_end_on, 1, 10) as end_date, + substring(course_runs.courserun_enrollment_start_on, 1, 10) as enrollment_start, + substring(course_runs.courserun_enrollment_end_on, 1, 10) as enrollment_end, + concat( + '', + course_runs.courserun_readable_id, + '' + ) as link, + concat(programs.program_readable_id, '+', course_runs.courserun_tag) as product_parent_run_id, + courses.cms_coursepage_duration as duration, + courses.cms_coursepage_format as courseware_format, + courses.cms_coursepage_time_commitment as time_commitment, + ecommerce_course_to_topics.coursetopic_name as coursetopic_names, + ecommerce_product.product_is_private, + courses.platform_name from ecommerce_product -left join ecommerce_productversion_latest - on ecommerce_product.product_id = ecommerce_productversion_latest.product_id -inner join course_runs - on ecommerce_product.courserun_id = course_runs.courserun_id -inner join courses - on course_runs.course_id = courses.course_id -left join programs - on courses.program_id = programs.program_id -left join ecommerce_course_to_topics - on course_runs.course_id = ecommerce_course_to_topics.course_id +left join ecommerce_productversion_latest on ecommerce_product.product_id = ecommerce_productversion_latest.product_id +inner join course_runs on ecommerce_product.courserun_id = course_runs.courserun_id +inner join courses on course_runs.course_id = courses.course_id +left join programs on courses.program_id = programs.program_id +left join ecommerce_course_to_topics on course_runs.course_id = ecommerce_course_to_topics.course_id where ecommerce_product.product_type = 'course run' union all select - 'xPRO' as product_platform - , ecommerce_product.product_id as productid - , program_runs.program_title as product_name - , program_runs.programrun_readable_id as product_readable_id - , ecommerce_productversion_latest.productversion_readable_id - , 'programrun' as product_type - , program_runs.short_program_code - , cast(ecommerce_productversion_latest.productversion_price as decimal(38, 2)) as list_price - , ecommerce_productversion_latest.productversion_description as product_description - , substring(program_runs.programrun_start_on, 1, 10) as start_date - , substring(program_runs.programrun_end_on, 1, 10) as end_date - , null as enrollment_start - , null as enrollment_end - , concat( - '', program_runs.programrun_readable_id, '' - ) as link - , null as product_parent_run_id - , programs.cms_programpage_duration as duration - , programs.cms_programpage_format as courseware_format - , programs.cms_programpage_time_commitment as time_commitment - , null as coursetopic_names - , ecommerce_product.product_is_private - , programs.platform_name + 'xPRO' as product_platform, + ecommerce_product.product_id as productid, + program_runs.program_title as product_name, + program_runs.programrun_readable_id as product_readable_id, + ecommerce_productversion_latest.productversion_readable_id, + 'programrun' as product_type, + program_runs.short_program_code, + cast(ecommerce_productversion_latest.productversion_price as decimal(38, 2)) as list_price, + ecommerce_productversion_latest.productversion_description as product_description, + substring(program_runs.programrun_start_on, 1, 10) as start_date, + substring(program_runs.programrun_end_on, 1, 10) as end_date, + null as enrollment_start, + null as enrollment_end, + concat( + '', + program_runs.programrun_readable_id, + '' + ) as link, + null as product_parent_run_id, + programs.cms_programpage_duration as duration, + programs.cms_programpage_format as courseware_format, + programs.cms_programpage_time_commitment as time_commitment, + null as coursetopic_names, + ecommerce_product.product_is_private, + programs.platform_name from ecommerce_product -left join ecommerce_productversion_latest - on ecommerce_product.product_id = ecommerce_productversion_latest.product_id -inner join programs - on ecommerce_product.program_id = programs.program_id -inner join program_runs - on programs.program_id = program_runs.program_id +left join ecommerce_productversion_latest on ecommerce_product.product_id = ecommerce_productversion_latest.product_id +inner join programs on ecommerce_product.program_id = programs.program_id +inner join program_runs on programs.program_id = program_runs.program_id where ecommerce_product.product_type = 'program' diff --git a/src/ol_dbt/models/marts/ocw/marts__ocw_courses.sql b/src/ol_dbt/models/marts/ocw/marts__ocw_courses.sql index ee63a4855..b0e9106b2 100644 --- a/src/ol_dbt/models/marts/ocw/marts__ocw_courses.sql +++ b/src/ol_dbt/models/marts/ocw/marts__ocw_courses.sql @@ -1,33 +1,29 @@ -with courses as ( - select * from {{ ref('int__ocw__courses') }} -) - -, instructors as ( - select - course_uuid - , array_join(array_agg(course_instructor_title), ', ') as course_instructors - from {{ ref('int__ocw__course_instructors') }} - group by course_uuid -) +with + courses as (select * from {{ ref("int__ocw__courses") }}), + instructors as ( + select course_uuid, array_join(array_agg(course_instructor_title), ', ') as course_instructors + from {{ ref("int__ocw__course_instructors") }} + group by course_uuid + ) select - courses.course_readable_id as course_short_id - , courses.course_name - , courses.course_title - , courses.course_term - , courses.course_year - , courses.course_level - , courses.course_primary_course_number - , courses.course_extra_course_numbers - , courses.course_is_live - , courses.course_is_unpublished - , courses.course_has_never_published - , courses.course_live_url - , courses.course_first_published_on - , courses.course_publish_date_updated_on - , courses.course_topics - , courses.course_learning_resource_types - , courses.course_department_numbers - , instructors.course_instructors + courses.course_readable_id as course_short_id, + courses.course_name, + courses.course_title, + courses.course_term, + courses.course_year, + courses.course_level, + courses.course_primary_course_number, + courses.course_extra_course_numbers, + courses.course_is_live, + courses.course_is_unpublished, + courses.course_has_never_published, + courses.course_live_url, + courses.course_first_published_on, + courses.course_publish_date_updated_on, + courses.course_topics, + courses.course_learning_resource_types, + courses.course_department_numbers, + instructors.course_instructors from courses left join instructors on courses.course_uuid = instructors.course_uuid diff --git a/src/ol_dbt/models/migration/edxorg_to_mitxonline_course_runs.sql b/src/ol_dbt/models/migration/edxorg_to_mitxonline_course_runs.sql index cf18db6a1..28bc9bbd5 100644 --- a/src/ol_dbt/models/migration/edxorg_to_mitxonline_course_runs.sql +++ b/src/ol_dbt/models/migration/edxorg_to_mitxonline_course_runs.sql @@ -1,92 +1,70 @@ -with mitx_courses as ( - select * from {{ ref('int__mitx__courses') }} -) - -, mitxonline_courseruns as ( - select * from {{ ref('int__mitxonline__course_runs') }} -) - -, mitxonline_course_department as ( - select - course_id - , array_join(array_agg(coursedepartment_name), ', ') as department_name - from {{ ref('int__mitxonline__course_to_departments') }} - group by course_id -) - -, edx_courseruns as ( - select - * - , element_at(split(courserun_readable_id, '/'), 3) as run_tag - , element_at(split(course_number, '.'), 1) as extracted_department_number - , {{ format_course_id('courserun_readable_id', false) }} as courseware_id - from {{ ref('int__edxorg__mitx_courseruns') }} -) - -, edx_signatories as ( - select - courserun_readable_id - , array_agg(signatory_normalized_name) as signatory_names - from {{ ref('stg__edxorg__s3__course_certificate_signatory') }} - group by courserun_readable_id -) - -, edx_enrollments as ( - select - courserun_readable_id - , count(*) as enrollment_count - from {{ ref('int__edxorg__mitx_courserun_enrollments') }} - group by courserun_readable_id -) - -, edx_certificates as ( - select - courserun_readable_id - , count(*) as certificate_count - from {{ ref('int__edxorg__mitx_courserun_certificates') }} - group by courserun_readable_id -) +with + mitx_courses as (select * from {{ ref("int__mitx__courses") }}), + mitxonline_courseruns as (select * from {{ ref("int__mitxonline__course_runs") }}), + mitxonline_course_department as ( + select course_id, array_join(array_agg(coursedepartment_name), ', ') as department_name + from {{ ref("int__mitxonline__course_to_departments") }} + group by course_id + ), + edx_courseruns as ( + select + *, + element_at(split(courserun_readable_id, '/'), 3) as run_tag, + element_at(split(course_number, '.'), 1) as extracted_department_number, + {{ format_course_id("courserun_readable_id", false) }} as courseware_id + from {{ ref("int__edxorg__mitx_courseruns") }} + ), + edx_signatories as ( + select courserun_readable_id, array_agg(signatory_normalized_name) as signatory_names + from {{ ref("stg__edxorg__s3__course_certificate_signatory") }} + group by courserun_readable_id + ), + edx_enrollments as ( + select courserun_readable_id, count(*) as enrollment_count + from {{ ref("int__edxorg__mitx_courserun_enrollments") }} + group by courserun_readable_id + ), + edx_certificates as ( + select courserun_readable_id, count(*) as certificate_count + from {{ ref("int__edxorg__mitx_courserun_certificates") }} + group by courserun_readable_id + ) select distinct - mitx_courses.mitxonline_course_id - , if( - mitx_courses.mitxonline_course_id is not null - , mitx_courses.course_readable_id - , 'course-v1:MITx+' || mitx_courses.course_number - ) as course_readable_id - , mitx_courses.course_title - , edx_courseruns.courserun_is_self_paced as is_self_paced - , edx_courseruns.courserun_is_published as is_published - , edx_courseruns.courserun_title - , edx_courseruns.courseware_id - , edx_courseruns.run_tag - , from_iso8601_timestamp(edx_courseruns.courserun_enrollment_start_date) as enrollment_start - , from_iso8601_timestamp(edx_courseruns.courserun_enrollment_end_date) as enrollment_end - , from_iso8601_timestamp(edx_courseruns.courserun_start_date) as start_date - , from_iso8601_timestamp(edx_courseruns.courserun_end_date) as end_date - , coalesce( - mitxonline_course_department.department_name - , edx_courseruns.coursedepartment_name - , {{ transform_edx_department_number('edx_courseruns.extracted_department_number') }} - ) as department_name - , edx_signatories.signatory_names - , edx_enrollments.enrollment_count - , edx_certificates.certificate_count + mitx_courses.mitxonline_course_id, + if( + mitx_courses.mitxonline_course_id is not null, + mitx_courses.course_readable_id, + 'course-v1:MITx+' || mitx_courses.course_number + ) as course_readable_id, + mitx_courses.course_title, + edx_courseruns.courserun_is_self_paced as is_self_paced, + edx_courseruns.courserun_is_published as is_published, + edx_courseruns.courserun_title, + edx_courseruns.courseware_id, + edx_courseruns.run_tag, + from_iso8601_timestamp(edx_courseruns.courserun_enrollment_start_date) as enrollment_start, + from_iso8601_timestamp(edx_courseruns.courserun_enrollment_end_date) as enrollment_end, + from_iso8601_timestamp(edx_courseruns.courserun_start_date) as start_date, + from_iso8601_timestamp(edx_courseruns.courserun_end_date) as end_date, + coalesce( + mitxonline_course_department.department_name, + edx_courseruns.coursedepartment_name, + {{ transform_edx_department_number("edx_courseruns.extracted_department_number") }} + ) as department_name, + edx_signatories.signatory_names, + edx_enrollments.enrollment_count, + edx_certificates.certificate_count from mitx_courses -inner join edx_courseruns - on mitx_courses.course_number = edx_courseruns.course_number -inner join edx_enrollments - on edx_courseruns.courserun_readable_id = edx_enrollments.courserun_readable_id -left join edx_certificates - on edx_courseruns.courserun_readable_id = edx_certificates.courserun_readable_id -left join mitxonline_course_department - on mitx_courses.mitxonline_course_id = mitxonline_course_department.course_id -left join mitxonline_courseruns - on - edx_courseruns.course_number = mitxonline_courseruns.course_number - and edx_courseruns.run_tag = mitxonline_courseruns.courserun_tag -left join edx_signatories - on edx_courseruns.courseware_id = edx_signatories.courserun_readable_id +inner join edx_courseruns on mitx_courses.course_number = edx_courseruns.course_number +inner join edx_enrollments on edx_courseruns.courserun_readable_id = edx_enrollments.courserun_readable_id +left join edx_certificates on edx_courseruns.courserun_readable_id = edx_certificates.courserun_readable_id +left join mitxonline_course_department on mitx_courses.mitxonline_course_id = mitxonline_course_department.course_id +left join + mitxonline_courseruns + on edx_courseruns.course_number = mitxonline_courseruns.course_number + and edx_courseruns.run_tag = mitxonline_courseruns.courserun_tag +left join edx_signatories on edx_courseruns.courseware_id = edx_signatories.courserun_readable_id where from_iso8601_timestamp(edx_courseruns.courserun_end_date) < current_date -- unenrollable runs and mitxonline_courseruns.course_number is null -- not already in mitxonline course runs diff --git a/src/ol_dbt/models/reporting/chatbot_usage_report.sql b/src/ol_dbt/models/reporting/chatbot_usage_report.sql index 75d5d54df..389c8cfe3 100644 --- a/src/ol_dbt/models/reporting/chatbot_usage_report.sql +++ b/src/ol_dbt/models/reporting/chatbot_usage_report.sql @@ -1,87 +1,78 @@ -with chatbot as ( - select - * - , row_number() over ( - partition by chatsession_thread_id - order by djangocheckpoint_id - ) as message_index - from ( - select * from {{ ref("int__learn_ai__chatbot") }} - where coalesce(agent_message, '') != '' or human_message is not null - ) -) - -, tutorbot as ( - select - * - , json_parse(json_extract_scalar(tutorbot_chat_json, '$')) as chat_json - from {{ ref("int__learn_ai__tutorbot") }} -) - -, tutorbot_flatten as ( - select - tutorbot.* - , t.idx as message_index - , case - when json_extract_scalar(t.element, '$.type') = 'HumanMessage' +with + chatbot as ( + select *, row_number() over (partition by chatsession_thread_id order by djangocheckpoint_id) as message_index + from + ( + select * + from {{ ref("int__learn_ai__chatbot") }} + where coalesce(agent_message, '') != '' or human_message is not null + ) + ), + tutorbot as ( + select *, json_parse(json_extract_scalar(tutorbot_chat_json, '$')) as chat_json + from {{ ref("int__learn_ai__tutorbot") }} + ), + tutorbot_flatten as ( + select + tutorbot.*, + t.idx as message_index, + case + when json_extract_scalar(t.element, '$.type') = 'HumanMessage' then json_extract_scalar(t.element, '$.content') - end as human_message - , case - when json_extract_scalar(t.element, '$.type') = 'AIMessage' + end as human_message, + case + when json_extract_scalar(t.element, '$.type') = 'AIMessage' then json_extract_scalar(t.element, '$.content') - end as agent_message - from tutorbot - cross join - unnest(cast(json_extract(tutorbot.chat_json, '$.chat_history') as array)) - with ordinality as t(element, idx) -- noqa: PRS -) - -, tutorbot_deduplicated as ( - select - tutorbotoutput_id - , chatsession_agent - , edx_module_id - , chatsession_thread_id - , human_message - , agent_message - , chatsession_created_on - , row_number() over ( - partition by chatsession_thread_id - order by tutorbotoutput_id, message_index - ) as message_index - from ( + end as agent_message + from tutorbot + cross join unnest(cast(json_extract(tutorbot.chat_json, '$.chat_history') as array)) + with ordinality as t(element, idx) -- noqa: PRS + ), + tutorbot_deduplicated as ( select - * - , row_number() over ( - partition by chatsession_thread_id, human_message, agent_message - order by tutorbotoutput_id, message_index - ) as message_occurrence - from tutorbot_flatten + tutorbotoutput_id, + chatsession_agent, + edx_module_id, + chatsession_thread_id, + human_message, + agent_message, + chatsession_created_on, + row_number() over ( + partition by chatsession_thread_id order by tutorbotoutput_id, message_index + ) as message_index + from + ( + select + *, + row_number() over ( + partition by chatsession_thread_id, human_message, agent_message + order by tutorbotoutput_id, message_index + ) as message_occurrence + from tutorbot_flatten + ) + where message_occurrence = 1 ) - where message_occurrence = 1 -) select - chatsession_agent as ai_agent - , chatsession_object_id as resource_id - , chatsession_thread_id as thread_id - , human_message - , agent_message as ai_message - , chatsession_created_on as created_on - , message_index + chatsession_agent as ai_agent, + chatsession_object_id as resource_id, + chatsession_thread_id as thread_id, + human_message, + agent_message as ai_message, + chatsession_created_on as created_on, + message_index from chatbot union all select - tutorbot_deduplicated.chatsession_agent as ai_agent - , tutorbot_deduplicated.edx_module_id as resource_id - , tutorbot_deduplicated.chatsession_thread_id as thread_id - , tutorbot_deduplicated.human_message - , tutorbot_deduplicated.agent_message as ai_message - , tutorbot_deduplicated.chatsession_created_on as created_on - , tutorbot_deduplicated.message_index + tutorbot_deduplicated.chatsession_agent as ai_agent, + tutorbot_deduplicated.edx_module_id as resource_id, + tutorbot_deduplicated.chatsession_thread_id as thread_id, + tutorbot_deduplicated.human_message, + tutorbot_deduplicated.agent_message as ai_message, + tutorbot_deduplicated.chatsession_created_on as created_on, + tutorbot_deduplicated.message_index from tutorbot_deduplicated -left join chatbot - on tutorbot_deduplicated.chatsession_thread_id = chatbot.chatsession_thread_id +left join chatbot on tutorbot_deduplicated.chatsession_thread_id = chatbot.chatsession_thread_id where chatbot.chatsession_thread_id is null diff --git a/src/ol_dbt/models/reporting/cheating_detection_report.sql b/src/ol_dbt/models/reporting/cheating_detection_report.sql index 1fc2db9a3..f3a5a84b8 100644 --- a/src/ol_dbt/models/reporting/cheating_detection_report.sql +++ b/src/ol_dbt/models/reporting/cheating_detection_report.sql @@ -1,138 +1,113 @@ -with problem_events as ( - select * from {{ ref('tfact_problem_events') }} -) - -, enrollment_detail as ( - select * from {{ ref('marts__combined_course_enrollment_detail') }} -) - -, course_content as ( - select * from {{ ref('dim_course_content') }} -) - -, overall_grade as ( - select - user_id - , courserun_readable_id - , max(courserungrade_grade) as courserungrade_grade - , sum(case when courserunenrollment_enrollment_mode = 'verified' then 1 else 0 end) as verified_cnt - from enrollment_detail - group by - user_id - , courserun_readable_id -) - -, problems_joined as ( - select - problem_events.platform - , problem_events.openedx_user_id - , problem_events.courserun_readable_id - , problem_events.problem_block_fk - , problem_events.event_timestamp - , problem_events.grade - , problem_events.max_grade - , problem_events.attempt - , overall_grade.courserungrade_grade - , unit.block_title as unit_name - , chapter.block_title as chapter_name - , sequential.block_title as sequential_name - , lag(problem_events.event_timestamp, 1) - over ( +with + problem_events as (select * from {{ ref("tfact_problem_events") }}), + enrollment_detail as (select * from {{ ref("marts__combined_course_enrollment_detail") }}), + course_content as (select * from {{ ref("dim_course_content") }}), + overall_grade as ( + select + user_id, + courserun_readable_id, + max(courserungrade_grade) as courserungrade_grade, + sum(case when courserunenrollment_enrollment_mode = 'verified' then 1 else 0 end) as verified_cnt + from enrollment_detail + group by user_id, courserun_readable_id + ), + problems_joined as ( + select + problem_events.platform, + problem_events.openedx_user_id, + problem_events.courserun_readable_id, + problem_events.problem_block_fk, + problem_events.event_timestamp, + problem_events.grade, + problem_events.max_grade, + problem_events.attempt, + overall_grade.courserungrade_grade, + unit.block_title as unit_name, + chapter.block_title as chapter_name, + sequential.block_title as sequential_name, + lag(problem_events.event_timestamp, 1) over ( partition by - problem_events.platform - , problem_events.openedx_user_id - , problem_events.courserun_readable_id - , cc.parent_block_id + problem_events.platform, + problem_events.openedx_user_id, + problem_events.courserun_readable_id, + cc.parent_block_id order by problem_events.event_timestamp ) as prev_event_timestamp - from problem_events - inner join overall_grade - on - cast(problem_events.openedx_user_id as varchar) = overall_grade.user_id + from problem_events + inner join + overall_grade + on cast(problem_events.openedx_user_id as varchar) = overall_grade.user_id and problem_events.courserun_readable_id = overall_grade.courserun_readable_id - left join course_content as cc - on - problem_events.problem_block_fk = cc.block_id - and cc.is_latest = true - left join course_content as unit - on - cc.parent_block_id = unit.block_id - and unit.is_latest = true - left join course_content as chapter - on - cc.chapter_block_id = chapter.block_id - and chapter.is_latest = true - left join course_content as sequential - on - cc.sequential_block_id = sequential.block_id - and sequential.is_latest = true - where overall_grade.verified_cnt > 0 -) + left join course_content as cc on problem_events.problem_block_fk = cc.block_id and cc.is_latest = true + left join course_content as unit on cc.parent_block_id = unit.block_id and unit.is_latest = true + left join course_content as chapter on cc.chapter_block_id = chapter.block_id and chapter.is_latest = true + left join + course_content as sequential on cc.sequential_block_id = sequential.block_id and sequential.is_latest = true + where overall_grade.verified_cnt > 0 + ) select - platform - , openedx_user_id - , courserun_readable_id - , problem_block_fk - , unit_name - , chapter_name - , sequential_name - , coalesce(( - upper(unit_name) like '%EXAM%' - and upper(unit_name) not like '%EXAMPLE%' - and upper(unit_name) not like '%EXAMINING%' - and upper(unit_name) not like '%PRACTICE%' - ) - or ( - upper(chapter_name) like '%EXAM%' - and upper(chapter_name) not like '%EXAMPLE%' - and upper(chapter_name) not like '%EXAMINING%' - and upper(chapter_name) not like '%PRACTICE%' - ) - or upper(chapter_name) like '%EXAM %' - or upper(unit_name) like '%EXAM %', true - ) as exam_indicator - , max(max_grade) as max_possible_grade - , max(attempt) as attempts_on_problem - , max(grade) as max_learner_grade - , array_agg(grade) as grades - , min( - case - when - prev_event_timestamp is not null - and date_diff('second', prev_event_timestamp, event_timestamp) < 600 - then date_diff('second', prev_event_timestamp, event_timestamp) - end - ) as time_spent_on_problem - , min( + platform, + openedx_user_id, + courserun_readable_id, + problem_block_fk, + unit_name, + chapter_name, + sequential_name, + coalesce( + ( + upper(unit_name) like '%EXAM%' + and upper(unit_name) not like '%EXAMPLE%' + and upper(unit_name) not like '%EXAMINING%' + and upper(unit_name) not like '%PRACTICE%' + ) + or ( + upper(chapter_name) like '%EXAM%' + and upper(chapter_name) not like '%EXAMPLE%' + and upper(chapter_name) not like '%EXAMINING%' + and upper(chapter_name) not like '%PRACTICE%' + ) + or upper(chapter_name) like '%EXAM %' + or upper(unit_name) like '%EXAM %', + true + ) as exam_indicator, + max(max_grade) as max_possible_grade, + max(attempt) as attempts_on_problem, + max(grade) as max_learner_grade, + array_agg(grade) as grades, + min( case - when - prev_event_timestamp is not null - then date_diff('second', prev_event_timestamp, event_timestamp) + when prev_event_timestamp is not null and date_diff('second', prev_event_timestamp, event_timestamp) < 600 + then date_diff('second', prev_event_timestamp, event_timestamp) end - ) as time_spent_on_problem_nolimit - , max(courserungrade_grade) as courserungrade_grade + ) as time_spent_on_problem, + min( + case when prev_event_timestamp is not null then date_diff('second', prev_event_timestamp, event_timestamp) end + ) as time_spent_on_problem_nolimit, + max(courserungrade_grade) as courserungrade_grade from problems_joined group by - platform - , openedx_user_id - , courserun_readable_id - , problem_block_fk - , unit_name - , chapter_name - , sequential_name - , coalesce(( - upper(unit_name) like '%EXAM%' - and upper(unit_name) not like '%EXAMPLE%' - and upper(unit_name) not like '%EXAMINING%' - and upper(unit_name) not like '%PRACTICE%' - ) - or ( - upper(chapter_name) like '%EXAM%' - and upper(chapter_name) not like '%EXAMPLE%' - and upper(chapter_name) not like '%EXAMINING%' - and upper(chapter_name) not like '%PRACTICE%' - ) - or upper(chapter_name) like '%EXAM %' - or upper(unit_name) like '%EXAM %', true + platform, + openedx_user_id, + courserun_readable_id, + problem_block_fk, + unit_name, + chapter_name, + sequential_name, + coalesce( + ( + upper(unit_name) like '%EXAM%' + and upper(unit_name) not like '%EXAMPLE%' + and upper(unit_name) not like '%EXAMINING%' + and upper(unit_name) not like '%PRACTICE%' + ) + or ( + upper(chapter_name) like '%EXAM%' + and upper(chapter_name) not like '%EXAMPLE%' + and upper(chapter_name) not like '%EXAMINING%' + and upper(chapter_name) not like '%PRACTICE%' + ) + or upper(chapter_name) like '%EXAM %' + or upper(unit_name) like '%EXAM %', + true ) diff --git a/src/ol_dbt/models/reporting/enrollment_detail_report.sql b/src/ol_dbt/models/reporting/enrollment_detail_report.sql index 04c5f25a7..3bca78a27 100644 --- a/src/ol_dbt/models/reporting/enrollment_detail_report.sql +++ b/src/ol_dbt/models/reporting/enrollment_detail_report.sql @@ -1,94 +1,78 @@ -with enrollments as ( - select * from {{ ref('marts__combined_course_enrollment_detail') }} -) - -, orders as ( - select * from {{ ref('marts__combined__orders') }} -) - -, program_enrollments as ( - select * from {{ ref('marts__combined_program_enrollment_detail') }} -) - -, coursesinprogram as ( - select * from {{ ref('marts__combined_coursesinprogram') }} -) - -, programs as ( - select distinct - program_enrollments.platform_name - , program_enrollments.program_name - , program_enrollments.user_email - , coursesinprogram.course_readable_id - from program_enrollments - inner join coursesinprogram - on - program_enrollments.platform_name = coursesinprogram.platform +with + enrollments as (select * from {{ ref("marts__combined_course_enrollment_detail") }}), + orders as (select * from {{ ref("marts__combined__orders") }}), + program_enrollments as (select * from {{ ref("marts__combined_program_enrollment_detail") }}), + coursesinprogram as (select * from {{ ref("marts__combined_coursesinprogram") }}), + programs as ( + select distinct + program_enrollments.platform_name, + program_enrollments.program_name, + program_enrollments.user_email, + coursesinprogram.course_readable_id + from program_enrollments + inner join + coursesinprogram + on program_enrollments.platform_name = coursesinprogram.platform and program_enrollments.program_name = coursesinprogram.program_name and program_enrollments.program_id = coursesinprogram.program_id - where coursesinprogram.program_name <> 'Computer Science' -) - -, course_passed_counts as ( - select - user_email - , count(distinct course_title) as num_of_course_passed - from enrollments - where courserungrade_is_passing = true - group by user_email -) + where coursesinprogram.program_name <> 'Computer Science' + ), + course_passed_counts as ( + select user_email, count(distinct course_title) as num_of_course_passed + from enrollments + where courserungrade_is_passing = true + group by user_email + ) select - enrollments.platform - , enrollments.courserunenrollment_id - , enrollments.course_readable_id - , enrollments.course_title - , enrollments.courserun_is_current - , enrollments.courserun_readable_id - , enrollments.courserun_start_on - , enrollments.courserun_end_on - , enrollments.courserun_title - , enrollments.courserunenrollment_created_on - , enrollments.courserunenrollment_enrollment_mode - , enrollments.courserunenrollment_enrollment_status - , enrollments.courserunenrollment_is_active - , enrollments.courserunenrollment_upgraded_on - , enrollments.courseruncertificate_created_on - , enrollments.courseruncertificate_is_earned - , enrollments.courseruncertificate_url - , enrollments.courserungrade_grade - , enrollments.courserungrade_is_passing - , enrollments.user_country_code - , enrollments.user_highest_education - , enrollments.user_full_name - , enrollments.user_username - , lower(enrollments.user_email) as user_email - , course_passed_counts.num_of_course_passed - , orders.coupon_code - , orders.coupon_name - , orders.discount - , enrollments.order_id - , enrollments.order_reference_number - , orders.order_created_on - , orders.order_state - , orders.receipt_payment_method - , orders.receipt_payment_amount - , orders.receipt_payer_email - , orders.receipt_payment_timestamp - , orders.unit_price - , programs.program_name - , if( - enrollments.platform = '{{ var("mitxonline") }}' - , concat('https://mitxonline.mit.edu/orders/receipt/', cast(orders.order_id as varchar)) - , null + enrollments.platform, + enrollments.courserunenrollment_id, + enrollments.course_readable_id, + enrollments.course_title, + enrollments.courserun_is_current, + enrollments.courserun_readable_id, + enrollments.courserun_start_on, + enrollments.courserun_end_on, + enrollments.courserun_title, + enrollments.courserunenrollment_created_on, + enrollments.courserunenrollment_enrollment_mode, + enrollments.courserunenrollment_enrollment_status, + enrollments.courserunenrollment_is_active, + enrollments.courserunenrollment_upgraded_on, + enrollments.courseruncertificate_created_on, + enrollments.courseruncertificate_is_earned, + enrollments.courseruncertificate_url, + enrollments.courserungrade_grade, + enrollments.courserungrade_is_passing, + enrollments.user_country_code, + enrollments.user_highest_education, + enrollments.user_full_name, + enrollments.user_username, + lower(enrollments.user_email) as user_email, + course_passed_counts.num_of_course_passed, + orders.coupon_code, + orders.coupon_name, + orders.discount, + enrollments.order_id, + enrollments.order_reference_number, + orders.order_created_on, + orders.order_state, + orders.receipt_payment_method, + orders.receipt_payment_amount, + orders.receipt_payer_email, + orders.receipt_payment_timestamp, + orders.unit_price, + programs.program_name, + if( + enrollments.platform = '{{ var("mitxonline") }}', + concat('https://mitxonline.mit.edu/orders/receipt/', cast(orders.order_id as varchar)), + null ) as receipt_url from enrollments -left join orders - on enrollments.order_id = orders.order_id and enrollments.platform = orders.platform -left join course_passed_counts - on enrollments.user_email = course_passed_counts.user_email -left join programs - on - enrollments.platform = programs.platform_name - and enrollments.course_readable_id = programs.course_readable_id - and enrollments.user_email = programs.user_email +left join orders on enrollments.order_id = orders.order_id and enrollments.platform = orders.platform +left join course_passed_counts on enrollments.user_email = course_passed_counts.user_email +left join + programs + on enrollments.platform = programs.platform_name + and enrollments.course_readable_id = programs.course_readable_id + and enrollments.user_email = programs.user_email diff --git a/src/ol_dbt/models/reporting/learner_engagement_report.sql b/src/ol_dbt/models/reporting/learner_engagement_report.sql index bd60090fe..dcea0e83c 100644 --- a/src/ol_dbt/models/reporting/learner_engagement_report.sql +++ b/src/ol_dbt/models/reporting/learner_engagement_report.sql @@ -1,299 +1,283 @@ -with video_pre_query as ( - select - platform - , openedx_user_id - , courserun_readable_id - , video_block_fk - , max(video_duration) as video_duration - , max(video_position) as end_time - , min(case when event_type = 'play_video' then video_position end) as start_time - from ol_warehouse_production_dimensional.tfact_video_events - where - event_type in ( - 'play_video' - , 'seek_video' - , 'pause_video' - , 'stop_video' - , 'complete_video' - ) - group by - platform - , openedx_user_id - , courserun_readable_id - , video_block_fk -) - -, discuss_table as ( - select - a.platform - , a.courserun_readable_id - , d.block_title as section_title - , c.block_title as subsection_title - , h2.course_title - , coalesce(u.email, ou.email) as email - , sum(a.post_created) as posts_created - , sum(a.post_replied) as posts_replied - from ol_warehouse_production_dimensional.afact_discussion_engagement as a - left join ol_warehouse_production_dimensional.dim_user as u - on a.platform = 'mitxonline' and a.openedx_user_id = u.mitxonline_openedx_user_id - left join ol_warehouse_production_dimensional.dim_user as ou - on a.platform = 'edxorg' and a.openedx_user_id = ou.edxorg_openedx_user_id - left join ol_warehouse_production_intermediate.int__combined__course_runs as h2 - on a.courserun_readable_id = h2.courserun_readable_id - left join ol_warehouse_production_dimensional.dim_course_content as c - on - a.sequential_block_fk = c.block_id +with + video_pre_query as ( + select + platform, + openedx_user_id, + courserun_readable_id, + video_block_fk, + max(video_duration) as video_duration, + max(video_position) as end_time, + min(case when event_type = 'play_video' then video_position end) as start_time + from ol_warehouse_production_dimensional.tfact_video_events + where event_type in ('play_video', 'seek_video', 'pause_video', 'stop_video', 'complete_video') + group by platform, openedx_user_id, courserun_readable_id, video_block_fk + ), + discuss_table as ( + select + a.platform, + a.courserun_readable_id, + d.block_title as section_title, + c.block_title as subsection_title, + h2.course_title, + coalesce(u.email, ou.email) as email, + sum(a.post_created) as posts_created, + sum(a.post_replied) as posts_replied + from ol_warehouse_production_dimensional.afact_discussion_engagement as a + left join + ol_warehouse_production_dimensional.dim_user as u + on a.platform = 'mitxonline' + and a.openedx_user_id = u.mitxonline_openedx_user_id + left join + ol_warehouse_production_dimensional.dim_user as ou + on a.platform = 'edxorg' + and a.openedx_user_id = ou.edxorg_openedx_user_id + left join + ol_warehouse_production_intermediate.int__combined__course_runs as h2 + on a.courserun_readable_id = h2.courserun_readable_id + left join + ol_warehouse_production_dimensional.dim_course_content as c + on a.sequential_block_fk = c.block_id and c.is_latest = true - left join ol_warehouse_production_dimensional.dim_course_content as d - on - a.chapter_block_fk = d.block_id + left join + ol_warehouse_production_dimensional.dim_course_content as d + on a.chapter_block_fk = d.block_id and d.is_latest = true - group by - a.platform - , a.courserun_readable_id - , d.block_title - , c.block_title - , h2.course_title - , coalesce(u.email, ou.email) -) - -, page_views_table as ( - select - a.platform - , a.courserun_readable_id - , d.block_title as section_title - , c.block_title as subsection_title - , h2.course_title - , coalesce(u.email, ou.email) as email - , sum(a.num_of_views) as num_of_page_views - from ol_warehouse_production_dimensional.afact_course_page_engagement as a - left join ol_warehouse_production_dimensional.dim_course_content as b - on - a.block_fk = b.block_id + group by + a.platform, + a.courserun_readable_id, + d.block_title, + c.block_title, + h2.course_title, + coalesce(u.email, ou.email) + ), + page_views_table as ( + select + a.platform, + a.courserun_readable_id, + d.block_title as section_title, + c.block_title as subsection_title, + h2.course_title, + coalesce(u.email, ou.email) as email, + sum(a.num_of_views) as num_of_page_views + from ol_warehouse_production_dimensional.afact_course_page_engagement as a + left join + ol_warehouse_production_dimensional.dim_course_content as b + on a.block_fk = b.block_id and b.is_latest = true - left join ol_warehouse_production_dimensional.dim_course_content as c - on - b.parent_block_id = c.block_id + left join + ol_warehouse_production_dimensional.dim_course_content as c + on b.parent_block_id = c.block_id and c.is_latest = true - left join ol_warehouse_production_dimensional.dim_course_content as d - on - c.parent_block_id = d.block_id + left join + ol_warehouse_production_dimensional.dim_course_content as d + on c.parent_block_id = d.block_id and d.is_latest = true - left join ol_warehouse_production_dimensional.dim_user as u - on - a.platform = 'mitxonline' + left join + ol_warehouse_production_dimensional.dim_user as u + on a.platform = 'mitxonline' and a.openedx_user_id = u.mitxonline_openedx_user_id - left join ol_warehouse_production_dimensional.dim_user as ou - on - a.platform = 'edxorg' + left join + ol_warehouse_production_dimensional.dim_user as ou + on a.platform = 'edxorg' and a.openedx_user_id = ou.edxorg_openedx_user_id - left join ol_warehouse_production_intermediate.int__combined__course_runs as h2 - on a.courserun_readable_id = h2.courserun_readable_id - group by - a.platform - , a.courserun_readable_id - , d.block_title - , c.block_title - , h2.course_title - , coalesce(u.email, ou.email) -) - -, video_views_table as ( - select - a.platform - , a.courserun_readable_id - , cc_section.block_title as section_title - , cc_subsection.block_title as subsection_title - , h2.course_title - , coalesce(b.email, ob.email) as email - , sum( - cast(case when a.end_time = 'null' then '0' else a.end_time end as decimal(30, 10)) - - cast(case when a.start_time = 'null' then '0' else a.start_time end as decimal(30, 10)) - ) - as estimated_time_played - , sum(a.video_duration) as video_duration - from video_pre_query as a - inner join ol_warehouse_production_dimensional.dim_video as c - on - a.courserun_readable_id = c.courserun_readable_id + left join + ol_warehouse_production_intermediate.int__combined__course_runs as h2 + on a.courserun_readable_id = h2.courserun_readable_id + group by + a.platform, + a.courserun_readable_id, + d.block_title, + c.block_title, + h2.course_title, + coalesce(u.email, ou.email) + ), + video_views_table as ( + select + a.platform, + a.courserun_readable_id, + cc_section.block_title as section_title, + cc_subsection.block_title as subsection_title, + h2.course_title, + coalesce(b.email, ob.email) as email, + sum( + cast(case when a.end_time = 'null' then '0' else a.end_time end as decimal(30, 10)) + - cast(case when a.start_time = 'null' then '0' else a.start_time end as decimal(30, 10)) + ) as estimated_time_played, + sum(a.video_duration) as video_duration + from video_pre_query as a + inner join + ol_warehouse_production_dimensional.dim_video as c + on a.courserun_readable_id = c.courserun_readable_id and a.video_block_fk = substring(c.video_block_pk, regexp_position(c.video_block_pk, 'block@') + 6) - left join ol_warehouse_production_dimensional.dim_user as b - on - a.platform = 'mitxonline' + left join + ol_warehouse_production_dimensional.dim_user as b + on a.platform = 'mitxonline' and a.openedx_user_id = b.mitxonline_openedx_user_id - left join ol_warehouse_production_dimensional.dim_user as ob - on - a.platform = 'edxorg' + left join + ol_warehouse_production_dimensional.dim_user as ob + on a.platform = 'edxorg' and a.openedx_user_id = ob.edxorg_openedx_user_id - left join ol_warehouse_production_intermediate.int__combined__course_runs as h2 - on a.courserun_readable_id = h2.courserun_readable_id - left join ol_warehouse_production_dimensional.dim_course_content as v - on - c.content_block_fk = v.content_block_pk + left join + ol_warehouse_production_intermediate.int__combined__course_runs as h2 + on a.courserun_readable_id = h2.courserun_readable_id + left join + ol_warehouse_production_dimensional.dim_course_content as v + on c.content_block_fk = v.content_block_pk and a.courserun_readable_id = v.courserun_readable_id - left join ol_warehouse_production_dimensional.dim_course_content as cc_subsection - on - v.parent_block_id = cc_subsection.block_id + left join + ol_warehouse_production_dimensional.dim_course_content as cc_subsection + on v.parent_block_id = cc_subsection.block_id and a.courserun_readable_id = cc_subsection.courserun_readable_id and cc_subsection.is_latest = true - left join ol_warehouse_production_dimensional.dim_course_content as cc_section - on - cc_subsection.parent_block_id = cc_section.block_id + left join + ol_warehouse_production_dimensional.dim_course_content as cc_section + on cc_subsection.parent_block_id = cc_section.block_id and a.courserun_readable_id = cc_section.courserun_readable_id and cc_section.is_latest = true - group by - a.platform - , a.courserun_readable_id - , cc_section.block_title - , cc_subsection.block_title - , h2.course_title - , coalesce(b.email, ob.email) -) - -, pre_problems_table as ( - select - c.sequential_block_id - , count(p.problem_block_pk) as problem_numb - from ol_warehouse_production_dimensional.dim_problem as p - inner join ol_warehouse_production_dimensional.dim_course_content as c - on p.content_block_fk = c.content_block_pk - group by c.sequential_block_id -) - -, problems_events as ( - select - problem_block_fk - , courserun_readable_id - , openedx_user_id - , avg( - cast(grade as decimal(30, 10)) - / nullif(cast(max_grade as decimal(30, 10)), 0) - ) - as avg_percent_grade - from ol_warehouse_production_dimensional.tfact_problem_events - group by - problem_block_fk - , courserun_readable_id - , openedx_user_id -) - -, problems_table as ( - select - a.platform - , a.courserun_readable_id - , sec.block_title as section_title - , d.block_title as subsection_title - , h2.course_title - , coalesce(u.email, ou.email) as email - , avg(g.avg_percent_grade) as avg_percent_grade - , count(distinct case when cast(a.num_of_attempts as int) > 0 then a.problem_block_fk end) as problems_attempted - , max(c.problem_numb) as number_of_problems - , cast(count(distinct case - when cast(a.num_of_attempts as int) > 0 - then a.problem_block_fk - end) as decimal(30, 10)) - / cast(max(c.problem_numb) as decimal(30, 10) - ) as percetage_problems_attempted - from ol_warehouse_production_dimensional.afact_problem_engagement as a - inner join pre_problems_table as c - on a.sequential_block_fk = c.sequential_block_id - left join ol_warehouse_production_dimensional.dim_user as u - on - a.platform = 'mitxonline' + group by + a.platform, + a.courserun_readable_id, + cc_section.block_title, + cc_subsection.block_title, + h2.course_title, + coalesce(b.email, ob.email) + ), + pre_problems_table as ( + select c.sequential_block_id, count(p.problem_block_pk) as problem_numb + from ol_warehouse_production_dimensional.dim_problem as p + inner join + ol_warehouse_production_dimensional.dim_course_content as c on p.content_block_fk = c.content_block_pk + group by c.sequential_block_id + ), + problems_events as ( + select + problem_block_fk, + courserun_readable_id, + openedx_user_id, + avg(cast(grade as decimal(30, 10)) / nullif(cast(max_grade as decimal(30, 10)), 0)) as avg_percent_grade + from ol_warehouse_production_dimensional.tfact_problem_events + group by problem_block_fk, courserun_readable_id, openedx_user_id + ), + problems_table as ( + select + a.platform, + a.courserun_readable_id, + sec.block_title as section_title, + d.block_title as subsection_title, + h2.course_title, + coalesce(u.email, ou.email) as email, + avg(g.avg_percent_grade) as avg_percent_grade, + count( + distinct case when cast(a.num_of_attempts as int) > 0 then a.problem_block_fk end + ) as problems_attempted, + max(c.problem_numb) as number_of_problems, + cast( + count(distinct case when cast(a.num_of_attempts as int) > 0 then a.problem_block_fk end) as decimal( + 30, 10 + ) + ) + / cast(max(c.problem_numb) as decimal(30, 10)) as percetage_problems_attempted + from ol_warehouse_production_dimensional.afact_problem_engagement as a + inner join pre_problems_table as c on a.sequential_block_fk = c.sequential_block_id + left join + ol_warehouse_production_dimensional.dim_user as u + on a.platform = 'mitxonline' and a.openedx_user_id = u.mitxonline_openedx_user_id - left join ol_warehouse_production_dimensional.dim_user as ou - on - a.platform = 'edxorg' + left join + ol_warehouse_production_dimensional.dim_user as ou + on a.platform = 'edxorg' and a.openedx_user_id = ou.edxorg_openedx_user_id - left join ol_warehouse_production_intermediate.int__combined__course_runs as h2 - on a.courserun_readable_id = h2.courserun_readable_id - inner join ol_warehouse_production_dimensional.dim_course_content as d - on - a.sequential_block_fk = d.block_id + left join + ol_warehouse_production_intermediate.int__combined__course_runs as h2 + on a.courserun_readable_id = h2.courserun_readable_id + inner join + ol_warehouse_production_dimensional.dim_course_content as d + on a.sequential_block_fk = d.block_id and d.is_latest = true - inner join ol_warehouse_production_dimensional.dim_course_content as sec - on - a.chapter_block_fk = sec.block_id + inner join + ol_warehouse_production_dimensional.dim_course_content as sec + on a.chapter_block_fk = sec.block_id and sec.is_latest = true - inner join problems_events as g - on - a.problem_block_fk = g.problem_block_fk + inner join + problems_events as g + on a.problem_block_fk = g.problem_block_fk and a.courserun_readable_id = g.courserun_readable_id and a.openedx_user_id = g.openedx_user_id - group by - a.platform - , a.courserun_readable_id - , sec.block_title - , d.block_title - , h2.course_title - , coalesce(u.email, ou.email) -) - -, page_and_video as ( - select - page_views_table.num_of_page_views - , video_views_table.estimated_time_played - , video_views_table.video_duration - , coalesce(video_views_table.platform, page_views_table.platform) as platform - , coalesce(video_views_table.email, page_views_table.email) as email - , coalesce(video_views_table.course_title, page_views_table.course_title) as course_title - , coalesce(video_views_table.courserun_readable_id, page_views_table.courserun_readable_id) - as courserun_readable_id - , coalesce(video_views_table.section_title, page_views_table.section_title) as section_title - , coalesce(video_views_table.subsection_title, page_views_table.subsection_title) as subsection_title - from page_views_table - full outer join video_views_table - on - page_views_table.email = video_views_table.email + group by + a.platform, + a.courserun_readable_id, + sec.block_title, + d.block_title, + h2.course_title, + coalesce(u.email, ou.email) + ), + page_and_video as ( + select + page_views_table.num_of_page_views, + video_views_table.estimated_time_played, + video_views_table.video_duration, + coalesce(video_views_table.platform, page_views_table.platform) as platform, + coalesce(video_views_table.email, page_views_table.email) as email, + coalesce(video_views_table.course_title, page_views_table.course_title) as course_title, + coalesce( + video_views_table.courserun_readable_id, page_views_table.courserun_readable_id + ) as courserun_readable_id, + coalesce(video_views_table.section_title, page_views_table.section_title) as section_title, + coalesce(video_views_table.subsection_title, page_views_table.subsection_title) as subsection_title + from page_views_table + full outer join + video_views_table + on page_views_table.email = video_views_table.email and page_views_table.courserun_readable_id = video_views_table.courserun_readable_id and page_views_table.section_title = video_views_table.section_title and page_views_table.subsection_title = video_views_table.subsection_title -) - -, page_video_problems as ( - select - page_and_video.num_of_page_views - , page_and_video.estimated_time_played - , page_and_video.video_duration - , problems_table.problems_attempted - , problems_table.number_of_problems - , problems_table.percetage_problems_attempted - , problems_table.avg_percent_grade - , coalesce(page_and_video.platform, problems_table.platform) as platform - , coalesce(page_and_video.email, problems_table.email) as email - , coalesce(page_and_video.course_title, problems_table.course_title) as course_title - , coalesce(page_and_video.courserun_readable_id, problems_table.courserun_readable_id) as courserun_readable_id - , coalesce(page_and_video.section_title, problems_table.section_title) as section_title - , coalesce(page_and_video.subsection_title, problems_table.subsection_title) as subsection_title - from page_and_video - full outer join problems_table - on - page_and_video.email = problems_table.email + ), + page_video_problems as ( + select + page_and_video.num_of_page_views, + page_and_video.estimated_time_played, + page_and_video.video_duration, + problems_table.problems_attempted, + problems_table.number_of_problems, + problems_table.percetage_problems_attempted, + problems_table.avg_percent_grade, + coalesce(page_and_video.platform, problems_table.platform) as platform, + coalesce(page_and_video.email, problems_table.email) as email, + coalesce(page_and_video.course_title, problems_table.course_title) as course_title, + coalesce( + page_and_video.courserun_readable_id, problems_table.courserun_readable_id + ) as courserun_readable_id, + coalesce(page_and_video.section_title, problems_table.section_title) as section_title, + coalesce(page_and_video.subsection_title, problems_table.subsection_title) as subsection_title + from page_and_video + full outer join + problems_table + on page_and_video.email = problems_table.email and page_and_video.courserun_readable_id = problems_table.courserun_readable_id and page_and_video.section_title = problems_table.section_title and page_and_video.subsection_title = problems_table.subsection_title -) + ) select - page_video_problems.num_of_page_views - , page_video_problems.estimated_time_played - , page_video_problems.video_duration - , page_video_problems.problems_attempted - , page_video_problems.number_of_problems - , page_video_problems.percetage_problems_attempted - , page_video_problems.avg_percent_grade - , discuss_table.posts_created - , discuss_table.posts_replied - , coalesce(page_video_problems.platform, discuss_table.platform) as platform - , coalesce(page_video_problems.email, discuss_table.email) as user_email - , coalesce(page_video_problems.course_title, discuss_table.course_title) as course_title - , coalesce(page_video_problems.courserun_readable_id, discuss_table.courserun_readable_id) as courserun_readable_id - , coalesce(page_video_problems.section_title, discuss_table.section_title) as section_title - , coalesce(page_video_problems.subsection_title, discuss_table.subsection_title) as subsection_title + page_video_problems.num_of_page_views, + page_video_problems.estimated_time_played, + page_video_problems.video_duration, + page_video_problems.problems_attempted, + page_video_problems.number_of_problems, + page_video_problems.percetage_problems_attempted, + page_video_problems.avg_percent_grade, + discuss_table.posts_created, + discuss_table.posts_replied, + coalesce(page_video_problems.platform, discuss_table.platform) as platform, + coalesce(page_video_problems.email, discuss_table.email) as user_email, + coalesce(page_video_problems.course_title, discuss_table.course_title) as course_title, + coalesce(page_video_problems.courserun_readable_id, discuss_table.courserun_readable_id) as courserun_readable_id, + coalesce(page_video_problems.section_title, discuss_table.section_title) as section_title, + coalesce(page_video_problems.subsection_title, discuss_table.subsection_title) as subsection_title from page_video_problems -full outer join discuss_table - on - page_video_problems.email = discuss_table.email - and page_video_problems.courserun_readable_id = discuss_table.courserun_readable_id - and page_video_problems.section_title = discuss_table.section_title - and page_video_problems.subsection_title = discuss_table.subsection_title +full outer join + discuss_table + on page_video_problems.email = discuss_table.email + and page_video_problems.courserun_readable_id = discuss_table.courserun_readable_id + and page_video_problems.section_title = discuss_table.section_title + and page_video_problems.subsection_title = discuss_table.subsection_title diff --git a/src/ol_dbt/models/reporting/program_summary_report.sql b/src/ol_dbt/models/reporting/program_summary_report.sql index 6e4062303..c73a3a8d3 100644 --- a/src/ol_dbt/models/reporting/program_summary_report.sql +++ b/src/ol_dbt/models/reporting/program_summary_report.sql @@ -1,69 +1,60 @@ -with program_enrollments as ( - select * from {{ ref('marts__combined_program_enrollment_detail') }} -) - -, course_enrollments as ( - select * from {{ ref('marts__combined_course_enrollment_detail') }} -) - -, program_courses as ( - select * from {{ ref('marts__combined_coursesinprogram') }} -) - -, aggregated_course_enrollments as ( - select - program_courses.program_name - , count(distinct course_enrollments.courserun_readable_id || course_enrollments.user_email) as total_enrollments - , count(distinct course_enrollments.user_email) as unique_users - , count(distinct course_enrollments.user_country_code) as unique_countries - , count( - distinct case - when course_enrollments.courserunenrollment_enrollment_mode = 'verified' +with + program_enrollments as (select * from {{ ref("marts__combined_program_enrollment_detail") }}), + course_enrollments as (select * from {{ ref("marts__combined_course_enrollment_detail") }}), + program_courses as (select * from {{ ref("marts__combined_coursesinprogram") }}), + aggregated_course_enrollments as ( + select + program_courses.program_name, + count( + distinct course_enrollments.courserun_readable_id || course_enrollments.user_email + ) as total_enrollments, + count(distinct course_enrollments.user_email) as unique_users, + count(distinct course_enrollments.user_country_code) as unique_countries, + count( + distinct case + when course_enrollments.courserunenrollment_enrollment_mode = 'verified' then course_enrollments.courserun_readable_id || course_enrollments.user_email - end - ) as verified_enrollments - , count( - distinct case - when course_enrollments.courserunenrollment_enrollment_mode = 'verified' + end + ) as verified_enrollments, + count( + distinct case + when course_enrollments.courserunenrollment_enrollment_mode = 'verified' then course_enrollments.user_email - end - ) as unique_verified_users - , count( - distinct case - when course_enrollments.courseruncertificate_is_earned = true + end + ) as unique_verified_users, + count( + distinct case + when course_enrollments.courseruncertificate_is_earned = true then course_enrollments.courserun_readable_id || course_enrollments.user_email - end - ) as course_certificates - , count( - distinct case - when course_enrollments.courseruncertificate_is_earned = true - then course_enrollments.user_email - end - ) as unique_course_certificate_earners - from course_enrollments - inner join program_courses on course_enrollments.course_readable_id = program_courses.course_readable_id - group by program_courses.program_name -) - -, aggregated_program_certificates as ( - select - program_name - , count(distinct user_email) as program_certificates - from program_enrollments - where user_has_completed_program = true - group by program_name -) + end + ) as course_certificates, + count( + distinct case + when course_enrollments.courseruncertificate_is_earned = true then course_enrollments.user_email + end + ) as unique_course_certificate_earners + from course_enrollments + inner join program_courses on course_enrollments.course_readable_id = program_courses.course_readable_id + group by program_courses.program_name + ), + aggregated_program_certificates as ( + select program_name, count(distinct user_email) as program_certificates + from program_enrollments + where user_has_completed_program = true + group by program_name + ) select - aggregated_course_enrollments.program_name - , aggregated_course_enrollments.total_enrollments - , aggregated_course_enrollments.unique_users - , aggregated_course_enrollments.unique_countries - , aggregated_course_enrollments.verified_enrollments - , aggregated_course_enrollments.unique_verified_users - , aggregated_course_enrollments.course_certificates - , aggregated_course_enrollments.unique_course_certificate_earners - , aggregated_program_certificates.program_certificates + aggregated_course_enrollments.program_name, + aggregated_course_enrollments.total_enrollments, + aggregated_course_enrollments.unique_users, + aggregated_course_enrollments.unique_countries, + aggregated_course_enrollments.verified_enrollments, + aggregated_course_enrollments.unique_verified_users, + aggregated_course_enrollments.course_certificates, + aggregated_course_enrollments.unique_course_certificate_earners, + aggregated_program_certificates.program_certificates from aggregated_course_enrollments -left join aggregated_program_certificates +left join + aggregated_program_certificates on aggregated_course_enrollments.program_name = aggregated_program_certificates.program_name diff --git a/src/ol_dbt/models/reporting/video_engagement_report.sql b/src/ol_dbt/models/reporting/video_engagement_report.sql index ead50f9af..f57541920 100644 --- a/src/ol_dbt/models/reporting/video_engagement_report.sql +++ b/src/ol_dbt/models/reporting/video_engagement_report.sql @@ -1,37 +1,32 @@ -with video_engagements as ( - select * from {{ ref('marts__combined_video_engagements') }} -) - -, video_engagements_total as ( - select - chapter_title - , courserun_readable_id - , count(distinct video_title) as total_video_count - from {{ ref('marts__combined_video_engagements') }} - group by chapter_title, courserun_readable_id -) +with + video_engagements as (select * from {{ ref("marts__combined_video_engagements") }}), + video_engagements_total as ( + select chapter_title, courserun_readable_id, count(distinct video_title) as total_video_count + from {{ ref("marts__combined_video_engagements") }} + group by chapter_title, courserun_readable_id + ) select - video_engagements.user_email - , video_engagements.chapter_title - , video_engagements.courserun_readable_id - , video_engagements_total.total_video_count - , video_engagements.platform - , video_engagements.courserun_is_current - , video_engagements.course_readable_id - , count(distinct video_engagements.video_title) as user_watched_video_count - , max(video_engagements.video_index) as max_video_index + video_engagements.user_email, + video_engagements.chapter_title, + video_engagements.courserun_readable_id, + video_engagements_total.total_video_count, + video_engagements.platform, + video_engagements.courserun_is_current, + video_engagements.course_readable_id, + count(distinct video_engagements.video_title) as user_watched_video_count, + max(video_engagements.video_index) as max_video_index from video_engagements -inner join video_engagements_total - on - video_engagements.chapter_title = video_engagements_total.chapter_title - and video_engagements.courserun_readable_id = video_engagements_total.courserun_readable_id +inner join + video_engagements_total + on video_engagements.chapter_title = video_engagements_total.chapter_title + and video_engagements.courserun_readable_id = video_engagements_total.courserun_readable_id where video_engagements.video_event_type = 'play_video' group by - video_engagements.user_email - , video_engagements.chapter_title - , video_engagements.courserun_readable_id - , video_engagements_total.total_video_count - , video_engagements.platform - , video_engagements.courserun_is_current - , video_engagements.course_readable_id + video_engagements.user_email, + video_engagements.chapter_title, + video_engagements.courserun_readable_id, + video_engagements_total.total_video_count, + video_engagements.platform, + video_engagements.courserun_is_current, + video_engagements.course_readable_id diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_applicationstep.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_applicationstep.sql index 84c66b741..6d41f4172 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_applicationstep.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_applicationstep.sql @@ -1,15 +1,17 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__applications_applicationstep') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__applications_applicationstep") }} + ), + cleaned as ( -, cleaned as ( + select + id as applicationstep_id, + bootcamp_id as course_id, + step_order as applicationstep_step_order, + submission_type as applicationstep_submission_type + from source + ) - select - id as applicationstep_id - , bootcamp_id as course_id - , step_order as applicationstep_step_order - , submission_type as applicationstep_submission_type - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_applicationstep_submission.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_applicationstep_submission.sql index 28f6de578..250033da0 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_applicationstep_submission.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_applicationstep_submission.sql @@ -1,24 +1,31 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__applications_applicationstepsubmission') }} -) +with + source as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__bootcamps__app__postgres__applications_applicationstepsubmission" + ) + }} + ) -{{ deduplicate_raw_table(order_by='id' , partition_columns = 'bootcamp_application_id, run_application_step_id') }} -, cleaned as ( + {{ deduplicate_raw_table(order_by="id", partition_columns="bootcamp_application_id, run_application_step_id") }}, + cleaned as ( - select - id as submission_id - , bootcamp_application_id as application_id - , content_type_id as contenttype_id - , object_id as submission_object_id - , run_application_step_id as courserun_applicationstep_id - , review_status as submission_review_status - , submission_status - , {{ cast_timestamp_to_iso8601('submitted_date') }} as submission_submitted_on - , {{ cast_timestamp_to_iso8601('review_status_date') }} as submission_reviewed_on - , {{ cast_timestamp_to_iso8601('created_on') }} as submission_created_on - , {{ cast_timestamp_to_iso8601('updated_on') }} as submission_updated_on - from most_recent_source -) + select + id as submission_id, + bootcamp_application_id as application_id, + content_type_id as contenttype_id, + object_id as submission_object_id, + run_application_step_id as courserun_applicationstep_id, + review_status as submission_review_status, + submission_status, + {{ cast_timestamp_to_iso8601("submitted_date") }} as submission_submitted_on, + {{ cast_timestamp_to_iso8601("review_status_date") }} as submission_reviewed_on, + {{ cast_timestamp_to_iso8601("created_on") }} as submission_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as submission_updated_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_courserun_application.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_courserun_application.sql index 3bab9af11..85791b8dd 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_courserun_application.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_courserun_application.sql @@ -1,21 +1,22 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__applications_bootcampapplication') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__applications_bootcampapplication") }} + ), + cleaned as ( -, cleaned as ( + select + id as application_id, + user_id, + bootcamp_run_id as courserun_id, + resume_file as application_resume_file, + linkedin_url as application_linkedin_url, + lower(state) as application_state, + {{ cast_timestamp_to_iso8601("resume_upload_date") }} as application_resume_uploaded_on, + {{ cast_timestamp_to_iso8601("created_on") }} as application_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as application_updated_on + from source + ) - select - id as application_id - , user_id - , bootcamp_run_id as courserun_id - , resume_file as application_resume_file - , linkedin_url as application_linkedin_url - , lower(state) as application_state - , {{ cast_timestamp_to_iso8601('resume_upload_date') }} as application_resume_uploaded_on - , {{ cast_timestamp_to_iso8601('created_on') }} as application_created_on - , {{ cast_timestamp_to_iso8601('updated_on') }} as application_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_courserun_applicationstep.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_courserun_applicationstep.sql index 8916f0752..358eb4af2 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_courserun_applicationstep.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__applications_courserun_applicationstep.sql @@ -1,16 +1,22 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__applications_bootcamprunapplicationstep') }} -) +with + source as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__bootcamps__app__postgres__applications_bootcamprunapplicationstep" + ) + }} + ), + cleaned as ( -, cleaned as ( + select + id as courserun_applicationstep_id, + bootcamp_run_id as courserun_id, + application_step_id as applicationstep_id, + {{ cast_timestamp_to_iso8601("due_date") }} as courserun_applicationstep_due_date + from source + ) - select - id as courserun_applicationstep_id - , bootcamp_run_id as courserun_id - , application_step_id as applicationstep_id - , {{ cast_timestamp_to_iso8601('due_date') }} as courserun_applicationstep_due_date - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__auth_user.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__auth_user.sql index 0823bc63b..1ffb227f2 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__auth_user.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__auth_user.sql @@ -1,17 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__auth_user') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__auth_user") }}), + cleaned as ( -, cleaned as ( + select + id as user_id, + username as user_username, + email as user_email, + is_active as user_is_active, + {{ cast_timestamp_to_iso8601("date_joined") }} as user_joined_on, + {{ cast_timestamp_to_iso8601("last_login") }} as user_last_login + from source + ) - select - id as user_id - , username as user_username - , email as user_email - , is_active as user_is_active - ,{{ cast_timestamp_to_iso8601('date_joined') }} as user_joined_on - ,{{ cast_timestamp_to_iso8601('last_login') }} as user_last_login - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courserunenrollment.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courserunenrollment.sql index aab540908..cafab41fd 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courserunenrollment.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courserunenrollment.sql @@ -1,21 +1,22 @@ -- Bootcamps Course Run Enrollment Information +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__klasses_bootcamprunenrollment") }} + ), + renamed as ( + select + id as courserunenrollment_id, + active as courserunenrollment_is_active, + user_id, + bootcamp_run_id as courserun_id, + change_status as courserunenrollment_enrollment_status, + user_certificate_is_blocked as courserunenrollment_is_certificate_blocked, + {{ cast_timestamp_to_iso8601("created_on") }} as courserunenrollment_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as courserunenrollment_updated_on, + {{ cast_timestamp_to_iso8601("novoed_sync_date") }} as courserunenrollment_novoed_sync_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__klasses_bootcamprunenrollment') }} -) - -, renamed as ( - select - id as courserunenrollment_id - , active as courserunenrollment_is_active - , user_id - , bootcamp_run_id as courserun_id - , change_status as courserunenrollment_enrollment_status - , user_certificate_is_blocked as courserunenrollment_is_certificate_blocked - ,{{ cast_timestamp_to_iso8601('created_on') }} as courserunenrollment_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as courserunenrollment_updated_on - ,{{ cast_timestamp_to_iso8601('novoed_sync_date') }} as courserunenrollment_novoed_sync_on - from source -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_course.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_course.sql index 8101b9558..50645c025 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_course.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_course.sql @@ -1,15 +1,7 @@ -- Bootcamps Course Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__klasses_bootcamp") }}), + cleaned as (select id as course_id, title as course_title, readable_id as course_readable_id from source) -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__klasses_bootcamp') }} -) - -, cleaned as ( - select - id as course_id - , title as course_title - , readable_id as course_readable_id - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_courserun.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_courserun.sql index 7081400f5..44267858c 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_courserun.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_courserun.sql @@ -1,18 +1,18 @@ -- Bootcamps Course Run Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__klasses_bootcamprun") }} + ), + cleaned as ( + select + id as courserun_id, + bootcamp_id as course_id, + title as courserun_title, + bootcamp_run_id as courserun_readable_id, + {{ cast_timestamp_to_iso8601("start_date") }} as courserun_start_on, + {{ cast_timestamp_to_iso8601("end_date") }} as courserun_end_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__klasses_bootcamprun') }} -) - -, cleaned as ( - select - id as courserun_id - , bootcamp_id as course_id - , title as courserun_title - , bootcamp_run_id as courserun_readable_id - ,{{ cast_timestamp_to_iso8601('start_date') }} as courserun_start_on - ,{{ cast_timestamp_to_iso8601('end_date') }} as courserun_end_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_courseruncertificate.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_courseruncertificate.sql index ecb40e514..c07904303 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_courseruncertificate.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_courseruncertificate.sql @@ -1,21 +1,23 @@ -- Bootcamps Users Course Certificate Information +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__klasses_bootcampruncertificate") }} + ), + cleaned as ( + select + id as courseruncertificate_id, + uuid as courseruncertificate_uuid, + bootcamp_run_id as courserun_id, + user_id, + is_revoked as courseruncertificate_is_revoked, + if( + is_revoked = false, concat('https://bootcamps.mit.edu/certificate/', uuid), null + ) as courseruncertificate_url, + {{ cast_timestamp_to_iso8601("created_on") }} as courseruncertificate_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as courseruncertificate_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__bootcamps__app__postgres__klasses_bootcampruncertificate') }} -) - -, cleaned as ( - select - id as courseruncertificate_id - , uuid as courseruncertificate_uuid - , bootcamp_run_id as courserun_id - , user_id - , is_revoked as courseruncertificate_is_revoked - , if(is_revoked = false, concat('https://bootcamps.mit.edu/certificate/', uuid), null) - as courseruncertificate_url - ,{{ cast_timestamp_to_iso8601('created_on') }} as courseruncertificate_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as courseruncertificate_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_installment.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_installment.sql index 9aed40af6..663e242a3 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_installment.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_installment.sql @@ -1,15 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__klasses_installment') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__klasses_installment") }} + ), + cleaned as ( -, cleaned as ( + select + id as installment_id, + bootcamp_run_id as courserun_id, + cast(amount as decimal(38, 2)) as installment_amount, + {{ cast_timestamp_to_iso8601("deadline") }} as installment_deadline + from source + ) - select - id as installment_id - , bootcamp_run_id as courserun_id - , cast(amount as decimal(38, 2)) as installment_amount - , {{ cast_timestamp_to_iso8601('deadline') }} as installment_deadline - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_personalprice.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_personalprice.sql index cf4cb0a17..8fb3a1f55 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_personalprice.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__courses_personalprice.sql @@ -1,15 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__klasses_personalprice') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__klasses_personalprice") }} + ), + cleaned as ( -, cleaned as ( + select + id as personalprice_id, + user_id, + bootcamp_run_id as courserun_id, + cast(price as decimal(38, 2)) as personalprice + from source + ) - select - id as personalprice_id - , user_id - , bootcamp_run_id as courserun_id - , cast(price as decimal(38, 2)) as personalprice - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__django_contenttype.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__django_contenttype.sql index cce105579..d20f197b7 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__django_contenttype.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__django_contenttype.sql @@ -1,20 +1,10 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__django_content_type') }} + select * from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__django_content_type") }} -) + ), + renamed as (select id as contenttype_id, concat_ws('_', app_label, model) as contenttype_full_name from source) -, renamed as ( - - select - id as contenttype_id - , concat_ws( - '_' - , app_label - , model - ) as contenttype_full_name - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_line.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_line.sql index b21917d91..9d9b89132 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_line.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_line.sql @@ -1,20 +1,17 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__ecommerce_line") }}), + renamed as ( + select + id as line_id, + cast(price as decimal(38, 2)) as line_price, + description as line_description, + order_id, + bootcamp_run_id as courserun_id, + {{ cast_timestamp_to_iso8601("created_on") }} as line_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as line_updated_on + from source - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__ecommerce_line') }} + ) -) - -, renamed as ( - select - id as line_id - , cast(price as decimal(38, 2)) as line_price - , description as line_description - , order_id - , bootcamp_run_id as courserun_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as line_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as line_updated_on - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_order.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_order.sql index c98f82ab5..95627730e 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_order.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_order.sql @@ -1,22 +1,19 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__ecommerce_order") }}), + renamed as ( + select + id as order_id, + status as order_state, + user_id as order_purchaser_user_id, + application_id, + payment_type as order_payment_type, + cast(total_price_paid as decimal(38, 2)) as order_total_price_paid, + concat('BOOTCAMP-prod-', cast(id as varchar)) as order_reference_number, + {{ cast_timestamp_to_iso8601("created_on") }} as order_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as order_updated_on + from source - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__ecommerce_order') }} + ) -) - -, renamed as ( - select - id as order_id - , status as order_state - , user_id as order_purchaser_user_id - , application_id - , payment_type as order_payment_type - , cast(total_price_paid as decimal(38, 2)) as order_total_price_paid - , concat('BOOTCAMP-prod-', cast(id as varchar)) as order_reference_number - ,{{ cast_timestamp_to_iso8601('created_on') }} as order_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as order_updated_on - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_orderaudit.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_orderaudit.sql index db5c21994..ca182e749 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_orderaudit.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_orderaudit.sql @@ -1,20 +1,21 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__ecommerce_orderaudit') }} + select * from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__ecommerce_orderaudit") }} -) + ), + renamed as ( + select + id as orderaudit_id, + order_id, + acting_user_id as orderaudit_acting_user_id, + data_before as orderaudit_data_before, + data_after as orderaudit_data_after, + {{ cast_timestamp_to_iso8601("created_on") }} as orderaudit_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as orderaudit_updated_on + from source -, renamed as ( - select - id as orderaudit_id - , order_id - , acting_user_id as orderaudit_acting_user_id - , data_before as orderaudit_data_before - , data_after as orderaudit_data_after - ,{{ cast_timestamp_to_iso8601('created_on') }} as orderaudit_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as orderaudit_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_receipt.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_receipt.sql index cebf8acd7..24bd39a42 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_receipt.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_receipt.sql @@ -1,18 +1,15 @@ -with source as ( - - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__ecommerce_receipt') }} - -) - -, renamed as ( - select - id as receipt_id - , order_id - , data as receipt_data - ,{{ cast_timestamp_to_iso8601('created_on') }} as receipt_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as receipt_updated_on - from source - -) - -select * from renamed +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__ecommerce_receipt") }}), + renamed as ( + select + id as receipt_id, + order_id, + data as receipt_data, + {{ cast_timestamp_to_iso8601("created_on") }} as receipt_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as receipt_updated_on + from source + + ) + +select * +from renamed diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_wiretransferreceipt.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_wiretransferreceipt.sql index e3834ac79..29d65af3c 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_wiretransferreceipt.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__ecommerce_wiretransferreceipt.sql @@ -1,19 +1,21 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__ecommerce_wiretransferreceipt') }} + select * + from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__ecommerce_wiretransferreceipt") }} -) + ), + renamed as ( + select + id as wiretransferreceipt_id, + order_id, + data as wiretransferreceipt_data, + {{ cast_timestamp_to_iso8601("created_on") }} as wiretransferreceipt_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as wiretransferreceipt_updated_on, + wire_transfer_id as wiretransferreceipt_import_id + from source -, renamed as ( - select - id as wiretransferreceipt_id - , order_id - , data as wiretransferreceipt_data - ,{{ cast_timestamp_to_iso8601('created_on') }} as wiretransferreceipt_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as wiretransferreceipt_updated_on - , wire_transfer_id as wiretransferreceipt_import_id - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__profiles_legaladdress.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__profiles_legaladdress.sql index 09c761831..0da21e523 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__profiles_legaladdress.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__profiles_legaladdress.sql @@ -1,27 +1,28 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__bootcamps__app__postgres__profiles_legaladdress') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__profiles_legaladdress") }} + ), + cleaned as ( -, cleaned as ( + select + id as user_address_id, + country as user_address_country, + user_id, + first_name as user_first_name, + last_name as user_last_name, + city as user_address_city, + state_or_territory as user_address_state_or_territory, + postal_code as user_address_postal_code, + concat_ws( + chr(10), + nullif(street_address_1, ''), + nullif(street_address_2, ''), + nullif(street_address_3, ''), + nullif(street_address_4, ''), + nullif(street_address_5, '') + ) as user_street_address + from source + ) - select - id as user_address_id - , country as user_address_country - , user_id - , first_name as user_first_name - , last_name as user_last_name - , city as user_address_city - , state_or_territory as user_address_state_or_territory - , postal_code as user_address_postal_code - , concat_ws( - chr(10) - , nullif(street_address_1, '') - , nullif(street_address_2, '') - , nullif(street_address_3, '') - , nullif(street_address_4, '') - , nullif(street_address_5, '') - ) as user_street_address - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__profiles_profile.sql b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__profiles_profile.sql index 98b916e2c..3fea49725 100644 --- a/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__profiles_profile.sql +++ b/src/ol_dbt/models/staging/bootcamps/stg__bootcamps__app__postgres__profiles_profile.sql @@ -1,40 +1,53 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__bootcamps__app__postgres__profiles_profile') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__bootcamps__app__postgres__profiles_profile") }}), + cleaned as ( + select + id as user_profile_id, + replace(replace(replace(name, ' ', '<>'), '><', ''), '<>', ' ') as user_full_name, + birth_year as user_birth_year, + company as user_company, + job_title as user_job_title, + industry as user_industry, + job_function as user_job_function, + user_id, + highest_education as user_highest_education, + {{ transform_gender_value("gender") }} as user_gender, + case + when company_size = 1 + then 'Small/Start-up (1+ employees)' + when company_size = 9 + then 'Small/Home office (1-9 employees)' + when company_size = 99 + then 'Small (10-99 employees)' + when company_size = 999 + then 'Small to medium-sized (100-999 employees)' + when company_size = 9999 + then 'Medium-sized (1000-9999 employees)' + when company_size = 10000 + then 'Large Enterprise (10,000+ employees)' + when company_size = 0 + then 'Other (N/A or Don''t know)' + else cast(company_size as varchar) + end as user_company_size, + case + when years_experience = 2 + then 'Less than 2 years' + when years_experience = 5 + then '2-5 years' + when years_experience = 10 + then '6 - 10 years' + when years_experience = 15 + then '11 - 15 years' + when years_experience = 20 + then '16 - 20 years' + when years_experience = 21 + then 'More than 20 years' + when years_experience = 0 + then 'Prefer not to say' + else cast(years_experience as varchar) + end as user_years_experience + from source + ) -, cleaned as ( - select - id as user_profile_id - , replace(replace(replace(name, ' ', '<>'), '><', ''), '<>', ' ') as user_full_name - , birth_year as user_birth_year - , company as user_company - , job_title as user_job_title - , industry as user_industry - , job_function as user_job_function - , user_id - , highest_education as user_highest_education - ,{{ transform_gender_value('gender') }} as user_gender - , case - when company_size = 1 then 'Small/Start-up (1+ employees)' - when company_size = 9 then 'Small/Home office (1-9 employees)' - when company_size = 99 then 'Small (10-99 employees)' - when company_size = 999 then 'Small to medium-sized (100-999 employees)' - when company_size = 9999 then 'Medium-sized (1000-9999 employees)' - when company_size = 10000 then 'Large Enterprise (10,000+ employees)' - when company_size = 0 then 'Other (N/A or Don''t know)' - else cast(company_size as varchar) - end as user_company_size - , case - when years_experience = 2 then 'Less than 2 years' - when years_experience = 5 then '2-5 years' - when years_experience = 10 then '6 - 10 years' - when years_experience = 15 then '11 - 15 years' - when years_experience = 20 then '16 - 20 years' - when years_experience = 21 then 'More than 20 years' - when years_experience = 0 then 'Prefer not to say' - else cast(years_experience as varchar) - end as user_years_experience - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__api__course.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__api__course.sql index 20e14cc38..d19f869be 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__api__course.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__api__course.sql @@ -1,28 +1,28 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__edxorg__s3__mitx_course') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__mitx_course") }}) -{{ deduplicate_raw_table(order_by='retrieved_at' , partition_columns = 'course_key') }} -, cleaned as ( - select - course_key as course_readable_id - , title as course_title - , short_description as course_description - , full_description as course_full_description - , level_type as course_level - , course_type - , owner as course_organizations - , marketing_url as course_marketing_url - , json_query(image, 'lax $.url' omit quotes) as course_image_url - , if( - subjects = '[]' - , null - , cast(json_parse(json_query(subjects, 'lax $.name' with array wrapper)) as array(varchar)) --noqa - ) as course_topics - , prerequisites_raw as course_prerequisites_text - , {{ cast_timestamp_to_iso8601('modified') }} as course_updated_at + {{ deduplicate_raw_table(order_by="retrieved_at", partition_columns="course_key") }}, + cleaned as ( + select + course_key as course_readable_id, + title as course_title, + short_description as course_description, + full_description as course_full_description, + level_type as course_level, + course_type, + owner as course_organizations, + marketing_url as course_marketing_url, + json_query(image, 'lax $.url' omit quotes) as course_image_url, + if( + subjects = '[]', + null, + cast(json_parse(json_query(subjects, 'lax $.name' with array wrapper)) as array(varchar)) -- noqa + ) as course_topics, + prerequisites_raw as course_prerequisites_text, + {{ cast_timestamp_to_iso8601("modified") }} as course_updated_at - from most_recent_source -) + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__api__courserun.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__api__courserun.sql index aff03ff43..bed21db25 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__api__courserun.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__api__courserun.sql @@ -1,55 +1,56 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__edxorg__s3__mitx_course_run') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__mitx_course_run") }}) -{{ deduplicate_raw_table(order_by='retrieved_at' , partition_columns = 'run_key') }} -, cleaned as ( - select - run_key as courserun_readable_id - , course_key as course_readable_id - , title as courserun_title - , short_description as courserun_short_description - , full_description as courserun_full_description - , level_type as courserun_level - , marketing_url as courserun_marketing_url - , languages as courserun_languages - , min_effort as courserun_min_weekly_hours - , max_effort as courserun_max_weekly_hours - , estimated_hours as courserun_estimated_hours - , enrollment_type as courserun_enrollment_mode - , availability as courserun_availability - , status as courserun_status - , is_enrollable as courserun_is_enrollable - , pacing_type as courserun_pace - , if(staff = '[]', null, staff) as courserun_instructors - , if(seats = '[]', null, seats) as courserun_enrollment_modes - , json_query(image, 'lax $.url' omit quotes) as courserun_image_url - , if(status = 'published' and is_enrollable, true, false) as courserun_is_published - , case - when pacing_type = 'self_paced' then true - when pacing_type = 'instructor_paced' then false - end as courserun_is_self_paced - , case - when weeks_to_complete = 1 then cast(weeks_to_complete as varchar) || ' week' - when weeks_to_complete > 1 then cast(weeks_to_complete as varchar) || ' weeks' - end as courserun_duration - , case - when min_effort is not null and max_effort is not null + {{ deduplicate_raw_table(order_by="retrieved_at", partition_columns="run_key") }}, + cleaned as ( + select + run_key as courserun_readable_id, + course_key as course_readable_id, + title as courserun_title, + short_description as courserun_short_description, + full_description as courserun_full_description, + level_type as courserun_level, + marketing_url as courserun_marketing_url, + languages as courserun_languages, + min_effort as courserun_min_weekly_hours, + max_effort as courserun_max_weekly_hours, + estimated_hours as courserun_estimated_hours, + enrollment_type as courserun_enrollment_mode, + availability as courserun_availability, + status as courserun_status, + is_enrollable as courserun_is_enrollable, + pacing_type as courserun_pace, + if(staff = '[]', null, staff) as courserun_instructors, + if(seats = '[]', null, seats) as courserun_enrollment_modes, + json_query(image, 'lax $.url' omit quotes) as courserun_image_url, + if(status = 'published' and is_enrollable, true, false) as courserun_is_published, + case + when pacing_type = 'self_paced' then true when pacing_type = 'instructor_paced' then false + end as courserun_is_self_paced, + case + when weeks_to_complete = 1 + then cast(weeks_to_complete as varchar) || ' week' + when weeks_to_complete > 1 + then cast(weeks_to_complete as varchar) || ' weeks' + end as courserun_duration, + case + when min_effort is not null and max_effort is not null then cast(min_effort as varchar) || '-' || cast(max_effort as varchar) || ' hours per week' - when min_effort is not null and max_effort is null + when min_effort is not null and max_effort is null then cast(min_effort as varchar) || ' hours per week' - when min_effort is null and max_effort is not null + when min_effort is null and max_effort is not null then cast(max_effort as varchar) || ' hours per week' - end as courserun_time_commitment - , {{ cast_timestamp_to_iso8601('start_on') }} as courserun_start_on - , {{ cast_timestamp_to_iso8601('end_on') }} as courserun_end_on - , {{ cast_timestamp_to_iso8601('enrollment_start') }} as courserun_enrollment_start_on - , {{ cast_timestamp_to_iso8601('enrollment_end') }} as courserun_enrollment_end_on - , {{ cast_timestamp_to_iso8601('announcement') }} as courserun_announce_on - , {{ cast_timestamp_to_iso8601('modified') }} as courserun_updated_at + end as courserun_time_commitment, + {{ cast_timestamp_to_iso8601("start_on") }} as courserun_start_on, + {{ cast_timestamp_to_iso8601("end_on") }} as courserun_end_on, + {{ cast_timestamp_to_iso8601("enrollment_start") }} as courserun_enrollment_start_on, + {{ cast_timestamp_to_iso8601("enrollment_end") }} as courserun_enrollment_end_on, + {{ cast_timestamp_to_iso8601("announcement") }} as courserun_announce_on, + {{ cast_timestamp_to_iso8601("modified") }} as courserun_updated_at - from most_recent_source -) + from most_recent_source + ) -select * from cleaned +select * +from cleaned where lower(courserun_title) not like 'delete%' diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_courserun.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_courserun.sql index 8fd0cfca2..691306a90 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_courserun.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_courserun.sql @@ -1,44 +1,43 @@ -- MITx Courses Information sourced from edx.org - -- Note that course id here is always in old format as {org}/{course number}/{run} as IR converts the new format -- to old format in order to preserve functionality of old apps -- e.g course_id: "MITx/14.73x/3T2020" is the same course run "course-v1:MITx+14.73x+3T2020" in MITxOnline - -with source as ( - - select * from {{ source('ol_warehouse_raw_data', 'raw__irx__edxorg__bigquery__mitx_course') }} - -) - -, renamed as ( - - select - course_title as courserun_title - , semester as courserun_semester - , url as courserun_url - , instructors as courserun_instructors - , registration_open as courserun_enrollment_start_date - , course_launch as courserun_start_date - , course_wrap as courserun_end_date - , department as coursedepartment_name - , dept_number as coursedepartment_number - , case course_id - when 'MITx/14.74x/3T2015' then '14.740x' - when 'MITx/ESD.SCM1x/3T2014' then 'CTL.SC1x' - when 'MITx/6.041x_3/2T2016' then '6.431x' - when 'MITx/6.041x_4/1T2017' then '6.431x' - else replace(course_number, ' ', '') - end as course_number - , case - when institution = 'MITx_PRO' then 'MITxPRO' - else institution - end as courserun_institution - , course_id as courserun_readable_id - , concat(element_at(split(course_id, '/'), 1), '/', element_at(split(course_id, '/'), 2)) as course_readable_id - , {{ translate_course_id_to_platform('course_id') }} as courserun_platform - , coalesce(self_paced = 1, false) as courserun_is_self_paced - from source - -) - -select * from renamed +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__irx__edxorg__bigquery__mitx_course") }}), + renamed as ( + + select + course_title as courserun_title, + semester as courserun_semester, + url as courserun_url, + instructors as courserun_instructors, + registration_open as courserun_enrollment_start_date, + course_launch as courserun_start_date, + course_wrap as courserun_end_date, + department as coursedepartment_name, + dept_number as coursedepartment_number, + case + course_id + when 'MITx/14.74x/3T2015' + then '14.740x' + when 'MITx/ESD.SCM1x/3T2014' + then 'CTL.SC1x' + when 'MITx/6.041x_3/2T2016' + then '6.431x' + when 'MITx/6.041x_4/1T2017' + then '6.431x' + else replace(course_number, ' ', '') + end as course_number, + case when institution = 'MITx_PRO' then 'MITxPRO' else institution end as courserun_institution, + course_id as courserun_readable_id, + concat( + element_at(split(course_id, '/'), 1), '/', element_at(split(course_id, '/'), 2) + ) as course_readable_id, + {{ translate_course_id_to_platform("course_id") }} as courserun_platform, + coalesce(self_paced = 1, false) as courserun_is_self_paced + from source + + ) + +select * +from renamed diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_person_course.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_person_course.sql index 7455a8e3a..6218219f3 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_person_course.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_person_course.sql @@ -1,69 +1,61 @@ -- Contains user profile, enrollments, grades and certificates Information for edx.org - - -with source as ( - - select * from {{ source('ol_warehouse_raw_data', 'raw__irx__edxorg__bigquery__mitx_person_course') }} - -) - -, renamed as ( - - select - user_id - , course_id as courserun_readable_id - ,{{ translate_course_id_to_platform('course_id') }} as courserun_platform - - --- users - , username as user_username - , yob as user_birth_year - , profile_country as user_profile_country - - , viewed as courseactivitiy_visited_once - , explored as courseactivitiy_viewed_half - -- user course activities from tracking logs - , nevents as courseactivitiy_num_events - , ndays_act as courseactivitiy_num_activity_days - , nprogcheck as courseactivitiy_num_progress_check - , nproblem_check as courseactivitiy_num_problem_check - , nshow_answer as courseactivitiy_num_show_answer - , ntranscript as courseactivitiy_num_show_transcript - , nseq_goto as courseactivitiy_num_seq_goto - , nplay_video as courseactivitiy_num_play_video - , nseek_video as courseactivitiy_num_seek_video - , npause_video as courseactivitiy_num_pause_video - , nvideo as courseactivitiy_num_video_interactions - , nvideos_unique_viewed as courseactivitiy_num_unique_videos_viewed - , nvideos_total_watched as courseactivitiy_percentage_total_videos_watched - , avg_dt as courseactivitiy_average_time_diff_in_sec - , sdv_dt as courseactivitiy_standard_deviation_in_sec - , max_dt as courseactivitiy_max_diff_in_sec - , n_dt as courseactivitiy_num_consecutive_events_used - , sum_dt as courseactivitiy_total_elapsed_time_in_sec - - --- enrollments, grades and certificates - , mode as courserunenrollment_enrollment_mode - , passing_grade as courserungrade_passing_grade - , grade as courserungrade_user_grade - , completed as courserungrade_is_passing - , certified as courseruncertificate_is_earned - , cert_status as courseruncertificate_status - , coalesce(is_active = 1, false) as courserunenrollment_is_active - --- trino doesn't have function to convert first letter to upper case - , regexp_replace( -- noqa: PRS - {{ transform_gender_value('gender') }}, '(^[a-z])(.)', x -> upper(x[1]) || x[2] -- noqa - ) as user_gender - ,{{ transform_education_value('loe') }} as user_highest_education - ,{{ cast_timestamp_to_iso8601('start_time') }} as courserunenrollment_created_on - ,{{ cast_timestamp_to_iso8601('verified_enroll_time') }} as courserunenrollment_enrolled_on - ,{{ cast_timestamp_to_iso8601('verified_unenroll_time') }} as courserunenrollment_unenrolled_on - ,{{ cast_timestamp_to_iso8601('cert_created_date') }} as courseruncertificate_created_on - ,{{ cast_timestamp_to_iso8601('cert_modified_date') }} as courseruncertificate_updated_on - ,{{ cast_timestamp_to_iso8601('first_event') }} as courseactivitiy_first_event_timestamp - ,{{ cast_timestamp_to_iso8601('last_event') }} as courseactivitiy_last_event_timestamp - from source - where user_id is not null --- temporary fix to filter the bad data - -) - -select * from renamed +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__irx__edxorg__bigquery__mitx_person_course") }}), + renamed as ( + + select + user_id, + course_id as courserun_readable_id, + {{ translate_course_id_to_platform("course_id") }} as courserun_platform, + -- - users + username as user_username, + yob as user_birth_year, + profile_country as user_profile_country, + viewed as courseactivitiy_visited_once, + explored as courseactivitiy_viewed_half, + -- user course activities from tracking logs + nevents as courseactivitiy_num_events, + ndays_act as courseactivitiy_num_activity_days, + nprogcheck as courseactivitiy_num_progress_check, + nproblem_check as courseactivitiy_num_problem_check, + nshow_answer as courseactivitiy_num_show_answer, + ntranscript as courseactivitiy_num_show_transcript, + nseq_goto as courseactivitiy_num_seq_goto, + nplay_video as courseactivitiy_num_play_video, + nseek_video as courseactivitiy_num_seek_video, + npause_video as courseactivitiy_num_pause_video, + nvideo as courseactivitiy_num_video_interactions, + nvideos_unique_viewed as courseactivitiy_num_unique_videos_viewed, + nvideos_total_watched as courseactivitiy_percentage_total_videos_watched, + avg_dt as courseactivitiy_average_time_diff_in_sec, + sdv_dt as courseactivitiy_standard_deviation_in_sec, + max_dt as courseactivitiy_max_diff_in_sec, + n_dt as courseactivitiy_num_consecutive_events_used, + sum_dt as courseactivitiy_total_elapsed_time_in_sec, + -- - enrollments, grades and certificates + mode as courserunenrollment_enrollment_mode, + passing_grade as courserungrade_passing_grade, + grade as courserungrade_user_grade, + completed as courserungrade_is_passing, + certified as courseruncertificate_is_earned, + cert_status as courseruncertificate_status, + coalesce(is_active = 1, false) as courserunenrollment_is_active, + -- - trino doesn't have function to convert first letter to upper case + regexp_replace( -- noqa: PRS + {{ transform_gender_value("gender") }}, '(^[a-z])(.)', x -> upper(x[1]) || x[2] -- noqa + ) as user_gender, + {{ transform_education_value("loe") }} as user_highest_education, + {{ cast_timestamp_to_iso8601("start_time") }} as courserunenrollment_created_on, + {{ cast_timestamp_to_iso8601("verified_enroll_time") }} as courserunenrollment_enrolled_on, + {{ cast_timestamp_to_iso8601("verified_unenroll_time") }} as courserunenrollment_unenrolled_on, + {{ cast_timestamp_to_iso8601("cert_created_date") }} as courseruncertificate_created_on, + {{ cast_timestamp_to_iso8601("cert_modified_date") }} as courseruncertificate_updated_on, + {{ cast_timestamp_to_iso8601("first_event") }} as courseactivitiy_first_event_timestamp, + {{ cast_timestamp_to_iso8601("last_event") }} as courseactivitiy_last_event_timestamp + from source + where user_id is not null -- - temporary fix to filter the bad data + + ) + +select * +from renamed diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_user_email_opt_in.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_user_email_opt_in.sql index 0cda93161..acaa1cfd5 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_user_email_opt_in.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_user_email_opt_in.sql @@ -1,21 +1,19 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__irx__edxorg__bigquery__email_opt_in') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__irx__edxorg__bigquery__email_opt_in") }}), + cleaned as ( -, cleaned as ( + select + user_id, + email as user_email, + username as user_username, + full_name as user_full_name, + is_opted_in_for_email as user_is_opted_in_for_email, + course_id as courserun_readable_id, + {{ translate_course_id_to_platform("course_id") }} as courserun_platform, + {{ cast_timestamp_to_iso8601("preference_set_datetime") }} as user_email_opt_in_updated_on + from source - select - user_id - , email as user_email - , username as user_username - , full_name as user_full_name - , is_opted_in_for_email as user_is_opted_in_for_email - , course_id as courserun_readable_id - ,{{ translate_course_id_to_platform('course_id') }} as courserun_platform - ,{{ cast_timestamp_to_iso8601('preference_set_datetime') }} as user_email_opt_in_updated_on - from source + ) -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_user_info_combo.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_user_info_combo.sql index 24ea9c6c3..bbcc447c6 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_user_info_combo.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__bigquery__mitx_user_info_combo.sql @@ -1,56 +1,53 @@ -- It contains user profiles, enrollments and certificate detail of learners in a given course -- There are some fields overlapping with person_course +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__irx__edxorg__bigquery__mitx_user_info_combo") }}), + cleaned as ( -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__irx__edxorg__bigquery__mitx_user_info_combo') }} -) + select + user_id, + {{ translate_course_id_to_platform("enrollment_course_id") }} as courserun_platform, + email as user_email, + username as user_username, + id_map_hash_id as user_map_hash_id, + replace(replace(replace(profile_name, ' ', '<>'), '><', ''), '<>', ' ') as user_full_name, + profile_mailing_address as user_mailing_address, + profile_goals as user_profile_goals, + profile_city as user_city, + -- filter outcountry values which are not valid country codes which consist of + -- two capital letters + case when regexp_like(profile_country, '^[A-Z]{2}$') then profile_country end as user_country, + profile_year_of_birth as user_birth_year, + profile_meta as user_profile_meta, + enrollment_is_active as courserunenrollment_is_active, + enrollment_mode as courserunenrollment_mode, + certificate_id as courseruncertificate_id, + certificate_user_id as courseruncertificate_user_id, + certificate_key as courseruncertificate_key, + certificate_mode as courseruncertificate_mode, + certificate_distinction as courseruncertificate_distinction, + certificate_grade as courseruncertificate_grade, + certificate_download_url as courseruncertificate_download_url, + certificate_download_uuid as courseruncertificate_download_uuid, + certificate_verify_uuid as courseruncertificate_verify_uuid, + certificate_name as courseruncertificate_name, + certificate_status as courseruncertificate_status, + -- - trino doesn't have function to convert first letter to upper case + regexp_replace( -- noqa: PRS + {{ transform_gender_value("profile_gender") }}, '(^[a-z])(.)', x -> upper(x[1]) || x[2] -- noqa + ) as user_gender, + {{ transform_education_value("profile_level_of_education") }} as user_highest_education, + {{ cast_timestamp_to_iso8601("date_joined") }} as user_joined_on, + {{ cast_timestamp_to_iso8601("last_login") }} as user_last_login, + {{ cast_timestamp_to_iso8601("enrollment_created") }} as courserunenrollment_created_on, + {{ cast_timestamp_to_iso8601("certificate_created_date") }} as courseruncertificate_created_on, + {{ cast_timestamp_to_iso8601("certificate_modified_date") }} as courseruncertificate_updated_on, + enrollment_course_id as courserunenrollment_courserun_readable_id, + certificate_course_id as courseruncertificate_courserun_readable_id + from source + -- - user_id could be blank due to parsing error on edx data so filter out these + where user_id is not null + ) -, cleaned as ( - - select - user_id - ,{{ translate_course_id_to_platform('enrollment_course_id') }} as courserun_platform - , email as user_email - , username as user_username - , id_map_hash_id as user_map_hash_id - , replace(replace(replace(profile_name, ' ', '<>'), '><', ''), '<>', ' ') as user_full_name - , profile_mailing_address as user_mailing_address - , profile_goals as user_profile_goals - , profile_city as user_city - -- filter outcountry values which are not valid country codes which consist of - -- two capital letters - , case when regexp_like(profile_country, '^[A-Z]{2}$') then profile_country end as user_country - , profile_year_of_birth as user_birth_year - , profile_meta as user_profile_meta - , enrollment_is_active as courserunenrollment_is_active - , enrollment_mode as courserunenrollment_mode - , certificate_id as courseruncertificate_id - , certificate_user_id as courseruncertificate_user_id - , certificate_key as courseruncertificate_key - , certificate_mode as courseruncertificate_mode - , certificate_distinction as courseruncertificate_distinction - , certificate_grade as courseruncertificate_grade - , certificate_download_url as courseruncertificate_download_url - , certificate_download_uuid as courseruncertificate_download_uuid - , certificate_verify_uuid as courseruncertificate_verify_uuid - , certificate_name as courseruncertificate_name - , certificate_status as courseruncertificate_status - --- trino doesn't have function to convert first letter to upper case - , regexp_replace( -- noqa: PRS - {{ transform_gender_value('profile_gender') }}, '(^[a-z])(.)', x -> upper(x[1]) || x[2] -- noqa - ) as user_gender - ,{{ transform_education_value('profile_level_of_education') }} as user_highest_education - ,{{ cast_timestamp_to_iso8601('date_joined') }} as user_joined_on - ,{{ cast_timestamp_to_iso8601('last_login') }} as user_last_login - ,{{ cast_timestamp_to_iso8601('enrollment_created') }} as courserunenrollment_created_on - ,{{ cast_timestamp_to_iso8601('certificate_created_date') }} as courseruncertificate_created_on - ,{{ cast_timestamp_to_iso8601('certificate_modified_date') }} as courseruncertificate_updated_on - , enrollment_course_id as courserunenrollment_courserun_readable_id - , certificate_course_id as courseruncertificate_courserun_readable_id - from source - --- user_id could be blank due to parsing error on edx data so filter out these - where user_id is not null -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_certificate_signatory.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_certificate_signatory.sql index 989fb802d..f2d0056ed 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_certificate_signatory.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_certificate_signatory.sql @@ -1,21 +1,23 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__edxorg__s3__course_structure__course_certificate_signatory') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__course_structure__course_certificate_signatory") }} + ) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id, course_id') }} -, cleaned as ( + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id, course_id") }}, + cleaned as ( - select - id as signatory_id - , course_id as courserun_readable_id - , title as signatory_title - , organization as signatory_organization - , certificate_id as signatory_certificate_id - , trim(name) as signatory_name - , regexp_replace(trim(name), '^(Professor |Dr\. )', '') as signatory_normalized_name - , concat('https://courses.edx.org', nullif(signature_image_path, '')) as signatory_image_url - from most_recent_source -) + select + id as signatory_id, + course_id as courserun_readable_id, + title as signatory_title, + organization as signatory_organization, + certificate_id as signatory_certificate_id, + trim(name) as signatory_name, + regexp_replace(trim(name), '^(Professor |Dr\. )', '') as signatory_normalized_name, + concat('https://courses.edx.org', nullif(signature_image_path, '')) as signatory_image_url + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_policy.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_policy.sql index 01c13e133..558d7fd76 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_policy.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_policy.sql @@ -1,18 +1,18 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__edxorg__s3__course_structure__course_policy') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__course_structure__course_policy") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'course_id') }} -, cleaned as ( + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="course_id") }}, + cleaned as ( - select - course_id as courserun_readable_id - , trim(both '"' from display_coursenumber) as display_coursenumber -- noqa: PRS - , self_paced as courserun_is_self_paced - , nullif(tabs, '[]') as courserun_tabs_array - , nullif(advanced_modules, '[]') as course_advanced_module_array - , discussions_settings as course_discussion_settings_object - from most_recent_source -) + select + course_id as courserun_readable_id, + trim(both '"' from display_coursenumber) as display_coursenumber, -- noqa: PRS + self_paced as courserun_is_self_paced, + nullif(tabs, '[]') as courserun_tabs_array, + nullif(advanced_modules, '[]') as course_advanced_module_array, + discussions_settings as course_discussion_settings_object + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_structure.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_structure.sql index 30f6cce91..d6ee29f4a 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_structure.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_structure.sql @@ -1,39 +1,36 @@ -with course_block_source as ( - select * from {{ source('ol_warehouse_raw_data','raw__edxorg__s3__course_structure__course_blocks') }} -) - -, course_block as ( - select * from ( +with + course_block_source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__course_structure__course_blocks") }} + ), + course_block as ( + select * + from + ( + select + course_block_source.*, + lag(course_block_source.course_content_hash) over ( + partition by course_block_source.block_id, course_block_source.course_id + order by course_block_source.retrieved_at asc + ) as previous_content_hash + from course_block_source + ) + where previous_content_hash is null or previous_content_hash != course_content_hash + ), + cleaned as ( select - course_block_source.* - , lag( - course_block_source.course_content_hash) - over - ( - partition by course_block_source.block_id, course_block_source.course_id - order by course_block_source.retrieved_at asc - ) - as previous_content_hash - from course_block_source + course_title as courserun_title, + block_index as coursestructure_block_index, + block_id as coursestructure_block_id, + block_parent as coursestructure_parent_block_id, + block_type as coursestructure_block_category, + block_title as coursestructure_block_title, + course_content_hash as coursestructure_content_hash, + block_content_hash as coursestructure_block_content_hash, + replace(replace(replace(replace(course_id, '[', ''), ']', ''), '"', ''), '-', '/') as courserun_readable_id, + json_query(block_details, 'lax $.metadata') as coursestructure_block_metadata, + {{ cast_timestamp_to_iso8601("retrieved_at") }} as coursestructure_retrieved_at + from course_block ) - where previous_content_hash is null or previous_content_hash != course_content_hash -) - -, cleaned as ( - select - course_title as courserun_title - , block_index as coursestructure_block_index - , block_id as coursestructure_block_id - , block_parent as coursestructure_parent_block_id - , block_type as coursestructure_block_category - , block_title as coursestructure_block_title - , course_content_hash as coursestructure_content_hash - , block_content_hash as coursestructure_block_content_hash - , replace(replace(replace(replace(course_id, '[', ''), ']', ''), '"', ''), '-', '/') - as courserun_readable_id - , json_query(block_details, 'lax $.metadata') as coursestructure_block_metadata - , {{ cast_timestamp_to_iso8601('retrieved_at') }} as coursestructure_retrieved_at - from course_block -) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_video.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_video.sql index b9f584d70..dcad88034 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_video.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__course_video.sql @@ -1,17 +1,17 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__edxorg__s3__course_structure__course_video') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__course_structure__course_video") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'course_id, video_block_id') }} -, cleaned as ( + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="course_id, video_block_id") }}, + cleaned as ( - select - course_id as courserun_readable_id - , video_block_id - , edx_video_id as video_edx_id - , cast(duration as decimal(38, 4)) as video_duration - , replace(replace(course_id, 'course-v1:', ''), '+', '/') as courserun_old_readable_id - from most_recent_source -) + select + course_id as courserun_readable_id, + video_block_id, + edx_video_id as video_edx_id, + cast(duration as decimal(38, 4)) as video_duration, + replace(replace(course_id, 'course-v1:', ''), '+', '/') as courserun_old_readable_id + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun.sql index b1306375d..eaced7fd2 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun.sql @@ -1,36 +1,34 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__edxorg__s3__course_structure__course_metadata') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__course_structure__course_metadata") }} + ) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'course_id') }} -, cleaned as ( - select - course_number - , institution as courserun_institution - , course_id as courserun_readable_id - , replace(replace(course_id, 'course-v1:', ''), '+', '/') as courserun_edx_readable_id - , title as courserun_title - , semester as courserun_tag - , self_paced as courserun_is_self_paced - , cast(json_parse(json_query(instructor_info, 'lax $.instructors.name' with array wrapper)) as array(varchar)) --noqa - as courserun_instructors - , case - when lower(course_start) = 'null' then null - else {{ cast_timestamp_to_iso8601('course_start') }} - end as courserun_start_on - , case - when lower(course_end) = 'null' then null - else {{ cast_timestamp_to_iso8601('course_end') }} - end as courserun_end_on - , case - when lower(enrollment_start) = 'null' then null - else {{ cast_timestamp_to_iso8601('enrollment_start') }} - end as courserun_enrollment_start_on - , case - when lower(enrollment_end) = 'null' then null - else {{ cast_timestamp_to_iso8601('enrollment_end') }} - end as courserun_enrollment_end_on - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="course_id") }}, + cleaned as ( + select + course_number, + institution as courserun_institution, + course_id as courserun_readable_id, + replace(replace(course_id, 'course-v1:', ''), '+', '/') as courserun_edx_readable_id, + title as courserun_title, + semester as courserun_tag, + self_paced as courserun_is_self_paced, + cast(json_parse(json_query(instructor_info, 'lax $.instructors.name' with array wrapper)) as array(varchar)) -- noqa + as courserun_instructors, + case + when lower(course_start) = 'null' then null else {{ cast_timestamp_to_iso8601("course_start") }} + end as courserun_start_on, + case + when lower(course_end) = 'null' then null else {{ cast_timestamp_to_iso8601("course_end") }} + end as courserun_end_on, + case + when lower(enrollment_start) = 'null' then null else {{ cast_timestamp_to_iso8601("enrollment_start") }} + end as courserun_enrollment_start_on, + case + when lower(enrollment_end) = 'null' then null else {{ cast_timestamp_to_iso8601("enrollment_end") }} + end as courserun_enrollment_end_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun_certificate.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun_certificate.sql index e732ef3a0..bcb31f9c1 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun_certificate.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun_certificate.sql @@ -1,31 +1,36 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__edxorg__s3__tables__certificates_generatedcertificate') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__tables__certificates_generatedcertificate") }} + ) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - --- all values are ingested as string, so we need to cast here to match other data sources - select - cast(id as integer) as courseruncertificate_id - , course_id as courserun_readable_id - , cast(user_id as integer) as user_id - , name as courseruncertificate_user_full_name - , key as courseruncertificate_key - , mode as courseruncertificate_mode - , download_url as courseruncertificate_download_url - , download_uuid as courseruncertificate_download_uuid - , verify_uuid as courseruncertificate_verify_uuid - , status as courseruncertificate_status - , try_cast(grade as decimal(38, 2)) as courseruncertificate_grade - , case - when lower(created_date) = 'null' or lower(created_date) = 'none' then null - else to_iso8601(date_parse(created_date, '%Y-%m-%d %H:%i:%s')) - end as courseruncertificate_created_on - , case - when lower(modified_date) = 'null' or lower(modified_date) = 'none' then null - else to_iso8601(date_parse(modified_date, '%Y-%m-%d %H:%i:%s')) - end as courseruncertificate_updated_on - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + -- - all values are ingested as string, so we need to cast here to match other data sources + select + cast(id as integer) as courseruncertificate_id, + course_id as courserun_readable_id, + cast(user_id as integer) as user_id, + name as courseruncertificate_user_full_name, + key as courseruncertificate_key, + mode as courseruncertificate_mode, + download_url as courseruncertificate_download_url, + download_uuid as courseruncertificate_download_uuid, + verify_uuid as courseruncertificate_verify_uuid, + status as courseruncertificate_status, + try_cast(grade as decimal(38, 2)) as courseruncertificate_grade, + case + when lower(created_date) = 'null' or lower(created_date) = 'none' + then null + else to_iso8601(date_parse(created_date, '%Y-%m-%d %H:%i:%s')) + end as courseruncertificate_created_on, + case + when lower(modified_date) = 'null' or lower(modified_date) = 'none' + then null + else to_iso8601(date_parse(modified_date, '%Y-%m-%d %H:%i:%s')) + end as courseruncertificate_updated_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun_enrollment.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun_enrollment.sql index 03cf2ca47..debb5d340 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun_enrollment.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun_enrollment.sql @@ -1,18 +1,18 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__edxorg__s3__tables__student_courseenrollment') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__tables__student_courseenrollment") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - --- all values are ingested as string, so we need to cast here to match other data sources - select - cast(id as integer) as courserunenrollment_id - , course_id as courserun_readable_id - , cast(user_id as integer) as user_id - , cast(is_active as boolean) as courserunenrollment_is_active - , mode as courserunenrollment_mode - , to_iso8601(date_parse(created, '%Y-%m-%d %H:%i:%s')) as courserunenrollment_created_on - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + -- - all values are ingested as string, so we need to cast here to match other data sources + select + cast(id as integer) as courserunenrollment_id, + course_id as courserun_readable_id, + cast(user_id as integer) as user_id, + cast(is_active as boolean) as courserunenrollment_is_active, + mode as courserunenrollment_mode, + to_iso8601(date_parse(created, '%Y-%m-%d %H:%i:%s')) as courserunenrollment_created_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun_grade.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun_grade.sql index ffd9ea195..53eb9392f 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun_grade.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courserun_grade.sql @@ -1,30 +1,42 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__edxorg__s3__tables__grades_persistentcoursegrade') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__tables__grades_persistentcoursegrade") }} + ) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'course_id, user_id') }} -, cleaned as ( - select - ---all values are ingested as string, so we need to cast here to match other data sources - course_id as courserun_readable_id - , cast(user_id as integer) as user_id - , cast(percent_grade as decimal(38, 2)) as courserungrade_grade - , letter_grade as courserungrade_letter_grade - , grading_policy_hash as courserungrade_grading_policy_hash - , case - when passed_timestamp = 'NULL' then null - else to_iso8601(from_iso8601_timestamp_nanos( - regexp_replace(passed_timestamp, '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2})(.*?)', '$1T$2$3') - )) - end as courserungrade_first_passed_on - --- use regex here to preserve the faction of second if value uses nanosecond - , to_iso8601(from_iso8601_timestamp_nanos( - regexp_replace(created, '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2})(.*?)', '$1T$2$3') - )) as courserungrade_created_on - , to_iso8601(from_iso8601_timestamp_nanos( - regexp_replace(modified, '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2})(.*?)', '$1T$2$3') - )) as courserungrade_updated_on - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="course_id, user_id") }}, + cleaned as ( + select + -- -all values are ingested as string, so we need to cast here to match other data sources + course_id as courserun_readable_id, + cast(user_id as integer) as user_id, + cast(percent_grade as decimal(38, 2)) as courserungrade_grade, + letter_grade as courserungrade_letter_grade, + grading_policy_hash as courserungrade_grading_policy_hash, + case + when passed_timestamp = 'NULL' + then null + else + to_iso8601( + from_iso8601_timestamp_nanos( + regexp_replace( + passed_timestamp, '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2})(.*?)', '$1T$2$3' + ) + ) + ) + end as courserungrade_first_passed_on, + -- - use regex here to preserve the faction of second if value uses nanosecond + to_iso8601( + from_iso8601_timestamp_nanos( + regexp_replace(created, '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2})(.*?)', '$1T$2$3') + ) + ) as courserungrade_created_on, + to_iso8601( + from_iso8601_timestamp_nanos( + regexp_replace(modified, '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2})(.*?)', '$1T$2$3') + ) + ) as courserungrade_updated_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courseware_studentmodule.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courseware_studentmodule.sql index 7d5a681fa..313144528 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courseware_studentmodule.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__courseware_studentmodule.sql @@ -1,22 +1,22 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__edxorg__s3__tables__courseware_studentmodule') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__tables__courseware_studentmodule") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'course_id, student_id, module_id') }} -, cleaned as ( + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="course_id, student_id, module_id") }}, + cleaned as ( - select - cast(id as bigint) as studentmodule_id - , course_id as courserun_readable_id - , module_id as coursestructure_block_id - , module_type as coursestructure_block_category - , cast(student_id as integer) as user_id - , state as studentmodule_state_data - , try_cast(grade as double) as studentmodule_problem_grade - , try_cast(max_grade as double) as studentmodule_problem_max_grade - , to_iso8601(date_parse(created, '%Y-%m-%d %H:%i:%s')) as studentmodule_created_on - , to_iso8601(date_parse(modified, '%Y-%m-%d %H:%i:%s')) as studentmodule_updated_on - from most_recent_source -) + select + cast(id as bigint) as studentmodule_id, + course_id as courserun_readable_id, + module_id as coursestructure_block_id, + module_type as coursestructure_block_category, + cast(student_id as integer) as user_id, + state as studentmodule_state_data, + try_cast(grade as double) as studentmodule_problem_grade, + try_cast(max_grade as double) as studentmodule_problem_max_grade, + to_iso8601(date_parse(created, '%Y-%m-%d %H:%i:%s')) as studentmodule_created_on, + to_iso8601(date_parse(modified, '%Y-%m-%d %H:%i:%s')) as studentmodule_updated_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__program_courses.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__program_courses.sql index 7a3fef886..dac2f3134 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__program_courses.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__program_courses.sql @@ -1,16 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__edxorg__s3__program_course') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__program_course") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'program_uuid, course_key') }} -, cleaned as ( - select - program_uuid - , {{ format_course_id('course_key') }} as course_readable_id - , course_title - , course_short_description as course_description - , course_type - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="program_uuid, course_key") }}, + cleaned as ( + select + program_uuid, + {{ format_course_id("course_key") }} as course_readable_id, + course_title, + course_short_description as course_description, + course_type + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__program_learner_report.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__program_learner_report.sql index b4516a515..a243f5397 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__program_learner_report.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__program_learner_report.sql @@ -1,121 +1,122 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__edxorg__program_learner_report') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__program_learner_report") }}), + source_sorted as ( + select + *, + case + when "date program certificate awarded" = 'null' + then null -- noqa: ST10 + else + -- Try parsing once and handle both formats + coalesce( + try(to_iso8601(date_parse("date program certificate awarded", '%Y-%m-%dT%H:%i:%sZ'))), + to_iso8601(date_parse("date program certificate awarded", '%Y-%m-%d %H:%i:%s Z')) + ) -, source_sorted as ( - select - * - , case - when "date program certificate awarded" = 'null' then null -- noqa: ST10 - else - -- Try parsing once and handle both formats - coalesce( - try(to_iso8601(date_parse("date program certificate awarded", '%Y-%m-%dT%H:%i:%sZ'))) - , to_iso8601(date_parse("date program certificate awarded", '%Y-%m-%d %H:%i:%s Z')) - ) + end as program_certificate_awarded_at, + row_number() over ( + partition by "user id", "course run key", "program uuid" + order by _airbyte_extracted_at desc, _ab_source_file_last_modified desc + ) as row_num + from source + ), + aggregated_program_certificate as ( + select + cast("user id" as integer) as user_id, + "program uuid" as program_uuid, + "course run key" as courserun_readable_id, + min(program_certificate_awarded_at) as earliest_program_cert_award_on, + max("completed program") as ever_completed_program + from source_sorted + group by 1, 2, 3 + ), + dedup_source as (select * from source_sorted where row_num = 1), + cleaned as ( - end as program_certificate_awarded_at - , row_number() over ( - partition by "user id", "course run key", "program uuid" - order by _airbyte_extracted_at desc, _ab_source_file_last_modified desc - ) as row_num - from source -) + select + "authoring institution" as org_id, + "program type" as program_type, + "program uuid" as program_uuid, + username as user_username, + name as user_full_name, + "course run key" as courserun_readable_id, + "course title" as course_title, + track as courserunenrollment_enrollment_mode, + cast("user id" as integer) as user_id, + cast(completed as boolean) as user_has_completed_course, + cast("completed program" as boolean) as user_has_completed_program, + cast("currently enrolled" as boolean) as courserunenrollment_is_active, + cast("purchased as bundle" as boolean) as user_has_purchased_as_bundle, + if("user roles" = 'null', null, "user roles") as user_roles, -- noqa: ST10 + if("letter grade" = 'null', null, "letter grade") as courserungrade_letter_grade, -- noqa: ST10 + if(grade = 'null', null, grade) as courserungrade_grade, + case + when "program uuid" like '%941d3eaf56966c7' + then 'Finance' + when "program uuid" like '%3173ff51e11a748' + then 'MIT Finance' + when "program uuid" like '%8c11bfd9c0d7b07' + then 'Statistics and Data Science (General Track)' + when "program uuid" like '%cd7c6461dd9b1d4' + then 'Statistics and Data Science (Social Sciences Track)' + else "program title" + end as program_title, + to_iso8601(date_parse("course run start date", '%Y-%m-%d %H:%i:%s Z')) as courserun_start_on, + to_iso8601(date_parse("date first enrolled", '%Y-%m-%d %H:%i:%s Z')) as courserunenrollment_created_on, + case + when "date completed" = 'null' + then null -- noqa: ST10 + else to_iso8601(date_parse("date completed", '%Y-%m-%d %H:%i:%s Z')) + end as completed_course_on, + case + when "last activity date" = 'null' + then null -- noqa: ST10 + else {{ cast_date_to_iso8601('"last activity date"') }} + end as courseactivity_last_activity_date, + case + when "date last unenrolled" = 'null' + then null -- noqa: ST10 + else to_iso8601(date_parse("date last unenrolled", '%Y-%m-%d %H:%i:%s Z')) + end as courserunenrollment_unenrolled_on, + case + when "date first upgraded to verified" = 'null' + then null -- noqa: ST10 + else to_iso8601(date_parse("date first upgraded to verified", '%Y-%m-%d %H:%i:%s Z')) + end as courserunenrollment_upgraded_on, + program_certificate_awarded_at as program_certificate_awarded_on + from dedup_source -, aggregated_program_certificate as ( - select - cast("user id" as integer) as user_id - , "program uuid" as program_uuid - , "course run key" as courserun_readable_id - , min(program_certificate_awarded_at) as earliest_program_cert_award_on - , max("completed program") as ever_completed_program - from source_sorted - group by 1, 2, 3 -) - -, dedup_source as ( - select * from source_sorted - where row_num = 1 -) - -, cleaned as ( - - select - "authoring institution" as org_id - , "program type" as program_type - , "program uuid" as program_uuid - , username as user_username - , name as user_full_name - , "course run key" as courserun_readable_id - , "course title" as course_title - , track as courserunenrollment_enrollment_mode - , cast("user id" as integer) as user_id - , cast(completed as boolean) as user_has_completed_course - , cast("completed program" as boolean) as user_has_completed_program - , cast("currently enrolled" as boolean) as courserunenrollment_is_active - , cast("purchased as bundle" as boolean) as user_has_purchased_as_bundle - , if("user roles" = 'null', null, "user roles") as user_roles -- noqa: ST10 - , if("letter grade" = 'null', null, "letter grade") as courserungrade_letter_grade -- noqa: ST10 - , if(grade = 'null', null, grade) as courserungrade_grade - , case - when "program uuid" like '%941d3eaf56966c7' then 'Finance' - when "program uuid" like '%3173ff51e11a748' then 'MIT Finance' - when "program uuid" like '%8c11bfd9c0d7b07' then 'Statistics and Data Science (General Track)' - when "program uuid" like '%cd7c6461dd9b1d4' then 'Statistics and Data Science (Social Sciences Track)' - else "program title" - end as program_title - , to_iso8601(date_parse("course run start date", '%Y-%m-%d %H:%i:%s Z')) as courserun_start_on - , to_iso8601(date_parse("date first enrolled", '%Y-%m-%d %H:%i:%s Z')) as courserunenrollment_created_on - , case - when "date completed" = 'null' then null -- noqa: ST10 - else to_iso8601(date_parse("date completed", '%Y-%m-%d %H:%i:%s Z')) - end as completed_course_on - , case - when "last activity date" = 'null' then null -- noqa: ST10 - else {{ cast_date_to_iso8601('"last activity date"') }} - end as courseactivity_last_activity_date - , case - when "date last unenrolled" = 'null' then null -- noqa: ST10 - else to_iso8601(date_parse("date last unenrolled", '%Y-%m-%d %H:%i:%s Z')) - end as courserunenrollment_unenrolled_on - , case - when "date first upgraded to verified" = 'null' then null -- noqa: ST10 - else to_iso8601(date_parse("date first upgraded to verified", '%Y-%m-%d %H:%i:%s Z')) - end as courserunenrollment_upgraded_on - , program_certificate_awarded_at as program_certificate_awarded_on - from dedup_source - -) + ) select - cleaned.org_id - , cleaned.program_type - , cleaned.program_uuid - , cleaned.program_title - , cleaned.user_username - , cleaned.user_full_name - , cleaned.courserun_readable_id - , cleaned.course_title - , cleaned.courserunenrollment_enrollment_mode - , cleaned.user_id - , cleaned.user_has_completed_course - , cast(aggregated_program_certificate.ever_completed_program as boolean) as user_has_completed_program - , cleaned.courserunenrollment_is_active - , cleaned.user_has_purchased_as_bundle - , cleaned.user_roles - , cleaned.courserungrade_letter_grade - , cleaned.courserungrade_grade - , cleaned.courserun_start_on - , cleaned.courserunenrollment_created_on - , cleaned.completed_course_on - , cleaned.courseactivity_last_activity_date - , cleaned.courserunenrollment_unenrolled_on - , cleaned.courserunenrollment_upgraded_on - , aggregated_program_certificate.earliest_program_cert_award_on as program_certificate_awarded_on - , regexp_extract(cleaned.program_title, '\((.*?)\)', 1) as program_track + cleaned.org_id, + cleaned.program_type, + cleaned.program_uuid, + cleaned.program_title, + cleaned.user_username, + cleaned.user_full_name, + cleaned.courserun_readable_id, + cleaned.course_title, + cleaned.courserunenrollment_enrollment_mode, + cleaned.user_id, + cleaned.user_has_completed_course, + cast(aggregated_program_certificate.ever_completed_program as boolean) as user_has_completed_program, + cleaned.courserunenrollment_is_active, + cleaned.user_has_purchased_as_bundle, + cleaned.user_roles, + cleaned.courserungrade_letter_grade, + cleaned.courserungrade_grade, + cleaned.courserun_start_on, + cleaned.courserunenrollment_created_on, + cleaned.completed_course_on, + cleaned.courseactivity_last_activity_date, + cleaned.courserunenrollment_unenrolled_on, + cleaned.courserunenrollment_upgraded_on, + aggregated_program_certificate.earliest_program_cert_award_on as program_certificate_awarded_on, + regexp_extract(cleaned.program_title, '\((.*?)\)', 1) as program_track from cleaned -left join aggregated_program_certificate - on - cleaned.user_id = aggregated_program_certificate.user_id - and cleaned.program_uuid = aggregated_program_certificate.program_uuid - and cleaned.courserun_readable_id = aggregated_program_certificate.courserun_readable_id +left join + aggregated_program_certificate + on cleaned.user_id = aggregated_program_certificate.user_id + and cleaned.program_uuid = aggregated_program_certificate.program_uuid + and cleaned.courserun_readable_id = aggregated_program_certificate.courserun_readable_id diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__programs.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__programs.sql index aa7fff292..9f251513c 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__programs.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__programs.sql @@ -1,18 +1,18 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__edxorg__s3__program') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__program") }}) -{{ deduplicate_raw_table(order_by='retrieved_at' , partition_columns = 'uuid') }} -, cleaned as ( - select - uuid as program_uuid - , title as program_title - , subtitle as program_subtitle - , type as program_type - , status as program_status - , authoring_organizations as program_organization - , {{ cast_timestamp_to_iso8601('data_modified_timestamp') }} as program_updated_on - from most_recent_source -) + {{ deduplicate_raw_table(order_by="retrieved_at", partition_columns="uuid") }}, + cleaned as ( + select + uuid as program_uuid, + title as program_title, + subtitle as program_subtitle, + type as program_type, + status as program_status, + authoring_organizations as program_organization, + {{ cast_timestamp_to_iso8601("data_modified_timestamp") }} as program_updated_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__tracking_logs__user_activity.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__tracking_logs__user_activity.sql index eb94ee026..62c6a3980 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__tracking_logs__user_activity.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__tracking_logs__user_activity.sql @@ -1,51 +1,65 @@ -{{ config( - materialized='incremental', - unique_key = ['user_username', 'useractivity_context_object', 'useractivity_event_source', - 'useractivity_event_type', 'useractivity_event_object', 'useractivity_timestamp'], - incremental_strategy='delete+insert', - views_enabled=false, - ) +{{ + config( + materialized="incremental", + unique_key=[ + "user_username", + "useractivity_context_object", + "useractivity_event_source", + "useractivity_event_type", + "useractivity_event_object", + "useractivity_timestamp", + ], + incremental_strategy="delete+insert", + views_enabled=false, + ) }} -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__edxorg__s3__tracking_logs') }} - where - username != '' - and json_query(context, 'lax $.user_id' omit quotes) is not null +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__tracking_logs") }} + where + username != '' and json_query(context, 'lax $.user_id' omit quotes) is not null - {% if is_incremental() %} - and "time" > (select max(this.useractivity_timestamp) from {{ this }} as this) - {% endif %} -) + {% if is_incremental() %} + and "time" > (select max(this.useractivity_timestamp) from {{ this }} as this) + {% endif %} + ) -{{ deduplicate_raw_table( - order_by='_airbyte_extracted_at desc, _ab_source_file_last_modified desc, "time"' - , partition_columns = 'username, context, event_source, event_type, event, "time"' -) }} -, cleaned as ( - select - username as user_username - , context as useractivity_context_object - , event as useractivity_event_object - , event_source as useractivity_event_source - , page as useractivity_page_url - , session as useractivity_session_id - , ip as useractivity_ip - , host as useractivity_http_host - , agent as useractivity_http_user_agent - , accept_language as useractivity_http_accept_language - , referer as useractivity_http_referer - , name as useractivity_event_name - , event_type as useractivity_event_type - , {{ extract_course_id_from_tracking_log(course_id_has_old_format=true) }} as courserun_readable_id - , cast(json_query(context, 'lax $.user_id' omit quotes) as integer) as user_id - , json_query(context, 'lax $.org_id' omit quotes) as org_id - , json_query(context, 'lax $.path' omit quotes) as useractivity_path - --- use regex here to preserve the nanoseconds as date_parse truncates the fraction of second to milliseconds - , to_iso8601(from_iso8601_timestamp_nanos( - regexp_replace("time", '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2}\.\d+)(.*?)', '$1T$2$3') - )) as useractivity_timestamp - from most_recent_source -) + {{ + deduplicate_raw_table( + order_by='_airbyte_extracted_at desc, _ab_source_file_last_modified desc, "time"', + partition_columns='username, context, event_source, event_type, event, "time"', + ) + }}, + cleaned as ( + select + username as user_username, + context as useractivity_context_object, + event as useractivity_event_object, + event_source as useractivity_event_source, + page as useractivity_page_url, + session as useractivity_session_id, + ip as useractivity_ip, + host as useractivity_http_host, + agent as useractivity_http_user_agent, + accept_language as useractivity_http_accept_language, + referer as useractivity_http_referer, + name as useractivity_event_name, + event_type as useractivity_event_type, + {{ extract_course_id_from_tracking_log(course_id_has_old_format=true) }} as courserun_readable_id, + cast(json_query(context, 'lax $.user_id' omit quotes) as integer) as user_id, + json_query(context, 'lax $.org_id' omit quotes) as org_id, + json_query(context, 'lax $.path' omit quotes) as useractivity_path, + -- - use regex here to preserve the nanoseconds as date_parse truncates the fraction of second to + -- milliseconds + to_iso8601( + from_iso8601_timestamp_nanos( + regexp_replace("time", '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2}\.\d+)(.*?)', '$1T$2$3') + ) + ) as useractivity_timestamp + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__user.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__user.sql index 66ac19abb..4b7b81f85 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__user.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__user.sql @@ -1,27 +1,33 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__edxorg__s3__tables__auth_user') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__tables__auth_user") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - select - ---all values are ingested as string, so we need to cast here to match other data sources - cast(id as integer) as user_id - , username as user_username - , email as user_email - , cast(is_active as boolean) as user_is_active - , cast(is_staff as boolean) as user_is_staff - , cast(is_superuser as boolean) as user_is_superuser - , to_iso8601(from_iso8601_timestamp_nanos( - regexp_replace(date_joined, '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2})(.*?)', '$1T$2$3') - )) as user_joined_on - , case - when lower(last_login) = 'null' or lower(last_login) = 'none' then null - else to_iso8601(from_iso8601_timestamp_nanos( - regexp_replace(last_login, '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2})(.*?)', '$1T$2$3') - )) - end as user_last_login - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + select + -- -all values are ingested as string, so we need to cast here to match other data sources + cast(id as integer) as user_id, + username as user_username, + email as user_email, + cast(is_active as boolean) as user_is_active, + cast(is_staff as boolean) as user_is_staff, + cast(is_superuser as boolean) as user_is_superuser, + to_iso8601( + from_iso8601_timestamp_nanos( + regexp_replace(date_joined, '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2})(.*?)', '$1T$2$3') + ) + ) as user_joined_on, + case + when lower(last_login) = 'null' or lower(last_login) = 'none' + then null + else + to_iso8601( + from_iso8601_timestamp_nanos( + regexp_replace(last_login, '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2})(.*?)', '$1T$2$3') + ) + ) + end as user_last_login + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__user_courseaccessrole.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__user_courseaccessrole.sql index e609933b5..6b1b4967b 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__user_courseaccessrole.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__user_courseaccessrole.sql @@ -1,16 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__edxorg__s3__tables__student_courseaccessrole') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__tables__student_courseaccessrole") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'course_id, user_id, role') }} -, cleaned as ( - select - course_id as courserun_readable_id - , cast(user_id as integer) as user_id - , role as courseaccess_role - , replace(replace(course_id, 'course-v1:', ''), '+', '/') as courserun_edx_readable_id - , if(lower(org) = 'mitxt', 'MITxT', org) as organization - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="course_id, user_id, role") }}, + cleaned as ( + select + course_id as courserun_readable_id, + cast(user_id as integer) as user_id, + role as courseaccess_role, + replace(replace(course_id, 'course-v1:', ''), '+', '/') as courserun_edx_readable_id, + if(lower(org) = 'mitxt', 'MITxT', org) as organization + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__user_profile.sql b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__user_profile.sql index d511e201c..fac59cc4e 100644 --- a/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__user_profile.sql +++ b/src/ol_dbt/models/staging/edxorg/stg__edxorg__s3__user_profile.sql @@ -1,44 +1,40 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__edxorg__s3__tables__auth_userprofile') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__edxorg__s3__tables__auth_userprofile") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - select - ---all values are ingested as string, so we need to cast here to match other data sources - cast(id as integer) as user_profile_id - , cast(user_id as integer) as user_id - , meta as user_profile_metadata - , case - when lower(name) = 'null' then null - when name = '' then null - when name like '><%' then null - else name - end as user_full_name - , case - when lower(bio) = 'null' then null - when bio = '' then null - else bio - end as user_bio - , case - when lower(goals) = 'null' then null - when goals = '' then null - when goals like '><%' then null - else goals - end as user_goals - , case - when lower(year_of_birth) = 'null' then null - when year_of_birth = '' then null - else try_cast(year_of_birth as integer) - end as user_year_of_birth - , case - when lower(country) = 'null' then null - when country = '' then null - else country - end as user_address_country - , {{ transform_gender_value('gender') }} as user_gender - , {{ transform_education_value('level_of_education') }} as user_highest_education - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + select + -- -all values are ingested as string, so we need to cast here to match other data sources + cast(id as integer) as user_profile_id, + cast(user_id as integer) as user_id, + meta as user_profile_metadata, + case + when lower(name) = 'null' then null when name = '' then null when name like '><%' then null else name + end as user_full_name, + case when lower(bio) = 'null' then null when bio = '' then null else bio end as user_bio, + case + when lower(goals) = 'null' + then null + when goals = '' + then null + when goals like '><%' + then null + else goals + end as user_goals, + case + when lower(year_of_birth) = 'null' + then null + when year_of_birth = '' + then null + else try_cast(year_of_birth as integer) + end as user_year_of_birth, + case + when lower(country) = 'null' then null when country = '' then null else country + end as user_address_country, + {{ transform_gender_value("gender") }} as user_gender, + {{ transform_education_value("level_of_education") }} as user_highest_education + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__chatbots_djangocheckpoint.sql b/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__chatbots_djangocheckpoint.sql index 7043bda94..930e7e12b 100644 --- a/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__chatbots_djangocheckpoint.sql +++ b/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__chatbots_djangocheckpoint.sql @@ -1,24 +1,27 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__learn_ai__app__postgres__ai_chatbots_djangocheckpoint') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__learn_ai__app__postgres__ai_chatbots_djangocheckpoint") }} + ) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - select - id as djangocheckpoint_id - , type as checkpoint_type - , session_id as chatsession_id - , thread_id as chatsession_thread_id - , checkpoint as checkpoint_json - , checkpoint_ns as checkpoint_namespace - , checkpoint_id - , parent_checkpoint_id - , metadata as checkpoint_metadata - , cast(json_query(metadata, 'lax $.step' omit quotes) as integer) as checkpoint_step - , json_query(metadata, 'lax $.writes.__start__.messages.kwargs.content' omit quotes) as human_message - , json_query(metadata, 'lax $.writes.agent.messages.kwargs.content' omit quotes) as agent_message - , json_query(metadata, 'lax $.source' omit quotes) as checkpoint_source - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + select + id as djangocheckpoint_id, + type as checkpoint_type, + session_id as chatsession_id, + thread_id as chatsession_thread_id, + checkpoint as checkpoint_json, + checkpoint_ns as checkpoint_namespace, + checkpoint_id, + parent_checkpoint_id, + metadata as checkpoint_metadata, + cast(json_query(metadata, 'lax $.step' omit quotes) as integer) as checkpoint_step, + json_query(metadata, 'lax $.writes.__start__.messages.kwargs.content' omit quotes) as human_message, + json_query(metadata, 'lax $.writes.agent.messages.kwargs.content' omit quotes) as agent_message, + json_query(metadata, 'lax $.source' omit quotes) as checkpoint_source + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__chatbots_tutorbotoutput.sql b/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__chatbots_tutorbotoutput.sql index 3590a4ad1..5f07965e0 100644 --- a/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__chatbots_tutorbotoutput.sql +++ b/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__chatbots_tutorbotoutput.sql @@ -1,14 +1,13 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__learn_ai__app__postgres__ai_chatbots_tutorbotoutput') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__learn_ai__app__postgres__ai_chatbots_tutorbotoutput") }} + ) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - select - id as tutorbotoutput_id - , thread_id as chatsession_thread_id - , chat_json as tutorbot_chat_json - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + select id as tutorbotoutput_id, thread_id as chatsession_thread_id, chat_json as tutorbot_chat_json + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__chatbots_userchatsession.sql b/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__chatbots_userchatsession.sql index 78043ddc6..7609d541a 100644 --- a/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__chatbots_userchatsession.sql +++ b/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__chatbots_userchatsession.sql @@ -1,20 +1,22 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__learn_ai__app__postgres__ai_chatbots_userchatsession') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__learn_ai__app__postgres__ai_chatbots_userchatsession") }} + ) -{{ deduplicate_raw_table(order_by='updated_on' , partition_columns = 'id') }} -, cleaned as ( - select - id as chatsession_id - , agent as chatsession_agent - , title as chatsession_title - , user_id - , thread_id as chatsession_thread_id - , object_id as chatsession_object_id - , dj_session_key as chatsession_django_session_key - , {{ cast_timestamp_to_iso8601('created_on') }} as chatsession_created_on - , {{ cast_timestamp_to_iso8601('updated_on') }} as chatsession_updated_on - from most_recent_source -) + {{ deduplicate_raw_table(order_by="updated_on", partition_columns="id") }}, + cleaned as ( + select + id as chatsession_id, + agent as chatsession_agent, + title as chatsession_title, + user_id, + thread_id as chatsession_thread_id, + object_id as chatsession_object_id, + dj_session_key as chatsession_django_session_key, + {{ cast_timestamp_to_iso8601("created_on") }} as chatsession_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as chatsession_updated_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__users_user.sql b/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__users_user.sql index 646fc226e..1450d1b7c 100644 --- a/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__users_user.sql +++ b/src/ol_dbt/models/staging/learn-ai/stg__learn_ai__app__postgres__users_user.sql @@ -1,22 +1,22 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__learn_ai__app__postgres__users_user') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__learn_ai__app__postgres__users_user") }}) -{{ deduplicate_raw_table(order_by='updated_on' , partition_columns = 'id') }} -, cleaned as ( - select - id as user_id - , global_id as user_global_id - , username as user_username - , email as user_email - , name as user_full_name - , is_active as user_is_active - , is_staff as user_is_staff - , is_superuser as user_is_superuser - , {{ cast_timestamp_to_iso8601('created_on') }} as user_created_on - , {{ cast_timestamp_to_iso8601('updated_on') }} as user_updated_on - , {{ cast_timestamp_to_iso8601('last_login') }} as user_last_login - from most_recent_source -) + {{ deduplicate_raw_table(order_by="updated_on", partition_columns="id") }}, + cleaned as ( + select + id as user_id, + global_id as user_global_id, + username as user_username, + email as user_email, + name as user_full_name, + is_active as user_is_active, + is_staff as user_is_staff, + is_superuser as user_is_superuser, + {{ cast_timestamp_to_iso8601("created_on") }} as user_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as user_updated_on, + {{ cast_timestamp_to_iso8601("last_login") }} as user_last_login + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__auth_user.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__auth_user.sql index e00572646..8b2cd1038 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__auth_user.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__auth_user.sql @@ -1,19 +1,17 @@ -- MicroMasters User Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__auth_user") }}), + cleaned as ( -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__auth_user') }} -) + select + id as user_id, + username as user_username, + email as user_email, + is_active as user_is_active, + {{ cast_timestamp_to_iso8601("date_joined") }} as user_joined_on, + {{ cast_timestamp_to_iso8601("last_login") }} as user_last_login + from source + ) -, cleaned as ( - - select - id as user_id - , username as user_username - , email as user_email - , is_active as user_is_active - ,{{ cast_timestamp_to_iso8601('date_joined') }} as user_joined_on - ,{{ cast_timestamp_to_iso8601('last_login') }} as user_last_login - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__auth_usersocialauth.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__auth_usersocialauth.sql index 9c5785643..881970bd2 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__auth_usersocialauth.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__auth_usersocialauth.sql @@ -2,19 +2,12 @@ -- Authentication backend can either be MITxOnline or edx.org -- Can be used to match MITxOnline users. However, -- not every user from MITxOnline DB has a matching record in this source table - -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__social_auth_usersocialauth') }} -) - -, cleaned as ( - - select - id as user_auth_id - , user_id - , uid as user_username - , provider as user_auth_provider - from source -) - -select * from cleaned +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__social_auth_usersocialauth") }} + ), + cleaned as (select id as user_auth_id, user_id, uid as user_username, provider as user_auth_provider from source) + +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_course.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_course.sql index 381d4f096..05fd84e9a 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_course.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_course.sql @@ -1,23 +1,21 @@ -- MicroMasters Course metadata Information -- It should be primarily used to link edx.org courses to programs in MicroMasters +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__courses_course") }}), + cleaned as ( + select + id as course_id, + title as course_title, + course_number, + program_id, + prerequisites as course_prerequisites, + description as course_description, + contact_email as course_contact_email, + should_display_progress as course_should_display_progress, + position_in_program as course_position_in_program, + replace(replace(replace(edx_key, 'course-v1:', ''), '+', '/'), '_1', '') as course_edx_key + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__courses_course') }} -) - -, cleaned as ( - select - id as course_id - , title as course_title - , course_number - , program_id - , prerequisites as course_prerequisites - , description as course_description - , contact_email as course_contact_email - , should_display_progress as course_should_display_progress - , position_in_program as course_position_in_program - , replace(replace(replace(edx_key, 'course-v1:', ''), '+', '/'), '_1', '') as course_edx_key - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_courserun.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_courserun.sql index 17975e7db..7e20e93c6 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_courserun.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_courserun.sql @@ -1,35 +1,39 @@ -- MicroMasters Course Run metadata Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__courses_courserun") }} + ), + cleaned as ( + select + id as courserun_id, + course_id, + title as courserun_title, + edx_course_key as courserun_readable_id, + replace(replace(edx_course_key, 'course-v1:', ''), '+', '/') as courserun_edxorg_readable_id, + case + when edx_course_key like 'MITx/%' + then split(edx_course_key, '/')[3] + when edx_course_key like 'course-v1:%' + then split(edx_course_key, '+')[3] + end as courserun_tag, + case + when courseware_backend = 'mitxonline' + then '{{ var("mitxonline") }}' + when courseware_backend = 'edxorg' + then '{{ var("edxorg") }}' + else courseware_backend + end as courserun_platform, + enrollment_url as courserun_enrollment_url, + is_discontinued as courserun_is_discontinued, + prerequisites as courserun_prerequisites, + {{ cast_timestamp_to_iso8601("start_date") }} as courserun_start_on, + {{ cast_timestamp_to_iso8601("end_date") }} as courserun_end_on, + {{ cast_timestamp_to_iso8601("enrollment_start") }} as courserun_enrollment_start_on, + {{ cast_timestamp_to_iso8601("enrollment_end") }} as courserun_enrollment_end_on, + {{ cast_timestamp_to_iso8601("freeze_grade_date") }} as courserun_grade_freeze_on, + {{ cast_timestamp_to_iso8601("upgrade_deadline") }} as courserun_upgrade_deadline + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__courses_courserun') }} -) - -, cleaned as ( - select - id as courserun_id - , course_id - , title as courserun_title - , edx_course_key as courserun_readable_id - , replace(replace(edx_course_key, 'course-v1:', ''), '+', '/') as courserun_edxorg_readable_id - , case - when edx_course_key like 'MITx/%' then split(edx_course_key, '/')[3] - when edx_course_key like 'course-v1:%' then split(edx_course_key, '+')[3] - end as courserun_tag - , case - when courseware_backend = 'mitxonline' then '{{ var("mitxonline") }}' - when courseware_backend = 'edxorg' then '{{ var("edxorg") }}' - else courseware_backend - end as courserun_platform - , enrollment_url as courserun_enrollment_url - , is_discontinued as courserun_is_discontinued - , prerequisites as courserun_prerequisites - ,{{ cast_timestamp_to_iso8601('start_date') }} as courserun_start_on - ,{{ cast_timestamp_to_iso8601('end_date') }} as courserun_end_on - ,{{ cast_timestamp_to_iso8601('enrollment_start') }} as courserun_enrollment_start_on - ,{{ cast_timestamp_to_iso8601('enrollment_end') }} as courserun_enrollment_end_on - ,{{ cast_timestamp_to_iso8601('freeze_grade_date') }} as courserun_grade_freeze_on - ,{{ cast_timestamp_to_iso8601('upgrade_deadline') }} as courserun_upgrade_deadline - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_electiveset.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_electiveset.sql index 08381ff8e..ba1b3113b 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_electiveset.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_electiveset.sql @@ -1,16 +1,13 @@ -- MicroMasters program elective set Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__courses_electivesset") }} + ), + cleaned as ( + select + id as electiveset_id, program_id, title as electiveset_title, required_number as electiveset_required_number + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__courses_electivesset') }} -) - -, cleaned as ( - select - id as electiveset_id - , program_id - , title as electiveset_title - , required_number as electiveset_required_number - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_electiveset_to_course.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_electiveset_to_course.sql index a1c762478..7237e2291 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_electiveset_to_course.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_electiveset_to_course.sql @@ -1,15 +1,9 @@ -- MicroMasters course to electives set Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__courses_electivecourse") }} + ), + cleaned as (select id as electivesettocourse_id, course_id, electives_set_id as electiveset_id from source) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__courses_electivecourse') }} -) - -, cleaned as ( - select - id as electivesettocourse_id - , course_id - , electives_set_id as electiveset_id - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_program.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_program.sql index 2d355eb34..24b8014b3 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_program.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__courses_program.sql @@ -1,22 +1,22 @@ -- MicroMasters Program Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__courses_program") }} + ), + cleaned as ( + select + id as program_id, + live as program_is_live, + title as program_title, + description as program_description, + price as program_price, + num_required_courses as program_num_required_courses, + ga_tracking_id as program_ga_tracking_id, + financial_aid_availability as program_is_financial_aid_available, + {{ cast_timestamp_to_iso8601("created_on") }} as program_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as program_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__courses_program') }} -) - -, cleaned as ( - select - id as program_id - , live as program_is_live - , title as program_title - , description as program_description - , price as program_price - , num_required_courses as program_num_required_courses - , ga_tracking_id as program_ga_tracking_id - , financial_aid_availability as program_is_financial_aid_available - ,{{ cast_timestamp_to_iso8601('created_on') }} as program_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as program_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__dashboard_programenrollment.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__dashboard_programenrollment.sql index e7e2f43e3..6ef6c8318 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__dashboard_programenrollment.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__dashboard_programenrollment.sql @@ -1,14 +1,9 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__dashboard_programenrollment') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__dashboard_programenrollment") }} + ), + cleaned as (select id as programenrollment_id, user_id, program_id, share_hash from source) -, cleaned as ( - select - id as programenrollment_id - , user_id - , program_id - , share_hash - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__django_contenttype.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__django_contenttype.sql index 11e564c44..099043f65 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__django_contenttype.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__django_contenttype.sql @@ -1,20 +1,10 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__django_content_type') }} + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__django_content_type") }} -) + ), + renamed as (select id as contenttype_id, concat_ws('_', app_label, model) as contenttype_full_name from source) -, renamed as ( - - select - id as contenttype_id - , concat_ws( - '_' - , app_label - , model - ) as contenttype_full_name - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_coupon.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_coupon.sql index f67ae4a66..c97cf6422 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_coupon.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_coupon.sql @@ -1,36 +1,37 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__micromasters__app__postgres__ecommerce_coupon') }} + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__ecommerce_coupon") }} -) + ), + renamed as ( -, renamed as ( - - select - id as coupon_id - , coupon_code - , coupon_type - , object_id as coupon_object_id - , amount_type as coupon_amount_type - , cast(amount as decimal(38, 2)) as coupon_amount - , enabled as coupon_is_active - , content_type_id as contenttype_id - , invoice_id as couponinvoice_id - , case - when amount_type = 'fixed-discount' + select + id as coupon_id, + coupon_code, + coupon_type, + object_id as coupon_object_id, + amount_type as coupon_amount_type, + cast(amount as decimal(38, 2)) as coupon_amount, + enabled as coupon_is_active, + content_type_id as contenttype_id, + invoice_id as couponinvoice_id, + case + when amount_type = 'fixed-discount' then concat('$', format('%.2f', amount)) - when amount_type = 'fixed-price' + when amount_type = 'fixed-price' then concat('Fixed Price: ', format('%.2f', amount)) - when amount_type = 'percent-discount' + when amount_type = 'percent-discount' then concat(format('%.2f', amount * 100), '%') - end as coupon_discount_amount_text - ,{{ cast_timestamp_to_iso8601('created_on') }} as coupon_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as coupon_updated_on - ,{{ cast_timestamp_to_iso8601('activation_date') }} as coupon_activated_on - ,{{ cast_timestamp_to_iso8601('expiration_date') }} as coupon_expires_on + end as coupon_discount_amount_text, + {{ cast_timestamp_to_iso8601("created_on") }} as coupon_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as coupon_updated_on, + {{ cast_timestamp_to_iso8601("activation_date") }} as coupon_activated_on, + {{ cast_timestamp_to_iso8601("expiration_date") }} as coupon_expires_on - from source + from source -) + ) -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_couponinvoice.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_couponinvoice.sql index 6def0e457..6d72ee1e3 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_couponinvoice.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_couponinvoice.sql @@ -1,17 +1,15 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__micromasters__app__postgres__ecommerce_couponinvoice') }} + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__ecommerce_couponinvoice") }} -) + ), + renamed as ( -, renamed as ( + select id as couponinvoice_id, invoice_number as couponinvoice_number, description as couponinvoice_description + from source - select - id as couponinvoice_id - , invoice_number as couponinvoice_number - , description as couponinvoice_description - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_line.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_line.sql index a0a739319..157e17f68 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_line.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_line.sql @@ -1,22 +1,19 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__ecommerce_line") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__micromasters__app__postgres__ecommerce_line') }} + select + id as line_id, + course_key as courserun_readable_id, + replace(replace(course_key, 'course-v1:', ''), '+', '/') as courserun_edxorg_readable_id, + cast(price as decimal(38, 2)) as line_price, + description as line_description, + {{ cast_timestamp_to_iso8601("created_at") }} as line_created_on, + {{ cast_timestamp_to_iso8601("modified_at") }} as line_updated_on, + order_id + from source -) + ) -, renamed as ( - - select - id as line_id - , course_key as courserun_readable_id - , replace(replace(course_key, 'course-v1:', ''), '+', '/') as courserun_edxorg_readable_id - , cast(price as decimal(38, 2)) as line_price - , description as line_description - ,{{ cast_timestamp_to_iso8601('created_at') }} as line_created_on - ,{{ cast_timestamp_to_iso8601('modified_at') }} as line_updated_on - , order_id - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_order.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_order.sql index 6fe8908aa..3c2002200 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_order.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_order.sql @@ -1,21 +1,22 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__micromasters__app__postgres__ecommerce_order') }} + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__ecommerce_order") }} -) + ), + renamed as ( -, renamed as ( + select + id as order_id, + status as order_state, + user_id, + reference_number as order_reference_number, + cast(total_price_paid as decimal(38, 2)) as order_total_price_paid, + {{ cast_timestamp_to_iso8601("created_at") }} as order_created_on, + {{ cast_timestamp_to_iso8601("modified_at") }} as order_updated_on + from source - select - id as order_id - , status as order_state - , user_id - , reference_number as order_reference_number - , cast(total_price_paid as decimal(38, 2)) as order_total_price_paid - ,{{ cast_timestamp_to_iso8601('created_at') }} as order_created_on - ,{{ cast_timestamp_to_iso8601('modified_at') }} as order_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_receipt.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_receipt.sql index dd116b110..809e6481d 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_receipt.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_receipt.sql @@ -1,39 +1,40 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__micromasters__app__postgres__ecommerce_receipt') }} + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__ecommerce_receipt") }} -) + ), + renamed as ( + select + id as receipt_id, + order_id, + json_query(data, 'lax $.decision' omit quotes) as receipt_transaction_status, + json_query(data, 'lax $.transaction_id' omit quotes) as receipt_transaction_id, + json_query(data, 'lax $.req_transaction_uuid' omit quotes) as receipt_transaction_uuid, + json_query(data, 'lax $.req_transaction_type' omit quotes) as receipt_transaction_type, + json_query(data, 'lax $.req_payment_method' omit quotes) as receipt_payment_method, + json_query(data, 'lax $.auth_code' omit quotes) as receipt_authorization_code, + json_query(data, 'lax $.req_reference_number' omit quotes) as receipt_reference_number, + json_query(data, 'lax $.req_bill_to_address_state' omit quotes) as receipt_bill_to_address_state, + json_query(data, 'lax $.req_bill_to_address_country' omit quotes) as receipt_bill_to_address_country, + cast(json_query(data, 'lax $.req_amount' omit quotes) as decimal(38, 2)) as receipt_payment_amount, + json_query(data, 'lax $.req_currency' omit quotes) as receipt_payment_currency, + json_query(data, 'lax $.req_bill_to_email' omit quotes) as receipt_payer_email, + json_query(data, 'lax $.req_card_number' omit quotes) as receipt_payment_card_number, + json_query(data, 'lax $.req_customer_ip_address' omit quotes) as receipt_payer_ip_address, + json_query(data, 'lax $.card_type_name' omit quotes) as receipt_payment_card_type, + concat( + json_query(data, 'lax $.req_bill_to_forename' omit quotes), + ' ', + json_query(data, 'lax $.req_bill_to_surname' omit quotes) + ) as receipt_payer_name, + data as receipt_data, + json_query(data, 'lax $.signed_date_time' omit quotes) as receipt_payment_timestamp, + {{ cast_timestamp_to_iso8601("created_at") }} as receipt_created_on, + {{ cast_timestamp_to_iso8601("modified_at") }} as receipt_updated_on + from source -, renamed as ( - select - id as receipt_id - , order_id - , json_query(data, 'lax $.decision' omit quotes) as receipt_transaction_status - , json_query(data, 'lax $.transaction_id' omit quotes) as receipt_transaction_id - , json_query(data, 'lax $.req_transaction_uuid' omit quotes) as receipt_transaction_uuid - , json_query(data, 'lax $.req_transaction_type' omit quotes) as receipt_transaction_type - , json_query(data, 'lax $.req_payment_method' omit quotes) as receipt_payment_method - , json_query(data, 'lax $.auth_code' omit quotes) as receipt_authorization_code - , json_query(data, 'lax $.req_reference_number' omit quotes) as receipt_reference_number - , json_query(data, 'lax $.req_bill_to_address_state' omit quotes) as receipt_bill_to_address_state - , json_query(data, 'lax $.req_bill_to_address_country' omit quotes) as receipt_bill_to_address_country - , cast(json_query(data, 'lax $.req_amount' omit quotes) as decimal(38, 2)) as receipt_payment_amount - , json_query(data, 'lax $.req_currency' omit quotes) as receipt_payment_currency - , json_query(data, 'lax $.req_bill_to_email' omit quotes) as receipt_payer_email - , json_query(data, 'lax $.req_card_number' omit quotes) as receipt_payment_card_number - , json_query(data, 'lax $.req_customer_ip_address' omit quotes) as receipt_payer_ip_address - , json_query(data, 'lax $.card_type_name' omit quotes) as receipt_payment_card_type - , concat( - json_query(data, 'lax $.req_bill_to_forename' omit quotes) - , ' ' - , json_query(data, 'lax $.req_bill_to_surname' omit quotes) - ) as receipt_payer_name - , data as receipt_data - , json_query(data, 'lax $.signed_date_time' omit quotes) as receipt_payment_timestamp - ,{{ cast_timestamp_to_iso8601('created_at') }} as receipt_created_on - ,{{ cast_timestamp_to_iso8601('modified_at') }} as receipt_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_redeemedcoupon.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_redeemedcoupon.sql index 2a2c87411..064d854f7 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_redeemedcoupon.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_redeemedcoupon.sql @@ -1,20 +1,22 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__micromasters__app__postgres__ecommerce_redeemedcoupon') }} + select * + from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__ecommerce_redeemedcoupon") }} -) + ), + renamed as ( -, renamed as ( + select + id as redeemedcoupon_id, + order_id, + coupon_id, + {{ cast_timestamp_to_iso8601("created_on") }} as redeemedcoupon_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as redeemedcoupon_updated_on - select - id as redeemedcoupon_id - , order_id - , coupon_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as redeemedcoupon_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as redeemedcoupon_updated_on + from source - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_usercoupon.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_usercoupon.sql index c52c7bcc0..fe72edc83 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_usercoupon.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__ecommerce_usercoupon.sql @@ -1,19 +1,20 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__micromasters__app__postgres__ecommerce_usercoupon') }} + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__ecommerce_usercoupon") }} -) + ), + renamed as ( -, renamed as ( + select + id as usercoupon_id, + user_id, + coupon_id, + {{ cast_timestamp_to_iso8601("created_on") }} as usercoupon_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as usercoupon_updated_on + from source - select - id as usercoupon_id - , user_id - , coupon_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as usercoupon_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as usercoupon_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__exams_examrun.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__exams_examrun.sql index 0d3baaf86..8f6edfe30 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__exams_examrun.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__exams_examrun.sql @@ -1,24 +1,21 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__exams_examrun") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__micromasters__app__postgres__exams_examrun') }} + select + id as examrun_id, + course_id, + exam_series_code, + edx_exam_course_key as examrun_readable_id, + semester as examrun_semester, + description as examrun_description, + authorized as examrun_is_authorized, + passing_score as examrun_passing_grade, + {{ cast_timestamp_to_iso8601("created_on") }} as examrun_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as examrun_updated_on + from source -) + ) -, renamed as ( - - select - id as examrun_id - , course_id - , exam_series_code - , edx_exam_course_key as examrun_readable_id - , semester as examrun_semester - , description as examrun_description - , authorized as examrun_is_authorized - , passing_score as examrun_passing_grade - ,{{ cast_timestamp_to_iso8601('created_on') }} as examrun_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as examrun_updated_on - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_combinedcoursegrade.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_combinedcoursegrade.sql index 8e4720e31..e7df39d93 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_combinedcoursegrade.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_combinedcoursegrade.sql @@ -1,20 +1,20 @@ -- DEDP course final grades combined from course run and exam from MicroMaster DB +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__grades_combinedfinalgrade") }} + ), + cleaned as ( + select + id as coursegrade_id, + user_id, + course_id, + grade as coursegrade_grade, + {{ cast_timestamp_to_iso8601("created_on") }} as coursegrade_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as coursegrade_updated_on -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__grades_combinedfinalgrade') }} -) + from source + ) -, cleaned as ( - select - id as coursegrade_id - , user_id - , course_id - , grade as coursegrade_grade - ,{{ cast_timestamp_to_iso8601('created_on') }} as coursegrade_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as coursegrade_updated_on - - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_coursecertificate.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_coursecertificate.sql index 4243b9f16..0948860a2 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_coursecertificate.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_coursecertificate.sql @@ -1,21 +1,26 @@ -- DEDP Course Certificate Information from MicroMaster DB +with + source as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__micromasters__app__postgres__grades_micromasterscoursecertificate" + ) + }} + ), + cleaned as ( + select + id as coursecertificate_id, + user_id, + course_id, + hash as coursecertificate_hash, + concat('https://micromasters.mit.edu/certificate/course/', hash) as coursecertificate_url, + {{ cast_timestamp_to_iso8601("created_on") }} as coursecertificate_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as coursecertificate_updated_on -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__grades_micromasterscoursecertificate') }} -) + from source + ) -, cleaned as ( - select - id as coursecertificate_id - , user_id - , course_id - , hash as coursecertificate_hash - , concat('https://micromasters.mit.edu/certificate/course/', hash) as coursecertificate_url - ,{{ cast_timestamp_to_iso8601('created_on') }} as coursecertificate_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as coursecertificate_updated_on - - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_courserungrade.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_courserungrade.sql index 5da2cc28a..431d7a64f 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_courserungrade.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_courserungrade.sql @@ -1,22 +1,21 @@ -- course run final grades from MicroMaster DB +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__grades_finalgrade") }} + ), + cleaned as ( + select + id as courserungrade_id, + user_id, + course_run_id as courserun_id, + grade as courserungrade_grade, + passed as courserungrade_is_passing, + status as courserungrade_status, + course_run_paid_on_edx as courserungrade_courserun_paid_on_edx, + {{ cast_timestamp_to_iso8601("created_on") }} as coursegrade_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as coursegrade_updated_on + from source + ) -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__grades_finalgrade') }} -) - -, cleaned as ( - select - id as courserungrade_id - , user_id - , course_run_id as courserun_id - , grade as courserungrade_grade - , passed as courserungrade_is_passing - , status as courserungrade_status - , course_run_paid_on_edx as courserungrade_courserun_paid_on_edx - ,{{ cast_timestamp_to_iso8601('created_on') }} as coursegrade_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as coursegrade_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_proctoredexamgrade.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_proctoredexamgrade.sql index 6c9e2fa1c..8901bd35c 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_proctoredexamgrade.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_proctoredexamgrade.sql @@ -1,26 +1,26 @@ -- DEDP proctor exam grades from MicroMasters DB +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__grades_proctoredexamgrade") }} + ), + cleaned as ( + select + id as proctoredexamgrade_id, + user_id, + course_id, + if(passing_score < 1, passing_score * 100, passing_score) as proctoredexamgrade_passing_score, + passed as proctoredexamgrade_is_passing, + score as proctoredexamgrade_score, + grade as proctoredexamgrade_letter_grade, + percentage_grade as proctoredexamgrade_percentage_grade, + exam_run_id as examrun_id, + {{ cast_timestamp_to_iso8601("exam_date") }} as proctoredexamgrade_exam_on, + {{ cast_timestamp_to_iso8601("created_on") }} as proctoredexamgrade_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as proctoredexamgrade_updated_on -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__grades_proctoredexamgrade') }} -) + from source + ) -, cleaned as ( - select - id as proctoredexamgrade_id - , user_id - , course_id - , if(passing_score < 1, passing_score * 100, passing_score) as proctoredexamgrade_passing_score - , passed as proctoredexamgrade_is_passing - , score as proctoredexamgrade_score - , grade as proctoredexamgrade_letter_grade - , percentage_grade as proctoredexamgrade_percentage_grade - , exam_run_id as examrun_id - ,{{ cast_timestamp_to_iso8601('exam_date') }} as proctoredexamgrade_exam_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as proctoredexamgrade_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as proctoredexamgrade_updated_on - - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_programcertificate.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_programcertificate.sql index 174941fc7..6a08e03f2 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_programcertificate.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__grades_programcertificate.sql @@ -1,20 +1,25 @@ -- MicroMasters Program Information +with + source as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__micromasters__app__postgres__grades_micromastersprogramcertificate" + ) + }} + ), + cleaned as ( + select + id as programcertificate_id, + {{ cast_timestamp_to_iso8601("created_on") }} as programcertificate_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as programcertificate_updated_on, + hash as programcertificate_hash, + program_id, + user_id -with source as ( - select * - from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__grades_micromastersprogramcertificate') }} -) + from source + ) -, cleaned as ( - select - id as programcertificate_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as programcertificate_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as programcertificate_updated_on - , hash as programcertificate_hash - , program_id - , user_id - - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__profiles_education.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__profiles_education.sql index 06836c0af..dde9053cf 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__profiles_education.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__profiles_education.sql @@ -1,22 +1,22 @@ -- MicroMasters User Educations Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__profiles_education") }} + ), + cleaned as ( + select + id as user_education_id, + profile_id as user_profile_id, + field_of_study as user_field_of_study, + online_degree as user_education_is_online_degree, + school_city as user_school_city, + school_country as user_school_country, + school_name as user_school_name, + school_state_or_territory as user_school_state, + {{ transform_education_value("degree_name") }} as user_education_degree, + {{ cast_date_to_iso8601("graduation_date") }} as user_graduation_date + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__profiles_education') }} -) - -, cleaned as ( - select - id as user_education_id - , profile_id as user_profile_id - , field_of_study as user_field_of_study - , online_degree as user_education_is_online_degree - , school_city as user_school_city - , school_country as user_school_country - , school_name as user_school_name - , school_state_or_territory as user_school_state - , {{ transform_education_value('degree_name') }} as user_education_degree - , {{ cast_date_to_iso8601('graduation_date') }} as user_graduation_date - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__profiles_employment.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__profiles_employment.sql index 0a492e3f4..e2a4b55dc 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__profiles_employment.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__profiles_employment.sql @@ -1,22 +1,22 @@ -- MicroMasters User Employment Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__profiles_employment") }} + ), + cleaned as ( + select + id as user_employment_id, + profile_id as user_profile_id, + position as user_job_position, + industry as user_company_industry, + company_name as user_company_name, + city as user_company_city, + state_or_territory as user_company_state, + country as user_company_country, + {{ cast_date_to_iso8601("start_date") }} as user_start_date, + {{ cast_date_to_iso8601("end_date") }} as user_end_date + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__profiles_employment') }} -) - -, cleaned as ( - select - id as user_employment_id - , profile_id as user_profile_id - , position as user_job_position - , industry as user_company_industry - , company_name as user_company_name - , city as user_company_city - , state_or_territory as user_company_state - , country as user_company_country - , {{ cast_date_to_iso8601('start_date') }} as user_start_date - , {{ cast_date_to_iso8601('end_date') }} as user_end_date - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__profiles_profile.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__profiles_profile.sql index ea00cebc2..363d98913 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__profiles_profile.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__postgres__profiles_profile.sql @@ -1,74 +1,69 @@ -- MicroMasters User Profile Information - -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__micromasters__app__postgres__profiles_profile') }} -) - -, cleaned as ( - select - id as user_profile_id - , user_id - , country as user_address_country - , birth_country as user_birth_country - , city as user_address_city - , state_or_territory as user_address_state_or_territory - , postal_code as user_address_postal_code - , address as user_street_address - , preferred_name as user_preferred_name - , mail_id as user_mail_id - , student_id as user_student_id - , email_optin as user_email_is_optin - , filled_out as user_profile_is_filled_out - , image as user_profile_image - , image_small as user_profile_image_small - , image_medium as user_profile_image_medium - , nationality as user_nationality - , phone_number as user_phone_number - , edx_employer as user_employer - , edx_job_title as user_job_title - , preferred_language as user_preferred_language - , edx_mailing_address as user_mailing_address - , verified_micromaster_user as user_is_verified - , agreed_to_terms_of_service as user_has_agreed_to_terms_of_service - , edx_language_proficiencies as user_language_proficiencies - , edx_requires_parental_consent as user_profile_parental_consent_is_required - , edx_bio as user_bio - , about_me as user_about_me - , edx_name as user_edx_name - , edx_goals as user_edx_goals - , fake_user as user_profile_is_fake - ,{{ cast_date_to_iso8601('date_of_birth') }} as user_birth_date - , coalesce(romanized_first_name, first_name) as user_first_name - , coalesce(romanized_last_name, last_name) as user_last_name - , replace( +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__micromasters__app__postgres__profiles_profile") }} + ), + cleaned as ( + select + id as user_profile_id, + user_id, + country as user_address_country, + birth_country as user_birth_country, + city as user_address_city, + state_or_territory as user_address_state_or_territory, + postal_code as user_address_postal_code, + address as user_street_address, + preferred_name as user_preferred_name, + mail_id as user_mail_id, + student_id as user_student_id, + email_optin as user_email_is_optin, + filled_out as user_profile_is_filled_out, + image as user_profile_image, + image_small as user_profile_image_small, + image_medium as user_profile_image_medium, + nationality as user_nationality, + phone_number as user_phone_number, + edx_employer as user_employer, + edx_job_title as user_job_title, + preferred_language as user_preferred_language, + edx_mailing_address as user_mailing_address, + verified_micromaster_user as user_is_verified, + agreed_to_terms_of_service as user_has_agreed_to_terms_of_service, + edx_language_proficiencies as user_language_proficiencies, + edx_requires_parental_consent as user_profile_parental_consent_is_required, + edx_bio as user_bio, + about_me as user_about_me, + edx_name as user_edx_name, + edx_goals as user_edx_goals, + fake_user as user_profile_is_fake, + {{ cast_date_to_iso8601("date_of_birth") }} as user_birth_date, + coalesce(romanized_first_name, first_name) as user_first_name, + coalesce(romanized_last_name, last_name) as user_last_name, replace( replace( - ( - concat_ws( - chr(32) - , nullif(first_name, '') - , nullif(last_name, '') - ) - ), ' ', '<>' - ), '><', '' - ), '<>', ' ' - ) as user_full_name - , concat_ws( - chr(32) - , nullif(romanized_first_name, '') - , nullif(romanized_last_name, '') - ) as user_romanized_full_name - ,{{ transform_education_value('edx_level_of_education') }} as user_highest_education - ,{{ transform_gender_value('gender') }} as user_gender - , case - when account_privacy = 'public' then 'Public to everyone' - when account_privacy = 'public_to_mm' then 'Public to logged in users' - when account_privacy = 'private' then 'Private' - else account_privacy - end as user_account_privacy - ,{{ cast_timestamp_to_iso8601('date_joined_micromasters') }} as user_joined_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as user_profile_updated_on - from source -) + replace((concat_ws(chr(32), nullif(first_name, ''), nullif(last_name, ''))), ' ', '<>'), '><', '' + ), + '<>', + ' ' + ) as user_full_name, + concat_ws( + chr(32), nullif(romanized_first_name, ''), nullif(romanized_last_name, '') + ) as user_romanized_full_name, + {{ transform_education_value("edx_level_of_education") }} as user_highest_education, + {{ transform_gender_value("gender") }} as user_gender, + case + when account_privacy = 'public' + then 'Public to everyone' + when account_privacy = 'public_to_mm' + then 'Public to logged in users' + when account_privacy = 'private' + then 'Private' + else account_privacy + end as user_account_privacy, + {{ cast_timestamp_to_iso8601("date_joined_micromasters") }} as user_joined_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as user_profile_updated_on + from source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__user_program_certificate_override_list.sql b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__user_program_certificate_override_list.sql index 2245782ca..80e5a4eb8 100644 --- a/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__user_program_certificate_override_list.sql +++ b/src/ol_dbt/models/staging/micromasters/stg__micromasters__app__user_program_certificate_override_list.sql @@ -1,4 +1,5 @@ -select * from ( - values - ('74869f589baf0522ec4341ee3b7970526b6de2918bc6bf29873eae040d6c2359', 11479768, 1) ---pragma: allowlist secret -) AS overwrite_list (program_certificate_hashed_id, user_edxorg_id, micromasters_program_id) -- noqa: L010,L025 +select * +from + ( + values ('74869f589baf0522ec4341ee3b7970526b6de2918bc6bf29873eae040d6c2359', 11479768, 1) -- pragma: allowlist secret + ) as overwrite_list(program_certificate_hashed_id, user_edxorg_id, micromasters_program_id) -- noqa: L010,L025 diff --git a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_learningresourcetopic.sql b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_learningresourcetopic.sql index 4d50d8767..b29b59152 100644 --- a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_learningresourcetopic.sql +++ b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_learningresourcetopic.sql @@ -1,15 +1,21 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__mitlearn__app__postgres__learning_resources_learningresourcetopic') }} -) +with + source as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitlearn__app__postgres__learning_resources_learningresourcetopic" + ) + }} + ), + cleaned as ( + select + id as learningresourcetopic_id, + name as learningresourcetopic_name, + parent_id as learningresourcetopic_parent_id, + topic_uuid as learningresourcetopic_uuid + from source + ) -, cleaned as ( - select - id as learningresourcetopic_id - , name as learningresourcetopic_name - , parent_id as learningresourcetopic_parent_id - , topic_uuid as learningresourcetopic_uuid - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_search_percolatequery.sql b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_search_percolatequery.sql index 3e1a749f4..aeac3a022 100644 --- a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_search_percolatequery.sql +++ b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_search_percolatequery.sql @@ -1,18 +1,24 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__mitlearn__app__postgres__learning_resources_search_percolatequery') }} -) +with + source as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitlearn__app__postgres__learning_resources_search_percolatequery" + ) + }} + ), + cleaned as ( + select + id as percolatequery_id, + query as percolatequery_query, + created_on as percolatequery_created_on, + updated_on as percolatequery_updated_on, + source_type as percolatequery_source_type, + display_label as percolatequery_display_label, + original_query as percolatequery_original_query + from source + ) -, cleaned as ( - select - id as percolatequery_id - , query as percolatequery_query - , created_on as percolatequery_created_on - , updated_on as percolatequery_updated_on - , source_type as percolatequery_source_type - , display_label as percolatequery_display_label - , original_query as percolatequery_original_query - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_search_percolatequery_users.sql b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_search_percolatequery_users.sql index 3851be8dc..5a2997a3f 100644 --- a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_search_percolatequery_users.sql +++ b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_search_percolatequery_users.sql @@ -1,15 +1,15 @@ -with source as ( - select * - from - {{ source('ol_warehouse_raw_data', 'raw__mitlearn__app__postgres__learning_resources_search_percolatequery_users') }} -- noqa: LT05 -) +with + source as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitlearn__app__postgres__learning_resources_search_percolatequery_users", + ) + }} -- noqa: LT05 + ), + cleaned as (select id as percolatequery_users_id, user_id, percolatequery_id from source) -, cleaned as ( - select - id as percolatequery_users_id - , user_id - , percolatequery_id - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_userlist.sql b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_userlist.sql index ab5fd0e26..a7904096e 100644 --- a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_userlist.sql +++ b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_userlist.sql @@ -1,18 +1,18 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__mitlearn__app__postgres__learning_resources_userlist') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitlearn__app__postgres__learning_resources_userlist") }} + ), + cleaned as ( + select + id as userlist_id, + title as userlist_title, + author_id as user_id, + description as userlist_description, + privacy_level as userlist_privacy_level, + {{ cast_timestamp_to_iso8601("created_on") }} as userlist_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as userlist_updated_on + from source + ) -, cleaned as ( - select - id as userlist_id - , title as userlist_title - , author_id as user_id - , description as userlist_description - , privacy_level as userlist_privacy_level - , {{ cast_timestamp_to_iso8601('created_on') }} as userlist_created_on - , {{ cast_timestamp_to_iso8601('updated_on') }} as userlist_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_userlist_topics.sql b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_userlist_topics.sql index 4b93712e9..d000ae351 100644 --- a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_userlist_topics.sql +++ b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_userlist_topics.sql @@ -1,14 +1,9 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__mitlearn__app__postgres__learning_resources_userlist_topics') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitlearn__app__postgres__learning_resources_userlist_topics") }} + ), + cleaned as (select id as userlisttopics_id, userlist_id, learningresourcetopic_id from source) -, cleaned as ( - select - id as userlisttopics_id - , userlist_id - , learningresourcetopic_id - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_userlistrelationship.sql b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_userlistrelationship.sql index fe0ce0900..89e35efe9 100644 --- a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_userlistrelationship.sql +++ b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__learning_resources_userlistrelationship.sql @@ -1,17 +1,23 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__mitlearn__app__postgres__learning_resources_userlistrelationship') }} -) +with + source as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitlearn__app__postgres__learning_resources_userlistrelationship" + ) + }} + ), + cleaned as ( + select + id as userlistrelationship_id, + child_id as userlistrelationship_child_id, + position as userlistrelationship_position, + parent_id as userlistrelationship_parent_id, + created_on as userlistrelationship_created_on, + updated_on as userlistrelationship_updated_on + from source + ) -, cleaned as ( - select - id as userlistrelationship_id - , child_id as userlistrelationship_child_id - , position as userlistrelationship_position - , parent_id as userlistrelationship_parent_id - , created_on as userlistrelationship_created_on - , updated_on as userlistrelationship_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__profiles_profile.sql b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__profiles_profile.sql index 95377104b..5a9d8d681 100644 --- a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__profiles_profile.sql +++ b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__profiles_profile.sql @@ -1,33 +1,31 @@ -- MIT Learn User Profile Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitlearn__app__postgres__profiles_profile") }}), + cleaned as ( + select + id as profile_id, + user_id, + email_optin as user_email_optin, + toc_optin as user_toc_optin, + certificate_desired as user_certificate_desired, + completed_onboarding as user_completed_onboarding, + nullif(name, '') as user_name, + nullif(bio, '') as user_bio, + nullif(goals, array[]) as user_goals, + nullif(headline, '') as user_headline, + nullif(location, '') as user_location, + nullif(delivery, array[]) as user_delivery_preference, + nullif(current_education, '') as user_current_education, + nullif(time_commitment, '') as user_time_commitment, + nullif(image, '') as user_image_url, + nullif(image_small, '') as user_image_small_url, + nullif(image_medium, '') as user_image_medium_url, + nullif(image_file, '') as user_image_file, + nullif(image_small_file, '') as user_image_small_file, + nullif(image_medium_file, '') as user_image_medium_file, + {{ cast_timestamp_to_iso8601("updated_at") }} as profile_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitlearn__app__postgres__profiles_profile') }} -) - -, cleaned as ( - select - id as profile_id - , user_id - , email_optin as user_email_optin - , toc_optin as user_toc_optin - , certificate_desired as user_certificate_desired - , completed_onboarding as user_completed_onboarding - , nullif(name, '') as user_name - , nullif(bio, '') as user_bio - , nullif(goals, array[]) as user_goals - , nullif(headline, '') as user_headline - , nullif(location, '') as user_location - , nullif(delivery, array[]) as user_delivery_preference - , nullif(current_education, '') as user_current_education - , nullif(time_commitment, '') as user_time_commitment - , nullif(image, '') as user_image_url - , nullif(image_small, '') as user_image_small_url - , nullif(image_medium, '') as user_image_medium_url - , nullif(image_file, '') as user_image_file - , nullif(image_small_file, '') as user_image_small_file - , nullif(image_medium_file, '') as user_image_medium_file - , {{ cast_timestamp_to_iso8601('updated_at') }} as profile_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__profiles_profile_topic_interests.sql b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__profiles_profile_topic_interests.sql index 8cd1ceccc..12f52cef6 100644 --- a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__profiles_profile_topic_interests.sql +++ b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__profiles_profile_topic_interests.sql @@ -1,14 +1,9 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__mitlearn__app__postgres__profiles_profile_topic_interests') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitlearn__app__postgres__profiles_profile_topic_interests") }} + ), + cleaned as (select id as profiletopicinterests_id, profile_id, learningresourcetopic_id from source) -, cleaned as ( - select - id as profiletopicinterests_id - , profile_id - , learningresourcetopic_id - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__users_user.sql b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__users_user.sql index 047f5f7f2..f15fbea59 100644 --- a/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__users_user.sql +++ b/src/ol_dbt/models/staging/mitlearn/stg__mitlearn__app__postgres__users_user.sql @@ -1,28 +1,26 @@ -- MIT Learn User Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitlearn__app__postgres__users_user") }}), + cleaned as ( + select + id as user_id, + username as user_username, + email as user_email, + is_active as user_is_active, + is_staff as user_is_staff, + is_superuser as user_is_superuser, + first_name as user_first_name, + last_name as user_last_name, + scim_id as user_scim_id, + scim_username as user_scim_username, + scim_external_id as user_scim_external_id, + coalesce(nullif(global_id, ''), scim_external_id) as user_global_id, + {{ cast_timestamp_to_iso8601("created_on") }} as user_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as user_updated_on, + {{ cast_timestamp_to_iso8601("date_joined") }} as user_joined_on, + {{ cast_timestamp_to_iso8601("last_login") }} as user_last_login + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitlearn__app__postgres__users_user') }} -) - -, cleaned as ( - select - id as user_id - , username as user_username - , email as user_email - , is_active as user_is_active - , is_staff as user_is_staff - , is_superuser as user_is_superuser - , first_name as user_first_name - , last_name as user_last_name - , scim_id as user_scim_id - , scim_username as user_scim_username - , scim_external_id as user_scim_external_id - , coalesce(nullif(global_id, ''), scim_external_id) as user_global_id - , {{ cast_timestamp_to_iso8601('created_on') }} as user_created_on - , {{ cast_timestamp_to_iso8601('updated_on') }} as user_updated_on - , {{ cast_timestamp_to_iso8601('date_joined') }} as user_joined_on - , {{ cast_timestamp_to_iso8601('last_login') }} as user_last_login - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_coursepage.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_coursepage.sql index d24077bbb..c3a2e6085 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_coursepage.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_coursepage.sql @@ -1,21 +1,20 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__cms_coursepage') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__cms_coursepage") }}), + cleaned as ( + select + page_ptr_id as wagtail_page_id, + course_id, + description as course_description, + length as course_length, + effort as course_effort, + prerequisites as course_prerequisites, + faq_url as course_faq_url, + about as course_about, + what_you_learn as course_what_you_learn, + video_url as course_video_url, + json_query(price, 'lax $.value.text' omit quotes) as course_price + from source + ) -, cleaned as ( - select - page_ptr_id as wagtail_page_id - , course_id - , description as course_description - , length as course_length - , effort as course_effort - , prerequisites as course_prerequisites - , faq_url as course_faq_url - , about as course_about - , what_you_learn as course_what_you_learn - , video_url as course_video_url - , json_query(price, 'lax $.value.text' omit quotes) as course_price - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_coursepage_topics.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_coursepage_topics.sql index afef76f9e..55e5cf39b 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_coursepage_topics.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_coursepage_topics.sql @@ -1,8 +1,7 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__cms_coursepage_topics') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__cms_coursepage_topics") }} + ) -select - coursepage_id as wagtail_page_id - , coursestopic_id as coursetopic_id +select coursepage_id as wagtail_page_id, coursestopic_id as coursetopic_id from source diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_instructorpage.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_instructorpage.sql index e312fab08..9cd2e541c 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_instructorpage.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_instructorpage.sql @@ -1,15 +1,12 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__cms_instructorpage') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__cms_instructorpage") }} + ), + cleaned as ( + select + page_ptr_id as wagtail_page_id, instructor_name, instructor_title, instructor_bio_short, instructor_bio_long + from source + ) -, cleaned as ( - select - page_ptr_id as wagtail_page_id - , instructor_name - , instructor_title - , instructor_bio_short - , instructor_bio_long - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_instructorpagelink.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_instructorpagelink.sql index fe93509fd..38cc3bad1 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_instructorpagelink.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_instructorpagelink.sql @@ -1,12 +1,8 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__cms_instructorpagelink') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__cms_instructorpagelink") }} + ), + cleaned as (select page_id as wagtail_page_id, linked_instructor_page_id as instructor_wagtail_page_id from source) -, cleaned as ( - select - page_id as wagtail_page_id - , linked_instructor_page_id as instructor_wagtail_page_id - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_programpage.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_programpage.sql index 764e291fc..10aa566bd 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_programpage.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_programpage.sql @@ -1,21 +1,20 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__cms_programpage') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__cms_programpage") }}), + cleaned as ( + select + page_ptr_id as wagtail_page_id, + program_id, + description as program_description, + length as program_length, + effort as program_effort, + prerequisites as program_prerequisites, + faq_url as program_faq_url, + about as program_about, + what_you_learn as program_what_you_learn, + video_url as program_video_url, + json_query(price, 'lax $.value.text' omit quotes) as program_price + from source + ) -, cleaned as ( - select - page_ptr_id as wagtail_page_id - , program_id - , description as program_description - , length as program_length - , effort as program_effort - , prerequisites as program_prerequisites - , faq_url as program_faq_url - , about as program_about - , what_you_learn as program_what_you_learn - , video_url as program_video_url - , json_query(price, 'lax $.value.text' omit quotes) as program_price - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_wagtail_page.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_wagtail_page.sql index d84d69565..0daff21e6 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_wagtail_page.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__cms_wagtail_page.sql @@ -1,28 +1,27 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__wagtailcore_page') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__wagtailcore_page") }}), + cleaned as ( + select + id as wagtail_page_id, + path as wagtail_page_path, + depth as wagtail_page_depth, + numchild as wagtail_page_num_children, + title as wagtail_page_title, + slug as wagtail_page_slug, + live as wagtail_page_is_live, + has_unpublished_changes as wagtail_page_has_unpublished_changes, + seo_title as wagtail_page_seo_title, + url_path as wagtail_page_url_path, + search_description as wagtail_page_search_description, + content_type_id as contenttype_id, + owner_id as owner_user_id, + live_revision_id as wagtail_page_live_pagerevision_id, + {{ cast_timestamp_to_iso8601("latest_revision_created_at") }} as wagtail_page_latest_revision_created_on, + {{ cast_timestamp_to_iso8601("first_published_at") }} as wagtail_page_first_published_on, + {{ cast_timestamp_to_iso8601("last_published_at") }} as wagtail_page_last_published_on -, cleaned as ( - select - id as wagtail_page_id - , path as wagtail_page_path - , depth as wagtail_page_depth - , numchild as wagtail_page_num_children - , title as wagtail_page_title - , slug as wagtail_page_slug - , live as wagtail_page_is_live - , has_unpublished_changes as wagtail_page_has_unpublished_changes - , seo_title as wagtail_page_seo_title - , url_path as wagtail_page_url_path - , search_description as wagtail_page_search_description - , content_type_id as contenttype_id - , owner_id as owner_user_id - , live_revision_id as wagtail_page_live_pagerevision_id - ,{{ cast_timestamp_to_iso8601('latest_revision_created_at') }} as wagtail_page_latest_revision_created_on - ,{{ cast_timestamp_to_iso8601('first_published_at') }} as wagtail_page_first_published_on - ,{{ cast_timestamp_to_iso8601('last_published_at') }} as wagtail_page_last_published_on + from source + ) - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_blockedcountry.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_blockedcountry.sql index 96474e57e..fb1ba5da4 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_blockedcountry.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_blockedcountry.sql @@ -1,17 +1,17 @@ -- MITx Online Course Blocked country Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_blockedcountry") }} + ), + cleaned as ( + select + id as blockedcountry_id, + country as blockedcountry_code, + course_id, + {{ cast_timestamp_to_iso8601("created_on") }} as blockedcountry_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as blockedcountry_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_blockedcountry') }} -) - -, cleaned as ( - select - id as blockedcountry_id - , country as blockedcountry_code - , course_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as blockedcountry_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as blockedcountry_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_course.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_course.sql index 918236107..12f9c550b 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_course.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_course.sql @@ -1,20 +1,18 @@ -- MITx Online Course Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_course") }}), + cleaned as ( + select + id as course_id, + live as course_is_live, + title as course_title, + readable_id as course_readable_id, + replace(replace(readable_id, 'course-v1:', ''), '+', '/') as course_edx_readable_id, + split(readable_id, '+')[2] as course_number, + {{ cast_timestamp_to_iso8601("created_on") }} as course_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as course_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_course') }} -) - -, cleaned as ( - select - id as course_id - , live as course_is_live - , title as course_title - , readable_id as course_readable_id - , replace(replace(readable_id, 'course-v1:', ''), '+', '/') as course_edx_readable_id - , split(readable_id, '+')[2] as course_number - ,{{ cast_timestamp_to_iso8601('created_on') }} as course_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as course_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_course_to_department.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_course_to_department.sql index d974913df..184ff4d30 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_course_to_department.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_course_to_department.sql @@ -1,13 +1,9 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_course_departments') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_course_departments") }} + ), + cleaned as (select id as coursetodepartment_id, department_id as coursedepartment_id, course_id from source) -, cleaned as ( - select - id as coursetodepartment_id - , department_id as coursedepartment_id - , course_id - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courserun.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courserun.sql index d9f72ceee..dce4c0077 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courserun.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courserun.sql @@ -1,34 +1,34 @@ -- MITx Online Course Run Information - -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_courserun') }} -) - -, cleaned as ( - select - id as courserun_id - , course_id - , live as courserun_is_live - , title as courserun_title - , courseware_id as courserun_readable_id - , courseware_url_path as courserun_url - , run_tag as courserun_tag - , is_self_paced as courserun_is_self_paced - , case - when courseware_id like 'MITx/%' or courseware_id like 'course-v1:MITx+%' +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_courserun") }} + ), + cleaned as ( + select + id as courserun_id, + course_id, + live as courserun_is_live, + title as courserun_title, + courseware_id as courserun_readable_id, + courseware_url_path as courserun_url, + run_tag as courserun_tag, + is_self_paced as courserun_is_self_paced, + case + when courseware_id like 'MITx/%' or courseware_id like 'course-v1:MITx+%' then '{{ var("edxorg") }}' - else '{{ var("mitxonline") }}' - end as courserun_platform - , replace(replace(courseware_id, 'course-v1:', ''), '+', '/') as courserun_edx_readable_id - ,{{ cast_timestamp_to_iso8601('start_date') }} as courserun_start_on - ,{{ cast_timestamp_to_iso8601('end_date') }} as courserun_end_on - ,{{ cast_timestamp_to_iso8601('enrollment_start') }} as courserun_enrollment_start_on - ,{{ cast_timestamp_to_iso8601('enrollment_end') }} as courserun_enrollment_end_on - ,{{ cast_timestamp_to_iso8601('expiration_date') }} as courserun_expired_on - ,{{ cast_timestamp_to_iso8601('upgrade_deadline') }} as courserun_upgrade_deadline - ,{{ cast_timestamp_to_iso8601('created_on') }} as courserun_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as courserun_updated_on - from source -) + else '{{ var("mitxonline") }}' + end as courserun_platform, + replace(replace(courseware_id, 'course-v1:', ''), '+', '/') as courserun_edx_readable_id, + {{ cast_timestamp_to_iso8601("start_date") }} as courserun_start_on, + {{ cast_timestamp_to_iso8601("end_date") }} as courserun_end_on, + {{ cast_timestamp_to_iso8601("enrollment_start") }} as courserun_enrollment_start_on, + {{ cast_timestamp_to_iso8601("enrollment_end") }} as courserun_enrollment_end_on, + {{ cast_timestamp_to_iso8601("expiration_date") }} as courserun_expired_on, + {{ cast_timestamp_to_iso8601("upgrade_deadline") }} as courserun_upgrade_deadline, + {{ cast_timestamp_to_iso8601("created_on") }} as courserun_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as courserun_updated_on + from source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courseruncertificate.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courseruncertificate.sql index 7883d2a87..f0146bc3e 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courseruncertificate.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courseruncertificate.sql @@ -1,22 +1,24 @@ -- MITx Online Users Course Certificate Information +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_courseruncertificate") }} + ), + cleaned as ( + select + id as courseruncertificate_id, + uuid as courseruncertificate_uuid, + course_run_id as courserun_id, + user_id, + certificate_page_revision_id, -- - rename it after the referenced model is created + is_revoked as courseruncertificate_is_revoked, + if( + is_revoked = false, concat('https://mitxonline.mit.edu/certificate/', uuid), null + ) as courseruncertificate_url, + {{ cast_timestamp_to_iso8601("created_on") }} as courseruncertificate_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as courseruncertificate_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_courseruncertificate') }} -) - -, cleaned as ( - select - id as courseruncertificate_id - , uuid as courseruncertificate_uuid - , course_run_id as courserun_id - , user_id - , certificate_page_revision_id --- rename it after the referenced model is created - , is_revoked as courseruncertificate_is_revoked - , if(is_revoked = false, concat('https://mitxonline.mit.edu/certificate/', uuid), null) - as courseruncertificate_url - ,{{ cast_timestamp_to_iso8601('created_on') }} as courseruncertificate_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as courseruncertificate_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courserunenrollment.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courserunenrollment.sql index bf3d72356..01a321c5a 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courserunenrollment.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courserunenrollment.sql @@ -1,26 +1,24 @@ -- MITx Online Course Run Enrollment Information +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_courserunenrollment") }} + ), + cleaned as ( + select + id as courserunenrollment_id, + -- -since raw data has both empty string and null, convert them to null for consistency + active as courserunenrollment_is_active, + edx_enrolled as courserunenrollment_is_edx_enrolled, + run_id as courserun_id, + user_id, + enrollment_mode as courserunenrollment_enrollment_mode, + edx_emails_subscription as courserunenrollment_has_edx_email_subscription, + case when change_status = '' then null else change_status end as courserunenrollment_enrollment_status, + {{ cast_timestamp_to_iso8601("created_on") }} as courserunenrollment_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as courserunenrollment_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_courserunenrollment') }} -) - -, cleaned as ( - select - id as courserunenrollment_id - ---since raw data has both empty string and null, convert them to null for consistency - , active as courserunenrollment_is_active - , edx_enrolled as courserunenrollment_is_edx_enrolled - , run_id as courserun_id - , user_id - , enrollment_mode as courserunenrollment_enrollment_mode - , edx_emails_subscription as courserunenrollment_has_edx_email_subscription - , case - when change_status = '' then null - else change_status - end as courserunenrollment_enrollment_status - ,{{ cast_timestamp_to_iso8601('created_on') }} as courserunenrollment_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as courserunenrollment_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courserungrade.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courserungrade.sql index d71fab67b..41b69a2f6 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courserungrade.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_courserungrade.sql @@ -1,21 +1,21 @@ -- MITx Online Users Course Grades Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_courserungrade") }} + ), + cleaned as ( + select + id as courserungrade_id, + course_run_id as courserun_id, + user_id, + grade as courserungrade_grade, + letter_grade as courserungrade_letter_grade, + set_by_admin as courserungrade_is_set_by_admin, + passed as courserungrade_is_passing, + {{ cast_timestamp_to_iso8601("created_on") }} as courserungrade_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as courserungrade_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_courserungrade') }} -) - -, cleaned as ( - select - id as courserungrade_id - , course_run_id as courserun_id - , user_id - , grade as courserungrade_grade - , letter_grade as courserungrade_letter_grade - , set_by_admin as courserungrade_is_set_by_admin - , passed as courserungrade_is_passing - ,{{ cast_timestamp_to_iso8601('created_on') }} as courserungrade_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as courserungrade_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_coursetopic.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_coursetopic.sql index 5df0a31ae..9bf1581f0 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_coursetopic.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_coursetopic.sql @@ -1,15 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_coursestopic') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_coursestopic") }} + ), + cleaned as ( + select + id as coursetopic_id, + name as coursetopic_name, + parent_id as coursetopic_parent_id, + {{ cast_timestamp_to_iso8601("created_on") }} as coursetopic_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as coursetopic_updated_on + from source + ) -, cleaned as ( - select - id as coursetopic_id - , name as coursetopic_name - , parent_id as coursetopic_parent_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as coursetopic_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as coursetopic_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_department.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_department.sql index d5cb4fc55..11a1a1006 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_department.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_department.sql @@ -1,16 +1,16 @@ -- MITx Online Course Department Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_department") }} + ), + cleaned as ( + select + id as coursedepartment_id, + name as coursedepartment_name, + {{ cast_timestamp_to_iso8601("created_on") }} as coursedepartment_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as coursedepartment_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_department') }} -) - -, cleaned as ( - select - id as coursedepartment_id - , name as coursedepartment_name - ,{{ cast_timestamp_to_iso8601('created_on') }} as coursedepartment_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as coursedepartment_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_program.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_program.sql index 2f39266c9..8746a1e58 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_program.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_program.sql @@ -1,26 +1,25 @@ -- MITx Online Program Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_program") }}), + cleaned as ( + select + id as program_id, + live as program_is_live, + title as program_title, + readable_id as program_readable_id, + availability as program_availability, + if(program_type like 'MicroMasters%', 'MicroMasters', program_type) as program_type, + if(program_type like 'MicroMasters%', true, false) as program_is_micromasters, + if( + program_type like 'MicroMasters%', 'MicroMasters Credential', 'Certificate of Completion' + ) as program_certification_type, + if(program_type like 'MicroMasters%', element_at(split(title, ': '), 1), title) as program_name, + if(program_type like 'MicroMasters%', element_at(split(title, ': '), 2), null) as program_track, + if(readable_id like '%DEDP%', true, false) as program_is_dedp, + {{ cast_timestamp_to_iso8601("created_on") }} as program_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as program_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_program') }} -) - -, cleaned as ( - select - id as program_id - , live as program_is_live - , title as program_title - , readable_id as program_readable_id - , availability as program_availability - , if(program_type like 'MicroMasters%', 'MicroMasters', program_type) as program_type - , if(program_type like 'MicroMasters%', true, false) as program_is_micromasters - , if(program_type like 'MicroMasters%', 'MicroMasters Credential', 'Certificate of Completion') - as program_certification_type - , if(program_type like 'MicroMasters%', element_at(split(title, ': '), 1), title) as program_name - , if(program_type like 'MicroMasters%', element_at(split(title, ': '), 2), null) as program_track - , if(readable_id like '%DEDP%', true, false) as program_is_dedp - ,{{ cast_timestamp_to_iso8601('created_on') }} as program_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as program_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programcertificate.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programcertificate.sql index f662696e9..a69fc7e82 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programcertificate.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programcertificate.sql @@ -1,18 +1,20 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_programcertificate') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_programcertificate") }} + ), + cleaned as ( + select + id as programcertificate_id, + uuid as programcertificate_uuid, + program_id, + user_id, + certificate_page_revision_id, -- - rename it after the referenced model is created + is_revoked as programcertificate_is_revoked, + {{ cast_timestamp_to_iso8601("created_on") }} as programcertificate_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as programcertificate_updated_on + from source + ) -, cleaned as ( - select - id as programcertificate_id - , uuid as programcertificate_uuid - , program_id - , user_id - , certificate_page_revision_id --- rename it after the referenced model is created - , is_revoked as programcertificate_is_revoked - ,{{ cast_timestamp_to_iso8601('created_on') }} as programcertificate_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as programcertificate_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programenrollment.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programenrollment.sql index 12d8de202..d183d531b 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programenrollment.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programenrollment.sql @@ -1,20 +1,20 @@ -- MITx Online Program Enrollment Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_programenrollment") }} + ), + cleaned as ( + select + id as programenrollment_id, + change_status as programenrollment_enrollment_status, + active as programenrollment_is_active, + program_id, + user_id, + enrollment_mode as programenrollment_enrollment_mode, + {{ cast_timestamp_to_iso8601("created_on") }} as programenrollment_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as programenrollment_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_programenrollment') }} -) - -, cleaned as ( - select - id as programenrollment_id - , change_status as programenrollment_enrollment_status - , active as programenrollment_is_active - , program_id - , user_id - , enrollment_mode as programenrollment_enrollment_mode - ,{{ cast_timestamp_to_iso8601('created_on') }} as programenrollment_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as programenrollment_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programrequirement.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programrequirement.sql index 3b1f2b60a..a7d76d337 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programrequirement.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programrequirement.sql @@ -1,22 +1,23 @@ -- MITx Online Program Requirement Information +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_programrequirement") }} + ), + cleaned as ( + select + id as programrequirement_id, + course_id, + program_id, + path as programrequirement_path, + depth as programrequirement_depth, + node_type as programrequirement_node_type, + numchild as programrequirement_numchild, + title as programrequirement_title, + operator as programrequirement_operator, + cast(operator_value as integer) as programrequirement_operator_value + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__courses_programrequirement') }} -) - -, cleaned as ( - select - id as programrequirement_id - , course_id - , program_id - , path as programrequirement_path - , depth as programrequirement_depth - , node_type as programrequirement_node_type - , numchild as programrequirement_numchild - , title as programrequirement_title - , operator as programrequirement_operator - , cast(operator_value as INTEGER) as programrequirement_operator_value - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programrun.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programrun.sql index a45f8e200..d90a15324 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programrun.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__courses_programrun.sql @@ -1,19 +1,20 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__courses_programrun') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__courses_programrun") }} + ), + renamed as ( -, renamed as ( + select + id as programrun_id, + run_tag as programrun_tag, + program_id, + {{ cast_timestamp_to_iso8601("start_date") }} as programrun_start_on, + {{ cast_timestamp_to_iso8601("end_date") }} as programrun_end_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as programrun_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as programrun_created_on + from source - select - id as programrun_id - , run_tag as programrun_tag - , program_id - ,{{ cast_timestamp_to_iso8601('start_date') }} as programrun_start_on - ,{{ cast_timestamp_to_iso8601('end_date') }} as programrun_end_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as programrun_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as programrun_created_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__django_contenttype.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__django_contenttype.sql index 11e564c44..099043f65 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__django_contenttype.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__django_contenttype.sql @@ -1,20 +1,10 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__django_content_type') }} + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__django_content_type") }} -) + ), + renamed as (select id as contenttype_id, concat_ws('_', app_label, model) as contenttype_full_name from source) -, renamed as ( - - select - id as contenttype_id - , concat_ws( - '_' - , app_label - , model - ) as contenttype_full_name - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basket.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basket.sql index d1ca73c0c..3937c63a8 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basket.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basket.sql @@ -1,20 +1,18 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__ecommerce_basket") }}) - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__ecommerce_basket') }} + {{ deduplicate_raw_table(order_by="id", partition_columns="user_id") }}, + renamed as ( -) + select + id as basket_id, + user_id, + {{ cast_timestamp_to_iso8601("created_on") }} as basket_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as basket_updated_on -{{ deduplicate_raw_table(order_by='id' , partition_columns = 'user_id') }} -, renamed as ( + from most_recent_source - select - id as basket_id - , user_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as basket_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as basket_updated_on + ) - from most_recent_source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basketdiscount.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basketdiscount.sql index cfdd61d98..bbce5ca71 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basketdiscount.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basketdiscount.sql @@ -1,22 +1,23 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__ecommerce_basketdiscount') }} + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__ecommerce_basketdiscount") }} -) + ), + renamed as ( -, renamed as ( + select + id as basketdiscount_id, + redeemed_by_id as user_id, + redeemed_basket_id as basket_id, + redeemed_discount_id as discount_id, + {{ cast_timestamp_to_iso8601("redemption_date") }} as basketdiscount_applied_on, + {{ cast_timestamp_to_iso8601("created_on") }} as basketdiscount_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as basketdiscount_updated_on - select - id as basketdiscount_id - , redeemed_by_id as user_id - , redeemed_basket_id as basket_id - , redeemed_discount_id as discount_id - ,{{ cast_timestamp_to_iso8601('redemption_date') }} as basketdiscount_applied_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as basketdiscount_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as basketdiscount_updated_on + from source - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basketitem.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basketitem.sql index 9c73d2e7c..0031e49af 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basketitem.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basketitem.sql @@ -1,21 +1,22 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__ecommerce_basketitem') }} + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__ecommerce_basketitem") }} -) + ), + renamed as ( -, renamed as ( + select + id as basketitem_id, + quantity as basketitem_quantity, + basket_id, + product_id, + {{ cast_timestamp_to_iso8601("created_on") }} as basketitem_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as basketitem_updated_on - select - id as basketitem_id - , quantity as basketitem_quantity - , basket_id - , product_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as basketitem_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as basketitem_updated_on + from source - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_discount.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_discount.sql index 242f2be2e..0abf97a75 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_discount.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_discount.sql @@ -1,34 +1,35 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__ecommerce_discount') }} + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__ecommerce_discount") }} -) + ), + renamed as ( -, renamed as ( - - select - id as discount_id - , cast(amount as decimal(38, 2)) as discount_amount - , discount_code - , discount_type - , max_redemptions as discount_max_redemptions - , redemption_type as discount_redemption_type - , payment_type as discount_source - , case - when discount_type = 'percent-off' + select + id as discount_id, + cast(amount as decimal(38, 2)) as discount_amount, + discount_code, + discount_type, + max_redemptions as discount_max_redemptions, + redemption_type as discount_redemption_type, + payment_type as discount_source, + case + when discount_type = 'percent-off' then concat(format('%.2f', amount), '%') - when discount_type = 'dollars-off' + when discount_type = 'dollars-off' then concat('$', format('%.2f', amount)) - when discount_type = 'fixed-price' + when discount_type = 'fixed-price' then concat('Fixed Price: ', format('%.2f', amount)) - end as discount_amount_text - ,{{ cast_timestamp_to_iso8601('created_on') }} as discount_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as discount_updated_on - ,{{ cast_timestamp_to_iso8601('activation_date') }} as discount_activated_on - ,{{ cast_timestamp_to_iso8601('expiration_date') }} as discount_expires_on + end as discount_amount_text, + {{ cast_timestamp_to_iso8601("created_on") }} as discount_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as discount_updated_on, + {{ cast_timestamp_to_iso8601("activation_date") }} as discount_activated_on, + {{ cast_timestamp_to_iso8601("expiration_date") }} as discount_expires_on - from source + from source -) + ) -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_discountproduct.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_discountproduct.sql index cb20a5dbc..07a33efa6 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_discountproduct.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_discountproduct.sql @@ -1,20 +1,21 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__ecommerce_discountproduct') }} + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__ecommerce_discountproduct") }} -) + ), + renamed as ( -, renamed as ( + select + id as discountproduct_id, + product_id, + discount_id, + {{ cast_timestamp_to_iso8601("created_on") }} as discountproduct_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as discountproduct_updated_on - select - id as discountproduct_id - , product_id - , discount_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as discountproduct_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as discountproduct_updated_on + from source - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_discountredemption.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_discountredemption.sql index 3b1755562..3b4fd3b68 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_discountredemption.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_discountredemption.sql @@ -1,20 +1,22 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__ecommerce_discountredemption') }} + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__ecommerce_discountredemption") }} -) + ), + renamed as ( -, renamed as ( + select + id as discountredemption_id, + redeemed_by_id as user_id, + redeemed_order_id as order_id, + redeemed_discount_id as discount_id, + {{ cast_timestamp_to_iso8601("redemption_date") }} as discountredemption_timestamp - select - id as discountredemption_id - , redeemed_by_id as user_id - , redeemed_order_id as order_id - , redeemed_discount_id as discount_id - ,{{ cast_timestamp_to_iso8601('redemption_date') }} as discountredemption_timestamp + from source - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_line.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_line.sql index 9564aa8cf..1e8cb8a13 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_line.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_line.sql @@ -1,21 +1,18 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__ecommerce_line") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__ecommerce_line') }} + select + id as line_id, + order_id, + product_version_id, + purchased_object_id as product_object_id, + purchased_content_type_id as contenttype_id, + {{ cast_timestamp_to_iso8601("created_on") }} as line_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as line_updated_on + from source -) + ) -, renamed as ( - - select - id as line_id - , order_id - , product_version_id - , purchased_object_id as product_object_id - , purchased_content_type_id as contenttype_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as line_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as line_updated_on - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_order.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_order.sql index 526213b46..f9c8bf96a 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_order.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_order.sql @@ -1,21 +1,18 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__ecommerce_order") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__ecommerce_order') }} + select + id as order_id, + state as order_state, + purchaser_id as order_purchaser_user_id, + reference_number as order_reference_number, + cast(total_price_paid as decimal(38, 2)) as order_total_price_paid, + {{ cast_timestamp_to_iso8601("created_on") }} as order_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as order_updated_on + from source -) + ) -, renamed as ( - - select - id as order_id - , state as order_state - , purchaser_id as order_purchaser_user_id - , reference_number as order_reference_number - , cast(total_price_paid as decimal(38, 2)) as order_total_price_paid - ,{{ cast_timestamp_to_iso8601('created_on') }} as order_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as order_updated_on - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_product.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_product.sql index 86d5a73ce..c3e71260d 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_product.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_product.sql @@ -1,21 +1,22 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__ecommerce_product') }} + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__ecommerce_product") }} -) + ), + renamed as ( -, renamed as ( + select + id as product_id, + cast(price as decimal(38, 2)) as product_price, + is_active as product_is_active, + object_id as product_object_id, + description as product_description, + content_type_id as contenttype_id, + {{ cast_timestamp_to_iso8601("created_on") }} as product_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as product_updated_on + from source + ) - select - id as product_id - , cast(price as decimal(38, 2)) as product_price - , is_active as product_is_active - , object_id as product_object_id - , description as product_description - , content_type_id as contenttype_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as product_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as product_updated_on - from source -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_transaction.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_transaction.sql index 78f4a98e9..fa64cbeef 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_transaction.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_transaction.sql @@ -1,22 +1,23 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__ecommerce_transaction') }} + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__ecommerce_transaction") }} -) + ), + renamed as ( -, renamed as ( + select + id as transaction_id, + data as transaction_data, + amount as transaction_amount, + order_id, + transaction_id as transaction_readable_identifier, + transaction_type, + {{ cast_timestamp_to_iso8601("created_on") }} as transaction_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as transaction_updated_on + from source - select - id as transaction_id - , data as transaction_data - , amount as transaction_amount - , order_id - , transaction_id as transaction_readable_identifier - , transaction_type - ,{{ cast_timestamp_to_iso8601('created_on') }} as transaction_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as transaction_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_userdiscount.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_userdiscount.sql index 6964d4b80..bbb574c15 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_userdiscount.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_userdiscount.sql @@ -1,19 +1,20 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__ecommerce_userdiscount') }} + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__ecommerce_userdiscount") }} -) + ), + renamed as ( -, renamed as ( + select + id as userdiscount_id, + user_id, + discount_id, + {{ cast_timestamp_to_iso8601("created_on") }} as userdiscount_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as userdiscount_updated_on + from source - select - id as userdiscount_id - , user_id - , discount_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as userdiscount_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as userdiscount_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_countryincomethreshold.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_countryincomethreshold.sql index 03b8e7e6f..1b74596c0 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_countryincomethreshold.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_countryincomethreshold.sql @@ -1,21 +1,27 @@ -with source as ( +with + source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__flexiblepricing_countryincomethreshold') }} + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitxonline__app__postgres__flexiblepricing_countryincomethreshold" + ) + }} -) + ), + renamed as ( -, renamed as ( + select + id as countryincomethreshold_id, + country_code as countryincomethreshold_country_code, + income_threshold as countryincomethreshold_income_threshold, + {{ cast_timestamp_to_iso8601("created_on") }} as countryincomethreshold_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as countryincomethreshold_updated_on - select - id as countryincomethreshold_id - , country_code as countryincomethreshold_country_code - , income_threshold as countryincomethreshold_income_threshold - ,{{ cast_timestamp_to_iso8601('created_on') }} as countryincomethreshold_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as countryincomethreshold_updated_on + from source - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_currencyexchangerate.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_currencyexchangerate.sql index 1b5be419b..609814769 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_currencyexchangerate.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_currencyexchangerate.sql @@ -1,23 +1,30 @@ -with source as ( +with + source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__flexiblepricing_currencyexchangerate') }} + select * + from + {{ + source( + "ol_warehouse_raw_data", "raw__mitxonline__app__postgres__flexiblepricing_currencyexchangerate" + ) + }} -) + ) -{{ deduplicate_raw_table(order_by='id' , partition_columns = 'currency_code') }} -, renamed as ( + {{ deduplicate_raw_table(order_by="id", partition_columns="currency_code") }}, + renamed as ( - select - id as currencyexchangerate_id - , description as currencyexchangerate_description - , currency_code as currencyexchangerate_currency_code - , exchange_rate as currencyexchangerate_exchange_rate - ,{{ cast_timestamp_to_iso8601('created_on') }} as currencyexchangerate_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as currencyexchangerate_updated_on + select + id as currencyexchangerate_id, + description as currencyexchangerate_description, + currency_code as currencyexchangerate_currency_code, + exchange_rate as currencyexchangerate_exchange_rate, + {{ cast_timestamp_to_iso8601("created_on") }} as currencyexchangerate_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as currencyexchangerate_updated_on - from most_recent_source + from most_recent_source -) + ) -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_flexiblepriceapplication.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_flexiblepriceapplication.sql index 483d6b38e..b09e61bd8 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_flexiblepriceapplication.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_flexiblepriceapplication.sql @@ -1,32 +1,34 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__flexiblepricing_flexibleprice') }} + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__flexiblepricing_flexibleprice") }} -) + ), + renamed as ( -, renamed as ( + select + id as flexiblepriceapplication_id, + status as flexiblepriceapplication_status, + tier_id as flexiblepricetier_id, + user_id, + income_usd as flexiblepriceapplication_income_usd, + justification as flexiblepriceapplication_justification, + original_income as flexiblepriceapplication_original_income, + cms_submission_id as flexiblepriceapplication_cms_submission_id, + country_of_income as flexiblepriceapplication_country_of_income, + original_currency as flexiblepriceapplication_original_currency, + country_of_residence as flexiblepriceapplication_country_of_residence, + courseware_object_id, + courseware_content_type_id as contenttype_id, + {{ cast_timestamp_to_iso8601("date_exchange_rate") }} as flexiblepriceapplication_exchange_rate_timestamp, + {{ cast_timestamp_to_iso8601("date_documents_sent") }} as flexiblepriceapplication_date_documents_sent, + {{ cast_timestamp_to_iso8601("created_on") }} as flexiblepriceapplication_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as flexiblepriceapplication_updated_on - select - id as flexiblepriceapplication_id - , status as flexiblepriceapplication_status - , tier_id as flexiblepricetier_id - , user_id - , income_usd as flexiblepriceapplication_income_usd - , justification as flexiblepriceapplication_justification - , original_income as flexiblepriceapplication_original_income - , cms_submission_id as flexiblepriceapplication_cms_submission_id - , country_of_income as flexiblepriceapplication_country_of_income - , original_currency as flexiblepriceapplication_original_currency - , country_of_residence as flexiblepriceapplication_country_of_residence - , courseware_object_id - , courseware_content_type_id as contenttype_id - ,{{ cast_timestamp_to_iso8601('date_exchange_rate') }} as flexiblepriceapplication_exchange_rate_timestamp - ,{{ cast_timestamp_to_iso8601('date_documents_sent') }} as flexiblepriceapplication_date_documents_sent - ,{{ cast_timestamp_to_iso8601('created_on') }} as flexiblepriceapplication_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as flexiblepriceapplication_updated_on + from source - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_flexiblepricetier.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_flexiblepricetier.sql index 02dfe0fa7..8d1e72aa4 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_flexiblepricetier.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_flexiblepricetier.sql @@ -1,20 +1,21 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__flexiblepricing_flexiblepricetier') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__flexiblepricing_flexiblepricetier") }} + ), + renamed as ( + select + source.id as flexiblepricetier_id, + source."current" as flexiblepricetier_is_current, + source.discount_id, + source.courseware_object_id, + source.income_threshold_usd as flexiblepricetier_income_threshold_usd, + source.courseware_content_type_id as contenttype_id, + {{ cast_timestamp_to_iso8601("source.created_on") }} as flexiblepricetier_created_on, + {{ cast_timestamp_to_iso8601("source.updated_on") }} as flexiblepricetier_updated_on -, renamed as ( - select - source.id as flexiblepricetier_id - , source."current" as flexiblepricetier_is_current - , source.discount_id - , source.courseware_object_id - , source.income_threshold_usd as flexiblepricetier_income_threshold_usd - , source.courseware_content_type_id as contenttype_id - ,{{ cast_timestamp_to_iso8601('source.created_on') }} as flexiblepricetier_created_on - ,{{ cast_timestamp_to_iso8601('source.updated_on') }} as flexiblepricetier_updated_on + from source + ) - from source -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__openedx_openedxuser.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__openedx_openedxuser.sql index 51849f1c0..3bd3b9360 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__openedx_openedxuser.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__openedx_openedxuser.sql @@ -1,19 +1,20 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__openedx_openedxuser') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__openedx_openedxuser") }} + ), + cleaned as ( -, cleaned as ( + select + id as openedxuser_id, + user_id, + platform as openedxuser_platform, + edx_username as openedxuser_username, + desired_edx_username as openedxuser_desired_username, + has_been_synced as openedxuser_has_been_synced, + {{ cast_timestamp_to_iso8601("created_on") }} as openedxuser_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as openedxuser_updated_on + from source + ) - select - id as openedxuser_id - , user_id - , platform as openedxuser_platform - , edx_username as openedxuser_username - , desired_edx_username as openedxuser_desired_username - , has_been_synced as openedxuser_has_been_synced - , {{ cast_timestamp_to_iso8601('created_on') }} as openedxuser_created_on - , {{ cast_timestamp_to_iso8601('updated_on') }} as openedxuser_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__reversion_revision.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__reversion_revision.sql index fbc211f6d..55e10a5a6 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__reversion_revision.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__reversion_revision.sql @@ -1,19 +1,20 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__reversion_revision') }} + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__reversion_revision") }} -) + ), + renamed as ( -, renamed as ( + select + id as revision_id, + comment as revision_comment, + user_id, + {{ cast_timestamp_to_iso8601("date_created") }} as revision_date_created - select - id as revision_id - , comment as revision_comment - , user_id - ,{{ cast_timestamp_to_iso8601('date_created') }} as revision_date_created + from source - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__reversion_version.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__reversion_version.sql index 208223d76..0ab9a15d0 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__reversion_version.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__reversion_version.sql @@ -1,19 +1,20 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__reversion_version') }} + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__reversion_version") }} -) + ), + renamed as ( -, renamed as ( + select + id as version_id, + cast(object_id as integer) as version_object_id, + revision_id, + content_type_id as contenttype_id, + serialized_data as version_object_serialized_data + from source - select - id as version_id - , cast(object_id as integer) as version_object_id - , revision_id - , content_type_id as contenttype_id - , serialized_data as version_object_serialized_data - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__users_legaladdress.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__users_legaladdress.sql index 68d959158..cf1850dea 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__users_legaladdress.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__users_legaladdress.sql @@ -1,19 +1,19 @@ -- MITx Online User Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__users_legaladdress") }} + ), + cleaned as ( -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__app__postgres__users_legaladdress') }} -) + select + id as user_address_id, + country as user_address_country, + state as user_address_state, + user_id, + first_name as user_first_name, + last_name as user_last_name + from source + ) -, cleaned as ( - - select - id as user_address_id - , country as user_address_country - , state as user_address_state - , user_id - , first_name as user_first_name - , last_name as user_last_name - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__users_user.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__users_user.sql index 422dc455c..3d5b1891e 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__users_user.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__users_user.sql @@ -1,24 +1,22 @@ -- MITx Online User Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__users_user") }}), + cleaned as ( -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__users_user') }} -) + select + id as user_id, + username as user_username, + email as user_email, + is_active as user_is_active, + nullif(global_id, '') as user_global_id, + nullif(scim_id, '') as user_scim_id, + nullif(scim_username, '') as user_scim_username, + nullif(scim_external_id, '') as user_scim_external_id, + replace(replace(replace(name, ' ', '<>'), '><', ''), '<>', ' ') as user_full_name, + {{ cast_timestamp_to_iso8601("created_on") }} as user_joined_on, + {{ cast_timestamp_to_iso8601("last_login") }} as user_last_login + from source + ) -, cleaned as ( - - select - id as user_id - , username as user_username - , email as user_email - , is_active as user_is_active - , nullif(global_id, '') as user_global_id - , nullif(scim_id, '') as user_scim_id - , nullif(scim_username, '') as user_scim_username - , nullif(scim_external_id, '') as user_scim_external_id - , replace(replace(replace(name, ' ', '<>'), '><', ''), '<>', ' ') as user_full_name - ,{{ cast_timestamp_to_iso8601('created_on') }} as user_joined_on - ,{{ cast_timestamp_to_iso8601('last_login') }} as user_last_login - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__users_userprofile.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__users_userprofile.sql index 51c71d75f..1430f37a6 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__users_userprofile.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__users_userprofile.sql @@ -1,28 +1,29 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__app__postgres__users_userprofile') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__app__postgres__users_userprofile") }} + ), + cleaned as ( + select + id as user_profile_id, + user_id, + year_of_birth as user_birth_year, + nullif(company, '') as user_company, + nullif(industry, '') as user_industry, + nullif(job_title, '') as user_job_title, + nullif(job_function, '') as user_job_function, + nullif(leadership_level, '') as user_leadership_level, + nullif(highest_education, '') as user_highest_education, + type_is_student as user_type_is_student, + type_is_professional as user_type_is_professional, + type_is_educator as user_type_is_educator, + type_is_other as user_type_is_other, + {{ transform_gender_value("gender") }} as user_gender, + {{ transform_company_size_value("company_size") }} as user_company_size, + {{ transform_years_experience_value("years_experience") }} as user_years_experience, + {{ cast_timestamp_to_iso8601("created_on") }} as user_profile_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as user_profile_updated_on + from source + ) -, cleaned as ( - select - id as user_profile_id - , user_id - , year_of_birth as user_birth_year - , nullif(company, '') as user_company - , nullif(industry, '') as user_industry - , nullif(job_title, '') as user_job_title - , nullif(job_function, '') as user_job_function - , nullif(leadership_level, '') as user_leadership_level - , nullif(highest_education, '') as user_highest_education - , type_is_student as user_type_is_student - , type_is_professional as user_type_is_professional - , type_is_educator as user_type_is_educator - , type_is_other as user_type_is_other - ,{{ transform_gender_value('gender') }} as user_gender - ,{{ transform_company_size_value('company_size') }} as user_company_size - ,{{ transform_years_experience_value('years_experience') }} as user_years_experience - ,{{ cast_timestamp_to_iso8601('created_on') }} as user_profile_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as user_profile_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__api__course_structure.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__api__course_structure.sql index 242f0434d..73dce66dc 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__api__course_structure.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__api__course_structure.sql @@ -1,43 +1,41 @@ ---- This table captures the changes made to course structure. It does so by comparing to course content from previous --- day. If no changes, then no new rows added to the table. If any changes to the course content, it adds all the --- blocks including the updated/new ones to the table. - ---- To get the course structure at any give time, use course_id + block_id + the last retrieved_at - -with course_block_source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__api__course_blocks') }} -) - -, course_block as ( - select * from ( +-- - This table captures the changes made to course structure. It does so by comparing to course content from previous +-- day. If no changes, then no new rows added to the table. If any changes to the course content, it adds all the +-- blocks including the updated/new ones to the table. +-- - To get the course structure at any give time, use course_id + block_id + the last retrieved_at +with + course_block_source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__api__course_blocks") }} + ), + course_block as ( + select * + from + ( + select + course_block_source.*, + lag(course_block_source.course_content_hash) over ( + partition by course_block_source.block_id, course_block_source.course_id + order by course_block_source.retrieved_at asc + ) as previous_content_hash + from course_block_source + ) + where previous_content_hash is null or previous_content_hash != course_content_hash + ), + cleaned as ( select - course_block_source.* - , lag(course_block_source.course_content_hash) over ( - partition by - course_block_source.block_id - , course_block_source.course_id - order by course_block_source.retrieved_at asc - ) as previous_content_hash - from course_block_source + course_id as courserun_readable_id, + course_title as courserun_title, + block_index as coursestructure_block_index, + block_id as coursestructure_block_id, + block_parent as coursestructure_parent_block_id, + block_type as coursestructure_block_category, + block_title as coursestructure_block_title, + course_content_hash as coursestructure_content_hash, + block_content_hash as coursestructure_block_content_hash, + json_query(block_details, 'lax $.metadata') as coursestructure_block_metadata, + {{ cast_timestamp_to_iso8601("course_start") }} as courserun_start_on, + {{ cast_timestamp_to_iso8601("retrieved_at") }} as coursestructure_retrieved_at + from course_block ) - where previous_content_hash is null or previous_content_hash != course_content_hash -) - -, cleaned as ( - select - course_id as courserun_readable_id - , course_title as courserun_title - , block_index as coursestructure_block_index - , block_id as coursestructure_block_id - , block_parent as coursestructure_parent_block_id - , block_type as coursestructure_block_category - , block_title as coursestructure_block_title - , course_content_hash as coursestructure_content_hash - , block_content_hash as coursestructure_block_content_hash - , json_query(block_details, 'lax $.metadata') as coursestructure_block_metadata - , {{ cast_timestamp_to_iso8601('course_start') }} as courserun_start_on - , {{ cast_timestamp_to_iso8601('retrieved_at') }} as coursestructure_retrieved_at - from course_block -) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__blockcompletion.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__blockcompletion.sql index 2c09524f9..ccefdc6f1 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__blockcompletion.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__blockcompletion.sql @@ -1,37 +1,37 @@ -{{ config( - materialized='incremental', - unique_key = 'blockcompletion_id', - incremental_strategy='delete+insert', - views_enabled=false, - ) +{{ + config( + materialized="incremental", + unique_key="blockcompletion_id", + incremental_strategy="delete+insert", + views_enabled=false, + ) }} -with source as ( - select * from - {{ - source('ol_warehouse_raw_data' - ,'raw__mitxonline__openedx__mysql__completion_blockcompletion') - }} +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__completion_blockcompletion") }} - {% if is_incremental() %} - where created >= (select max(this.blockcompletion_created_on) from {{ this }} as this) - {% endif %} -) + {% if is_incremental() %} + where created >= (select max(this.blockcompletion_created_on) from {{ this }} as this) + {% endif %} + ) ---- this is needed for the initial dbt run to deduplicate the data from raw table -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( + -- - this is needed for the initial dbt run to deduplicate the data from raw table + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( - select - id as blockcompletion_id - , user_id - , created as blockcompletion_created_on - , modified as blockcompletion_updated_on - , block_key as block_fk - , block_type as block_category - , completion as block_completed - , course_key as courserun_readable_id - from most_recent_source -) + select + id as blockcompletion_id, + user_id, + created as blockcompletion_created_on, + modified as blockcompletion_updated_on, + block_key as block_fk, + block_type as block_category, + completion as block_completed, + course_key as courserun_readable_id + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__courseware_studentmodulehistoryextended.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__courseware_studentmodulehistoryextended.sql index f979f8210..265bac803 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__courseware_studentmodulehistoryextended.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__courseware_studentmodulehistoryextended.sql @@ -1,35 +1,41 @@ -{{ config( - materialized='incremental', - unique_key = 'studentmodulehistoryextended_id', - incremental_strategy='delete+insert', - views_enabled=false, - ) +{{ + config( + materialized="incremental", + unique_key="studentmodulehistoryextended_id", + incremental_strategy="delete+insert", + views_enabled=false, + ) }} -with source as ( - select * from - {{ - source('ol_warehouse_raw_data' - ,'raw__mitxonline__openedx__mysql__coursewarehistoryextended_studentmodulehistoryextended') - }} +with + source as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitxonline__openedx__mysql__coursewarehistoryextended_studentmodulehistoryextended", + ) + }} - {% if is_incremental() %} - where created >= (select max(this.studentmodule_created_on) from {{ this }} as this) - {% endif %} -) + {% if is_incremental() %} + where created >= (select max(this.studentmodule_created_on) from {{ this }} as this) + {% endif %} + ) ---- this is needed for the initial dbt run to deduplicate the data from raw table -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( + -- - this is needed for the initial dbt run to deduplicate the data from raw table + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( - select - id as studentmodulehistoryextended_id - , student_module_id as studentmodule_id - , state as studentmodule_state_data - , grade as studentmodule_problem_grade - , max_grade as studentmodule_problem_max_grade - , created as studentmodule_created_on - from most_recent_source -) + select + id as studentmodulehistoryextended_id, + student_module_id as studentmodule_id, + state as studentmodule_state_data, + grade as studentmodule_problem_grade, + max_grade as studentmodule_problem_max_grade, + created as studentmodule_created_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__auth_user.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__auth_user.sql index 8fb46acd3..8ec393d59 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__auth_user.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__auth_user.sql @@ -1,19 +1,17 @@ -- MITx Online open edX User Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__auth_user") }}), + cleaned as ( -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__openedx__mysql__auth_user') }} -) + select + id as openedx_user_id, + username as user_username, + email as user_email, + is_active as user_is_active, + is_staff as user_is_staff, + is_superuser as user_is_superuser + from source + ) -, cleaned as ( - - select - id as openedx_user_id - , username as user_username - , email as user_email - , is_active as user_is_active - , is_staff as user_is_staff - , is_superuser as user_is_superuser - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__bulk_email_optout.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__bulk_email_optout.sql index eef41edc3..bc50a1d6d 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__bulk_email_optout.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__bulk_email_optout.sql @@ -1,15 +1,11 @@ -- MITx Online Openedx mySQL Bulk Email Optout +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__bulk_email_optout") }} + ), + cleaned as ( + select id as email_optout_id, user_id as openedx_user_id, course_id as courserun_readable_id from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__bulk_email_optout') }} -) - -, cleaned as ( - select - id as email_optout_id - , user_id as openedx_user_id - , course_id as courserun_readable_id - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__courseware_studentmodule.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__courseware_studentmodule.sql index ca01c0d81..025256aa1 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__courseware_studentmodule.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__courseware_studentmodule.sql @@ -1,22 +1,24 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__openedx__mysql__courseware_studentmodule') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__courseware_studentmodule") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'course_id, module_id, student_id') }} -, cleaned as ( + {{ deduplicate_raw_table(order_by="modified", partition_columns="course_id, module_id, student_id") }}, + cleaned as ( - select - id as studentmodule_id - , course_id as courserun_readable_id - , module_id as coursestructure_block_id - , module_type as coursestructure_block_category - , student_id as openedx_user_id - , state as studentmodule_state_data - , grade as studentmodule_problem_grade - , max_grade as studentmodule_problem_max_grade - , to_iso8601(created) as studentmodule_created_on - , to_iso8601(modified) as studentmodule_updated_on - from most_recent_source -) + select + id as studentmodule_id, + course_id as courserun_readable_id, + module_id as coursestructure_block_id, + module_type as coursestructure_block_category, + student_id as openedx_user_id, + state as studentmodule_state_data, + grade as studentmodule_problem_grade, + max_grade as studentmodule_problem_max_grade, + to_iso8601(created) as studentmodule_created_on, + to_iso8601(modified) as studentmodule_updated_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__edxval_coursevideo.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__edxval_coursevideo.sql index ddd5735ef..36ca98785 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__edxval_coursevideo.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__edxval_coursevideo.sql @@ -1,16 +1,13 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__openedx__mysql__edxval_coursevideo') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__edxval_coursevideo") }} + ), + cleaned as ( -, cleaned as ( + select id as coursevideo_id, course_id as courserun_readable_id, video_id, is_hidden as coursevideo_is_hidden - select - id as coursevideo_id - , course_id as courserun_readable_id - , video_id - , is_hidden as coursevideo_is_hidden + from source + ) - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__edxval_video.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__edxval_video.sql index 9e0488f72..cbefd7015 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__edxval_video.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__edxval_video.sql @@ -1,18 +1,17 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__openedx__mysql__edxval_video') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__edxval_video") }}), + cleaned as ( -, cleaned as ( + select + id as video_id, + edx_video_id as video_edx_uuid, + client_video_id as video_client_id, + status as video_status, + duration as video_duration, + {{ cast_timestamp_to_iso8601("created") }} as video_created_on - select - id as video_id - , edx_video_id as video_edx_uuid - , client_video_id as video_client_id - , status as video_status - , duration as video_duration - , {{ cast_timestamp_to_iso8601('created') }} as video_created_on + from source + ) - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_subsectiongrade.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_subsectiongrade.sql index e9ca59aa1..c937c48a3 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_subsectiongrade.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_subsectiongrade.sql @@ -1,25 +1,27 @@ -with source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__openedx__mysql__grades_persistentsubsectiongrade') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__grades_persistentsubsectiongrade") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }} -, cleaned as ( + {{ deduplicate_raw_table(order_by="modified", partition_columns="id") }}, + cleaned as ( - select - id as subsectiongrade_id - , course_id as courserun_readable_id - , user_id as openedx_user_id - , usage_key as coursestructure_block_id - , visible_blocks_hash as visibleblocks_hash - , possible_all as subsectiongrade_total_score - , possible_graded as subsectiongrade_total_graded_score - , earned_all as subsectiongrade_total_earned_score - , earned_graded as subsectiongrade_total_earned_graded_score - , to_iso8601(first_attempted) as subsectiongrade_first_attempted_on - , to_iso8601(created) as subsectiongrade_created_on - , to_iso8601(modified) as subsectiongrade_updated_on - from most_recent_source -) + select + id as subsectiongrade_id, + course_id as courserun_readable_id, + user_id as openedx_user_id, + usage_key as coursestructure_block_id, + visible_blocks_hash as visibleblocks_hash, + possible_all as subsectiongrade_total_score, + possible_graded as subsectiongrade_total_graded_score, + earned_all as subsectiongrade_total_earned_score, + earned_graded as subsectiongrade_total_earned_graded_score, + to_iso8601(first_attempted) as subsectiongrade_first_attempted_on, + to_iso8601(created) as subsectiongrade_created_on, + to_iso8601(modified) as subsectiongrade_updated_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_subsectiongradeoverride.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_subsectiongradeoverride.sql index 5d4f60bb9..560e1cb84 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_subsectiongradeoverride.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_subsectiongradeoverride.sql @@ -1,24 +1,31 @@ -with source as ( - select * from {{ - source('ol_warehouse_raw_data', 'raw__mitxonline__openedx__mysql__grades_persistentsubsectiongradeoverride') - }} -) +with + source as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitxonline__openedx__mysql__grades_persistentsubsectiongradeoverride", + ) + }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }} -, cleaned as ( + {{ deduplicate_raw_table(order_by="modified", partition_columns="id") }}, + cleaned as ( - select - id as subsectiongradeoverride_id - , grade_id as subsectiongrade_id - , possible_all_override as subsectiongradeoverride_total_score - , possible_graded_override as subsectiongradeoverride_total_graded_score - , earned_all_override as subsectiongradeoverride_total_earned_score - , earned_graded_override as subsectiongradeoverride_total_earned_graded_score - , override_reason as subsectiongradeoverride_reason - , system as subsectiongradeoverride_system - , to_iso8601(created) as subsectiongradeoverride_created_on - , to_iso8601(modified) as subsectiongradeoverride_updated_on - from most_recent_source -) + select + id as subsectiongradeoverride_id, + grade_id as subsectiongrade_id, + possible_all_override as subsectiongradeoverride_total_score, + possible_graded_override as subsectiongradeoverride_total_graded_score, + earned_all_override as subsectiongradeoverride_total_earned_score, + earned_graded_override as subsectiongradeoverride_total_earned_graded_score, + override_reason as subsectiongradeoverride_reason, + system as subsectiongradeoverride_system, + to_iso8601(created) as subsectiongradeoverride_created_on, + to_iso8601(modified) as subsectiongradeoverride_updated_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_visibleblocks.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_visibleblocks.sql index b415f6cf4..d42844e3d 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_visibleblocks.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_visibleblocks.sql @@ -1,15 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__openedx__mysql__grades_visibleblocks') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__grades_visibleblocks") }} + ), + cleaned as ( -, cleaned as ( + select + id as visibleblocks_id, + course_id as courserun_readable_id, + hashed as visibleblocks_hash, + blocks_json as visibleblocks_json + from source + ) - select - id as visibleblocks_id - , course_id as courserun_readable_id - , hashed as visibleblocks_hash - , blocks_json as visibleblocks_json - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__user_courseaccessrole.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__user_courseaccessrole.sql index 66870597d..6770b4080 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__user_courseaccessrole.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__user_courseaccessrole.sql @@ -1,15 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__student_courseaccessrole') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__mysql__student_courseaccessrole") }} + ), + cleaned as ( + select + id as courseaccessrole_id, + course_id as courserun_readable_id, + user_id as openedx_user_id, + role as courseaccess_role, + if(lower(org) = 'mitxt', 'MITxT', org) as organization + from source + ) -, cleaned as ( - select - id as courseaccessrole_id - , course_id as courserun_readable_id - , user_id as openedx_user_id - , role as courseaccess_role - , if(lower(org) = 'mitxt', 'MITxT', org) as organization - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__tracking_logs__user_activity.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__tracking_logs__user_activity.sql index 2804d5979..b1e7398c8 100644 --- a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__tracking_logs__user_activity.sql +++ b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__tracking_logs__user_activity.sql @@ -1,55 +1,69 @@ -- MITx Online user activities from tracking logs -- Due to size of the raw table, build this model as incremental and apply dedup on new rows -{{ config( - materialized='incremental', - unique_key = ['user_username', 'useractivity_context_object', 'useractivity_event_source', - 'useractivity_event_type', 'useractivity_event_object', 'useractivity_timestamp'], - incremental_strategy='delete+insert', - views_enabled=false, - ) +{{ + config( + materialized="incremental", + unique_key=[ + "user_username", + "useractivity_context_object", + "useractivity_event_source", + "useractivity_event_type", + "useractivity_event_object", + "useractivity_timestamp", + ], + incremental_strategy="delete+insert", + views_enabled=false, + ) }} -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__tracking_logs') }} - where - username != '' - and json_query(context, 'lax $.user_id' omit quotes) != 'null' - and json_query(event, 'lax $.exception' omit quotes) is null +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitxonline__openedx__tracking_logs") }} + where + username != '' + and json_query(context, 'lax $.user_id' omit quotes) != 'null' + and json_query(event, 'lax $.exception' omit quotes) is null - {% if is_incremental() %} - and "time" > (select max(this.useractivity_timestamp) from {{ this }} as this) --noqa - {% endif %} -) + {% if is_incremental() %} + and "time" > (select max(this.useractivity_timestamp) from {{ this }} as this) -- noqa + {% endif %} + ) -{{ deduplicate_raw_table( - order_by='_airbyte_extracted_at desc, _ab_source_file_last_modified desc, "time"' - , partition_columns = 'username, context, event_source, event_type, event, "time"' -) }} -, cleaned as ( - select - username as user_username - , context as useractivity_context_object - , event as useractivity_event_object - , event_source as useractivity_event_source - , page as useractivity_page_url - , session as useractivity_session_id - , ip as useractivity_ip - , host as useractivity_http_host - , agent as useractivity_http_user_agent - , accept_language as useractivity_http_accept_language - , referer as useractivity_http_referer - , name as useractivity_event_name - , event_type as useractivity_event_type - , {{ extract_course_id_from_tracking_log() }} as courserun_readable_id - --- extract common fields from context object - , cast(json_query(context, 'lax $.user_id' omit quotes) as integer) as openedx_user_id - , json_query(context, 'lax $.org_id' omit quotes) as org_id - , json_query(context, 'lax $.path' omit quotes) as useractivity_path - --- due to log collector changes, values of time field come with different formats - , to_iso8601(from_iso8601_timestamp_nanos( - regexp_replace(time, '(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}:\d{2}\.\d+)(.*?)', '$1T$2$3') -- noqa - )) as useractivity_timestamp - from most_recent_source -) + {{ + deduplicate_raw_table( + order_by='_airbyte_extracted_at desc, _ab_source_file_last_modified desc, "time"', + partition_columns='username, context, event_source, event_type, event, "time"', + ) + }}, + cleaned as ( + select + username as user_username, + context as useractivity_context_object, + event as useractivity_event_object, + event_source as useractivity_event_source, + page as useractivity_page_url, + session as useractivity_session_id, + ip as useractivity_ip, + host as useractivity_http_host, + agent as useractivity_http_user_agent, + accept_language as useractivity_http_accept_language, + referer as useractivity_http_referer, + name as useractivity_event_name, + event_type as useractivity_event_type, + {{ extract_course_id_from_tracking_log() }} as courserun_readable_id, + -- - extract common fields from context object + cast(json_query(context, 'lax $.user_id' omit quotes) as integer) as openedx_user_id, + json_query(context, 'lax $.org_id' omit quotes) as org_id, + json_query(context, 'lax $.path' omit quotes) as useractivity_path, + -- - due to log collector changes, values of time field come with different formats + to_iso8601( + from_iso8601_timestamp_nanos( + regexp_replace(time, '(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}:\d{2}\.\d+)(.*?)', '$1T$2$3') -- noqa + ) + ) as useractivity_timestamp + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__emeritus__api__bigquery__user_enrollments.sql b/src/ol_dbt/models/staging/mitxpro/stg__emeritus__api__bigquery__user_enrollments.sql index b33f1dbc5..9c433de94 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__emeritus__api__bigquery__user_enrollments.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__emeritus__api__bigquery__user_enrollments.sql @@ -1,58 +1,59 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__emeritus__bigquery__api_enrollments') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__emeritus__bigquery__api_enrollments") }}) -{{ deduplicate_raw_table( - order_by='opportunity_last_modified_date' - , partition_columns='batch_id, email, first_name, last_name' -) }} -, cleaned as ( - select - ---course run - batch_id as courserun_id - , batch as courserun_title - , {{ cast_timestamp_to_iso8601('batch_start_date') }} as courserun_start_on - , {{ cast_timestamp_to_iso8601('batch_end_date') }} as courserun_end_on - --wrike_run_code maps to external_course_run_id in xPro app - , wrike_run_code as courserun_external_readable_id + {{ + deduplicate_raw_table( + order_by="opportunity_last_modified_date", partition_columns="batch_id, email, first_name, last_name" + ) + }}, + cleaned as ( + select + -- -course run + batch_id as courserun_id, + batch as courserun_title, + {{ cast_timestamp_to_iso8601("batch_start_date") }} as courserun_start_on, + {{ cast_timestamp_to_iso8601("batch_end_date") }} as courserun_end_on, + -- wrike_run_code maps to external_course_run_id in xPro app + wrike_run_code as courserun_external_readable_id, + -- -user + student_id as user_id, + first_name as user_first_name, + last_name as user_last_name, + company_name as user_company, + work_exp_slab as user_years_experience, + nationality as user_nationality, + residential_zip_code as user_address_postal_code, + country as user_address_country, + gdpr_agree as user_gdpr_agree, + {{ cast_timestamp_to_iso8601("gdpr_consent_date") }} as user_gdpr_consent_date, + -- -enrollment + enrollment_type, + {{ cast_timestamp_to_iso8601("opportunity_created_date") }} as enrollment_created_on, + {{ cast_timestamp_to_iso8601("opportunity_last_modified_date") }} as enrollment_updated_on, + if(lower(status) = 'enrolled', true, false) as is_enrolled, + if("deferred" = true, 'deferred', null) as enrollment_status, -- noqa: ST10 + if(email = '(blank)', null, email) as user_email, + if("function" = 'No Response', null, "function") as user_job_function, -- noqa: ST10 + if(job_title in ('No Response', '(Blank)'), null, job_title) as user_job_title, + if(industry in ('No Response', '(blank)'), null, industry) as user_industry, + if(residential_address = 'NO RESPONSE', null, residential_address) as user_address_street, + if(residential_city = 'NO RESPONSE', null, residential_city) as user_address_city, + if(residential_state = 'NO RESPONSE', null, residential_state) as user_address_state, + concat(first_name, ' ', last_name) as user_full_name, + -- - match these values to our existing values from other platforms + case + when gender = 'Prefer not to answer' + then 'Other/Prefer Not to Say' + when gender = 'Nonbinary' + then 'Non-binary/non-conforming' + when gender = 'Prefer not to answer' + then 'Other/Prefer Not to Say' + when gender = 'No Response' + then null + else gender + end as user_gender + from most_recent_source + ) - ---user - , student_id as user_id - , first_name as user_first_name - , last_name as user_last_name - , company_name as user_company - , work_exp_slab as user_years_experience - , nationality as user_nationality - , residential_zip_code as user_address_postal_code - , country as user_address_country - , gdpr_agree as user_gdpr_agree - , {{ cast_timestamp_to_iso8601('gdpr_consent_date') }} as user_gdpr_consent_date - - ---enrollment - , enrollment_type - , {{ cast_timestamp_to_iso8601('opportunity_created_date') }} as enrollment_created_on - , {{ cast_timestamp_to_iso8601('opportunity_last_modified_date') }} as enrollment_updated_on - , if(lower(status) = 'enrolled', true, false) as is_enrolled - , if("deferred" = true, 'deferred', null) as enrollment_status -- noqa: ST10 - - , if(email = '(blank)', null, email) as user_email - , if("function" = 'No Response', null, "function") as user_job_function -- noqa: ST10 - , if(job_title in ('No Response', '(Blank)'), null, job_title) as user_job_title - , if(industry in ('No Response', '(blank)'), null, industry) as user_industry - , if(residential_address = 'NO RESPONSE', null, residential_address) as user_address_street - , if(residential_city = 'NO RESPONSE', null, residential_city) as user_address_city - , if(residential_state = 'NO RESPONSE', null, residential_state) as user_address_state - - , concat(first_name, ' ', last_name) as user_full_name - --- match these values to our existing values from other platforms - , case - when gender = 'Prefer not to answer' then 'Other/Prefer Not to Say' - when gender = 'Nonbinary' then 'Non-binary/non-conforming' - when gender = 'Prefer not to answer' then 'Other/Prefer Not to Say' - when gender = 'No Response' then null - else gender - end as user_gender - from most_recent_source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__global_alumni__api__bigquery__user_enrollments.sql b/src/ol_dbt/models/staging/mitxpro/stg__global_alumni__api__bigquery__user_enrollments.sql index 85852e95f..6c33064b6 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__global_alumni__api__bigquery__user_enrollments.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__global_alumni__api__bigquery__user_enrollments.sql @@ -1,49 +1,49 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__global_alumni__bigquery__api_enrollments') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__global_alumni__bigquery__api_enrollments") }}) --- The raw table is loaded using full refresh + appends sync mode. --- As a result, every sync will create duplicates of already existing data, --- so we need to deduplicate the data. -{{ deduplicate_raw_table(order_by='_airbyte_generation_id', partition_columns='batch_id, email') }} -, cleaned as ( - select - ---course run - batch_id as courserun_id - , batch as courserun_title - , {{ cast_timestamp_to_iso8601('batch_start_date') }} as courserun_start_on - , {{ cast_timestamp_to_iso8601('batch_end_date') }} as courserun_end_on - , wrike_run_code as courserun_external_readable_id + -- The raw table is loaded using full refresh + appends sync mode. + -- As a result, every sync will create duplicates of already existing data, + -- so we need to deduplicate the data. + {{ deduplicate_raw_table(order_by="_airbyte_generation_id", partition_columns="batch_id, email") }}, + cleaned as ( + select + -- -course run + batch_id as courserun_id, + batch as courserun_title, + {{ cast_timestamp_to_iso8601("batch_start_date") }} as courserun_start_on, + {{ cast_timestamp_to_iso8601("batch_end_date") }} as courserun_end_on, + wrike_run_code as courserun_external_readable_id, + -- -user + student_id as user_id, + first_name as user_first_name, + last_name as user_last_name, + email as user_email, + company_name as user_company, + work_exp_slab as user_years_experience, + residential_zip_code as user_address_postal_code, + country as user_address_country, + gdpr_agree as user_gdpr_agree, + if(residential_address = 'NO RESPONSE', null, residential_address) as user_address_street, + if(residential_city = 'NO RESPONSE', null, residential_city) as user_address_city, + if(residential_state = 'NO RESPONSE', null, residential_state) as user_address_state, + if("function" = 'No Response', null, "function") as user_job_function, -- noqa: ST10 + if(job_title = 'No Response', null, job_title) as user_job_title, + if(industry = 'No Response', null, industry) as user_industry, + {{ cast_timestamp_to_iso8601("gdpr_consent_date") }} as user_gdpr_consent_date, + concat(first_name, ' ', last_name) as user_full_name, + -- - match these values to our existing values from other platforms + case + when gender = 'Nonbinary' + then 'Non-binary/non-conforming' + when gender = 'No Response' + then null + else gender + end as user_gender, + -- -enrollment + if(lower(status) = 'enrolled', true, false) as is_enrolled, + if("deferred" = true, 'deferred', null) as enrollment_status -- noqa: ST10 + from most_recent_source + ) - ---user - , student_id as user_id - , first_name as user_first_name - , last_name as user_last_name - , email as user_email - , company_name as user_company - , work_exp_slab as user_years_experience - , residential_zip_code as user_address_postal_code - , country as user_address_country - , gdpr_agree as user_gdpr_agree - , if(residential_address = 'NO RESPONSE', null, residential_address) as user_address_street - , if(residential_city = 'NO RESPONSE', null, residential_city) as user_address_city - , if(residential_state = 'NO RESPONSE', null, residential_state) as user_address_state - , if("function" = 'No Response', null, "function") as user_job_function -- noqa: ST10 - , if(job_title = 'No Response', null, job_title) as user_job_title - , if(industry = 'No Response', null, industry) as user_industry - , {{ cast_timestamp_to_iso8601('gdpr_consent_date') }} as user_gdpr_consent_date - , concat(first_name, ' ', last_name) as user_full_name - --- match these values to our existing values from other platforms - , case - when gender = 'Nonbinary' then 'Non-binary/non-conforming' - when gender = 'No Response' then null - else gender - end as user_gender - - ---enrollment - , if(lower(status) = 'enrolled', true, false) as is_enrolled - , if("deferred" = true, 'deferred', null) as enrollment_status -- noqa: ST10 - from most_recent_source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2bcoupon.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2bcoupon.sql index 5c89a060b..2fb67b686 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2bcoupon.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2bcoupon.sql @@ -1,25 +1,26 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__b2b_ecommerce_b2bcoupon') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__b2b_ecommerce_b2bcoupon") }} -) + ), + renamed as ( -, renamed as ( + select + id as b2bcoupon_id, + company_id, + product_id, + enabled as b2bcoupon_is_enabled, + discount_percent as b2bcoupon_discount_percent, + name as b2bcoupon_name, + reusable as b2bcoupon_is_reusable, + coupon_code as b2bcoupon_coupon_code, + {{ cast_timestamp_to_iso8601("updated_on") }} as b2bcoupon_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as b2bcoupon_created_on, + {{ cast_timestamp_to_iso8601("expiration_date") }} as b2bcoupon_expires_on, + {{ cast_timestamp_to_iso8601("activation_date") }} as b2bcoupon_activated_on + from source + ) - select - id as b2bcoupon_id - , company_id - , product_id - , enabled as b2bcoupon_is_enabled - , discount_percent as b2bcoupon_discount_percent - , name as b2bcoupon_name - , reusable as b2bcoupon_is_reusable - , coupon_code as b2bcoupon_coupon_code - ,{{ cast_timestamp_to_iso8601('updated_on') }} as b2bcoupon_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as b2bcoupon_created_on - ,{{ cast_timestamp_to_iso8601('expiration_date') }} as b2bcoupon_expires_on - ,{{ cast_timestamp_to_iso8601('activation_date') }} as b2bcoupon_activated_on - from source -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2bcouponaudit.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2bcouponaudit.sql index 7e5d7fab8..c830eb285 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2bcouponaudit.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2bcouponaudit.sql @@ -1,21 +1,22 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__b2b_ecommerce_b2bcouponaudit') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__b2b_ecommerce_b2bcouponaudit") }} -) + ), + renamed as ( -, renamed as ( + select + id as b2bcouponaudit_id, + coupon_id as b2bcoupon_id, + acting_user_id as b2bcouponaudit_acting_user_id, + data_before as b2bcouponaudit_data_before, + data_after as b2bcouponaudit_data_after, + {{ cast_timestamp_to_iso8601("created_on") }} as b2bcouponaudit_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as b2bcouponaudit_updated_on + from source - select - id as b2bcouponaudit_id - , coupon_id as b2bcoupon_id - , acting_user_id as b2bcouponaudit_acting_user_id - , data_before as b2bcouponaudit_data_before - , data_after as b2bcouponaudit_data_after - ,{{ cast_timestamp_to_iso8601('created_on') }} as b2bcouponaudit_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as b2bcouponaudit_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2bcouponredemption.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2bcouponredemption.sql index 39356adc9..77a1021a8 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2bcouponredemption.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2bcouponredemption.sql @@ -1,18 +1,20 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__b2b_ecommerce_b2bcouponredemption') }} + select * + from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__b2b_ecommerce_b2bcouponredemption") }} -) + ), + renamed as ( -, renamed as ( + select + id as b2bcouponredemption_id, + order_id as b2border_id, + coupon_id as b2bcoupon_id, + {{ cast_timestamp_to_iso8601("updated_on") }} as b2bcouponredemption_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as b2bcouponredemption_created_on + from source + ) - select - id as b2bcouponredemption_id - , order_id as b2border_id - , coupon_id as b2bcoupon_id - ,{{ cast_timestamp_to_iso8601('updated_on') }} as b2bcouponredemption_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as b2bcouponredemption_created_on - from source -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2border.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2border.sql index a6b0a3444..daf62a3d2 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2border.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2border.sql @@ -1,29 +1,26 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__b2b_ecommerce_b2border") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__b2b_ecommerce_b2border') }} + select + id as b2border_id, + total_price as b2border_total_price, + status as b2border_status, + per_item_price as b2border_per_item_price, + unique_id as b2border_unique_uuid, + num_seats as b2border_num_seats, + coupon_id as b2bcoupon_id, + product_version_id as productversion_id, + coupon_payment_version_id as couponpaymentversion_id, + contract_number as b2border_contract_number, + discount as b2border_discount, + program_run_id as programrun_id, + email as b2border_email, + {{ cast_timestamp_to_iso8601("updated_on") }} as b2border_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as b2border_created_on + from source -) + ) -, renamed as ( - - select - id as b2border_id - , total_price as b2border_total_price - , status as b2border_status - , per_item_price as b2border_per_item_price - , unique_id as b2border_unique_uuid - , num_seats as b2border_num_seats - , coupon_id as b2bcoupon_id - , product_version_id as productversion_id - , coupon_payment_version_id as couponpaymentversion_id - , contract_number as b2border_contract_number - , discount as b2border_discount - , program_run_id as programrun_id - , email as b2border_email - ,{{ cast_timestamp_to_iso8601('updated_on') }} as b2border_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as b2border_created_on - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2borderaudit.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2borderaudit.sql index 6281a84a3..a808dc790 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2borderaudit.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2borderaudit.sql @@ -1,21 +1,22 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__b2b_ecommerce_b2borderaudit') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__b2b_ecommerce_b2borderaudit") }} -) + ), + renamed as ( -, renamed as ( + select + id as b2borderaudit_id, + order_id as b2border_id, + acting_user_id as b2borderaudit_acting_user_id, + data_before as b2borderaudit_data_before, + data_after as b2borderaudit_data_after, + {{ cast_timestamp_to_iso8601("created_on") }} as b2borderaudit_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as b2borderaudit_updated_on + from source - select - id as b2borderaudit_id - , order_id as b2border_id - , acting_user_id as b2borderaudit_acting_user_id - , data_before as b2borderaudit_data_before - , data_after as b2borderaudit_data_after - ,{{ cast_timestamp_to_iso8601('created_on') }} as b2borderaudit_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as b2borderaudit_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2breceipt.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2breceipt.sql index e20a03ca0..575d2cbc3 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2breceipt.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__b2becommerce_b2breceipt.sql @@ -1,26 +1,27 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__b2b_ecommerce_b2breceipt') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__b2b_ecommerce_b2breceipt") }} -) + ), + renamed as ( -, renamed as ( + select + id as b2breceipt_id, + order_id as b2border_id, + data as b2breceipt_data, + json_query(data, 'lax $.decision' omit quotes) as b2breceipt_transaction_status, + json_query(data, 'lax $.transaction_id' omit quotes) as b2breceipt_transaction_id, + json_query(data, 'lax $.auth_code' omit quotes) as b2breceipt_authorization_code, + json_query(data, 'lax $.req_reference_number' omit quotes) as b2breceipt_reference_number, + json_query(data, 'lax $.req_payment_method' omit quotes) as b2breceipt_payment_method, + json_query(data, 'lax $.req_bill_to_address_state' omit quotes) as b2breceipt_bill_to_address_state, + json_query(data, 'lax $.req_bill_to_address_country' omit quotes) as b2breceipt_bill_to_address_country, + {{ cast_timestamp_to_iso8601("created_on") }} as b2breceipt_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as b2breceipt_updated_on + from source - select - id as b2breceipt_id - , order_id as b2border_id - , data as b2breceipt_data - , json_query(data, 'lax $.decision' omit quotes) as b2breceipt_transaction_status - , json_query(data, 'lax $.transaction_id' omit quotes) as b2breceipt_transaction_id - , json_query(data, 'lax $.auth_code' omit quotes) as b2breceipt_authorization_code - , json_query(data, 'lax $.req_reference_number' omit quotes) as b2breceipt_reference_number - , json_query(data, 'lax $.req_payment_method' omit quotes) as b2breceipt_payment_method - , json_query(data, 'lax $.req_bill_to_address_state' omit quotes) as b2breceipt_bill_to_address_state - , json_query(data, 'lax $.req_bill_to_address_country' omit quotes) as b2breceipt_bill_to_address_country - ,{{ cast_timestamp_to_iso8601('created_on') }} as b2breceipt_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as b2breceipt_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_certificatepage.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_certificatepage.sql index e5e5532f4..e36a9eb59 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_certificatepage.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_certificatepage.sql @@ -1,17 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__cms_certificatepage') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__cms_certificatepage") }}), + cleaned as ( + select + page_ptr_id as wagtail_page_id, + product_name as cms_certificate_product_name, + ceus as cms_certificate_ceus, -- noqa + cast(json_parse(json_query(signatories, 'lax $[*].value' with array wrapper)) as array(integer)) -- noqa + as cms_certificate_signitory_ids, + institute_text as cms_certificate_institute_text, + overrides as cms_certificate_overrides + from source + ) -, cleaned as ( - select - page_ptr_id as wagtail_page_id - , product_name as cms_certificate_product_name - , CEUs as cms_certificate_ceus --noqa - , cast(json_parse(json_query(signatories, 'lax $[*].value' with array wrapper)) as array(integer)) --noqa - as cms_certificate_signitory_ids - , institute_text as cms_certificate_institute_text - , overrides as cms_certificate_overrides - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_coursepage.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_coursepage.sql index 4be2ab2aa..780ce2b8a 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_coursepage.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_coursepage.sql @@ -1,63 +1,46 @@ -with coursepage as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__cms_coursepage') }} -) - -, coursepage_sorted as ( - select - * - , row_number() over (partition by course_id order by _airbyte_extracted_at desc) as row_num - from coursepage -) - -, most_recent_coursepage as ( - select * from coursepage_sorted - where row_num = 1 -) - -, externalcoursepage as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__cms_externalcoursepage') }} -) - -, externalcoursepage_sorted as ( - select - * - , row_number() over (partition by course_id order by _airbyte_extracted_at desc) as row_num - from externalcoursepage -) - -, most_recent_externalcoursepage as ( - select * from externalcoursepage_sorted - where row_num = 1 -) +with + coursepage as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__cms_coursepage") }}), + coursepage_sorted as ( + select *, row_number() over (partition by course_id order by _airbyte_extracted_at desc) as row_num + from coursepage + ), + most_recent_coursepage as (select * from coursepage_sorted where row_num = 1), + externalcoursepage as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__cms_externalcoursepage") }} + ), + externalcoursepage_sorted as ( + select *, row_number() over (partition by course_id order by _airbyte_extracted_at desc) as row_num + from externalcoursepage + ), + most_recent_externalcoursepage as (select * from externalcoursepage_sorted where row_num = 1) select - page_ptr_id as wagtail_page_id - , description as cms_coursepage_description - , course_id - , duration as cms_coursepage_duration - , format as cms_coursepage_format - , subhead as cms_coursepage_subhead - , time_commitment as cms_coursepage_time_commitment - , catalog_details as cms_coursepage_catalog_details - , external_marketing_url as cms_coursepage_external_marketing_url - , 'cms_coursepage' as cms_coursepage_model + page_ptr_id as wagtail_page_id, + description as cms_coursepage_description, + course_id, + duration as cms_coursepage_duration, + format as cms_coursepage_format, + subhead as cms_coursepage_subhead, + time_commitment as cms_coursepage_time_commitment, + catalog_details as cms_coursepage_catalog_details, + external_marketing_url as cms_coursepage_external_marketing_url, + 'cms_coursepage' as cms_coursepage_model from most_recent_coursepage union all select - most_recent_externalcoursepage.page_ptr_id as wagtail_page_id - , most_recent_externalcoursepage.description as cms_coursepage_description - , most_recent_externalcoursepage.course_id - , most_recent_externalcoursepage.duration as cms_coursepage_duration - , most_recent_externalcoursepage.format as cms_coursepage_format - , most_recent_externalcoursepage.subhead as cms_coursepage_subhead - , most_recent_externalcoursepage.time_commitment as cms_coursepage_time_commitment - , most_recent_externalcoursepage.catalog_details as cms_coursepage_catalog_details - , most_recent_externalcoursepage.external_marketing_url as cms_coursepage_external_marketing_url - , 'cms_externalcoursepage' as cms_coursepage_model + most_recent_externalcoursepage.page_ptr_id as wagtail_page_id, + most_recent_externalcoursepage.description as cms_coursepage_description, + most_recent_externalcoursepage.course_id, + most_recent_externalcoursepage.duration as cms_coursepage_duration, + most_recent_externalcoursepage.format as cms_coursepage_format, + most_recent_externalcoursepage.subhead as cms_coursepage_subhead, + most_recent_externalcoursepage.time_commitment as cms_coursepage_time_commitment, + most_recent_externalcoursepage.catalog_details as cms_coursepage_catalog_details, + most_recent_externalcoursepage.external_marketing_url as cms_coursepage_external_marketing_url, + 'cms_externalcoursepage' as cms_coursepage_model from most_recent_externalcoursepage -- There are currently courses that has both a course page and an external course page but -- adding a dedup in case any are added in the future and to match cms_programpage -left join coursepage - on most_recent_externalcoursepage.course_id = coursepage.course_id +left join coursepage on most_recent_externalcoursepage.course_id = coursepage.course_id where coursepage.course_id is null diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_coursepage_topics.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_coursepage_topics.sql index 2b9ab4737..b0674e0d6 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_coursepage_topics.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_coursepage_topics.sql @@ -1,17 +1,13 @@ -with coursepage_topics as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__cms_coursepage_topics') }} -) +with + coursepage_topics as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__cms_coursepage_topics") }} + ), + externalcoursepage_topics as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__cms_externalcoursepage_topics") }} + ) -, externalcoursepage_topics as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__cms_externalcoursepage_topics') }} -) - -select - coursepage_id as wagtail_page_id - , coursetopic_id +select coursepage_id as wagtail_page_id, coursetopic_id from coursepage_topics union all -select - externalcoursepage_id as wagtail_page_id - , coursetopic_id +select externalcoursepage_id as wagtail_page_id, coursetopic_id from externalcoursepage_topics diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_coursesinprogrampage.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_coursesinprogrampage.sql index 6c43ef62f..534f58a01 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_coursesinprogrampage.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_coursesinprogrampage.sql @@ -1,15 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__cms_coursesinprogrampage') }} -) - -, cleaned as ( - select - page_ptr_id as wagtail_page_id - , heading as cms_coursesinprogrampage_heading - , body as cms_coursesinprogrampage_body - , cast(json_parse(json_query(contents, 'lax $[*].value' with array wrapper)) as array(integer)) --noqa +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__cms_coursesinprogrampage") }} + ), + cleaned as ( + select + page_ptr_id as wagtail_page_id, + heading as cms_coursesinprogrampage_heading, + body as cms_coursesinprogrampage_body, + cast(json_parse(json_query(contents, 'lax $[*].value' with array wrapper)) as array(integer)) -- noqa as cms_coursesinprogrampage_coursepage_wagtail_page_ids - from source -) + from source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_facultymemberspage.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_facultymemberspage.sql index 576222867..f8b4d398f 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_facultymemberspage.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_facultymemberspage.sql @@ -1,14 +1,13 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__cms_facultymemberspage') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__cms_facultymemberspage") }}), + cleaned as ( + select + page_ptr_id as wagtail_page_id, + heading as cms_facultymemberspage_heading, + subhead as cms_facultymemberspage_subhead, + members as cms_facultymemberspage_faculty + from source + ) -, cleaned as ( - select - page_ptr_id as wagtail_page_id - , heading as cms_facultymemberspage_heading - , subhead as cms_facultymemberspage_subhead - , members as cms_facultymemberspage_faculty - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_programpage.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_programpage.sql index 13452cb1f..d718915aa 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_programpage.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_programpage.sql @@ -1,40 +1,37 @@ -with programpage as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__cms_programpage') }} -) - -, externalprogrampage as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__cms_externalprogrampage') }} -) +with + programpage as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__cms_programpage") }}), + externalprogrampage as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__cms_externalprogrampage") }} + ) select - page_ptr_id as wagtail_page_id - , description as cms_programpage_description - , program_id - , duration as cms_programpage_duration - , format as cms_programpage_format - , subhead as cms_programpage_subhead - , time_commitment as cms_programpage_time_commitment - , catalog_details as cms_programpage_catalog_details - , external_marketing_url as cms_programpage_external_marketing_url - , featured as cms_programpage_is_featured - , 'cms_programpage' as cms_programpage_model + page_ptr_id as wagtail_page_id, + description as cms_programpage_description, + program_id, + duration as cms_programpage_duration, + format as cms_programpage_format, + subhead as cms_programpage_subhead, + time_commitment as cms_programpage_time_commitment, + catalog_details as cms_programpage_catalog_details, + external_marketing_url as cms_programpage_external_marketing_url, + featured as cms_programpage_is_featured, + 'cms_programpage' as cms_programpage_model from programpage union all select - externalprogrampage.page_ptr_id as wagtail_page_id - , externalprogrampage.description as cms_cprogrampage_description - , externalprogrampage.program_id - , externalprogrampage.duration as cms_programpage_duration - , externalprogrampage.format as cms_programpage_format - , externalprogrampage.subhead as cms_programpage_subhead - , externalprogrampage.time_commitment as cms_programpage_time_commitment - , externalprogrampage.catalog_details as cms_programpage_catalog_details - , externalprogrampage.external_marketing_url as cms_programpage_external_marketing_url - , externalprogrampage.featured as cms_programpage_is_featured - , 'cms_externalprogrampage' as cms_programpage_model + externalprogrampage.page_ptr_id as wagtail_page_id, + externalprogrampage.description as cms_cprogrampage_description, + externalprogrampage.program_id, + externalprogrampage.duration as cms_programpage_duration, + externalprogrampage.format as cms_programpage_format, + externalprogrampage.subhead as cms_programpage_subhead, + externalprogrampage.time_commitment as cms_programpage_time_commitment, + externalprogrampage.catalog_details as cms_programpage_catalog_details, + externalprogrampage.external_marketing_url as cms_programpage_external_marketing_url, + externalprogrampage.featured as cms_programpage_is_featured, + 'cms_externalprogrampage' as cms_programpage_model from externalprogrampage -- There is one program that has both a program page and an external program page. The external program page is not live -left join programpage - on externalprogrampage.program_id = programpage.program_id +left join programpage on externalprogrampage.program_id = programpage.program_id where programpage.program_id is null diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_signatorypage.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_signatorypage.sql index a782049c3..bee32cf85 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_signatorypage.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__cms_signatorypage.sql @@ -1,15 +1,14 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__cms_signatorypage') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__cms_signatorypage") }}), + cleaned as ( + select + page_ptr_id as wagtail_page_id, + name as cms_signatorypage_name, + title_1 as cms_signatorypage_title_1, + title_2 as cms_signatorypage_title_2, + organization as cms_signatorypage_organization + from source + ) -, cleaned as ( - select - page_ptr_id as wagtail_page_id - , name as cms_signatorypage_name - , title_1 as cms_signatorypage_title_1 - , title_2 as cms_signatorypage_title_2 - , organization as cms_signatorypage_organization - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_course.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_course.sql index ad00f2ad9..a64acf671 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_course.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_course.sql @@ -1,27 +1,26 @@ ---MITxPro Online Course Information +-- MITxPro Online Course Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__courses_course") }}), + cleaned as ( + select + id as course_id, + live as course_is_live, + title as course_title, + program_id, + readable_id as course_readable_id, + substring( + readable_id, + ((position('+' in readable_id)) + 1), + (position('x' in substring(readable_id, position('+' in readable_id))) - 2) + ) as short_program_code, + position_in_program, + platform_id, + replace(replace(readable_id, 'course-v1:', ''), '+', '/') as course_edx_readable_id, + {{ cast_timestamp_to_iso8601("created_on") }} as course_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as course_updated_on, + is_external as course_is_external + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__courses_course') }} -) - -, cleaned as ( - select - id as course_id - , live as course_is_live - , title as course_title - , program_id - , readable_id as course_readable_id - , substring( - readable_id, ((position('+' in readable_id)) + 1) - , (position('x' in substring(readable_id, position('+' in readable_id))) - 2) - ) as short_program_code - , position_in_program - , platform_id - , replace(replace(readable_id, 'course-v1:', ''), '+', '/') as course_edx_readable_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as course_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as course_updated_on - , is_external as course_is_external - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courserun.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courserun.sql index 597fa5a81..5fafdbfc1 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courserun.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courserun.sql @@ -1,28 +1,26 @@ -- MITxPro Course Run Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__courses_courserun") }}), + cleaned as ( + select + id as courserun_id, + course_id, + external_course_run_id as courserun_external_readable_id, + live as courserun_is_live, + title as courserun_title, + courseware_id as courserun_readable_id, + concat('https://courses.xpro.mit.edu', courseware_url_path) as courserun_url, + run_tag as courserun_tag, + replace(replace(courseware_id, 'course-v1:', ''), '+', '/') as courserun_edx_readable_id, + {{ cast_timestamp_to_iso8601("start_date") }} as courserun_start_on, + {{ cast_timestamp_to_iso8601("end_date") }} as courserun_end_on, + {{ cast_timestamp_to_iso8601("enrollment_start") }} as courserun_enrollment_start_on, + {{ cast_timestamp_to_iso8601("enrollment_end") }} as courserun_enrollment_end_on, + {{ cast_timestamp_to_iso8601("expiration_date") }} as courserun_expired_on, + {{ cast_timestamp_to_iso8601("created_on") }} as courserun_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as courserun_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__courses_courserun') }} -) - -, cleaned as ( - select - id as courserun_id - , course_id - , external_course_run_id as courserun_external_readable_id - , live as courserun_is_live - , title as courserun_title - , courseware_id as courserun_readable_id - , concat('https://courses.xpro.mit.edu', courseware_url_path) as courserun_url - , run_tag as courserun_tag - , replace(replace(courseware_id, 'course-v1:', ''), '+', '/') as courserun_edx_readable_id - ,{{ cast_timestamp_to_iso8601('start_date') }} as courserun_start_on - ,{{ cast_timestamp_to_iso8601('end_date') }} as courserun_end_on - ,{{ cast_timestamp_to_iso8601('enrollment_start') }} as courserun_enrollment_start_on - ,{{ cast_timestamp_to_iso8601('enrollment_end') }} as courserun_enrollment_end_on - ,{{ cast_timestamp_to_iso8601('expiration_date') }} as courserun_expired_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as courserun_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as courserun_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courseruncertificate.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courseruncertificate.sql index 0bb3c2623..e219d3b7e 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courseruncertificate.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courseruncertificate.sql @@ -1,21 +1,21 @@ -- MIT xPro Users Course Certificate Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__courses_courseruncertificate") }} + ), + cleaned as ( + select + id as courseruncertificate_id, + uuid as courseruncertificate_uuid, + course_run_id as courserun_id, + user_id, + certificate_page_revision_id, -- - rename it after the referenced model is created + is_revoked as courseruncertificate_is_revoked, + if(is_revoked = false, concat('https://xpro.mit.edu/certificate/', uuid), null) as courseruncertificate_url, + {{ cast_timestamp_to_iso8601("created_on") }} as courseruncertificate_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as courseruncertificate_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__courses_courseruncertificate') }} -) - -, cleaned as ( - select - id as courseruncertificate_id - , uuid as courseruncertificate_uuid - , course_run_id as courserun_id - , user_id - , certificate_page_revision_id --- rename it after the referenced model is created - , is_revoked as courseruncertificate_is_revoked - , if(is_revoked = false, concat('https://xpro.mit.edu/certificate/', uuid), null) as courseruncertificate_url - ,{{ cast_timestamp_to_iso8601('created_on') }} as courseruncertificate_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as courseruncertificate_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courserunenrollment.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courserunenrollment.sql index c4843d7b1..aa208bc7f 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courserunenrollment.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courserunenrollment.sql @@ -1,22 +1,22 @@ -- xPro Course Run Enrollment Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__courses_courserunenrollment") }} + ), + cleaned as ( + select + id as courserunenrollment_id, + change_status as courserunenrollment_enrollment_status, + active as courserunenrollment_is_active, + edx_enrolled as courserunenrollment_is_edx_enrolled, + run_id as courserun_id, + user_id, + company_id as ecommerce_company_id, + order_id as ecommerce_order_id, + {{ cast_timestamp_to_iso8601("created_on") }} as courserunenrollment_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as courserunenrollment_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__courses_courserunenrollment') }} -) - -, cleaned as ( - select - id as courserunenrollment_id - , change_status as courserunenrollment_enrollment_status - , active as courserunenrollment_is_active - , edx_enrolled as courserunenrollment_is_edx_enrolled - , run_id as courserun_id - , user_id - , company_id as ecommerce_company_id - , order_id as ecommerce_order_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as courserunenrollment_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as courserunenrollment_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courserungrade.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courserungrade.sql index aa79de096..3287d3d1c 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courserungrade.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_courserungrade.sql @@ -1,21 +1,19 @@ -- MIT xPro Users Course Grades Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__courses_courserungrade") }}), + cleaned as ( + select + id as courserungrade_id, + course_run_id as courserun_id, + user_id, + grade as courserungrade_grade, + letter_grade as courserungrade_letter_grade, + set_by_admin as courserungrade_is_set_by_admin, + passed as courserungrade_is_passing, + {{ cast_timestamp_to_iso8601("created_on") }} as courserungrade_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as courserungrade_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__courses_courserungrade') }} -) - -, cleaned as ( - select - id as courserungrade_id - , course_run_id as courserun_id - , user_id - , grade as courserungrade_grade - , letter_grade as courserungrade_letter_grade - , set_by_admin as courserungrade_is_set_by_admin - , passed as courserungrade_is_passing - ,{{ cast_timestamp_to_iso8601('created_on') }} as courserungrade_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as courserungrade_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_coursetopic.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_coursetopic.sql index 4a849b89c..eab05aca8 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_coursetopic.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_coursetopic.sql @@ -1,17 +1,15 @@ -- MITxPro Course Topic Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__courses_coursetopic") }}), + cleaned as ( + select + id as coursetopic_id, + name as coursetopic_name, + parent_id as coursetopic_parent_coursetopic_id, + {{ cast_timestamp_to_iso8601("created_on") }} as coursetopic_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as coursetopic_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__courses_coursetopic') }} -) - -, cleaned as ( - select - id as coursetopic_id - , name as coursetopic_name - , parent_id as coursetopic_parent_coursetopic_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as coursetopic_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as coursetopic_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_platform.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_platform.sql index 43f3010c3..a87642211 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_platform.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_platform.sql @@ -1,14 +1,7 @@ ---MITxPro Platform Information +-- MITxPro Platform Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__courses_platform") }}), + cleaned as (select id as platform_id, name as platform_name from source) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__courses_platform') }} -) - -, cleaned as ( - select - id as platform_id - , name as platform_name - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_program.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_program.sql index 4ae083999..0cd4b1d37 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_program.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_program.sql @@ -1,31 +1,24 @@ -- MITxPro Program Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__courses_program") }}), + cleaned as ( + select + id as program_id, + live as program_is_live, + title as program_title, + readable_id as program_readable_id, + substring( + readable_id, + ((position('+' in readable_id)) + 1), + (position('x' in substring(readable_id, position('+' in readable_id))) - 2) + ) as short_program_code, + platform_id, + {{ cast_timestamp_to_iso8601("created_on") }} as program_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as program_updated_on, + is_external as program_is_external -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__courses_program') }} -) + from source + ) -, cleaned as ( - select - id as program_id - , live as program_is_live - , title as program_title - , readable_id as program_readable_id - , substring( - readable_id, ((position('+' in readable_id)) + 1) - , ( - position( - 'x' in substring( - readable_id, position('+' in readable_id) - ) - ) - 2 - ) - ) as short_program_code - , platform_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as program_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as program_updated_on - , is_external as program_is_external - - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_programcertificate.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_programcertificate.sql index eff6d3913..6f36080a6 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_programcertificate.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_programcertificate.sql @@ -1,20 +1,20 @@ -- MIT xPro Users Program Certificate Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__courses_programcertificate") }} + ), + cleaned as ( + select + id as programcertificate_id, + uuid as programcertificate_uuid, + program_id, + user_id, + certificate_page_revision_id, -- - rename it after the referenced model is created + is_revoked as programcertificate_is_revoked, + {{ cast_timestamp_to_iso8601("created_on") }} as programcertificate_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as programcertificate_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__courses_programcertificate') }} -) - -, cleaned as ( - select - id as programcertificate_id - , uuid as programcertificate_uuid - , program_id - , user_id - , certificate_page_revision_id --- rename it after the referenced model is created - , is_revoked as programcertificate_is_revoked - ,{{ cast_timestamp_to_iso8601('created_on') }} as programcertificate_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as programcertificate_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_programenrollment.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_programenrollment.sql index 7129055ab..94774c9d8 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_programenrollment.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_programenrollment.sql @@ -1,21 +1,21 @@ -- xPro Program Enrollment Information +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__courses_programenrollment") }} + ), + cleaned as ( + select + id as programenrollment_id, + change_status as programenrollment_enrollment_status, + active as programenrollment_is_active, + program_id, + user_id, + company_id as ecommerce_company_id, + order_id as ecommerce_order_id, + {{ cast_timestamp_to_iso8601("created_on") }} as programenrollment_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as programenrollment_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__courses_programenrollment') }} -) - -, cleaned as ( - select - id as programenrollment_id - , change_status as programenrollment_enrollment_status - , active as programenrollment_is_active - , program_id - , user_id - , company_id as ecommerce_company_id - , order_id as ecommerce_order_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as programenrollment_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as programenrollment_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_programrun.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_programrun.sql index 9d6feb520..9172a1f41 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_programrun.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__courses_programrun.sql @@ -1,19 +1,17 @@ -- MITxPro Program Run Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__courses_programrun") }}), + cleaned as ( + select + id as programrun_id, + program_id, + run_tag as programrun_tag, + {{ cast_timestamp_to_iso8601("start_date") }} as programrun_start_on, + {{ cast_timestamp_to_iso8601("end_date") }} as programrun_end_on, + {{ cast_timestamp_to_iso8601("created_on") }} as programrun_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as programrun_updated_on + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__courses_programrun') }} -) - -, cleaned as ( - select - id as programrun_id - , program_id - , run_tag as programrun_tag - ,{{ cast_timestamp_to_iso8601('start_date') }} as programrun_start_on - ,{{ cast_timestamp_to_iso8601('end_date') }} as programrun_end_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as programrun_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as programrun_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__django_contenttype.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__django_contenttype.sql index ceefb58cf..97ec01ff2 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__django_contenttype.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__django_contenttype.sql @@ -1,20 +1,6 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__django_content_type") }}), + renamed as (select id as contenttype_id, concat_ws('_', app_label, model) as contenttype_full_name from source) - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__django_content_type') }} - -) - -, renamed as ( - - select - id as contenttype_id - , concat_ws( - '_' - , app_label - , model - ) as contenttype_full_name - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_basket.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_basket.sql index 46568f8e5..d59022d61 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_basket.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_basket.sql @@ -1,20 +1,18 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_basket") }}) - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_basket') }} + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="user_id") }}, + renamed as ( -) + select + id as basket_id, + user_id, + {{ cast_timestamp_to_iso8601("created_on") }} as basket_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as basket_updated_on -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'user_id') }} -, renamed as ( + from most_recent_source - select - id as basket_id - , user_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as basket_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as basket_updated_on + ) - from most_recent_source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_basketitem.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_basketitem.sql index dc88bdfae..847025c8f 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_basketitem.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_basketitem.sql @@ -1,21 +1,18 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_basketitem") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_basketitem') }} + select + id as basketitem_id, + quantity as basketitem_quantity, + basket_id, + product_id, + program_run_id as programrun_id, + {{ cast_timestamp_to_iso8601("created_on") }} as basketitem_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as basketitem_updated_on + from source -) + ) -, renamed as ( - - select - id as basketitem_id - , quantity as basketitem_quantity - , basket_id - , product_id - , program_run_id as programrun_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as basketitem_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as basketitem_updated_on - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_basketrunselection.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_basketrunselection.sql index b7ecd4407..76b513ccb 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_basketrunselection.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_basketrunselection.sql @@ -1,19 +1,20 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_courserunselection') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_courserunselection") }} -) + ), + renamed as ( -, renamed as ( + select + id as basketrunselection_id, + basket_id, + run_id as courserun_id, + {{ cast_timestamp_to_iso8601("created_on") }} as basketrunselection_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as basketrunselection_updated_on + from source - select - id as basketrunselection_id - , basket_id - , run_id as courserun_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as basketrunselection_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as basketrunselection_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_bulkcouponassignment.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_bulkcouponassignment.sql index 0beab1a24..118e427dc 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_bulkcouponassignment.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_bulkcouponassignment.sql @@ -1,24 +1,25 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_bulkcouponassignment') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_bulkcouponassignment") }} -) + ), + renamed as ( -, renamed as ( + select + assignment_sheet_id as bulkcouponassignment_assignment_sheet_id, + id as bulkcouponassignment_id, + {{ cast_timestamp_to_iso8601("assignments_started_date") }} as bulkcouponassignment_assignments_started_on, + {{ cast_timestamp_to_iso8601("last_assignment_date") }} as bulkcouponassignment_last_assignment_on, + {{ cast_timestamp_to_iso8601("message_delivery_completed_date") }} + as bulkcouponassignment_message_delivery_completed_on, + {{ cast_timestamp_to_iso8601("sheet_last_modified_date") }} as bulkcouponassignment_sheet_last_modified_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as bulkcouponassignment_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as bulkcouponassignment_created_on - select - assignment_sheet_id as bulkcouponassignment_assignment_sheet_id - , id as bulkcouponassignment_id - ,{{ cast_timestamp_to_iso8601('assignments_started_date') }} as bulkcouponassignment_assignments_started_on - ,{{ cast_timestamp_to_iso8601('last_assignment_date') }} as bulkcouponassignment_last_assignment_on - ,{{ cast_timestamp_to_iso8601('message_delivery_completed_date') }} - as bulkcouponassignment_message_delivery_completed_on - ,{{ cast_timestamp_to_iso8601('sheet_last_modified_date') }} as bulkcouponassignment_sheet_last_modified_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as bulkcouponassignment_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as bulkcouponassignment_created_on + from source - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_company.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_company.sql index e8b5e3c18..fa0a42794 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_company.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_company.sql @@ -1,16 +1,13 @@ -with source as ( - - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_company') }} - -) - -, renamed as ( - select - id as company_id - , name as company_name - ,{{ cast_timestamp_to_iso8601('updated_on') }} as company_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as company_created_on - from source -) - -select * from renamed +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_company") }}), + renamed as ( + select + id as company_id, + name as company_name, + {{ cast_timestamp_to_iso8601("updated_on") }} as company_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as company_created_on + from source + ) + +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_coupon.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_coupon.sql index cc37cc944..c417878b2 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_coupon.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_coupon.sql @@ -1,20 +1,17 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_coupon") }}), + renamed as ( + select + id as coupon_id, + coupon_code, + payment_id as couponpayment_id, + enabled as coupon_is_active, + include_future_runs as coupon_applies_to_future_runs, + is_global as coupon_is_global, + {{ cast_timestamp_to_iso8601("updated_on") }} as coupon_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as coupon_created_on + from source + ) - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_coupon') }} - -) - -, renamed as ( - select - id as coupon_id - , coupon_code - , payment_id as couponpayment_id - , enabled as coupon_is_active - , include_future_runs as coupon_applies_to_future_runs - , is_global as coupon_is_global - ,{{ cast_timestamp_to_iso8601('updated_on') }} as coupon_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as coupon_created_on - from source -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponbasket.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponbasket.sql index abed59d45..4a471a5f2 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponbasket.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponbasket.sql @@ -1,20 +1,22 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_couponselection') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_couponselection") }} -) + ) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'basket_id') }} -, renamed as ( + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="basket_id") }}, + renamed as ( - select - id as couponbasket_id - , basket_id - , coupon_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as couponbasket_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as couponbasket_updated_on - from most_recent_source + select + id as couponbasket_id, + basket_id, + coupon_id, + {{ cast_timestamp_to_iso8601("created_on") }} as couponbasket_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as couponbasket_updated_on + from most_recent_source -) + ) -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponpayment.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponpayment.sql index 9b73c316d..d33450ff6 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponpayment.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponpayment.sql @@ -1,16 +1,17 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_couponpayment') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_couponpayment") }} -) + ), + renamed as ( + select + id as couponpayment_id, + name as couponpayment_name, + {{ cast_timestamp_to_iso8601("updated_on") }} as couponpayment_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as couponpayment_created_on + from source + ) -, renamed as ( - select - id as couponpayment_id - , name as couponpayment_name - ,{{ cast_timestamp_to_iso8601('updated_on') }} as couponpayment_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as couponpayment_created_on - from source -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponpaymentversion.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponpaymentversion.sql index da863fcb4..79e7156cb 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponpaymentversion.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponpaymentversion.sql @@ -1,38 +1,39 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_couponpaymentversion') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_couponpaymentversion") }} -) + ), + renamed as ( -, renamed as ( - - select - id as couponpaymentversion_id - , company_id - , num_coupon_codes as couponpaymentversion_num_coupon_codes - , coupon_type as couponpaymentversion_coupon_type - , cast(amount as decimal(38, 2)) as couponpaymentversion_discount_amount - , max_redemptions_per_user as couponpaymentversion_max_redemptions_per_user - , payment_id as couponpayment_id - , automatic as couponpaymentversion_is_automatic - , payment_type as couponpaymentversion_discount_source - , payment_transaction as couponpaymentversion_payment_transaction - , tag as couponpaymentversion_tag - , discount_type as couponpaymentversion_discount_type - , max_redemptions as couponpaymentversion_max_redemptions - , case - when discount_type = 'dollars-off' + select + id as couponpaymentversion_id, + company_id, + num_coupon_codes as couponpaymentversion_num_coupon_codes, + coupon_type as couponpaymentversion_coupon_type, + cast(amount as decimal(38, 2)) as couponpaymentversion_discount_amount, + max_redemptions_per_user as couponpaymentversion_max_redemptions_per_user, + payment_id as couponpayment_id, + automatic as couponpaymentversion_is_automatic, + payment_type as couponpaymentversion_discount_source, + payment_transaction as couponpaymentversion_payment_transaction, + tag as couponpaymentversion_tag, + discount_type as couponpaymentversion_discount_type, + max_redemptions as couponpaymentversion_max_redemptions, + case + when discount_type = 'dollars-off' then concat('$', format('%.2f', amount)) - when discount_type = 'percent-off' + when discount_type = 'percent-off' then concat(format('%.2f', amount * 100), '%') - end as couponpaymentversion_discount_amount_text - ,{{ cast_timestamp_to_iso8601('expiration_date') }} as couponpaymentversion_expires_on - ,{{ cast_timestamp_to_iso8601('activation_date') }} as couponpaymentversion_activated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as couponpaymentversion_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as couponpaymentversion_updated_on + end as couponpaymentversion_discount_amount_text, + {{ cast_timestamp_to_iso8601("expiration_date") }} as couponpaymentversion_expires_on, + {{ cast_timestamp_to_iso8601("activation_date") }} as couponpaymentversion_activated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as couponpaymentversion_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as couponpaymentversion_updated_on - from source + from source -) + ) -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponproduct.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponproduct.sql index 5a6591a0f..65cbd43a3 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponproduct.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponproduct.sql @@ -1,20 +1,21 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_couponeligibility') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_couponeligibility") }} -) + ), + renamed as ( -, renamed as ( + select + id as couponproduct_id, + product_id, + coupon_id, + program_run_id as programrun_id, + {{ cast_timestamp_to_iso8601("created_on") }} as couponproduct_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as couponproduct_updated_on + from source - select - id as couponproduct_id - , product_id - , coupon_id - , program_run_id as programrun_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as couponproduct_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as couponproduct_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponredemption.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponredemption.sql index 67ffec46f..8e31d7cea 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponredemption.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponredemption.sql @@ -1,19 +1,20 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_couponredemption') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_couponredemption") }} -) + ), + renamed as ( -, renamed as ( + select + id as couponredemption_id, + order_id, + coupon_version_id as couponversion_id, + {{ cast_timestamp_to_iso8601("created_on") }} as couponredemption_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as couponredemption_updated_on + from source - select - id as couponredemption_id - , order_id - , coupon_version_id as couponversion_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as couponredemption_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as couponredemption_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponversion.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponversion.sql index d9905c27f..8af27dcc4 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponversion.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_couponversion.sql @@ -1,17 +1,18 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_couponversion') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_couponversion") }} -) + ), + renamed as ( + select + id as couponversion_id, + coupon_id, + payment_version_id as couponpaymentversion_id, + {{ cast_timestamp_to_iso8601("updated_on") }} as couponversion_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as couponversion_created_on + from source + ) -, renamed as ( - select - id as couponversion_id - , coupon_id - , payment_version_id as couponpaymentversion_id - ,{{ cast_timestamp_to_iso8601('updated_on') }} as couponversion_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as couponversion_created_on - from source -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_line.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_line.sql index e49cf36ca..c8afc4c45 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_line.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_line.sql @@ -1,19 +1,16 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_line") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_line') }} + select + id as line_id, + order_id, + product_version_id as productversion_id, + {{ cast_timestamp_to_iso8601("created_on") }} as line_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as line_updated_on + from source -) + ) -, renamed as ( - - select - id as line_id - , order_id - , product_version_id as productversion_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as line_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as line_updated_on - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_linerunselection.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_linerunselection.sql index 13aad8cc8..c5e25e965 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_linerunselection.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_linerunselection.sql @@ -1,19 +1,20 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_linerunselection') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_linerunselection") }} -) + ), + renamed as ( -, renamed as ( + select + id as linerunselection_id, + line_id, + run_id as courserun_id, + {{ cast_timestamp_to_iso8601("created_on") }} as linerunselection_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as linerunselection_updated_on + from source - select - id as linerunselection_id - , line_id - , run_id as courserun_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as linerunselection_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as linerunselection_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_order.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_order.sql index 98a8d75f6..590f6eca4 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_order.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_order.sql @@ -1,26 +1,22 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_order") }}), + renamed as ( + select + id as order_id, + status as order_state, + cast(total_price_paid as decimal(38, 2)) as order_total_price_paid, + purchaser_id as order_purchaser_user_id, + tax_country_code as order_tax_country_code, + cast(tax_rate as decimal(38, 2)) as order_tax_rate, + tax_rate_name as order_tax_rate_name, + cast((total_price_paid * (tax_rate / 100)) as decimal(38, 2)) as order_tax_amount, + cast( + (total_price_paid * (tax_rate / 100)) + total_price_paid as decimal(38, 2) + ) as order_total_price_paid_plus_tax, + {{ cast_timestamp_to_iso8601("updated_on") }} as order_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as order_created_on + from source + ) - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_order') }} - -) - -, renamed as ( - select - id as order_id - , status as order_state - , cast(total_price_paid as decimal(38, 2)) as order_total_price_paid - , purchaser_id as order_purchaser_user_id - , tax_country_code as order_tax_country_code - , cast(tax_rate as decimal(38, 2)) as order_tax_rate - , tax_rate_name as order_tax_rate_name - , cast((total_price_paid * (tax_rate / 100)) as decimal(38, 2)) as order_tax_amount - , cast( - (total_price_paid * (tax_rate / 100)) + total_price_paid - as decimal(38, 2) - ) as order_total_price_paid_plus_tax - ,{{ cast_timestamp_to_iso8601('updated_on') }} as order_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as order_created_on - from source -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_orderaudit.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_orderaudit.sql index 9af2ed8c6..2605dd981 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_orderaudit.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_orderaudit.sql @@ -1,21 +1,18 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_orderaudit") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_orderaudit') }} + select + id as orderaudit_id, + order_id, + acting_user_id as orderaudit_acting_user_id, + data_before as orderaudit_data_before, + data_after as orderaudit_data_after, + {{ cast_timestamp_to_iso8601("created_on") }} as orderaudit_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as orderaudit_updated_on + from source -) + ) -, renamed as ( - - select - id as orderaudit_id - , order_id - , acting_user_id as orderaudit_acting_user_id - , data_before as orderaudit_data_before - , data_after as orderaudit_data_after - ,{{ cast_timestamp_to_iso8601('created_on') }} as orderaudit_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as orderaudit_updated_on - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_product.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_product.sql index 111c72f03..ebe4d74d5 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_product.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_product.sql @@ -1,21 +1,18 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_product") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_product') }} + select + id as product_id, + is_active as product_is_active, + object_id as product_object_id, + is_private as product_is_private, + content_type_id as contenttype_id, + {{ cast_timestamp_to_iso8601("updated_on") }} as product_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as product_created_on + from source -) + ) -, renamed as ( - - select - id as product_id - , is_active as product_is_active - , object_id as product_object_id - , is_private as product_is_private - , content_type_id as contenttype_id - ,{{ cast_timestamp_to_iso8601('updated_on') }} as product_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as product_created_on - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_productcouponassignment.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_productcouponassignment.sql index dd2d5fe24..dd6c2a6d2 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_productcouponassignment.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_productcouponassignment.sql @@ -1,25 +1,27 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_productcouponassignment') }} + select * + from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_productcouponassignment") }} -) + ), + renamed as ( -, renamed as ( + select + id as productcouponassignment_id, + bulk_assignment_id as bulkcouponassignment_id, + product_coupon_id as couponproduct_id, + redeemed as productcouponassignment_is_redeemed, + message_status as productcouponassignment_message_status, + email as productcouponassignment_email, + original_email as productcouponassignment_original_email, + {{ cast_timestamp_to_iso8601("message_status_date") }} as productcouponassignment_message_status_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as productcouponassignment_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as productcouponassignment_updated_on - select - id as productcouponassignment_id - , bulk_assignment_id as bulkcouponassignment_id - , product_coupon_id as couponproduct_id - , redeemed as productcouponassignment_is_redeemed - , message_status as productcouponassignment_message_status - , email as productcouponassignment_email - , original_email as productcouponassignment_original_email - ,{{ cast_timestamp_to_iso8601('message_status_date') }} as productcouponassignment_message_status_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as productcouponassignment_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as productcouponassignment_updated_on + from source - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_productversion.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_productversion.sql index 53d25e2c9..3beb611bc 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_productversion.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_productversion.sql @@ -1,22 +1,23 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_productversion') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_productversion") }} -) + ), + renamed as ( -, renamed as ( + select + id as productversion_id, + text_id as productversion_readable_id, + cast(price as decimal(38, 2)) as productversion_price, + description as productversion_description, + product_id, + requires_enrollment_code as productversion_requires_enrollment_code, + {{ cast_timestamp_to_iso8601("updated_on") }} as productversion_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as productversion_created_on + from source - select - id as productversion_id - , text_id as productversion_readable_id - , cast(price as decimal(38, 2)) as productversion_price - , description as productversion_description - , product_id - , requires_enrollment_code as productversion_requires_enrollment_code - ,{{ cast_timestamp_to_iso8601('updated_on') }} as productversion_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as productversion_created_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_programrunline.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_programrunline.sql index 57a2c47a6..635894282 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_programrunline.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_programrunline.sql @@ -1,19 +1,20 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_programrunline') }} + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_programrunline") }} -) + ), + renamed as ( -, renamed as ( + select + id as programrunline_id, + line_id, + program_run_id as programrun_id, + {{ cast_timestamp_to_iso8601("created_on") }} as programrunline_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as programrunline_updated_on + from source - select - id as programrunline_id - , line_id - , program_run_id as programrun_id - ,{{ cast_timestamp_to_iso8601('created_on') }} as programrunline_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as programrunline_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_receipt.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_receipt.sql index fdaab95a1..043926e13 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_receipt.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__ecommerce_receipt.sql @@ -1,19 +1,16 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__ecommerce_receipt") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__ecommerce_receipt') }} + select + id as receipt_id, + order_id, + data as receipt_data, + {{ cast_timestamp_to_iso8601("created_on") }} as receipt_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as receipt_updated_on + from source -) + ) -, renamed as ( - - select - id as receipt_id - , order_id - , data as receipt_data - ,{{ cast_timestamp_to_iso8601('created_on') }} as receipt_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as receipt_updated_on - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__users_legaladdress.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__users_legaladdress.sql index 6ced29b9f..fdbb3d3ae 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__users_legaladdress.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__users_legaladdress.sql @@ -1,30 +1,28 @@ -- MITx Online User Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__users_legaladdress") }}), + cleaned as ( -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__app__postgres__users_legaladdress') }} -) + select + id as user_address_id, + country as user_address_country, + user_id, + first_name as user_first_name, + last_name as user_last_name, + city as user_address_city, + state_or_territory as user_address_state_or_territory, + postal_code as user_address_postal_code, + vat_id as user_vat_id, + concat_ws( + chr(10), + nullif(street_address_1, ''), + nullif(street_address_2, ''), + nullif(street_address_3, ''), + nullif(street_address_4, ''), + nullif(street_address_5, '') + ) as user_street_address + from source + ) -, cleaned as ( - - select - id as user_address_id - , country as user_address_country - , user_id - , first_name as user_first_name - , last_name as user_last_name - , city as user_address_city - , state_or_territory as user_address_state_or_territory - , postal_code as user_address_postal_code - , vat_id as user_vat_id - , concat_ws( - chr(10) - , nullif(street_address_1, '') - , nullif(street_address_2, '') - , nullif(street_address_3, '') - , nullif(street_address_4, '') - , nullif(street_address_5, '') - ) as user_street_address - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__users_profile.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__users_profile.sql index 0bc8c3560..494d1c498 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__users_profile.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__users_profile.sql @@ -1,24 +1,22 @@ -- xPro User Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__users_profile") }}), + cleaned as ( + select + id as user_profile_id, + birth_year as user_birth_year, + company as user_company, + job_title as user_job_title, + industry as user_industry, + job_function as user_job_function, + leadership_level as user_leadership_level, + user_id, + highest_education as user_highest_education, + {{ transform_gender_value("gender") }} as user_gender, + {{ transform_company_size_value("company_size") }} as user_company_size, + {{ transform_years_experience_value("years_experience") }} as user_years_experience + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__users_profile') }} -) - -, cleaned as ( - select - id as user_profile_id - , birth_year as user_birth_year - , company as user_company - , job_title as user_job_title - , industry as user_industry - , job_function as user_job_function - , leadership_level as user_leadership_level - , user_id - , highest_education as user_highest_education - ,{{ transform_gender_value('gender') }} as user_gender - ,{{ transform_company_size_value('company_size') }} as user_company_size - ,{{ transform_years_experience_value('years_experience') }} as user_years_experience - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__users_user.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__users_user.sql index eadc5124a..20c63dd88 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__users_user.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__users_user.sql @@ -1,19 +1,17 @@ -- xPro User Information +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__users_user") }}), + cleaned as ( + select + id as user_id, + username as user_username, + email as user_email, + is_active as user_is_active, + replace(replace(replace(name, ' ', '<>'), '><', ''), '<>', ' ') as user_full_name, + {{ cast_timestamp_to_iso8601("created_on") }} as user_joined_on, + {{ cast_timestamp_to_iso8601("last_login") }} as user_last_login + from source + ) -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__users_user') }} -) - -, cleaned as ( - select - id as user_id - , username as user_username - , email as user_email - , is_active as user_is_active - , replace(replace(replace(name, ' ', '<>'), '><', ''), '<>', ' ') as user_full_name - ,{{ cast_timestamp_to_iso8601('created_on') }} as user_joined_on - ,{{ cast_timestamp_to_iso8601('last_login') }} as user_last_login - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__wagtail_page.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__wagtail_page.sql index 42122a426..b3168ee39 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__wagtail_page.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__app__postgres__wagtail_page.sql @@ -1,29 +1,27 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__app__postgres__wagtailcore_page') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__app__postgres__wagtailcore_page") }}), + cleaned as ( + select + id as wagtail_page_id, + path as wagtail_page_path, + depth as wagtail_page_depth, + numchild as wagtail_page_num_children, + title as wagtail_page_title, + slug as wagtail_page_slug, + live as wagtail_page_is_live, + has_unpublished_changes as wagtail_page_has_unpublished_changes, + seo_title as wagtail_page_seo_title, + url_path as wagtail_page_url_path, + search_description as wagtail_page_search_description, + content_type_id as contenttype_id, + owner_id as owner_user_id, + {{ cast_timestamp_to_iso8601("latest_revision_created_at") }} as wagtail_page_latest_revision_created_on, + {{ cast_timestamp_to_iso8601("first_published_at") }} as wagtail_page_first_published_on, + live_revision_id as wagtail_page_live_pagerevision_id, + {{ cast_timestamp_to_iso8601("last_published_at") }} as wagtail_page_last_published_on -, cleaned as ( - select - id as wagtail_page_id - , path as wagtail_page_path - , depth as wagtail_page_depth - , numchild as wagtail_page_num_children - , title as wagtail_page_title - , slug as wagtail_page_slug - , live as wagtail_page_is_live - , has_unpublished_changes as wagtail_page_has_unpublished_changes - , seo_title as wagtail_page_seo_title + from source + ) - , url_path as wagtail_page_url_path - , search_description as wagtail_page_search_description - , content_type_id as contenttype_id - , owner_id as owner_user_id - ,{{ cast_timestamp_to_iso8601('latest_revision_created_at') }} as wagtail_page_latest_revision_created_on - ,{{ cast_timestamp_to_iso8601('first_published_at') }} as wagtail_page_first_published_on - , live_revision_id as wagtail_page_live_pagerevision_id - ,{{ cast_timestamp_to_iso8601('last_published_at') }} as wagtail_page_last_published_on - - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__api__course_structure.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__api__course_structure.sql index d2edece62..f832bfb1d 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__api__course_structure.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__api__course_structure.sql @@ -1,39 +1,37 @@ -with course_block_source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__api__course_blocks') }} -) - -, course_block as ( - select * from ( +with + course_block_source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__api__course_blocks") }} + ), + course_block as ( + select * + from + ( + select + course_block_source.*, + lag(course_block_source.course_content_hash) over ( + partition by course_block_source.block_id, course_block_source.course_id + order by course_block_source.retrieved_at asc + ) as previous_content_hash + from course_block_source + ) + where previous_content_hash is null or previous_content_hash != course_content_hash + ), + cleaned as ( select - course_block_source.* - , lag(course_block_source.course_content_hash) - over ( - partition by - course_block_source.block_id - , course_block_source.course_id - order by course_block_source.retrieved_at asc - ) - as previous_content_hash - from course_block_source + course_id as courserun_readable_id, + course_title as courserun_title, + block_index as coursestructure_block_index, + block_id as coursestructure_block_id, + block_parent as coursestructure_parent_block_id, + block_type as coursestructure_block_category, + block_title as coursestructure_block_title, + course_content_hash as coursestructure_content_hash, + block_content_hash as coursestructure_block_content_hash, + json_query(block_details, 'lax $.metadata') as coursestructure_block_metadata, + {{ cast_timestamp_to_iso8601("course_start") }} as courserun_start_on, + {{ cast_timestamp_to_iso8601("retrieved_at") }} as coursestructure_retrieved_at + from course_block ) - where previous_content_hash is null or previous_content_hash != course_content_hash -) - -, cleaned as ( - select - course_id as courserun_readable_id - , course_title as courserun_title - , block_index as coursestructure_block_index - , block_id as coursestructure_block_id - , block_parent as coursestructure_parent_block_id - , block_type as coursestructure_block_category - , block_title as coursestructure_block_title - , course_content_hash as coursestructure_content_hash - , block_content_hash as coursestructure_block_content_hash - , json_query(block_details, 'lax $.metadata') as coursestructure_block_metadata - , {{ cast_timestamp_to_iso8601('course_start') }} as courserun_start_on - , {{ cast_timestamp_to_iso8601('retrieved_at') }} as coursestructure_retrieved_at - from course_block -) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__blockcompletion.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__blockcompletion.sql index 9bdb1544a..2560f2f8c 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__blockcompletion.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__blockcompletion.sql @@ -1,37 +1,37 @@ -{{ config( - materialized='incremental', - unique_key = 'blockcompletion_id', - incremental_strategy='delete+insert', - views_enabled=false, - ) +{{ + config( + materialized="incremental", + unique_key="blockcompletion_id", + incremental_strategy="delete+insert", + views_enabled=false, + ) }} -with source as ( - select * from - {{ - source('ol_warehouse_raw_data' - ,'raw__xpro__openedx__mysql__completion_blockcompletion') - }} +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__completion_blockcompletion") }} - {% if is_incremental() %} - where created >= (select max(this.blockcompletion_created_on) from {{ this }} as this) - {% endif %} -) + {% if is_incremental() %} + where created >= (select max(this.blockcompletion_created_on) from {{ this }} as this) + {% endif %} + ) ---- this is needed for the initial dbt run to deduplicate the data from raw table -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( + -- - this is needed for the initial dbt run to deduplicate the data from raw table + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( - select - id as blockcompletion_id - , user_id - , created as blockcompletion_created_on - , modified as blockcompletion_updated_on - , block_key as block_fk - , block_type as block_category - , completion as block_completed - , course_key as courserun_readable_id - from most_recent_source -) + select + id as blockcompletion_id, + user_id, + created as blockcompletion_created_on, + modified as blockcompletion_updated_on, + block_key as block_fk, + block_type as block_category, + completion as block_completed, + course_key as courserun_readable_id + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__courseware_studentmodulehistoryextended.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__courseware_studentmodulehistoryextended.sql index 083b7c1b2..4b6bf2199 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__courseware_studentmodulehistoryextended.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__courseware_studentmodulehistoryextended.sql @@ -1,35 +1,41 @@ -{{ config( - materialized='incremental', - unique_key = 'studentmodulehistoryextended_id', - incremental_strategy='delete+insert', - views_enabled=false, - ) +{{ + config( + materialized="incremental", + unique_key="studentmodulehistoryextended_id", + incremental_strategy="delete+insert", + views_enabled=false, + ) }} -with source as ( - select * from - {{ - source('ol_warehouse_raw_data' - ,'raw__xpro__openedx__mysql__coursewarehistoryextended_studentmodulehistoryextended') - }} +with + source as ( + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__xpro__openedx__mysql__coursewarehistoryextended_studentmodulehistoryextended", + ) + }} - {% if is_incremental() %} - where created >= (select max(this.studentmodule_created_on) from {{ this }} as this) - {% endif %} -) + {% if is_incremental() %} + where created >= (select max(this.studentmodule_created_on) from {{ this }} as this) + {% endif %} + ) ---- this is needed for the initial dbt run to deduplicate the data from raw table -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( + -- - this is needed for the initial dbt run to deduplicate the data from raw table + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( - select - id as studentmodulehistoryextended_id - , student_module_id as studentmodule_id - , state as studentmodule_state_data - , grade as studentmodule_problem_grade - , max_grade as studentmodule_problem_max_grade - , created as studentmodule_created_on - from most_recent_source -) + select + id as studentmodulehistoryextended_id, + student_module_id as studentmodule_id, + state as studentmodule_state_data, + grade as studentmodule_problem_grade, + max_grade as studentmodule_problem_max_grade, + created as studentmodule_created_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__auth_user.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__auth_user.sql index 1f4133909..ced6f4b65 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__auth_user.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__auth_user.sql @@ -1,17 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__openedx__mysql__auth_user') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__auth_user") }}), + cleaned as ( -, cleaned as ( + select + id as openedx_user_id, + username as user_username, + email as user_email, + is_active as user_is_active, + is_staff as user_is_staff, + is_superuser as user_is_superuser + from source + ) - select - id as openedx_user_id - , username as user_username - , email as user_email - , is_active as user_is_active - , is_staff as user_is_staff - , is_superuser as user_is_superuser - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__courserun_enrollment.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__courserun_enrollment.sql index 813593768..14acc83ba 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__courserun_enrollment.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__courserun_enrollment.sql @@ -1,15 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__student_courseenrollment') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__student_courseenrollment") }} + ), + cleaned as ( + select + course_id as courserun_readable_id, + user_id as openedx_user_id, + is_active as courserunenrollment_is_active, + mode as courserunenrollment_enrollment_mode, + to_iso8601(created) as courserunenrollment_created_on + from source + ) -, cleaned as ( - select - course_id as courserun_readable_id - , user_id as openedx_user_id - , is_active as courserunenrollment_is_active - , mode as courserunenrollment_enrollment_mode - , to_iso8601(created) as courserunenrollment_created_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__courseware_studentmodule.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__courseware_studentmodule.sql index bac465f04..200d4fdeb 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__courseware_studentmodule.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__courseware_studentmodule.sql @@ -1,22 +1,24 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__openedx__mysql__courseware_studentmodule') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__courseware_studentmodule") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'course_id, module_id, student_id') }} -, cleaned as ( + {{ deduplicate_raw_table(order_by="modified", partition_columns="course_id, module_id, student_id") }}, + cleaned as ( - select - id as studentmodule_id - , course_id as courserun_readable_id - , module_id as coursestructure_block_id - , module_type as coursestructure_block_category - , student_id as openedx_user_id - , state as studentmodule_state_data - , grade as studentmodule_problem_grade - , max_grade as studentmodule_problem_max_grade - , to_iso8601(created) as studentmodule_created_on - , to_iso8601(modified) as studentmodule_updated_on - from most_recent_source -) + select + id as studentmodule_id, + course_id as courserun_readable_id, + module_id as coursestructure_block_id, + module_type as coursestructure_block_category, + student_id as openedx_user_id, + state as studentmodule_state_data, + grade as studentmodule_problem_grade, + max_grade as studentmodule_problem_max_grade, + to_iso8601(created) as studentmodule_created_on, + to_iso8601(modified) as studentmodule_updated_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__edxval_coursevideo.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__edxval_coursevideo.sql index 0731ffd58..a0d9d6bff 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__edxval_coursevideo.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__edxval_coursevideo.sql @@ -1,16 +1,11 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__openedx__mysql__edxval_coursevideo') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__edxval_coursevideo") }}), + cleaned as ( -, cleaned as ( + select id as coursevideo_id, course_id as courserun_readable_id, video_id, is_hidden as coursevideo_is_hidden - select - id as coursevideo_id - , course_id as courserun_readable_id - , video_id - , is_hidden as coursevideo_is_hidden + from source + ) - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__edxval_video.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__edxval_video.sql index 5e086db8b..c583520da 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__edxval_video.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__edxval_video.sql @@ -1,18 +1,17 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__xpro__openedx__mysql__edxval_video') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__edxval_video") }}), + cleaned as ( -, cleaned as ( + select + id as video_id, + edx_video_id as video_edx_uuid, + client_video_id as video_client_id, + status as video_status, + duration as video_duration, + {{ cast_timestamp_to_iso8601("created") }} as video_created_on - select - id as video_id - , edx_video_id as video_edx_uuid - , client_video_id as video_client_id - , status as video_status - , duration as video_duration - , {{ cast_timestamp_to_iso8601('created') }} as video_created_on + from source + ) - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__user_courseaccessrole.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__user_courseaccessrole.sql index 579af3322..0ee0caa0f 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__user_courseaccessrole.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__mysql__user_courseaccessrole.sql @@ -1,15 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__mysql__student_courseaccessrole') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__mysql__student_courseaccessrole") }} + ), + cleaned as ( + select + id as courseaccessrole_id, + course_id as courserun_readable_id, + user_id as openedx_user_id, + role as courseaccess_role, + org as organization + from source + ) -, cleaned as ( - select - id as courseaccessrole_id - , course_id as courserun_readable_id - , user_id as openedx_user_id - , role as courseaccess_role - , org as organization - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__tracking_logs__user_activity.sql b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__tracking_logs__user_activity.sql index 6eee4ccc4..d69c3be7d 100644 --- a/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__tracking_logs__user_activity.sql +++ b/src/ol_dbt/models/staging/mitxpro/stg__mitxpro__openedx__tracking_logs__user_activity.sql @@ -1,55 +1,69 @@ -- xPro user activities from tracking logs -- Due to size of the raw table, build this model as incremental and apply dedup on new rows -{{ config( - materialized='incremental', - unique_key = ['user_username', 'useractivity_context_object', 'useractivity_event_source', - 'useractivity_event_type', 'useractivity_event_object', 'useractivity_timestamp'], - incremental_strategy='delete+insert', - views_enabled=false, - ) +{{ + config( + materialized="incremental", + unique_key=[ + "user_username", + "useractivity_context_object", + "useractivity_event_source", + "useractivity_event_type", + "useractivity_event_object", + "useractivity_timestamp", + ], + incremental_strategy="delete+insert", + views_enabled=false, + ) }} -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__xpro__openedx__tracking_logs') }} - where - username != '' - and json_query(context, 'lax $.user_id' omit quotes) is not null - and json_query(event, 'lax $.exception' omit quotes) is null +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__xpro__openedx__tracking_logs") }} + where + username != '' + and json_query(context, 'lax $.user_id' omit quotes) is not null + and json_query(event, 'lax $.exception' omit quotes) is null - {% if is_incremental() %} - and "time" > (select max(this.useractivity_timestamp) from {{ this }} as this) --noqa - {% endif %} -) + {% if is_incremental() %} + and "time" > (select max(this.useractivity_timestamp) from {{ this }} as this) -- noqa + {% endif %} + ) -{{ deduplicate_raw_table( - order_by='_airbyte_extracted_at desc, _ab_source_file_last_modified desc, "time"' - , partition_columns = 'username, context, event_source, event_type, event, "time"' -) }} -, cleaned as ( - select - username as user_username - , context as useractivity_context_object - , event as useractivity_event_object - , event_source as useractivity_event_source - , page as useractivity_page_url - , session as useractivity_session_id - , ip as useractivity_ip - , host as useractivity_http_host - , agent as useractivity_http_user_agent - , accept_language as useractivity_http_accept_language - , referer as useractivity_http_referer - , name as useractivity_event_name - , event_type as useractivity_event_type - , {{ extract_course_id_from_tracking_log() }} as courserun_readable_id - --- extract common fields from context object - , cast(json_query(context, 'lax $.user_id' omit quotes) as integer) as openedx_user_id - , json_query(context, 'lax $.org_id' omit quotes) as org_id - , json_query(context, 'lax $.path' omit quotes) as useractivity_path - --- due to log collector changes, values of time field come with different formats - , to_iso8601(from_iso8601_timestamp_nanos( - regexp_replace(time, '(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}:\d{2}\.\d+)(.*?)', '$1T$2$3') -- noqa - )) as useractivity_timestamp - from most_recent_source -) + {{ + deduplicate_raw_table( + order_by='_airbyte_extracted_at desc, _ab_source_file_last_modified desc, "time"', + partition_columns='username, context, event_source, event_type, event, "time"', + ) + }}, + cleaned as ( + select + username as user_username, + context as useractivity_context_object, + event as useractivity_event_object, + event_source as useractivity_event_source, + page as useractivity_page_url, + session as useractivity_session_id, + ip as useractivity_ip, + host as useractivity_http_host, + agent as useractivity_http_user_agent, + accept_language as useractivity_http_accept_language, + referer as useractivity_http_referer, + name as useractivity_event_name, + event_type as useractivity_event_type, + {{ extract_course_id_from_tracking_log() }} as courserun_readable_id, + -- - extract common fields from context object + cast(json_query(context, 'lax $.user_id' omit quotes) as integer) as openedx_user_id, + json_query(context, 'lax $.org_id' omit quotes) as org_id, + json_query(context, 'lax $.path' omit quotes) as useractivity_path, + -- - due to log collector changes, values of time field come with different formats + to_iso8601( + from_iso8601_timestamp_nanos( + regexp_replace(time, '(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}:\d{2}\.\d+)(.*?)', '$1T$2$3') -- noqa + ) + ) as useractivity_timestamp + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__api__course_structure.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__api__course_structure.sql index 4630d40c4..8bdd950fd 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__api__course_structure.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__api__course_structure.sql @@ -1,39 +1,37 @@ -with course_block_source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__api__course_blocks') }} -) - -, course_block as ( - select * from ( +with + course_block_source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__api__course_blocks") }} + ), + course_block as ( + select * + from + ( + select + course_block_source.*, + lag(course_block_source.course_content_hash) over ( + partition by course_block_source.block_id, course_block_source.course_id + order by course_block_source.retrieved_at asc + ) as previous_content_hash + from course_block_source + ) + where previous_content_hash is null or previous_content_hash != course_content_hash + ), + cleaned as ( select - course_block_source.* - , lag(course_block_source.course_content_hash) - over ( - partition by - course_block_source.block_id - , course_block_source.course_id - order by course_block_source.retrieved_at asc - ) - as previous_content_hash - from course_block_source + course_id as courserun_readable_id, + course_title as courserun_title, + block_index as coursestructure_block_index, + block_id as coursestructure_block_id, + block_parent as coursestructure_parent_block_id, + block_type as coursestructure_block_category, + block_title as coursestructure_block_title, + course_content_hash as coursestructure_content_hash, + block_content_hash as coursestructure_block_content_hash, + json_query(block_details, 'lax $.metadata') as coursestructure_block_metadata, + {{ cast_timestamp_to_iso8601("course_start") }} as courserun_start_on, + {{ cast_timestamp_to_iso8601("retrieved_at") }} as coursestructure_retrieved_at + from course_block ) - where previous_content_hash is null or previous_content_hash != course_content_hash -) - -, cleaned as ( - select - course_id as courserun_readable_id - , course_title as courserun_title - , block_index as coursestructure_block_index - , block_id as coursestructure_block_id - , block_parent as coursestructure_parent_block_id - , block_type as coursestructure_block_category - , block_title as coursestructure_block_title - , course_content_hash as coursestructure_content_hash - , block_content_hash as coursestructure_block_content_hash - , json_query(block_details, 'lax $.metadata') as coursestructure_block_metadata - , {{ cast_timestamp_to_iso8601('course_start') }} as courserun_start_on - , {{ cast_timestamp_to_iso8601('retrieved_at') }} as coursestructure_retrieved_at - from course_block -) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__auth_user.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__auth_user.sql index cbb775f25..8813eb613 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__auth_user.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__auth_user.sql @@ -1,28 +1,24 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__auth_user') }} -) - -, cleaned as ( - select - id as user_id - , username as user_username - , email as user_email - , is_active as user_is_active - , is_staff as user_is_staff - , is_superuser as user_is_superuser - , nullif(first_name, '') as user_first_name - , nullif(last_name, '') as user_last_name - , replace( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__auth_user") }}), + cleaned as ( + select + id as user_id, + username as user_username, + email as user_email, + is_active as user_is_active, + is_staff as user_is_staff, + is_superuser as user_is_superuser, + nullif(first_name, '') as user_first_name, + nullif(last_name, '') as user_last_name, replace( - replace( - (concat(nullif(first_name, ''), ' ', nullif(last_name, ''))) - , ' ', '<>' - ), '><', '' - ), '<>', ' ' - ) as user_full_name - , to_iso8601(date_joined) as user_joined_on - , to_iso8601(last_login) as user_last_login - from source -) + replace(replace((concat(nullif(first_name, ''), ' ', nullif(last_name, ''))), ' ', '<>'), '><', ''), + '<>', + ' ' + ) as user_full_name, + to_iso8601(date_joined) as user_joined_on, + to_iso8601(last_login) as user_last_login + from source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__auth_userprofile.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__auth_userprofile.sql index fbb299634..54fe5ab28 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__auth_userprofile.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__auth_userprofile.sql @@ -1,18 +1,17 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__auth_userprofile') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__auth_userprofile") }}), + cleaned as ( + select + id as user_profile_id, + user_id, + replace(replace(replace(name, ' ', '<>'), '><', ''), '<>', ' ') as user_full_name, + year_of_birth as user_birth_year, + nullif(city, '') as user_address_city, + nullif(country, '') as user_address_country, + {{ transform_gender_value("gender") }} as user_gender, + {{ transform_education_value("level_of_education") }} as user_highest_education + from source + ) -, cleaned as ( - select - id as user_profile_id - , user_id - , replace(replace(replace(name, ' ', '<>'), '><', ''), '<>', ' ') as user_full_name - , year_of_birth as user_birth_year - , nullif(city, '') as user_address_city - , nullif(country, '') as user_address_country - , {{ transform_gender_value('gender') }} as user_gender - , {{ transform_education_value('level_of_education') }} as user_highest_education - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__blockcompletion.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__blockcompletion.sql index 2c91a05b6..e9dcff244 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__blockcompletion.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__blockcompletion.sql @@ -1,37 +1,37 @@ -{{ config( - materialized='incremental', - unique_key = 'blockcompletion_id', - incremental_strategy='delete+insert', - views_enabled=false, - ) +{{ + config( + materialized="incremental", + unique_key="blockcompletion_id", + incremental_strategy="delete+insert", + views_enabled=false, + ) }} -with source as ( - select * from - {{ - source('ol_warehouse_raw_data' - ,'raw__mitx__openedx__mysql__completion_blockcompletion') - }} +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__completion_blockcompletion") }} - {% if is_incremental() %} - where created >= (select max(this.blockcompletion_created_on) from {{ this }} as this) - {% endif %} -) + {% if is_incremental() %} + where created >= (select max(this.blockcompletion_created_on) from {{ this }} as this) + {% endif %} + ) ---- this is needed for the initial dbt run to deduplicate the data from raw table -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( + -- - this is needed for the initial dbt run to deduplicate the data from raw table + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( - select - id as blockcompletion_id - , user_id - , created as blockcompletion_created_on - , modified as blockcompletion_updated_on - , block_key as block_fk - , block_type as block_category - , completion as block_completed - , course_key as courserun_readable_id - from most_recent_source -) + select + id as blockcompletion_id, + user_id, + created as blockcompletion_created_on, + modified as blockcompletion_updated_on, + block_key as block_fk, + block_type as block_category, + completion as block_completed, + course_key as courserun_readable_id + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun.sql index 1e61092c0..bc355e683 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun.sql @@ -1,26 +1,28 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__course_overviews_courseoverview') }} -) +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__course_overviews_courseoverview") }} + ), + cleaned as ( + select + id as courserun_readable_id, + display_name as courserun_title, + org as courserun_org, + display_number_with_default as courserun_course_number, + course_video_url as courserun_video_url, + lowest_passing_grade as courserun_passing_grade, + self_paced as courserun_is_self_paced, + eligible_for_financial_aid as courserun_is_eligible_for_financial_aid, + {{ cast_timestamp_to_iso8601('"start"') }} as courserun_start_on, + {{ cast_timestamp_to_iso8601('"end"') }} as courserun_end_on, + {{ cast_timestamp_to_iso8601("enrollment_start") }} as courserun_enrollment_start_on, + {{ cast_timestamp_to_iso8601("enrollment_end") }} as courserun_enrollment_end_on, + {{ cast_timestamp_to_iso8601("announcement") }} as courserun_announce_on, + {{ cast_timestamp_to_iso8601("certificate_available_date") }} as courserun_certificate_available_on, + {{ cast_timestamp_to_iso8601("created") }} as courserun_created_on, + {{ cast_timestamp_to_iso8601("modified") }} as courserun_updated_on + from source + ) -, cleaned as ( - select - id as courserun_readable_id - , display_name as courserun_title - , org as courserun_org - , display_number_with_default as courserun_course_number - , course_video_url as courserun_video_url - , lowest_passing_grade as courserun_passing_grade - , self_paced as courserun_is_self_paced - , eligible_for_financial_aid as courserun_is_eligible_for_financial_aid - ,{{ cast_timestamp_to_iso8601('"start"') }} as courserun_start_on - ,{{ cast_timestamp_to_iso8601('"end"') }} as courserun_end_on - ,{{ cast_timestamp_to_iso8601('enrollment_start') }} as courserun_enrollment_start_on - ,{{ cast_timestamp_to_iso8601('enrollment_end') }} as courserun_enrollment_end_on - ,{{ cast_timestamp_to_iso8601('announcement') }} as courserun_announce_on - ,{{ cast_timestamp_to_iso8601('certificate_available_date') }} as courserun_certificate_available_on - ,{{ cast_timestamp_to_iso8601('created') }} as courserun_created_on - ,{{ cast_timestamp_to_iso8601('modified') }} as courserun_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun_enrollment.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun_enrollment.sql index f16de2f76..23697f406 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun_enrollment.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun_enrollment.sql @@ -1,16 +1,17 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__student_courseenrollment') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__student_courseenrollment") }} + ), + cleaned as ( + select + id as courserunenrollment_id, + course_id as courserun_readable_id, + user_id, + is_active as courserunenrollment_is_active, + mode as courserunenrollment_enrollment_mode, + to_iso8601(created) as courserunenrollment_created_on + from source + ) -, cleaned as ( - select - id as courserunenrollment_id - , course_id as courserun_readable_id - , user_id - , is_active as courserunenrollment_is_active - , mode as courserunenrollment_enrollment_mode - , to_iso8601(created) as courserunenrollment_created_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun_grade.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun_grade.sql index 87c802258..917e14003 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun_grade.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun_grade.sql @@ -1,18 +1,19 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__grades_persistentcoursegrade') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__grades_persistentcoursegrade") }} + ), + cleaned as ( + select + id as courserungrade_id, + course_id as courserun_readable_id, + user_id, + percent_grade as courserungrade_grade, + letter_grade as courserungrade_letter_grade, + {{ cast_timestamp_to_iso8601("passed_timestamp") }} as courserungrade_passed_on, + {{ cast_timestamp_to_iso8601("created") }} as courserungrade_created_on, + {{ cast_timestamp_to_iso8601("modified") }} as courserungrade_updated_on + from source + ) -, cleaned as ( - select - id as courserungrade_id - , course_id as courserun_readable_id - , user_id - , percent_grade as courserungrade_grade - , letter_grade as courserungrade_letter_grade - ,{{ cast_timestamp_to_iso8601('passed_timestamp') }} as courserungrade_passed_on - ,{{ cast_timestamp_to_iso8601('created') }} as courserungrade_created_on - ,{{ cast_timestamp_to_iso8601('modified') }} as courserungrade_updated_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courseware_studentmodule.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courseware_studentmodule.sql index 0f6205d45..f4db45ec7 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courseware_studentmodule.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courseware_studentmodule.sql @@ -1,22 +1,24 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitx__openedx__mysql__courseware_studentmodule') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__courseware_studentmodule") }} + ) -{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'course_id, student_id, module_id') }} -, cleaned as ( + {{ deduplicate_raw_table(order_by="modified", partition_columns="course_id, student_id, module_id") }}, + cleaned as ( - select - id as studentmodule_id - , course_id as courserun_readable_id - , module_id as coursestructure_block_id - , module_type as coursestructure_block_category - , student_id as user_id - , state as studentmodule_state_data - , grade as studentmodule_problem_grade - , max_grade as studentmodule_problem_max_grade - , to_iso8601(created) as studentmodule_created_on - , to_iso8601(modified) as studentmodule_updated_on - from most_recent_source -) + select + id as studentmodule_id, + course_id as courserun_readable_id, + module_id as coursestructure_block_id, + module_type as coursestructure_block_category, + student_id as user_id, + state as studentmodule_state_data, + grade as studentmodule_problem_grade, + max_grade as studentmodule_problem_max_grade, + to_iso8601(created) as studentmodule_created_on, + to_iso8601(modified) as studentmodule_updated_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courseware_studentmodulehistoryextended.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courseware_studentmodulehistoryextended.sql index e41fe21e0..033f39cbe 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courseware_studentmodulehistoryextended.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courseware_studentmodulehistoryextended.sql @@ -1,43 +1,46 @@ -{{ config( - materialized='incremental', - unique_key = 'studentmodulehistoryextended_id', - incremental_strategy='delete+insert', - views_enabled=false, - ) +{{ + config( + materialized="incremental", + unique_key="studentmodulehistoryextended_id", + incremental_strategy="delete+insert", + views_enabled=false, + ) }} -with source as ( - select * from - {{ - source('ol_warehouse_raw_data' - ,'raw__mitx__openedx__mysql__courseware_studentmodulehistory') - }} +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__courseware_studentmodulehistory") }} - union all + union all - select * from - {{ - source('ol_warehouse_raw_data' - ,'raw__mitx__openedx__mysql__coursewarehistoryextended_studentmodulehistoryextended') - }} + select * + from + {{ + source( + "ol_warehouse_raw_data", + "raw__mitx__openedx__mysql__coursewarehistoryextended_studentmodulehistoryextended", + ) + }} - {% if is_incremental() %} - where created >= (select max(this.studentmodule_created_on) from {{ this }} as this) - {% endif %} -) + {% if is_incremental() %} + where created >= (select max(this.studentmodule_created_on) from {{ this }} as this) + {% endif %} + ) ---- this is needed for the initial dbt run to deduplicate the data from raw table -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( + -- - this is needed for the initial dbt run to deduplicate the data from raw table + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( - select - id as studentmodulehistoryextended_id - , student_module_id as studentmodule_id - , state as studentmodule_state_data - , grade as studentmodule_problem_grade - , max_grade as studentmodule_problem_max_grade - , created as studentmodule_created_on - from most_recent_source -) + select + id as studentmodulehistoryextended_id, + student_module_id as studentmodule_id, + state as studentmodule_state_data, + grade as studentmodule_problem_grade, + max_grade as studentmodule_problem_max_grade, + created as studentmodule_created_on + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__edxval_coursevideo.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__edxval_coursevideo.sql index 868419207..71e3e94fd 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__edxval_coursevideo.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__edxval_coursevideo.sql @@ -1,16 +1,11 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitx__openedx__mysql__edxval_coursevideo') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__edxval_coursevideo") }}), + cleaned as ( -, cleaned as ( + select id as coursevideo_id, course_id as courserun_readable_id, video_id, is_hidden as coursevideo_is_hidden - select - id as coursevideo_id - , course_id as courserun_readable_id - , video_id - , is_hidden as coursevideo_is_hidden + from source + ) - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__edxval_video.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__edxval_video.sql index d448033ff..787c75193 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__edxval_video.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__edxval_video.sql @@ -1,17 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__mitx__openedx__mysql__edxval_video') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__edxval_video") }}), + cleaned as ( -, cleaned as ( + select + id as video_id, + edx_video_id as video_edx_uuid, + client_video_id as video_client_id, + status as video_status, + duration as video_duration, + to_iso8601(created) as video_created_on + from source + ) - select - id as video_id - , edx_video_id as video_edx_uuid - , client_video_id as video_client_id - , status as video_status - , duration as video_duration - , to_iso8601(created) as video_created_on - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__tracking_logs__user_activity.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__tracking_logs__user_activity.sql index 25c8680fd..95ed39b74 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__tracking_logs__user_activity.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__tracking_logs__user_activity.sql @@ -1,49 +1,62 @@ -{{ config( - materialized='incremental', - unique_key = ['user_username', 'useractivity_context_object', 'useractivity_event_source', - 'useractivity_event_type', 'useractivity_event_object', 'useractivity_timestamp'], - incremental_strategy='delete+insert', - views_enabled=false, - ) +{{ + config( + materialized="incremental", + unique_key=[ + "user_username", + "useractivity_context_object", + "useractivity_event_source", + "useractivity_event_type", + "useractivity_event_object", + "useractivity_timestamp", + ], + incremental_strategy="delete+insert", + views_enabled=false, + ) }} -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__tracking_logs') }} - where - username != '' - and json_query(context, 'lax $.user_id' omit quotes) is not null +with + source as ( + select * + from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__tracking_logs") }} + where + username != '' and json_query(context, 'lax $.user_id' omit quotes) is not null - {% if is_incremental() %} - and "time" > (select max(this.useractivity_timestamp) from {{ this }} as this) - {% endif %} -) + {% if is_incremental() %} + and "time" > (select max(this.useractivity_timestamp) from {{ this }} as this) + {% endif %} + ) -{{ deduplicate_raw_table( - order_by='_airbyte_extracted_at desc, _ab_source_file_last_modified desc, "time"' - , partition_columns = 'username, context, event_source, event_type, event, "time"' -) }} -, cleaned as ( - select - username as user_username - , context as useractivity_context_object - , event as useractivity_event_object - , event_source as useractivity_event_source - , page as useractivity_page_url - , session as useractivity_session_id - , ip as useractivity_ip - , host as useractivity_http_host - , agent as useractivity_http_user_agent - , accept_language as useractivity_http_accept_language - , referer as useractivity_http_referer - , coalesce(name, event_type) as useractivity_event_type - , {{ extract_course_id_from_tracking_log(course_id_has_old_format=false) }} as courserun_readable_id - , cast(json_query(context, 'lax $.user_id' omit quotes) as integer) as user_id - , json_query(context, 'lax $.org_id' omit quotes) as org_id - , json_query(context, 'lax $.path' omit quotes) as useractivity_path - , to_iso8601(from_iso8601_timestamp_nanos( - regexp_replace("time", '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2}\.\d+)(.*?)', '$1T$2$3') - )) as useractivity_timestamp - from most_recent_source -) + {{ + deduplicate_raw_table( + order_by='_airbyte_extracted_at desc, _ab_source_file_last_modified desc, "time"', + partition_columns='username, context, event_source, event_type, event, "time"', + ) + }}, + cleaned as ( + select + username as user_username, + context as useractivity_context_object, + event as useractivity_event_object, + event_source as useractivity_event_source, + page as useractivity_page_url, + session as useractivity_session_id, + ip as useractivity_ip, + host as useractivity_http_host, + agent as useractivity_http_user_agent, + accept_language as useractivity_http_accept_language, + referer as useractivity_http_referer, + coalesce(name, event_type) as useractivity_event_type, + {{ extract_course_id_from_tracking_log(course_id_has_old_format=false) }} as courserun_readable_id, + cast(json_query(context, 'lax $.user_id' omit quotes) as integer) as user_id, + json_query(context, 'lax $.org_id' omit quotes) as org_id, + json_query(context, 'lax $.path' omit quotes) as useractivity_path, + to_iso8601( + from_iso8601_timestamp_nanos( + regexp_replace("time", '(\d{4}-\d{2}-\d{2})[ ](\d{2}:\d{2}:\d{2}\.\d+)(.*?)', '$1T$2$3') + ) + ) as useractivity_timestamp + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__user_courseaccessrole.sql b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__user_courseaccessrole.sql index 46d87f4f9..eae37a79f 100644 --- a/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__user_courseaccessrole.sql +++ b/src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__user_courseaccessrole.sql @@ -1,15 +1,16 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__student_courseaccessrole') }} -) +with + source as ( + select * from {{ source("ol_warehouse_raw_data", "raw__mitx__openedx__mysql__student_courseaccessrole") }} + ), + cleaned as ( + select + id as courseaccessrole_id, + course_id as courserun_readable_id, + user_id, + role as courseaccess_role, + if(lower(org) = 'mitx', 'MITx', org) as organization + from source + ) -, cleaned as ( - select - id as courseaccessrole_id - , course_id as courserun_readable_id - , user_id - , role as courseaccess_role - , if(lower(org) = 'mitx', 'MITx', org) as organization - from source -) - -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/ocw/stg__ocw__studio__postgres__websites_website.sql b/src/ol_dbt/models/staging/ocw/stg__ocw__studio__postgres__websites_website.sql index 962f4995c..37d166227 100644 --- a/src/ol_dbt/models/staging/ocw/stg__ocw__studio__postgres__websites_website.sql +++ b/src/ol_dbt/models/staging/ocw/stg__ocw__studio__postgres__websites_website.sql @@ -1,44 +1,42 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__ocw__studio__postgres__websites_website") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__ocw__studio__postgres__websites_website') }} + select + uuid as website_uuid, + name as website_name, + title as website_title, + short_id as website_short_id, + owner_id as website_owner_user_id, + starter_id as websitestarter_id, + source as website_source, + metadata as website_metadata, + unpublish_status as website_unpublish_status, + live_publish_status as website_publish_status, + live_last_published_by_id as website_last_published_by_user_id, + last_unpublished_by_id as website_last_unpublished_by_user_id, + url_path as website_url_path, + if(publish_date is not null and unpublish_status is null, true, false) as website_is_live, + if(publish_date is not null and unpublish_status = 'succeeded', true, false) as website_is_unpublished, + if(publish_date is null, true, false) as website_has_never_published, + if( + publish_date is not null and unpublish_status is null, + concat('{{ var("ocw_production_url") }}', url_path), + null + ) as website_live_url, + {{ cast_timestamp_to_iso8601("first_published_to_production") }} as website_first_published_on, + {{ cast_timestamp_to_iso8601("publish_date") }} as website_publish_date_updated_on, + {{ cast_timestamp_to_iso8601("created_on") }} as website_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as website_updated_on, + nullif(json_query(metadata, 'lax $.primary_course_number' omit quotes), '') as primary_course_number, + nullif(json_query(metadata, 'lax $.term' omit quotes), '') as metadata_course_term, + nullif(json_query(metadata, 'lax $.course_title' omit quotes), '') as metadata_course_title, + nullif(json_query(metadata, 'lax $.year' omit quotes), '') as metadata_course_year + from source -) + ) -, renamed as ( - - select - uuid as website_uuid - , name as website_name - , title as website_title - , short_id as website_short_id - , owner_id as website_owner_user_id - , starter_id as websitestarter_id - , source as website_source - , metadata as website_metadata - , unpublish_status as website_unpublish_status - , live_publish_status as website_publish_status - , live_last_published_by_id as website_last_published_by_user_id - , last_unpublished_by_id as website_last_unpublished_by_user_id - , url_path as website_url_path - , if(publish_date is not null and unpublish_status is null, true, false) as website_is_live - , if(publish_date is not null and unpublish_status = 'succeeded', true, false) as website_is_unpublished - , if(publish_date is null, true, false) as website_has_never_published - , if( - publish_date is not null and unpublish_status is null - , concat('{{ var("ocw_production_url") }}', url_path), null - ) as website_live_url - ,{{ cast_timestamp_to_iso8601('first_published_to_production') }} as website_first_published_on - ,{{ cast_timestamp_to_iso8601('publish_date') }} as website_publish_date_updated_on - ,{{ cast_timestamp_to_iso8601('created_on') }} as website_created_on - ,{{ cast_timestamp_to_iso8601('updated_on') }} as website_updated_on - , nullif(json_query(metadata, 'lax $.primary_course_number' omit quotes), '') as primary_course_number - , nullif(json_query(metadata, 'lax $.term' omit quotes), '') as metadata_course_term - , nullif(json_query(metadata, 'lax $.course_title' omit quotes), '') as metadata_course_title - , nullif(json_query(metadata, 'lax $.year' omit quotes), '') as metadata_course_year - from source - -) - -select * from renamed ---exclude test course from the data +select * +from renamed +-- exclude test course from the data where website_name != 'ocw-ci-test-course' diff --git a/src/ol_dbt/models/staging/ocw/stg__ocw__studio__postgres__websites_websitecontent.sql b/src/ol_dbt/models/staging/ocw/stg__ocw__studio__postgres__websites_websitecontent.sql index a5d91f115..631924003 100644 --- a/src/ol_dbt/models/staging/ocw/stg__ocw__studio__postgres__websites_websitecontent.sql +++ b/src/ol_dbt/models/staging/ocw/stg__ocw__studio__postgres__websites_websitecontent.sql @@ -1,69 +1,72 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__ocw__studio__postgres__websites_websitecontent') }} + select * from {{ source("ol_warehouse_raw_data", "raw__ocw__studio__postgres__websites_websitecontent") }} -) + ), + renamed as ( -, renamed as ( + select + id as websitecontent_id, + website_id as website_uuid, + title as websitecontent_title, + text_id as websitecontent_text_id, + type as websitecontent_type, + is_page_content as websitecontent_is_page, + filename as websitecontent_filename, + dirpath as websitecontent_dirpath, + file as websitecontent_file, + parent_id as websitecontent_parent_id, + owner_id as websitecontent_owner_user_id, + updated_by_id as websitecontent_updated_by_user_id, + metadata as websitecontent_metadata, + -- extract website metadata from sitemetadata for courses + json_query(metadata, 'lax $.course_description' omit quotes) as course_description, + nullif(json_query(metadata, 'lax $.course_title' omit quotes), '') as course_title, + nullif(json_query(metadata, 'lax $.term' omit quotes), '') as course_term, + nullif(json_query(metadata, 'lax $.year' omit quotes), '') as course_year, + -- convert to comma-separated list to be consistent with extra_course_numbers + array_join( + cast(json_parse(json_query(metadata, 'lax $.level')) as array(varchar)), ', ' -- noqa + ) as course_level, + array_join( + cast( + json_parse( + case + when json_query(metadata, 'lax $.learning_resource_types') = '[]' + then null + else nullif(json_query(metadata, 'lax $.learning_resource_types'), '') + end + ) as array(varchar) -- noqa + ), + ', ' + ) as learning_resource_types, + array_join( + cast(json_parse(json_query(metadata, 'lax $.department_numbers')) as array(varchar)), ', ' -- noqa + ) as course_department_numbers, + nullif(json_query(metadata, 'lax $.primary_course_number' omit quotes), '') as course_primary_course_number, + json_query(metadata, 'lax $.extra_course_numbers' omit quotes) as course_extra_course_numbers, + json_query(metadata, 'lax $.instructors.content') as course_instructor_uuids, + json_query(metadata, 'lax $.topics') as course_topics, + json_query(metadata, 'lax $.department_numbers') as course_department_numbers_json, + {{ cast_timestamp_to_iso8601("deleted") }} as websitecontent_deleted_on, + {{ cast_timestamp_to_iso8601("created_on") }} as websitecontent_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as websitecontent_updated_on, + -- miscellaneous metadata + json_query(metadata, 'lax $.body' omit quotes) as metadata_body, + json_query(metadata, 'lax $.description' omit quotes) as metadata_description, + cast(nullif(json_query(metadata, 'lax $.draft' omit quotes), '') as boolean) as metadata_draft, + json_query(metadata, 'lax $.file' omit quotes) as metadata_file, + json_query(metadata, 'lax $.file_size' omit quotes) as metadata_file_size, + json_query(metadata, 'lax $.license' omit quotes) as metadata_license, + json_query(metadata, 'lax $.ocw_type' omit quotes) as metadata_legacy_type, + json_query(metadata, 'lax $.resourcetype' omit quotes) as metadata_resource_type, + json_query(metadata, 'lax $.title' omit quotes) as metadata_title, + json_query(metadata, 'lax $.uid' omit quotes) as metadata_uid - select - id as websitecontent_id - , website_id as website_uuid - , title as websitecontent_title - , text_id as websitecontent_text_id - , type as websitecontent_type - , is_page_content as websitecontent_is_page - , filename as websitecontent_filename - , dirpath as websitecontent_dirpath - , file as websitecontent_file - , parent_id as websitecontent_parent_id - , owner_id as websitecontent_owner_user_id - , updated_by_id as websitecontent_updated_by_user_id - , metadata as websitecontent_metadata - -- extract website metadata from sitemetadata for courses - , json_query(metadata, 'lax $.course_description' omit quotes) as course_description - , nullif(json_query(metadata, 'lax $.course_title' omit quotes), '') as course_title - , nullif(json_query(metadata, 'lax $.term' omit quotes), '') as course_term - , nullif(json_query(metadata, 'lax $.year' omit quotes), '') as course_year - -- convert to comma-separated list to be consistent with extra_course_numbers - , array_join( - cast(json_parse(json_query(metadata, 'lax $.level')) as array (varchar)), ', ' --noqa - ) as course_level - , array_join( - cast( - json_parse( - case - when json_query(metadata, 'lax $.learning_resource_types') = '[]' then null - else nullif(json_query(metadata, 'lax $.learning_resource_types'), '') - end - ) as array(varchar) --noqa - ), ', ' - ) as learning_resource_types - , array_join( - cast(json_parse(json_query(metadata, 'lax $.department_numbers')) as array (varchar)), ', ' --noqa - ) as course_department_numbers - , nullif(json_query(metadata, 'lax $.primary_course_number' omit quotes), '') as course_primary_course_number - , json_query(metadata, 'lax $.extra_course_numbers' omit quotes) as course_extra_course_numbers - , json_query(metadata, 'lax $.instructors.content') as course_instructor_uuids - , json_query(metadata, 'lax $.topics') as course_topics - , json_query(metadata, 'lax $.department_numbers') as course_department_numbers_json - , {{ cast_timestamp_to_iso8601('deleted') }} as websitecontent_deleted_on - , {{ cast_timestamp_to_iso8601('created_on') }} as websitecontent_created_on - , {{ cast_timestamp_to_iso8601('updated_on') }} as websitecontent_updated_on - -- miscellaneous metadata - , json_query(metadata, 'lax $.body' omit quotes) as metadata_body - , json_query(metadata, 'lax $.description' omit quotes) as metadata_description - , cast(nullif(json_query(metadata, 'lax $.draft' omit quotes), '') as boolean) as metadata_draft - , json_query(metadata, 'lax $.file' omit quotes) as metadata_file - , json_query(metadata, 'lax $.file_size' omit quotes) as metadata_file_size - , json_query(metadata, 'lax $.license' omit quotes) as metadata_license - , json_query(metadata, 'lax $.ocw_type' omit quotes) as metadata_legacy_type - , json_query(metadata, 'lax $.resourcetype' omit quotes) as metadata_resource_type - , json_query(metadata, 'lax $.title' omit quotes) as metadata_title - , json_query(metadata, 'lax $.uid' omit quotes) as metadata_uid + from source - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/ocw/stg__ocw__studio__postgres__websites_websitestarter.sql b/src/ol_dbt/models/staging/ocw/stg__ocw__studio__postgres__websites_websitestarter.sql index d31b1655a..e004d6e69 100644 --- a/src/ol_dbt/models/staging/ocw/stg__ocw__studio__postgres__websites_websitestarter.sql +++ b/src/ol_dbt/models/staging/ocw/stg__ocw__studio__postgres__websites_websitestarter.sql @@ -1,22 +1,23 @@ -with source as ( +with + source as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__ocw__studio__postgres__websites_websitestarter') }} + select * from {{ source("ol_warehouse_raw_data", "raw__ocw__studio__postgres__websites_websitestarter") }} -) + ), + renamed as ( -, renamed as ( + select + id as websitestarter_id, + name as websitestarter_name, + slug as websitestarter_slug, + source as websitestarter_source, + path as websitestarter_path, + status as websitestarter_status, + {{ cast_timestamp_to_iso8601("created_on") }} as websitestarter_created_on, + {{ cast_timestamp_to_iso8601("updated_on") }} as websitestarter_updated_on + from source - select - id as websitestarter_id - , name as websitestarter_name - , slug as websitestarter_slug - , source as websitestarter_source - , path as websitestarter_path - , status as websitestarter_status - , {{ cast_timestamp_to_iso8601('created_on') }} as websitestarter_created_on - , {{ cast_timestamp_to_iso8601('updated_on') }} as websitestarter_updated_on - from source + ) -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_collection.sql b/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_collection.sql index b831df699..fe71b26d0 100644 --- a/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_collection.sql +++ b/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_collection.sql @@ -1,27 +1,24 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__ovs__postgres__ui_collection") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__ovs__postgres__ui_collection') }} + select + id as collection_id, + key as collection_uuid, + title as collection_title, + slug as collection_slug, + description as collection_description, + edx_course_id as courserun_readable_id, + owner_id as collection_user_id, + is_logged_in_only as collection_is_logged_in_only, + allow_share_openedx as collection_is_allowed_to_share, + stream_source as collection_stream_source, + {{ cast_timestamp_to_iso8601("created_at") }} as collection_created_on, + {{ cast_timestamp_to_iso8601("updated_at") }} as collection_updated_on -) + from source -, renamed as ( + ) - select - id as collection_id - , key as collection_uuid - , title as collection_title - , slug as collection_slug - , description as collection_description - , edx_course_id as courserun_readable_id - , owner_id as collection_user_id - , is_logged_in_only as collection_is_logged_in_only - , allow_share_openedx as collection_is_allowed_to_share - , stream_source as collection_stream_source - ,{{ cast_timestamp_to_iso8601('created_at') }} as collection_created_on - ,{{ cast_timestamp_to_iso8601('updated_at') }} as collection_updated_on - - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_collectionedxendpoint.sql b/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_collectionedxendpoint.sql index c6960797e..4148f1aa0 100644 --- a/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_collectionedxendpoint.sql +++ b/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_collectionedxendpoint.sql @@ -1,17 +1,6 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__ovs__postgres__ui_collectionedxendpoint") }}), + renamed as (select id as collectionedxendpoint_id, collection_id, edx_endpoint_id as edxendpoint_id from source) - select * from {{ source('ol_warehouse_raw_data', 'raw__ovs__postgres__ui_collectionedxendpoint') }} - -) - -, renamed as ( - - select - id as collectionedxendpoint_id - , collection_id - , edx_endpoint_id as edxendpoint_id - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_edxendpoint.sql b/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_edxendpoint.sql index 30d969d5e..e826c3a4b 100644 --- a/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_edxendpoint.sql +++ b/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_edxendpoint.sql @@ -1,20 +1,17 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__ovs__postgres__ui_edxendpoint") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__ovs__postgres__ui_edxendpoint') }} + select + id as edxendpoint_id, + name as edxendpoint_name, + base_url as edxendpoint_base_url, + edx_video_api_path as edxendpoint_edx_video_api_path, + {{ cast_timestamp_to_iso8601("created_at") }} as edxendpoint_created_on, + {{ cast_timestamp_to_iso8601("updated_at") }} as edxendpoint_updated_on + from source -) + ) -, renamed as ( - - select - id as edxendpoint_id - , name as edxendpoint_name - , base_url as edxendpoint_base_url - , edx_video_api_path as edxendpoint_edx_video_api_path - ,{{ cast_timestamp_to_iso8601('created_at') }} as edxendpoint_created_on - ,{{ cast_timestamp_to_iso8601('updated_at') }} as edxendpoint_updated_on - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_encodejob.sql b/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_encodejob.sql index c7a068f48..97c59a79f 100644 --- a/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_encodejob.sql +++ b/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_encodejob.sql @@ -1,63 +1,66 @@ -with source as ( +with + source as ( + select + id, + object_id, + content_type_id, + message, + state, + created_at, + last_modified, + cast( + json_query(replace(message, '''', '"'), 'lax $.Output.Duration' omit quotes) as decimal(38, 4) + ) as video_duration + from {{ source("ol_warehouse_raw_data", "raw__ovs__postgres__dj_elastictranscoder_encodejob") }} - select - id - , object_id - , content_type_id - , message - , state - , created_at - , last_modified - , cast(json_query(replace(message, '''', '"'), 'lax $.Output.Duration' omit quotes) as decimal(38, 4)) - as video_duration - from {{ source('ol_warehouse_raw_data', 'raw__ovs__postgres__dj_elastictranscoder_encodejob') }} + union all - union all + select + id, + object_id, + content_type_id, + message, + state, + created_at, + last_modified, + cast( + json_extract( + cast( + json_extract( + cast(json_extract(json_parse(message), '$.outputGroupDetails[0]') as json), + '$.outputDetails[0]' + ) as json + ), + '$.durationInMs' + ) as decimal(38, 4) + ) + / 1000.0 as video_duration + from {{ source("ol_warehouse_raw_data", "raw__ovs__postgres__ui_encodejob") }} - select - id - , object_id - , content_type_id - , message - , state - , created_at - , last_modified - , cast( - json_extract( - cast( - json_extract( - cast( - json_extract(json_parse(message), '$.outputGroupDetails[0]') - as json - ), '$.outputDetails[0]' - ) as json - ), '$.durationInMs' - ) as decimal(38, 4) - ) / 1000.0 as video_duration - from {{ source('ol_warehouse_raw_data', 'raw__ovs__postgres__ui_encodejob') }} + ), + renamed as ( -) - -, renamed as ( - - select - id as encodejob_id - , object_id as video_id - , content_type_id as contenttype_id - , video_duration - , replace(message, '''', '"') as encodejob_message - , case - when state = 0 then 'Submitted' - when state = 1 then 'Progressing' - when state = 2 then 'Error' - when state = 3 then 'Warning' - when state = 4 then 'Complete' - end as encodejob_state - ,{{ cast_timestamp_to_iso8601('created_at') }} as encodejob_created_on - ,{{ cast_timestamp_to_iso8601('last_modified') }} as encodejob_updated_on - - from source - -) - -select * from renamed + select + id as encodejob_id, + object_id as video_id, + content_type_id as contenttype_id, + video_duration, + replace(message, '''', '"') as encodejob_message, + case + when state = 0 + then 'Submitted' + when state = 1 + then 'Progressing' + when state = 2 + then 'Error' + when state = 3 + then 'Warning' + when state = 4 + then 'Complete' + end as encodejob_state, + {{ cast_timestamp_to_iso8601("created_at") }} as encodejob_created_on, + {{ cast_timestamp_to_iso8601("last_modified") }} as encodejob_updated_on + from source + ) +select * +from renamed diff --git a/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_video.sql b/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_video.sql index 6c2d71613..e705b8be5 100644 --- a/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_video.sql +++ b/src/ol_dbt/models/staging/ovs/stg__ovs__studio__postgres__ui_video.sql @@ -1,28 +1,25 @@ -with source as ( +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__ovs__postgres__ui_video") }}), + renamed as ( - select * from {{ source('ol_warehouse_raw_data', 'raw__ovs__postgres__ui_video') }} + select + id as video_id, + key as video_uuid, + title as video_title, + description as video_description, + collection_id, + source_url as video_source_url, + status as video_status, + multiangle as video_is_multiangle, + is_logged_in_only as video_is_logged_in_only, + is_public as video_is_public, + is_private as video_is_private, + {{ cast_timestamp_to_iso8601("created_at") }} as video_created_on, + {{ cast_timestamp_to_iso8601("updated_at") }} as video_updated_on -) + from source -, renamed as ( + ) - select - id as video_id - , key as video_uuid - , title as video_title - , description as video_description - , collection_id - , source_url as video_source_url - , status as video_status - , multiangle as video_is_multiangle - , is_logged_in_only as video_is_logged_in_only - , is_public as video_is_public - , is_private as video_is_private - ,{{ cast_timestamp_to_iso8601('created_at') }} as video_created_on - ,{{ cast_timestamp_to_iso8601('updated_at') }} as video_updated_on - - from source - -) - -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/salesforce/stg__salesforce__opportunity.sql b/src/ol_dbt/models/staging/salesforce/stg__salesforce__opportunity.sql index 5c2003968..dc760cd6c 100644 --- a/src/ol_dbt/models/staging/salesforce/stg__salesforce__opportunity.sql +++ b/src/ol_dbt/models/staging/salesforce/stg__salesforce__opportunity.sql @@ -1,40 +1,42 @@ -with source as ( +with + source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__thirdparty__salesforce___destination_v2__Opportunity') }} - where isdeleted = false + select * + from {{ source("ol_warehouse_raw_data", "raw__thirdparty__salesforce___destination_v2__Opportunity") }} + where isdeleted = false -) + ) -{{ deduplicate_raw_table(order_by='systemmodstamp', partition_columns = 'id') }} -, renamed as ( + {{ deduplicate_raw_table(order_by="systemmodstamp", partition_columns="id") }}, + renamed as ( - select - id as opportunity_id - , name as opportunity_name - , type as opportunity_type - , business_type__c as opportunity_business_type - , agreement_type__c as opportunity_agreement_type - , mit_is_sales_lead__c as opportunity_mit_is_sales_lead - , revenue_share_or_commission_contract__c as opportunity_revenue_share_or_commission_contract - , revenue_share_or_commission_amount__c as opportunity_revenue_share_or_commission_amount - , stagename as opportunity_stage - , amount as opportunity_amount - , probability as opportunity_probability - , iswon as opportunity_is_won - , isclosed as opportunity_is_closed - , enrollment_codes_redeemed__c as opportunity_num_enrollment_codes_redeemed - , of_seats_purchased__c as opportunity_num_of_seats_purchased - , price_per_seat__c as opportunity_price_per_seat - , leadsource as opportunity_leadsource - , nextstep as opportunity_nextstep - , hasopportunitylineitem as opportunity_has_lineitem - , data_quality_score__c as opportunity_data_quality_score - , data_quality_description__c as opportunity_data_quality_description - ,{{ cast_date_to_iso8601('closedate') }} as opportunity_close_date - ,{{ cast_timestamp_to_iso8601('createddate') }} as opportunity_created_on - ,{{ cast_timestamp_to_iso8601('systemmodstamp') }} as opportunity_modified_on - from most_recent_source -) + select + id as opportunity_id, + name as opportunity_name, + type as opportunity_type, + business_type__c as opportunity_business_type, + agreement_type__c as opportunity_agreement_type, + mit_is_sales_lead__c as opportunity_mit_is_sales_lead, + revenue_share_or_commission_contract__c as opportunity_revenue_share_or_commission_contract, + revenue_share_or_commission_amount__c as opportunity_revenue_share_or_commission_amount, + stagename as opportunity_stage, + amount as opportunity_amount, + probability as opportunity_probability, + iswon as opportunity_is_won, + isclosed as opportunity_is_closed, + enrollment_codes_redeemed__c as opportunity_num_enrollment_codes_redeemed, + of_seats_purchased__c as opportunity_num_of_seats_purchased, + price_per_seat__c as opportunity_price_per_seat, + leadsource as opportunity_leadsource, + nextstep as opportunity_nextstep, + hasopportunitylineitem as opportunity_has_lineitem, + data_quality_score__c as opportunity_data_quality_score, + data_quality_description__c as opportunity_data_quality_description, + {{ cast_date_to_iso8601("closedate") }} as opportunity_close_date, + {{ cast_timestamp_to_iso8601("createddate") }} as opportunity_created_on, + {{ cast_timestamp_to_iso8601("systemmodstamp") }} as opportunity_modified_on + from most_recent_source + ) -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/salesforce/stg__salesforce__opportunitylineitem.sql b/src/ol_dbt/models/staging/salesforce/stg__salesforce__opportunitylineitem.sql index 1db9fc3db..37af81155 100644 --- a/src/ol_dbt/models/staging/salesforce/stg__salesforce__opportunitylineitem.sql +++ b/src/ol_dbt/models/staging/salesforce/stg__salesforce__opportunitylineitem.sql @@ -1,29 +1,31 @@ -with source as ( +with + source as ( - select * - from {{ source('ol_warehouse_raw_data', 'raw__thirdparty__salesforce___destination_v2__OpportunityLineItem') }} - where isdeleted = false + select * + from {{ source("ol_warehouse_raw_data", "raw__thirdparty__salesforce___destination_v2__OpportunityLineItem") }} + where isdeleted = false -) + ) -{{ deduplicate_raw_table(order_by='systemmodstamp', partition_columns = 'id') }} -, renamed as ( + {{ deduplicate_raw_table(order_by="systemmodstamp", partition_columns="id") }}, + renamed as ( - select - id as opportunitylineitem_id - , opportunityid as opportunity_id - , name as opportunitylineitem_product_name - , description as opportunitylineitem_description - , productcode as opportunitylineitem_product_code - , listprice as opportunitylineitem_list_price - , unitprice as opportunitylineitem_sales_price - , discount as opportunitylineitem_discount_percent - , quantity as opportunitylineitem_quantity - , totalprice as opportunitylineitem_total_price - ,{{ cast_date_to_iso8601('servicedate') }} as opportunitylineitem_service_date - ,{{ cast_timestamp_to_iso8601('createddate') }} as opportunitylineitem_created_on - ,{{ cast_timestamp_to_iso8601('systemmodstamp') }} as opportunitylineitem_modified_on - from most_recent_source -) + select + id as opportunitylineitem_id, + opportunityid as opportunity_id, + name as opportunitylineitem_product_name, + description as opportunitylineitem_description, + productcode as opportunitylineitem_product_code, + listprice as opportunitylineitem_list_price, + unitprice as opportunitylineitem_sales_price, + discount as opportunitylineitem_discount_percent, + quantity as opportunitylineitem_quantity, + totalprice as opportunitylineitem_total_price, + {{ cast_date_to_iso8601("servicedate") }} as opportunitylineitem_service_date, + {{ cast_timestamp_to_iso8601("createddate") }} as opportunitylineitem_created_on, + {{ cast_timestamp_to_iso8601("systemmodstamp") }} as opportunitylineitem_modified_on + from most_recent_source + ) -select * from renamed +select * +from renamed diff --git a/src/ol_dbt/models/staging/zendesk/stg__zendesk__brand.sql b/src/ol_dbt/models/staging/zendesk/stg__zendesk__brand.sql index e4cf57854..b969cb0e2 100644 --- a/src/ol_dbt/models/staging/zendesk/stg__zendesk__brand.sql +++ b/src/ol_dbt/models/staging/zendesk/stg__zendesk__brand.sql @@ -1,23 +1,23 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__thirdparty__zendesk_support__brands') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__thirdparty__zendesk_support__brands") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - select - id as brand_id - , url as brand_api_url - , brand_url - , name as brand_name - , active as brand_is_active - , default as brand_is_default - , subdomain as brand_subdomain - , is_deleted as brand_is_deleted - , has_help_center as brand_has_help_center - , help_center_state as brand_help_center_state - , {{ cast_timestamp_to_iso8601('created_at') }} as brand_created_at - , {{ cast_timestamp_to_iso8601('updated_at') }} as brand_updated_at - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + select + id as brand_id, + url as brand_api_url, + brand_url, + name as brand_name, + active as brand_is_active, + default as brand_is_default, + subdomain as brand_subdomain, + is_deleted as brand_is_deleted, + has_help_center as brand_has_help_center, + help_center_state as brand_help_center_state, + {{ cast_timestamp_to_iso8601("created_at") }} as brand_created_at, + {{ cast_timestamp_to_iso8601("updated_at") }} as brand_updated_at + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/zendesk/stg__zendesk__group.sql b/src/ol_dbt/models/staging/zendesk/stg__zendesk__group.sql index fe49a56e5..4769aa753 100644 --- a/src/ol_dbt/models/staging/zendesk/stg__zendesk__group.sql +++ b/src/ol_dbt/models/staging/zendesk/stg__zendesk__group.sql @@ -1,20 +1,20 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__thirdparty__zendesk_support__groups') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__thirdparty__zendesk_support__groups") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - select - id as group_id - , url as group_api_url - , name as group_name - , description as group_description - , default as group_is_default - , deleted as group_is_deleted - , is_public as group_is_public - , {{ cast_timestamp_to_iso8601('created_at') }} as group_created_at - , {{ cast_timestamp_to_iso8601('updated_at') }} as group_updated_at - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + select + id as group_id, + url as group_api_url, + name as group_name, + description as group_description, + default as group_is_default, + deleted as group_is_deleted, + is_public as group_is_public, + {{ cast_timestamp_to_iso8601("created_at") }} as group_created_at, + {{ cast_timestamp_to_iso8601("updated_at") }} as group_updated_at + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/zendesk/stg__zendesk__organization.sql b/src/ol_dbt/models/staging/zendesk/stg__zendesk__organization.sql index ed92c0e63..3c78502b6 100644 --- a/src/ol_dbt/models/staging/zendesk/stg__zendesk__organization.sql +++ b/src/ol_dbt/models/staging/zendesk/stg__zendesk__organization.sql @@ -1,23 +1,23 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__thirdparty__zendesk_support__organizations') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__thirdparty__zendesk_support__organizations") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - select - id as organization_id - , url as organization_api_url - , name as organization_name - , nullif(tags, array[]) as organization_tags - , nullif(notes, '') as organization_notes - , group_id - , nullif(domain_names, array[]) as organization_domain_names - , shared_tickets as organization_has_shared_tickets - , shared_comments as organization_has_shared_comments - , {{ cast_timestamp_to_iso8601('deleted_at') }} as organization_deleted_at - , {{ cast_timestamp_to_iso8601('created_at') }} as organization_created_at - , {{ cast_timestamp_to_iso8601('updated_at') }} as organization_updated_at - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + select + id as organization_id, + url as organization_api_url, + name as organization_name, + nullif(tags, array[]) as organization_tags, + nullif(notes, '') as organization_notes, + group_id, + nullif(domain_names, array[]) as organization_domain_names, + shared_tickets as organization_has_shared_tickets, + shared_comments as organization_has_shared_comments, + {{ cast_timestamp_to_iso8601("deleted_at") }} as organization_deleted_at, + {{ cast_timestamp_to_iso8601("created_at") }} as organization_created_at, + {{ cast_timestamp_to_iso8601("updated_at") }} as organization_updated_at + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/zendesk/stg__zendesk__ticket.sql b/src/ol_dbt/models/staging/zendesk/stg__zendesk__ticket.sql index e1b341658..cab879dbc 100644 --- a/src/ol_dbt/models/staging/zendesk/stg__zendesk__ticket.sql +++ b/src/ol_dbt/models/staging/zendesk/stg__zendesk__ticket.sql @@ -1,56 +1,59 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__thirdparty__zendesk_support__tickets') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__thirdparty__zendesk_support__tickets") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - select - id as ticket_id - , via as ticket_via_object - , url as ticket_api_url - , type as ticket_type - , status as ticket_status - , description as ticket_description - , subject as ticket_subject - , raw_subject as ticket_raw_subject - , brand_id - , group_id - , priority as ticket_priority - , is_public as ticket_is_public - , recipient as ticket_recipient_email - , problem_id - , requester_id as ticket_requester_user_id - , submitter_id as ticket_submitter_user_id - , assignee_id as ticket_assignee_user_id - , ticket_form_id - , organization_id - , custom_status_id - , allow_attachments as ticket_allow_attachments - , allow_channelback as ticket_allow_channelback - , has_incidents as ticket_has_incidents - , satisfaction_rating as ticket_satisfaction_rating_object - , from_messaging_channel as ticket_is_from_messaging_channel - , due_at as ticket_due_at - , custom_fields as ticket_custom_fields - , generated_timestamp as ticket_unix_timestamp - , nullif(tags, array[]) as ticket_tags - , nullif(email_cc_ids, array[]) as ticket_email_cc_user_ids - , nullif(follower_ids, array[]) as ticket_follower_user_ids - , nullif(collaborator_ids, array[]) as ticket_collaborator_user_ids - , nullif(sharing_agreement_ids, array[]) as sharing_agreement_ids - , nullif(json_query(satisfaction_rating, 'lax $.score' omit quotes), 'null') - as ticket_satisfaction_rating_score - , nullif(json_query(satisfaction_rating, 'lax $.comment' omit quotes), 'null') - as ticket_satisfaction_rating_comment - , nullif(json_query(satisfaction_rating, 'lax $.reason' omit quotes), 'null') - as ticket_satisfaction_rating_reason - , nullif(json_query(via, 'lax $.channel' omit quotes), 'null') as ticket_source_channel - , nullif(json_query(via, 'lax $.source.from.address' omit quotes), 'null') as ticket_source_email - , nullif(json_query(via, 'lax $.source.from.ticket_id' omit quotes), 'null') as ticket_source_ticket_id - , nullif(json_query(via, 'lax $.source.rel' omit quotes), 'null') as ticket_source_rel - , {{ cast_timestamp_to_iso8601('created_at') }} as ticket_created_at - , {{ cast_timestamp_to_iso8601('updated_at') }} as ticket_updated_at - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + select + id as ticket_id, + via as ticket_via_object, + url as ticket_api_url, + type as ticket_type, + status as ticket_status, + description as ticket_description, + subject as ticket_subject, + raw_subject as ticket_raw_subject, + brand_id, + group_id, + priority as ticket_priority, + is_public as ticket_is_public, + recipient as ticket_recipient_email, + problem_id, + requester_id as ticket_requester_user_id, + submitter_id as ticket_submitter_user_id, + assignee_id as ticket_assignee_user_id, + ticket_form_id, + organization_id, + custom_status_id, + allow_attachments as ticket_allow_attachments, + allow_channelback as ticket_allow_channelback, + has_incidents as ticket_has_incidents, + satisfaction_rating as ticket_satisfaction_rating_object, + from_messaging_channel as ticket_is_from_messaging_channel, + due_at as ticket_due_at, + custom_fields as ticket_custom_fields, + generated_timestamp as ticket_unix_timestamp, + nullif(tags, array[]) as ticket_tags, + nullif(email_cc_ids, array[]) as ticket_email_cc_user_ids, + nullif(follower_ids, array[]) as ticket_follower_user_ids, + nullif(collaborator_ids, array[]) as ticket_collaborator_user_ids, + nullif(sharing_agreement_ids, array[]) as sharing_agreement_ids, + nullif( + json_query(satisfaction_rating, 'lax $.score' omit quotes), 'null' + ) as ticket_satisfaction_rating_score, + nullif( + json_query(satisfaction_rating, 'lax $.comment' omit quotes), 'null' + ) as ticket_satisfaction_rating_comment, + nullif( + json_query(satisfaction_rating, 'lax $.reason' omit quotes), 'null' + ) as ticket_satisfaction_rating_reason, + nullif(json_query(via, 'lax $.channel' omit quotes), 'null') as ticket_source_channel, + nullif(json_query(via, 'lax $.source.from.address' omit quotes), 'null') as ticket_source_email, + nullif(json_query(via, 'lax $.source.from.ticket_id' omit quotes), 'null') as ticket_source_ticket_id, + nullif(json_query(via, 'lax $.source.rel' omit quotes), 'null') as ticket_source_rel, + {{ cast_timestamp_to_iso8601("created_at") }} as ticket_created_at, + {{ cast_timestamp_to_iso8601("updated_at") }} as ticket_updated_at + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/zendesk/stg__zendesk__ticket_comment.sql b/src/ol_dbt/models/staging/zendesk/stg__zendesk__ticket_comment.sql index a82e65a14..74e6b98c8 100644 --- a/src/ol_dbt/models/staging/zendesk/stg__zendesk__ticket_comment.sql +++ b/src/ol_dbt/models/staging/zendesk/stg__zendesk__ticket_comment.sql @@ -1,28 +1,28 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__thirdparty__zendesk_support__ticket_comments') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__thirdparty__zendesk_support__ticket_comments") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - select - id as comment_id - , ticket_id - , via as comment_via_object - , type as comment_type - , public as comment_is_public - , uploads as comment_uploads - , author_id as comment_author_user_id - , audit_id - , body as comment_body_string - , html_body as comment_html_body - , plain_body as comment_plain_body - , "timestamp" as comment_unix_timestamp - , nullif(attachments, array[]) as comment_attachments - , nullif(json_query(via, 'lax $.channel' omit quotes), 'null') as comment_source_channel - , nullif(json_query(via, 'lax $.source.from.address' omit quotes), 'null') as comment_source_email - , nullif(json_query(via, 'lax $.source.rel' omit quotes), 'null') as comment_source_rel - , {{ cast_timestamp_to_iso8601('created_at') }} as comment_created_at - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + select + id as comment_id, + ticket_id, + via as comment_via_object, + type as comment_type, + public as comment_is_public, + uploads as comment_uploads, + author_id as comment_author_user_id, + audit_id, + body as comment_body_string, + html_body as comment_html_body, + plain_body as comment_plain_body, + "timestamp" as comment_unix_timestamp, + nullif(attachments, array[]) as comment_attachments, + nullif(json_query(via, 'lax $.channel' omit quotes), 'null') as comment_source_channel, + nullif(json_query(via, 'lax $.source.from.address' omit quotes), 'null') as comment_source_email, + nullif(json_query(via, 'lax $.source.rel' omit quotes), 'null') as comment_source_rel, + {{ cast_timestamp_to_iso8601("created_at") }} as comment_created_at + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/zendesk/stg__zendesk__ticket_field.sql b/src/ol_dbt/models/staging/zendesk/stg__zendesk__ticket_field.sql index 899374330..9deb5a255 100644 --- a/src/ol_dbt/models/staging/zendesk/stg__zendesk__ticket_field.sql +++ b/src/ol_dbt/models/staging/zendesk/stg__zendesk__ticket_field.sql @@ -1,18 +1,18 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__thirdparty__zendesk_support__ticket_fields') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__thirdparty__zendesk_support__ticket_fields") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - select - id as field_id - , url as field_api_url - , description as field_description - , title as field_title - , active as field_is_active - , {{ cast_timestamp_to_iso8601('created_at') }} as field_created_at - , {{ cast_timestamp_to_iso8601('updated_at') }} as field_updated_at - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + select + id as field_id, + url as field_api_url, + description as field_description, + title as field_title, + active as field_is_active, + {{ cast_timestamp_to_iso8601("created_at") }} as field_created_at, + {{ cast_timestamp_to_iso8601("updated_at") }} as field_updated_at + from most_recent_source + ) -select * from cleaned +select * +from cleaned diff --git a/src/ol_dbt/models/staging/zendesk/stg__zendesk__user.sql b/src/ol_dbt/models/staging/zendesk/stg__zendesk__user.sql index d2eab4ba6..2f6ea839d 100644 --- a/src/ol_dbt/models/staging/zendesk/stg__zendesk__user.sql +++ b/src/ol_dbt/models/staging/zendesk/stg__zendesk__user.sql @@ -1,52 +1,56 @@ -with source as ( - select * from {{ source('ol_warehouse_raw_data','raw__thirdparty__zendesk_support__users') }} -) +with + source as (select * from {{ source("ol_warehouse_raw_data", "raw__thirdparty__zendesk_support__users") }}) -{{ deduplicate_raw_table(order_by='_airbyte_extracted_at' , partition_columns = 'id') }} -, cleaned as ( - select - id as user_id - , url as user_api_url - , name as user_name - , alias as user_alias - , email as user_email - , phone as user_phone_number - , photo as user_photo_object - , active as user_is_active - , permanently_deleted as user_is_permanently_deleted - , suspended as user_is_suspended - , shared as user_is_shared - , verified as user_is_verified - , details as user_details - , notes as user_notes - , chat_only as user_is_chat_only - , moderator as user_is_moderator - , restricted_agent as user_is_restricted_agent - , signature as user_signature - , locale as user_locale - , locale_id as user_locale_id - , time_zone as user_time_zone - , iana_time_zone as user_iana_time_zone - , user_fields - , shared_agent as user_is_shared_agent - , organization_id - , default_group_id as user_default_group_id - , ticket_restriction as user_ticket_restriction - , only_private_comments as user_only_private_comments - , two_factor_auth_enabled as user_has_two_factor_auth_enabled - , nullif(tags, array[]) as user_tags - , role as user_role - , case - when role_type = 0 then 'custom agent' - when role_type = 1 then 'light agent' - when role_type = 3 then 'chat agent' - when role_type = 4 then 'admin' - else cast(role_type as varchar) - end as user_role_type - , {{ cast_timestamp_to_iso8601('last_login_at') }} as user_last_login_at - , {{ cast_timestamp_to_iso8601('created_at') }} as user_created_at - , {{ cast_timestamp_to_iso8601('updated_at') }} as user_updated_at - from most_recent_source -) + {{ deduplicate_raw_table(order_by="_airbyte_extracted_at", partition_columns="id") }}, + cleaned as ( + select + id as user_id, + url as user_api_url, + name as user_name, + alias as user_alias, + email as user_email, + phone as user_phone_number, + photo as user_photo_object, + active as user_is_active, + permanently_deleted as user_is_permanently_deleted, + suspended as user_is_suspended, + shared as user_is_shared, + verified as user_is_verified, + details as user_details, + notes as user_notes, + chat_only as user_is_chat_only, + moderator as user_is_moderator, + restricted_agent as user_is_restricted_agent, + signature as user_signature, + locale as user_locale, + locale_id as user_locale_id, + time_zone as user_time_zone, + iana_time_zone as user_iana_time_zone, + user_fields, + shared_agent as user_is_shared_agent, + organization_id, + default_group_id as user_default_group_id, + ticket_restriction as user_ticket_restriction, + only_private_comments as user_only_private_comments, + two_factor_auth_enabled as user_has_two_factor_auth_enabled, + nullif(tags, array[]) as user_tags, + role as user_role, + case + when role_type = 0 + then 'custom agent' + when role_type = 1 + then 'light agent' + when role_type = 3 + then 'chat agent' + when role_type = 4 + then 'admin' + else cast(role_type as varchar) + end as user_role_type, + {{ cast_timestamp_to_iso8601("last_login_at") }} as user_last_login_at, + {{ cast_timestamp_to_iso8601("created_at") }} as user_created_at, + {{ cast_timestamp_to_iso8601("updated_at") }} as user_updated_at + from most_recent_source + ) -select * from cleaned +select * +from cleaned