From 36f6469f70fada5dc5433edfe09f77b25555d731 Mon Sep 17 00:00:00 2001 From: Jim Laney Date: Tue, 4 Feb 2025 14:23:41 -0800 Subject: [PATCH 1/2] add weekly job backfill_courses --- docker/prod-values.yml | 13 +++++++++- .../management/commands/backfill_courses.py | 25 ++++++++----------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/docker/prod-values.yml b/docker/prod-values.yml index cff6e80a..d863d6ed 100644 --- a/docker/prod-values.yml +++ b/docker/prod-values.yml @@ -172,7 +172,7 @@ cronjob: cpu: 25m memory: 64Mi - name: import-users - schedule: "15,45 12-23 * * 1-6" + schedule: "3,15,27,39,51 12-23 * * 1-6" command: ["/scripts/management_command.sh"] args: ["import_users", "2"] resources: @@ -215,6 +215,17 @@ cronjob: requests: cpu: 25m memory: 128Mi + - name: backfill-courses + schedule: "30 22 * * 3" + command: ["/scripts/management_command.sh"] + args: ["backfill_courses", "--commit"] + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 50m + memory: 256Mi - name: import-groups schedule: "*/15 12-23 * * 1-6" command: ["/scripts/management_command.sh"] diff --git a/sis_provisioner/management/commands/backfill_courses.py b/sis_provisioner/management/commands/backfill_courses.py index 98db1af2..d5a1638c 100644 --- a/sis_provisioner/management/commands/backfill_courses.py +++ b/sis_provisioner/management/commands/backfill_courses.py @@ -28,7 +28,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): term_sis_id = options.get('term-sis-id') commit = options.get('commit') - logger.info(f'Term: {term_sis_id}, Commit: {commit}') + logger.debug(f'Term: {term_sis_id}, Commit: {commit}') report_data = get_course_report_data(term_sis_id) header = report_data.pop(0) @@ -74,8 +74,8 @@ def handle(self, *args, **options): needs_save = True except Course.MultipleObjectsReturned: - logger.info(f'ERROR Multiple courses for {canvas_course_id}, ' - f'{course_sis_id}') + logger.error(f'ERROR Multiple courses for {canvas_course_id}, ' + f'{course_sis_id}') continue except Course.DoesNotExist: @@ -99,19 +99,14 @@ def handle(self, *args, **options): course.expiration_date = course.default_expiration_date needs_save = True except DataFailureException as err: - logger.info(f'ERROR {canvas_course_id} ' - f'{course_sis_id}, {err}') + logger.error(f'ERROR {canvas_course_id} ' + f'{course_sis_id}, {err}') continue - # Logic for first round of expirations - if course.expiration_date.year == 2023: - course.expiration_date = course.expiration_date.replace( - month=12, day=18) - if needs_save and commit: course.save() - logger.info(f'BACKFILL Canvas ID: {course.canvas_course_id}, ' - f'SIS ID: {course.course_id}, ' - f'Term ID: {course.term_id}, ' - f'Created: {course.created_date}, ' - f'Expires: {course.expiration_date}') + logger.debug(f'BACKFILL Canvas ID: {course.canvas_course_id}, ' + f'SIS ID: {course.course_id}, ' + f'Term ID: {course.term_id}, ' + f'Created: {course.created_date}, ' + f'Expires: {course.expiration_date}') From c4dcbc5c2aa71ae6f38911ff0b65faa953428631 Mon Sep 17 00:00:00 2001 From: Jim Laney Date: Tue, 4 Feb 2025 15:38:03 -0800 Subject: [PATCH 2/2] handle duplicate course records --- .../management/commands/backfill_courses.py | 5 ----- sis_provisioner/models/course.py | 17 +++++++++++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/sis_provisioner/management/commands/backfill_courses.py b/sis_provisioner/management/commands/backfill_courses.py index d5a1638c..31faad14 100644 --- a/sis_provisioner/management/commands/backfill_courses.py +++ b/sis_provisioner/management/commands/backfill_courses.py @@ -73,11 +73,6 @@ def handle(self, *args, **options): course.deleted_date = None needs_save = True - except Course.MultipleObjectsReturned: - logger.error(f'ERROR Multiple courses for {canvas_course_id}, ' - f'{course_sis_id}') - continue - except Course.DoesNotExist: course = Course(course_id=course_sis_id, canvas_course_id=canvas_course_id, diff --git a/sis_provisioner/models/course.py b/sis_provisioner/models/course.py index 0b2b2087..c5cc7ac1 100644 --- a/sis_provisioner/models/course.py +++ b/sis_provisioner/models/course.py @@ -12,7 +12,7 @@ from sis_provisioner.models.term import Term from sis_provisioner.dao.course import ( valid_canvas_course_id, valid_course_sis_id, valid_canvas_section, - get_new_sections_by_term) + valid_academic_course_sis_id, get_new_sections_by_term) from sis_provisioner.dao.canvas import create_course, delete_course from sis_provisioner.dao.term import get_current_active_term from sis_provisioner.exceptions import ( @@ -37,7 +37,17 @@ def find_course(self, canvas_course_id, sis_course_id): if len(courses) == 1: return courses[0] elif len(courses) > 1: - raise Course.MultipleObjectsReturned() + sis_course = None + for course in courses: + try: + valid_academic_course_sis_id(course.course_id) + sis_course = course + except CoursePolicyException: + course.delete() + if sis_course is not None: + return sis_course + else: + raise Course.DoesNotExist() else: raise Course.DoesNotExist() except CoursePolicyException: @@ -49,8 +59,7 @@ def find_course(self, canvas_course_id, sis_course_id): valid_course_sis_id(sis_course_id) return Course.objects.get(course_id=sis_course_id) except CoursePolicyException: - pass - raise Course.DoesNotExist() + raise Course.DoesNotExist() def create_user_course(self, sis_user_id, name, account_id=None, sis_term_id=None):