diff --git a/compass/admin.py b/compass/admin.py index 9c61c425..bb19924c 100644 --- a/compass/admin.py +++ b/compass/admin.py @@ -4,7 +4,8 @@ from compass.models import ( AppUser, Student, AccessGroup, Affiliation, Contact, - ContactType, ContactMethod, ContactTopic, EligibilityType) + ContactType, ContactMethod, ContactTopic, EligibilityType, + OMADContactQueue) from compass.dao.group import is_admin_user from django.contrib import admin from django.http import HttpResponseRedirect @@ -75,5 +76,6 @@ def _session_data(self, obj): admin_site.register(ContactMethod, SAMLAdminModel) admin_site.register(ContactTopic, SAMLAdminModel) admin_site.register(Contact, SAMLAdminModel) +admin_site.register(OMADContactQueue, SAMLAdminModel) admin_site.register(TokenProxy, SAMLAdminModel) admin_site.register(Session, SessionAdminModel) diff --git a/compass/dao/contact.py b/compass/dao/contact.py new file mode 100644 index 00000000..abf089ec --- /dev/null +++ b/compass/dao/contact.py @@ -0,0 +1,70 @@ +# Copyright 2024 UW-IT, University of Washington +# SPDX-License-Identifier: Apache-2.0 + +from compass.models import ContactType, AccessGroup, AppUser +from dateutil import parser + + +def validate_contact_post_data(contact_dict): + access_group = AccessGroup.objects.by_name("OMAD") + # check that adviser netid is defined + validate_adviser_netid(contact_dict.get("adviser_netid")) + # check that system key is defined + validate_student_systemkey( + contact_dict.get("student_systemkey")) + # parse checkin date to ensure it is in the correct format + contact_dict["checkin_date"] = parse_checkin_date_str( + contact_dict.get("checkin_date")) + # verify that the specified contact type exists in OMAD + contact_dict["contact_type"] = parse_contact_type_str( + contact_dict.get("contact_type"), access_group) + + # if the adviser is a member of the omad group and the contact record + # was successfully parsed, create an app-user and a student record for + # them if one doesn't already exist + app_user = AppUser.objects.upsert_appuser( + contact_dict["adviser_netid"]) + + +def validate_adviser_netid(adviser_netid): + if adviser_netid is None: + raise ValueError("Missing adviser netid") + + +def validate_student_systemkey(student_systemkey): + if student_systemkey is None: + raise ValueError("Missing student systemkey") + + try: + if not student_systemkey.isdigit(): + raise ValueError("Student systemkey is not a positive integer") + except AttributeError as e: + raise ValueError(f"Invalid student systemkey: {e}") + + +def pad_student_systemkey(student_systemkey): + return student_systemkey.zfill(9) + + +def parse_checkin_date_str(checkin_date_str): + # parse checkin date + if checkin_date_str is None: + raise ValueError("Check-in date not specified") + else: + try: + dt = parser.parse(checkin_date_str) + if dt.tzinfo is None: + raise ValueError("Invalid check-in date, missing timezone") + return dt + except parser.ParserError as e: + raise ValueError(f"Invalid check-in date: {e}") + + +def parse_contact_type_str(contact_type_str, access_group): + try: + return ContactType.objects.get(access_group=access_group, + slug=contact_type_str) + except ContactType.DoesNotExist: + raise ValueError( + f"Contact type '{contact_type_str}' does not exist " + f"for the {access_group.name} access group.") diff --git a/compass/management/commands/process_omad_contacts.py b/compass/management/commands/process_omad_contacts.py new file mode 100644 index 00000000..e6a2f629 --- /dev/null +++ b/compass/management/commands/process_omad_contacts.py @@ -0,0 +1,75 @@ +# Copyright 2024 UW-IT, University of Washington +# SPDX-License-Identifier: Apache-2.0 + +from django.core.management.base import BaseCommand +from compass.models import OMADContactQueue +from datetime import datetime +import json +from compass.models import ( + AccessGroup, AppUser, Contact, Student) +from logging import getLogger +from compass.dao.contact import (validate_contact_post_data, + pad_student_systemkey) +import traceback +logger = getLogger(__name__) + + +class Command(BaseCommand): + help = "process OMAD contacts" + + def add_arguments(self, parser): + parser.add_argument('--reprocess', + action='store_true', + help="reprocess all contacts regardless of status") + + def handle(self, *args, **options): + if options['reprocess']: + contacts = OMADContactQueue.objects.all() + else: + contacts = OMADContactQueue.objects.filter(processing_attempts=0) + + for contact in contacts: + try: + self.process_contact(contact) + except Exception as e: + logger.exception(f"Error processing contact {contact.id}") + contact.processing_attempts += 1 + contact.process_attempted_date = datetime.now() + contact.processing_error = repr(e) + contact.stack_trace = traceback.format_exc() + contact.save() + continue + contact.delete() + + @staticmethod + def process_contact(contact): + contact_dict = json.loads(contact.json) + validate_contact_post_data(contact_dict) + + # Get additional objects + access_group = AccessGroup.objects.by_name("OMAD") + app_user = AppUser.objects.upsert_appuser( + contact_dict["adviser_netid"]) + + # Parse/format data + student_systemkey = pad_student_systemkey( + contact_dict["student_systemkey"]) + + student, _ = Student.objects.get_or_create( + system_key=student_systemkey) + + # create the new contact record + contact = Contact() + contact.app_user = app_user + contact.student = student + contact.contact_type = contact_dict["contact_type"] + contact.checkin_date = contact_dict["checkin_date"] + contact.source = "Checkin" + try: + contact.trans_id = contact_dict["trans_id"] + except KeyError: + pass + contact.save() + contact.access_group.add(access_group) + logger.info(f"Checkin contact {contact.contact_type} processed for " + f"student {student.system_key}") diff --git a/compass/migrations/0014_omadcontactqueue.py b/compass/migrations/0014_omadcontactqueue.py new file mode 100644 index 00000000..4ccfb08d --- /dev/null +++ b/compass/migrations/0014_omadcontactqueue.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.9 on 2024-01-09 05:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('compass', '0013_contact_trans_id_historicalcontact_trans_id'), + ] + + operations = [ + migrations.CreateModel( + name='OMADContactQueue', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('json', models.TextField()), + ('created', models.DateTimeField(auto_now_add=True)), + ('process_attempted_date', models.DateTimeField(null=True)), + ('processing_error', models.TextField(null=True)), + ('stack_trace', models.TextField(null=True)), + ('processing_attempts', models.IntegerField(default=0)), + ], + ), + ] diff --git a/compass/models.py b/compass/models.py index 7fc08d0d..2a9332ee 100644 --- a/compass/models.py +++ b/compass/models.py @@ -427,3 +427,22 @@ class VisitType(BaseAccessGroupContent): name = models.CharField(unique=True, max_length=50) slug = models.SlugField(unique=True, max_length=50) editable = models.BooleanField(default=False) + + +class OMADContactQueue(models.Model): + """ + Stores raw contacts from the OMAD check-in system for processing into + Contacts. Successfully created contacts will be removed from this table. + """ + + json = models.TextField() + created = models.DateTimeField(auto_now_add=True) + process_attempted_date = models.DateTimeField(null=True) + processing_error = models.TextField(null=True) + stack_trace = models.TextField(null=True) + processing_attempts = models.IntegerField(default=0) + + def __str__(self): + return f"#{self.id} created: {self.created} " \ + f"attempts: {self.processing_attempts} " \ + f"err: {self.processing_error}" diff --git a/compass/templates/omad_contact_admin.html b/compass/templates/omad_contact_admin.html new file mode 100644 index 00000000..766d8509 --- /dev/null +++ b/compass/templates/omad_contact_admin.html @@ -0,0 +1,73 @@ +{% extends 'supporttools/base.html' %} +{% load static %} + +{% block content %} + +

OMAD Contact Processing Queue

+ + + + + + + + + + + + + + + + + + {% for contact in contacts %} + + + + + + + + + {% endfor %} +
IDCreatedProcessing attemptsLast attemptedLast errorContact JSON
{{ contact.id }}{{ contact.created }}{{ contact.processing_attempts }}{{ contact.process_attempted_date }}{{ contact.processing_error }} + {{ contact.json }} +
+ Show traceback + 
{{ contact.stack_trace }}
+
+ +
+{% endblock content %} diff --git a/compass/templates/supporttools/custom_sidebar_links.html b/compass/templates/supporttools/custom_sidebar_links.html new file mode 100644 index 00000000..7482f9cf --- /dev/null +++ b/compass/templates/supporttools/custom_sidebar_links.html @@ -0,0 +1,6 @@ +{% include 'supporttools/default_sidelinks.html' %} + +

Compass Tools

+ diff --git a/compass/tests/commands/test_process_omad_contacts.py b/compass/tests/commands/test_process_omad_contacts.py new file mode 100644 index 00000000..0aae7dea --- /dev/null +++ b/compass/tests/commands/test_process_omad_contacts.py @@ -0,0 +1,60 @@ +# Copyright 2024 UW-IT, University of Washington +# SPDX-License-Identifier: Apache-2.0 + + +from django.core.management import call_command +from django.test import TestCase +from compass.models import OMADContactQueue, Contact, AccessGroup +import json +from rest_framework.authtoken.models import Token +from django.contrib.auth.models import User + + +class TestOMADContactProcessing(TestCase): + API_TOKEN = None + + def setUp(self): + super(TestOMADContactProcessing, self).setUp() + AccessGroup(name="OMAD", access_group_id="u_astra_group1").save() + user = User.objects.create_user(username='testuser', password='12345') + token = Token.objects.create(user=user) + self.API_TOKEN = token.key + + def test_success(self): + # Create a contact queue entry + test_body = { + "adviser_netid": "javerage", + "student_systemkey": "001234567", + "contact_type": "appointment", + "checkin_date": "2012-01-19 17:21:00 PDT", + "source": "Compass", + "trans_id": 1234567890 + } + OMADContactQueue.objects.create(json=json.dumps(test_body)) + call_command('process_omad_contacts') + self.assertEqual(OMADContactQueue.objects.count(), 0) + self.assertEqual(Contact.objects.count(), 1) + + def test_failure(self): + test_body = { + "adviser_netid": "javerage123", + "student_systemkey": "001234567", + "contact_type": "appointment", + "checkin_date": "2012-01-19 17:21:00 PDT", + "source": "Compass", + "trans_id": 1234567890 + } + OMADContactQueue.objects.create(json=json.dumps(test_body)) + call_command('process_omad_contacts') + self.assertEqual(OMADContactQueue.objects.count(), 1) + self.assertEqual(Contact.objects.count(), 0) + # don't reprocess without flag + call_command('process_omad_contacts') + contact = OMADContactQueue.objects.all()[0] + self.assertEqual(contact.processing_attempts, 1) + self.assertEqual(contact.processing_error, "PersonNotFoundException()") + + # reprocess with flag + call_command('process_omad_contacts', reprocess=True) + contact = OMADContactQueue.objects.all()[0] + self.assertEqual(contact.processing_attempts, 2) diff --git a/compass/tests/dao/test_contact.py b/compass/tests/dao/test_contact.py new file mode 100644 index 00000000..e0927993 --- /dev/null +++ b/compass/tests/dao/test_contact.py @@ -0,0 +1,39 @@ +# Copyright 2024 UW-IT, University of Washington +# SPDX-License-Identifier: Apache-2.0 + + +from django.test import TestCase +from datetime import datetime +from compass.dao.contact import ( + parse_checkin_date_str, validate_adviser_netid, validate_student_systemkey) + + +class ContactDaoTest(TestCase): + def test_parse_checkin_date_str(self): + # no checkin date specified + with self.assertRaises(ValueError): + parse_checkin_date_str(None) + # bad checkin format + with self.assertRaises(ValueError): + parse_checkin_date_str("2022-09-T::") + # Missing TZ info + with self.assertRaises(ValueError): + parse_checkin_date_str("2022-09-19T06:15:04") + # correct checkin format + checkin_date = parse_checkin_date_str( + "2022-09-19T06:15:04Z") + self.assertEqual(type(checkin_date), datetime) + + def test_validate_adviser_netid(self): + with self.assertRaises(ValueError): + validate_adviser_netid(None) + validate_adviser_netid("foo") + + def test_validate_student_systemkey(self): + with self.assertRaises(ValueError): + validate_student_systemkey(None) + with self.assertRaises(ValueError): + validate_student_systemkey("badsyskey") + with self.assertRaises(ValueError): + validate_student_systemkey(1234) + validate_student_systemkey("1234") diff --git a/compass/tests/views/api/test_contact.py b/compass/tests/views/api/test_contact.py index fcaab501..dfa367cd 100644 --- a/compass/tests/views/api/test_contact.py +++ b/compass/tests/views/api/test_contact.py @@ -2,13 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 -from datetime import datetime from django.test import Client from django.contrib.auth.models import User from unittest.mock import MagicMock, patch from compass.views.api.contact import ContactOMADView from compass.tests import ApiTest from compass.models import AccessGroup, Contact, AppUser +from django.core.management import call_command from rest_framework.authtoken.models import Token @@ -45,35 +45,6 @@ def test_api_auth(self): test_request) self.assertEqual(response.status_code, 401) - def test_parse_checkin_date_str(self): - # no checkin date specified - with self.assertRaises(ValueError): - ContactOMADView().parse_checkin_date_str(None) - # bad checkin format - with self.assertRaises(ValueError): - ContactOMADView().parse_checkin_date_str("2022-09-T::") - # Missing TZ info - with self.assertRaises(ValueError): - ContactOMADView().parse_checkin_date_str("2022-09-19T06:15:04") - # correct checkin format - checkin_date = ContactOMADView().parse_checkin_date_str( - "2022-09-19T06:15:04Z") - self.assertEqual(type(checkin_date), datetime) - - def test_validate_adviser_netid(self): - with self.assertRaises(ValueError): - ContactOMADView().validate_adviser_netid(None) - ContactOMADView().validate_adviser_netid("foo") - - def test_validate_student_systemkey(self): - with self.assertRaises(ValueError): - ContactOMADView().validate_student_systemkey(None) - with self.assertRaises(ValueError): - ContactOMADView().validate_student_systemkey("badsyskey") - with self.assertRaises(ValueError): - ContactOMADView().validate_student_systemkey(1234) - ContactOMADView().validate_student_systemkey("1234") - @patch('compass.views.api.contact.Student') @patch('compass.views.api.contact.AppUser') @patch('compass.views.api.contact.Contact') @@ -158,12 +129,14 @@ def test_syskey_leading_zero(self): # create without padding self.post_response('contact_omad', test_nopad) + call_command('process_omad_contacts') contacts = Contact.objects.all() self.assertEqual(contacts[0].student.system_key, "001234567") # create with padding self.post_response('contact_omad', test_pad) + call_command('process_omad_contacts') contacts = Contact.objects.all() self.assertEqual(contacts[1].student.system_key, "001234567") @@ -193,6 +166,7 @@ def test_trans_id(self): test_noid) self.post_response('contact_omad', test_id) + call_command('process_omad_contacts') contacts = Contact.objects.all() self.assertEqual(len(contacts), 2) self.assertIsNone(contacts[0].trans_id) @@ -214,6 +188,7 @@ def test_delete(self, mock_is_member, return_value=True): HTTP_AUTHORIZATION=token_str) self.post_response('contact_omad', test_id) + call_command('process_omad_contacts') contacts = Contact.objects.all() c_id = contacts[0].id @@ -238,6 +213,7 @@ def test_put(self, mock_is_member): HTTP_AUTHORIZATION=token_str) resp = self.post_response('contact_omad', test_checkin) + call_command('process_omad_contacts') contact = Contact.objects.get(id=1) self.assertIsNone(contact.notes) diff --git a/compass/urls.py b/compass/urls.py index 7a6a6bc5..2ba7834d 100644 --- a/compass/urls.py +++ b/compass/urls.py @@ -7,6 +7,7 @@ from django.conf import settings from django.views.generic import TemplateView from compass.views.pages import LandingView +from compass.views.admin.contact import OMADContactAdminView from compass.views.api.student import ( StudentContactsView, StudentSchedulesView, @@ -134,6 +135,11 @@ PhotoView.as_view(), name="photo", ), + re_path( + r"^api/internal/support/omad_contact/$", + OMADContactAdminView.as_view(), + name="omad_contact_admin", + ), re_path( r"^api/internal/support/$", SupportView.as_view(), diff --git a/compass/views/admin/__init__.py b/compass/views/admin/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/compass/views/admin/contact.py b/compass/views/admin/contact.py new file mode 100644 index 00000000..11b7c8cd --- /dev/null +++ b/compass/views/admin/contact.py @@ -0,0 +1,19 @@ +# Copyright 2024 UW-IT, University of Washington +# SPDX-License-Identifier: Apache-2.0 + + +from django.contrib.auth.decorators import login_required +from django.utils.decorators import method_decorator +from django.views.generic import TemplateView +from compass.models import OMADContactQueue + + +@method_decorator(login_required, name='dispatch') +class OMADContactAdminView(TemplateView): + template_name = 'omad_contact_admin.html' + + def get_context_data(self, **kwargs): + context = super(OMADContactAdminView, self).get_context_data(**kwargs) + contacts = OMADContactQueue.objects.all() + context['contacts'] = contacts + return context diff --git a/compass/views/api/contact.py b/compass/views/api/contact.py index f5423943..23a0b91f 100644 --- a/compass/views/api/contact.py +++ b/compass/views/api/contact.py @@ -7,11 +7,10 @@ BaseAPIView, JSONClientContentNegotiation, TokenAPIView) from compass.models import ( AccessGroup, AppUser, Contact, ContactTopic, ContactType, ContactMethod, - Student) + Student, OMADContactQueue) from compass.serializers import ( ContactReadSerializer, ContactWriteSerializer, ContactTopicSerializer, ContactTypeSerializer, ContactMethodSerializer) -from dateutil import parser from django.utils.text import slugify from django.utils.decorators import method_decorator from django.views.decorators.csrf import csrf_exempt @@ -20,6 +19,8 @@ from rest_framework import status from userservice.user import UserService from logging import getLogger +import json +from compass.dao.contact import validate_contact_post_data from uw_person_client.exceptions import PersonNotFoundException @@ -242,95 +243,19 @@ class ContactOMADView(TokenAPIView): # Force JSON so clients aren't required to send ContentType header content_negotiation_class = JSONClientContentNegotiation - def parse_contact_type_str(self, contact_type_str, access_group): - try: - return ContactType.objects.get(access_group=access_group, - slug=contact_type_str) - except ContactType.DoesNotExist: - raise ValueError( - f"Contact type '{contact_type_str}' does not exist " - f"for the {access_group.name} access group.") - - def parse_checkin_date_str(self, checkin_date_str): - # parse checkin date - if checkin_date_str is None: - raise ValueError("Check-in date not specified") - else: - try: - dt = parser.parse(checkin_date_str) - if dt.tzinfo is None: - raise ValueError("Invalid check-in date, missing timezone") - return dt - except parser.ParserError as e: - raise ValueError(f"Invalid check-in date: {e}") - - def validate_adviser_netid(self, adviser_netid): - if adviser_netid is None: - raise ValueError("Missing adviser netid") - - def validate_student_systemkey(self, student_systemkey): - if student_systemkey is None: - raise ValueError("Missing student systemkey") - - try: - if not student_systemkey.isdigit(): - raise ValueError("Student systemkey is not a positive integer") - except AttributeError as e: - raise ValueError(f"Invalid student systemkey: {e}") - - @staticmethod - def pad_student_systemkey(student_systemkey): - return student_systemkey.zfill(9) - def post(self, request): contact_dict = request.data + queued_contact = OMADContactQueue.objects.create( + json=json.dumps(contact_dict) + ) + logger.info(f"OMAD contact queued, id: {queued_contact.id}") try: - access_group = AccessGroup.objects.by_name("OMAD") - # check that adviser netid is defined - self.validate_adviser_netid(contact_dict.get("adviser_netid")) - # check that system key is defined - self.validate_student_systemkey( - contact_dict.get("student_systemkey")) - # parse checkin date to ensure it is in the correct format - contact_dict["checkin_date"] = self.parse_checkin_date_str( - contact_dict.get("checkin_date")) - # verify that the specified contact type exists in OMAD - contact_dict["contact_type"] = self.parse_contact_type_str( - contact_dict.get("contact_type"), access_group) + validate_contact_post_data(contact_dict) except AccessGroup.DoesNotExist as e: return Response(repr(e), status=status.HTTP_501_NOT_IMPLEMENTED) except ValueError as e: return Response(repr(e), status=status.HTTP_400_BAD_REQUEST) - # if the adviser is a member of the omad group and the contact record - # was successfully parsed, create an app-user and a student record for - # them if one doesn't already exist - try: - app_user = AppUser.objects.upsert_appuser( - contact_dict["adviser_netid"]) except PersonNotFoundException as e: - logger.error("ContactOMADView: Person not found for " - "adviser_netid: %s" % contact_dict["adviser_netid"]) return Response("Person record for adviser not found", status=status.HTTP_400_BAD_REQUEST) - - student_systemkey = self.pad_student_systemkey( - contact_dict["student_systemkey"]) - # student_systemkey = contact_dict["student_systemkey"] - student, _ = Student.objects.get_or_create( - system_key=student_systemkey) - # create the new contact record - contact = Contact() - contact.app_user = app_user - contact.student = student - contact.contact_type = contact_dict["contact_type"] - contact.checkin_date = contact_dict["checkin_date"] - contact.source = "Checkin" - try: - contact.trans_id = contact_dict["trans_id"] - except KeyError: - pass - contact.save() - contact.access_group.add(access_group) - logger.info(f"Checkin contact {contact.contact_type} added for " - f"student {student.system_key}") return Response(status=status.HTTP_201_CREATED) diff --git a/compass_vue/components/add-contact.vue b/compass_vue/components/add-contact.vue index ec6551ff..f182b119 100644 --- a/compass_vue/components/add-contact.vue +++ b/compass_vue/components/add-contact.vue @@ -238,12 +238,19 @@