Merge pull request #373 from uw-it-aca/develop

Develop

compass/admin.py

@@ -4,7 +4,8 @@
 
 from compass.models import (
     AppUser, Student, AccessGroup, Affiliation, Contact,
-    ContactType, ContactMethod, ContactTopic, EligibilityType)
+    ContactType, ContactMethod, ContactTopic, EligibilityType,
+    OMADContactQueue)
 from compass.dao.group import is_admin_user
 from django.contrib import admin
 from django.http import HttpResponseRedirect
@@ -75,5 +76,6 @@ def _session_data(self, obj):
 admin_site.register(ContactMethod, SAMLAdminModel)
 admin_site.register(ContactTopic, SAMLAdminModel)
 admin_site.register(Contact, SAMLAdminModel)
+admin_site.register(OMADContactQueue, SAMLAdminModel)
 admin_site.register(TokenProxy, SAMLAdminModel)
 admin_site.register(Session, SessionAdminModel)

compass/dao/contact.py

@@ -0,0 +1,70 @@
+# Copyright 2024 UW-IT, University of Washington
+# SPDX-License-Identifier: Apache-2.0
+
+from compass.models import ContactType, AccessGroup, AppUser
+from dateutil import parser
+
+
+def validate_contact_post_data(contact_dict):
+    access_group = AccessGroup.objects.by_name("OMAD")
+    # check that adviser netid is defined
+    validate_adviser_netid(contact_dict.get("adviser_netid"))
+    # check that system key is defined
+    validate_student_systemkey(
+        contact_dict.get("student_systemkey"))
+    # parse checkin date to ensure it is in the correct format
+    contact_dict["checkin_date"] = parse_checkin_date_str(
+        contact_dict.get("checkin_date"))
+    # verify that the specified contact type exists in OMAD
+    contact_dict["contact_type"] = parse_contact_type_str(
+        contact_dict.get("contact_type"), access_group)
+
+    # if the adviser is a member of the omad group and the contact record
+    # was successfully parsed, create an app-user and a student record for
+    # them if one doesn't already exist
+    app_user = AppUser.objects.upsert_appuser(
+        contact_dict["adviser_netid"])
+
+
+def validate_adviser_netid(adviser_netid):
+    if adviser_netid is None:
+        raise ValueError("Missing adviser netid")
+
+
+def validate_student_systemkey(student_systemkey):
+    if student_systemkey is None:
+        raise ValueError("Missing student systemkey")
+
+    try:
+        if not student_systemkey.isdigit():
+            raise ValueError("Student systemkey is not a positive integer")
+    except AttributeError as e:
+        raise ValueError(f"Invalid student systemkey: {e}")
+
+
+def pad_student_systemkey(student_systemkey):
+    return student_systemkey.zfill(9)
+
+
+def parse_checkin_date_str(checkin_date_str):
+    # parse checkin date
+    if checkin_date_str is None:
+        raise ValueError("Check-in date not specified")
+    else:
+        try:
+            dt = parser.parse(checkin_date_str)
+            if dt.tzinfo is None:
+                raise ValueError("Invalid check-in date, missing timezone")
+            return dt
+        except parser.ParserError as e:
+            raise ValueError(f"Invalid check-in date: {e}")
+
+
+def parse_contact_type_str(contact_type_str, access_group):
+    try:
+        return ContactType.objects.get(access_group=access_group,
+                                       slug=contact_type_str)
+    except ContactType.DoesNotExist:
+        raise ValueError(
+            f"Contact type '{contact_type_str}' does not exist "
+            f"for the {access_group.name} access group.")

compass/management/commands/process_omad_contacts.py

@@ -0,0 +1,75 @@
+# Copyright 2024 UW-IT, University of Washington
+# SPDX-License-Identifier: Apache-2.0
+
+from django.core.management.base import BaseCommand
+from compass.models import OMADContactQueue
+from datetime import datetime
+import json
+from compass.models import (
+    AccessGroup, AppUser, Contact, Student)
+from logging import getLogger
+from compass.dao.contact import (validate_contact_post_data,
+                                 pad_student_systemkey)
+import traceback
+logger = getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "process OMAD contacts"
+
+    def add_arguments(self, parser):
+        parser.add_argument('--reprocess',
+                            action='store_true',
+                            help="reprocess all contacts regardless of status")
+
+    def handle(self, *args, **options):
+        if options['reprocess']:
+            contacts = OMADContactQueue.objects.all()
+        else:
+            contacts = OMADContactQueue.objects.filter(processing_attempts=0)
+
+        for contact in contacts:
+            try:
+                self.process_contact(contact)
+            except Exception as e:
+                logger.exception(f"Error processing contact {contact.id}")
+                contact.processing_attempts += 1
+                contact.process_attempted_date = datetime.now()
+                contact.processing_error = repr(e)
+                contact.stack_trace = traceback.format_exc()
+                contact.save()
+                continue
+            contact.delete()
+
+    @staticmethod
+    def process_contact(contact):
+        contact_dict = json.loads(contact.json)
+        validate_contact_post_data(contact_dict)
+
+        # Get additional objects
+        access_group = AccessGroup.objects.by_name("OMAD")
+        app_user = AppUser.objects.upsert_appuser(
+            contact_dict["adviser_netid"])
+
+        # Parse/format data
+        student_systemkey = pad_student_systemkey(
+            contact_dict["student_systemkey"])
+
+        student, _ = Student.objects.get_or_create(
+            system_key=student_systemkey)
+
+        # create the new contact record
+        contact = Contact()
+        contact.app_user = app_user
+        contact.student = student
+        contact.contact_type = contact_dict["contact_type"]
+        contact.checkin_date = contact_dict["checkin_date"]
+        contact.source = "Checkin"
+        try:
+            contact.trans_id = contact_dict["trans_id"]
+        except KeyError:
+            pass
+        contact.save()
+        contact.access_group.add(access_group)
+        logger.info(f"Checkin contact {contact.contact_type} processed for "
+                    f"student {student.system_key}")

compass/migrations/0014_omadcontactqueue.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.9 on 2024-01-09 05:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('compass', '0013_contact_trans_id_historicalcontact_trans_id'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='OMADContactQueue',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('json', models.TextField()),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('process_attempted_date', models.DateTimeField(null=True)),
+                ('processing_error', models.TextField(null=True)),
+                ('stack_trace', models.TextField(null=True)),
+                ('processing_attempts', models.IntegerField(default=0)),
+            ],
+        ),
+    ]

compass/models.py

@@ -427,3 +427,22 @@ class VisitType(BaseAccessGroupContent):
     name = models.CharField(unique=True, max_length=50)
     slug = models.SlugField(unique=True, max_length=50)
     editable = models.BooleanField(default=False)
+
+
+class OMADContactQueue(models.Model):
+    """
+    Stores raw contacts from the OMAD check-in system for processing into
+    Contacts.  Successfully created contacts will be removed from this table.
+    """
+
+    json = models.TextField()
+    created = models.DateTimeField(auto_now_add=True)
+    process_attempted_date = models.DateTimeField(null=True)
+    processing_error = models.TextField(null=True)
+    stack_trace = models.TextField(null=True)
+    processing_attempts = models.IntegerField(default=0)
+
+    def __str__(self):
+        return f"#{self.id} created: {self.created} " \
+               f"attempts: {self.processing_attempts} " \
+               f"err: {self.processing_error}"

compass/templates/omad_contact_admin.html

@@ -0,0 +1,73 @@
+{% extends 'supporttools/base.html' %}
+{% load static %}
+
+{% block content %}
+  <style>
+    table {
+      border-collapse: collapse;
+      table-layout: fixed;
+      width: 100%;
+    }
+    th, td {
+      text-align: left;
+      padding: 8px;
+    }
+    tr:nth-child(even) {background-color: #f2f2f2;}
+    details {
+      border: 1px solid #aaa;
+      border-radius: 4px;
+      padding: 0.5em 0.5em 0;
+    }
+
+    summary {
+      font-weight: bold;
+      margin: -0.5em -0.5em 0;
+      padding: 0.5em;
+    }
+
+    details[open] {
+      padding: 0.5em;
+    }
+
+    details[open] summary {
+      border-bottom: 1px solid #aaa;
+      margin-bottom: 0.5em;
+    }
+  </style>
+  <h1>OMAD Contact Processing Queue</h1>
+  <table>
+    <colgroup>
+      <col style="width: auto">
+      <col style="width: auto">
+      <col style="width: auto">
+      <col style="width: auto">
+      <col style="width: auto">
+      <col style="width: 60%">
+    </colgroup>
+    <tr>
+      <th>ID</th>
+      <th>Created</th>
+      <th>Processing attempts</th>
+      <th>Last attempted</th>
+      <th>Last error</th>
+      <th>Contact JSON</th>
+    </tr>
+    {% for contact in contacts %}
+      <tr>
+        <td>{{ contact.id }}</td>
+        <td>{{ contact.created }}</td>
+        <td>{{ contact.processing_attempts }}</td>
+        <td>{{ contact.process_attempted_date }}</td>
+        <td>{{ contact.processing_error }}</td>
+        <td>
+          {{ contact.json }}
+          <details>
+            <summary>Show traceback</summary>
+            <pre>{{ contact.stack_trace }}</pre>
+          </details>
+
+        </td>
+      </tr>
+    {% endfor %}
+  </table>
+{% endblock content %}

compass/templates/supporttools/custom_sidebar_links.html

@@ -0,0 +1,6 @@
+{% include 'supporttools/default_sidelinks.html' %}
+
+<h3>Compass Tools</h3>
+<ul>
+    <li><a href="{% url 'omad_contact_admin' %}">OMAD Contact Queue</a></li>
+</ul>

compass/tests/commands/test_process_omad_contacts.py

@@ -0,0 +1,60 @@
+# Copyright 2024 UW-IT, University of Washington
+# SPDX-License-Identifier: Apache-2.0
+
+
+from django.core.management import call_command
+from django.test import TestCase
+from compass.models import OMADContactQueue, Contact, AccessGroup
+import json
+from rest_framework.authtoken.models import Token
+from django.contrib.auth.models import User
+
+
+class TestOMADContactProcessing(TestCase):
+    API_TOKEN = None
+
+    def setUp(self):
+        super(TestOMADContactProcessing, self).setUp()
+        AccessGroup(name="OMAD", access_group_id="u_astra_group1").save()
+        user = User.objects.create_user(username='testuser', password='12345')
+        token = Token.objects.create(user=user)
+        self.API_TOKEN = token.key
+
+    def test_success(self):
+        # Create a contact queue entry
+        test_body = {
+            "adviser_netid": "javerage",
+            "student_systemkey": "001234567",
+            "contact_type": "appointment",
+            "checkin_date": "2012-01-19 17:21:00 PDT",
+            "source": "Compass",
+            "trans_id": 1234567890
+        }
+        OMADContactQueue.objects.create(json=json.dumps(test_body))
+        call_command('process_omad_contacts')
+        self.assertEqual(OMADContactQueue.objects.count(), 0)
+        self.assertEqual(Contact.objects.count(), 1)
+
+    def test_failure(self):
+        test_body = {
+            "adviser_netid": "javerage123",
+            "student_systemkey": "001234567",
+            "contact_type": "appointment",
+            "checkin_date": "2012-01-19 17:21:00 PDT",
+            "source": "Compass",
+            "trans_id": 1234567890
+        }
+        OMADContactQueue.objects.create(json=json.dumps(test_body))
+        call_command('process_omad_contacts')
+        self.assertEqual(OMADContactQueue.objects.count(), 1)
+        self.assertEqual(Contact.objects.count(), 0)
+        # don't reprocess without flag
+        call_command('process_omad_contacts')
+        contact = OMADContactQueue.objects.all()[0]
+        self.assertEqual(contact.processing_attempts, 1)
+        self.assertEqual(contact.processing_error, "PersonNotFoundException()")
+
+        # reprocess with flag
+        call_command('process_omad_contacts', reprocess=True)
+        contact = OMADContactQueue.objects.all()[0]
+        self.assertEqual(contact.processing_attempts, 2)

compass/tests/dao/test_contact.py

@@ -0,0 +1,39 @@
+# Copyright 2024 UW-IT, University of Washington
+# SPDX-License-Identifier: Apache-2.0
+
+
+from django.test import TestCase
+from datetime import datetime
+from compass.dao.contact import (
+    parse_checkin_date_str, validate_adviser_netid, validate_student_systemkey)
+
+
+class ContactDaoTest(TestCase):
+    def test_parse_checkin_date_str(self):
+        # no checkin date specified
+        with self.assertRaises(ValueError):
+            parse_checkin_date_str(None)
+        # bad checkin format
+        with self.assertRaises(ValueError):
+            parse_checkin_date_str("2022-09-T::")
+        # Missing TZ info
+        with self.assertRaises(ValueError):
+            parse_checkin_date_str("2022-09-19T06:15:04")
+        # correct checkin format
+        checkin_date = parse_checkin_date_str(
+            "2022-09-19T06:15:04Z")
+        self.assertEqual(type(checkin_date), datetime)
+
+    def test_validate_adviser_netid(self):
+        with self.assertRaises(ValueError):
+            validate_adviser_netid(None)
+        validate_adviser_netid("foo")
+
+    def test_validate_student_systemkey(self):
+        with self.assertRaises(ValueError):
+            validate_student_systemkey(None)
+        with self.assertRaises(ValueError):
+            validate_student_systemkey("badsyskey")
+        with self.assertRaises(ValueError):
+            validate_student_systemkey(1234)
+        validate_student_systemkey("1234")