Merge pull request #426 from uw-it-aca/develop

Develop

compass/dao/contact.py

@@ -43,10 +43,6 @@ def validate_student_systemkey(student_systemkey):
         raise ValueError(f"Invalid student systemkey: {e}")
 
 
-def pad_student_systemkey(student_systemkey):
-    return student_systemkey.zfill(9)
-
-
 def parse_checkin_date_str(checkin_date_str):
     # parse checkin date
     if checkin_date_str is None:

compass/dao/csv.py

@@ -0,0 +1,125 @@
+# Copyright 2024 UW-IT, University of Washington
+# SPDX-License-Identifier: Apache-2.0
+
+from compass.dao.person import (
+    get_students_by_system_keys, get_students_by_student_numbers)
+from compass.utils import format_system_key, format_student_number
+from compass.exceptions import InvalidCSV
+from logging import getLogger
+import chardet
+import csv
+import os
+
+logger = getLogger(__name__)
+
+
+def normalize(s):
+    return s.replace(' ', '').replace('_', '').lower()
+
+
+class InsensitiveDict(dict):
+    """
+    Override get() to strip() and lower() the input key, and strip() the
+    returned value.
+    """
+    def get(self, *k, default=None):
+        for i in k:
+            if normalize(i) in self:
+                try:
+                    return super().get(normalize(i)).strip()
+                except AttributeError:
+                    break
+        return default
+
+
+class InsensitiveDictReader(csv.DictReader):
+    """
+    Override the csv.fieldnames property to strip() and lower() the fieldnames.
+    """
+    @property
+    def fieldnames(self):
+        return [normalize(field) for field in super().fieldnames]
+
+    def __next__(self):
+        return InsensitiveDict(super().__next__())
+
+
+class StudentCSV():
+    def __init__(self):
+        self.encoding = None
+        self.has_header = False
+        self.dialect = None
+        self.student_id_col = None
+        self.system_key_cols = ['systemkey', 'syskey']
+        self.student_num_cols = ['studentid', 'studentno', 'studentnum',
+                                 'studentnumber', 'studentidnumber']
+
+    def decode_file(self, csvfile):
+        if not self.encoding:
+            result = chardet.detect(csvfile)
+            self.encoding = result['encoding']
+        return csvfile.decode(self.encoding)
+
+    def validate(self, fileobj):
+        # Read the first line of the file to validate the header
+        try:
+            decoded_file = self.decode_file(fileobj.readline())
+            self.has_header = csv.Sniffer().has_header(decoded_file)
+            self.dialect = csv.Sniffer().sniff(decoded_file)
+        except Exception as err:
+            raise InvalidCSV(str(err))
+
+        reader = InsensitiveDictReader(decoded_file.splitlines(),
+                                       dialect=self.dialect)
+
+        self.student_id_col = next((
+            s for s in (self.system_key_cols + self.student_num_cols) if s in (
+                reader.fieldnames)), None)
+
+        if self.student_id_col is None:
+            raise InvalidCSV('Missing header row or student identifier')
+
+        fileobj.seek(0, 0)
+
+    def students_from_file(self, fileobj):
+        """
+        Reads a CSV file object, and returns a list of person JSON objects
+
+        Supported column names are contained in self.system_key_cols and
+        self.student_num_cols, with priority given to system_keys.
+
+        All other field names are ignored.
+        """
+        self.validate(fileobj)
+        decoded_file = self.decode_file(fileobj.read()).splitlines()
+
+        student_ids = []
+        for row in InsensitiveDictReader(decoded_file, dialect=self.dialect):
+            if self.student_id_col in self.system_key_cols:
+                student_id = format_system_key(row.get(self.student_id_col))
+            else:
+                student_id = format_student_number(
+                    row.get(self.student_id_col))
+
+            if student_id is not None:
+                student_ids.append(student_id)
+
+        students = []
+        if len(student_ids):
+            if self.student_id_col in self.system_key_cols:
+                identifier = 'system_key'
+                students_dict = get_students_by_system_keys(student_ids)
+            else:
+                identifier = 'student_number'
+                students_dict = get_students_by_student_numbers(student_ids)
+
+            for sid in student_ids:
+                if sid in students_dict:
+                    students.append(students_dict[sid])
+                else:
+                    students.append({
+                        identifier: sid,
+                        'error': f'Student not found',
+                    })
+
+        return students

compass/dao/person.py

@@ -101,17 +101,23 @@ def get_appuser_by_uwnetid(uwnetid):
 
 def get_students_by_system_keys(system_keys, **kwargs):
     photo_key = PhotoDAO().generate_photo_key()
-    students = Student.objects.filter(system_key__in=system_keys).values(
+    students = Student.objects.filter(
+            system_key__in=system_keys
+        ).values(
             'system_key',
             'student_number',
             'person__uwnetid',
             'person__uwregid',
             'person__pronouns',
-            'person__display_name'
+            'person__display_name',
+            'person__first_name',
+            'person__surname',
         ).annotate(
             uwnetid=F('person__uwnetid'),
             pronouns=F('person__pronouns'),
             display_name=F('person__display_name'),
+            first_name=F('person__first_name'),
+            surname=F('person__surname'),
             photo_url=Concat(
                 Value('/api/internal/photo/'), F('person__uwregid'),
                 Value(f'/{photo_key}/'), output_field=CharField()),
@@ -124,6 +130,32 @@ def get_students_by_system_keys(system_keys, **kwargs):
     return students_dict
 
 
+def get_students_by_student_numbers(student_numbers, **kwargs):
+    students = Student.objects.filter(
+            student_number__in=student_numbers
+        ).values(
+            'system_key',
+            'student_number',
+            'person__uwnetid',
+            'person__uwregid',
+            'person__pronouns',
+            'person__display_name',
+            'person__first_name',
+            'person__surname',
+        ).annotate(
+            uwnetid=F('person__uwnetid'),
+            pronouns=F('person__pronouns'),
+            display_name=F('person__display_name'),
+            first_name=F('person__first_name'),
+            surname=F('person__surname')
+        )
+
+    students_dict = {}
+    for student in students:
+        students_dict[student['student_number']] = student
+    return students_dict
+
+
 def get_adviser_caseload(uwnetid):
     adviser = Adviser.objects.get_adviser_by_uwnetid(uwnetid)
     photo_key = PhotoDAO().generate_photo_key()

compass/exceptions.py

@@ -2,16 +2,19 @@
 # SPDX-License-Identifier: Apache-2.0
 
 
-"""
-Custom exceptions used by Compass.
-"""
-
-
 class OverrideNotPermitted(Exception):
     def __str__(self):
         return "Action not permitted while using admin override"
 
 
 class InvalidSystemKey(Exception):
     def __str__(self):
-        return "system_ey is invalid"
+        return "system_key is invalid"
+
+
+class InvalidCSV(Exception):
+    def __init__(self, error="Invalid CSV file"):
+        self.error = error
+
+    def __str__(self):
+        return self.error

compass/fixtures/uw_person/student.json

@@ -47,7 +47,7 @@
             "ethnic_group_desc": "White",
             "exemption_code": "0",
             "exemption_desc": "NONE",
-            "external_email": "javerage@gmail.com",
+            "external_email": "lisa@test.com",
             "first_generation_4yr_ind": false,
             "first_generation_ind": false,
             "gender": "F",
@@ -111,7 +111,7 @@
             "spp_category": "1",
             "spp_category_dt": "2022-07-01 04:00:09.613-08:00",
             "sr_col_gpa": "0.00",
-            "student_email": "javerage@uw.edu",
+            "student_email": "lisa@uw.edu",
             "student_number": "1233338",
             "system_key": "888777333",
             "total_credits": "79.00",

compass/management/commands/process_omad_contacts.py

@@ -2,15 +2,15 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from django.core.management.base import BaseCommand
-from compass.models import OMADContactQueue
+from compass.models import (
+    AccessGroup, AppUser, Contact, Student, OMADContactQueue)
+from compass.dao.contact import validate_contact_post_data
+from compass.utils import format_system_key
 from datetime import datetime, timezone
 import json
-from compass.models import (
-    AccessGroup, AppUser, Contact, Student)
 from logging import getLogger
-from compass.dao.contact import (validate_contact_post_data,
-                                 pad_student_systemkey)
 import traceback
+
 logger = getLogger(__name__)
 
 
@@ -52,7 +52,7 @@ def process_contact(contact):
             contact_dict["adviser_netid"])
 
         # Parse/format data
-        student_systemkey = pad_student_systemkey(
+        student_systemkey = format_system_key(
             contact_dict["student_systemkey"])
 
         student, _ = Student.objects.get_or_create(

compass/resources/csv/insensitive.csv

@@ -0,0 +1,2 @@
+"field1","  field2","Field3",FIELD4,"field5","field 6","field7",Field_8
+"ök1",øk2,ok3,ok4,"   ok5  ","   ",,ok_8

compass/resources/csv/missing_header.csv

@@ -0,0 +1,6 @@
+800000,"javerage","Average","J","AB","2.0","GRADUATE","Biology","[email protected]"
+40000,"jjulius","Julius","J","AB","2.0","GRADUATE","Oceanography","[email protected]"
+1000000,"baverage","Average","Bill P","AC","2.0","GRADUATE","Oceanography","[email protected]"
+7000000,"student1","Student 1","","AA","2.0","UNDERGRADUATE","Health Services","[email protected]"
+9000000,"student2","Student","2","AA","2.0","NON_MATRIC","Non Matriculated","[email protected]"
+2000000,"student3","Student","3","AC","2.0","NON_MATRIC","Non Matriculated","[email protected]"

compass/resources/csv/missing_student_id.csv

@@ -0,0 +1,7 @@
+UWNetID,LastName,FirstName,Section,Credits,Class,Major,Email
+"javerage","Average","J","AB","2.0","GRADUATE","Biology","[email protected]"
+"jjulius","Julius","J","AB","2.0","GRADUATE","Oceanography","[email protected]"
+"baverage","Average","Bill P","AC","2.0","GRADUATE","Oceanography","[email protected]"
+"student1","Student 1","","AA","2.0","UNDERGRADUATE","Health Services","[email protected]"
+"student2","Student","2","AA","2.0","NON_MATRIC","Non Matriculated","[email protected]"
+"student3","Student","3","AC","2.0","NON_MATRIC","Non Matriculated","[email protected]"

compass/resources/csv/valid_multiple_ids.csv

@@ -0,0 +1,6 @@
+Student_No,UWNetID,LastName,FirstName,System_Key,Section,Credits,Class,Major,Email
+"1033334","javerage","Average","J","532353230", "AB","2.0","GRADUATE","Biology","[email protected]"
+"1233338","lisa","Simpson","Lisa","888777333", "AB","2.0","GRADUATE","Oceanography","[email protected]"
+"1233334","jbothell","Bothell","J","820582050","AC","2.0","GRADUATE","Oceanography","[email protected]"
+"7000000","student1","Student 1","","700000001","AA","2.0","UNDERGRADUATE","Health Services","[email protected]"
+"","student3","Student","3","","AC","2.0","NON_MATRIC","Non Matriculated","[email protected]"

compass/resources/csv/valid_student_num.csv

@@ -0,0 +1,6 @@
+Student_No,UWNetID,LastName,FirstName,Section,Credits,Class,Major,Email
+"1033334","javerage","Average","J","AB","2.0","GRADUATE","Biology","[email protected]"
+"1233338","lisa","Simpson","Lisa","AB","2.0","GRADUATE","Oceanography","[email protected]"
+"1233334","jbothell","Bothell","J","AC","2.0","GRADUATE","Oceanography","[email protected]"
+"7000000","student1","Student 1","","AA","2.0","UNDERGRADUATE","Health Services","[email protected]"
+"","student3","Student","3","AC","2.0","NON_MATRIC","Non Matriculated","[email protected]"

compass/resources/csv/valid_system_key.csv

@@ -0,0 +1,6 @@
+SystemKey,UWNetID,LastName,FirstName,Section,Credits,Class,Major,Email
+"532353230","javerage","Average","J","AB","2.0","GRADUATE","Biology","[email protected]"
+"888777333","lisa","Simpson","Lisa","AB","2.0","GRADUATE","Oceanography","[email protected]"
+"820582050","jbothell","Bothell","J","AC","2.0","GRADUATE","Oceanography","[email protected]"
+"700000001","student1","Student 1","","AA","2.0","UNDERGRADUATE","Health Services","[email protected]"
+"","student3","Student","3","AC","2.0","NON_MATRIC","Non Matriculated","[email protected]"