From eda96351b64d7b200b97fe73500c55cac2d9dd95 Mon Sep 17 00:00:00 2001 From: Charlon Palacay Date: Tue, 7 May 2024 09:41:43 -0700 Subject: [PATCH 01/36] Adds affiliations nav menu. --- compass_vue/components/nav-menu.vue | 31 ++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/compass_vue/components/nav-menu.vue b/compass_vue/components/nav-menu.vue index cab8262b..6cf1130b 100644 --- a/compass_vue/components/nav-menu.vue +++ b/compass_vue/components/nav-menu.vue @@ -29,7 +29,36 @@ - +
  • ENROLLMENT SERVICES
    • +
  • + Reports +
    • +
  • + Affiliations +
    • + +
  • ADVISOR RESOURCES
  • Date: Tue, 7 May 2024 09:42:43 -0700 Subject: [PATCH 02/36] Adds affiliations nav menu. --- compass_vue/components/nav-menu.vue | 31 ++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/compass_vue/components/nav-menu.vue b/compass_vue/components/nav-menu.vue index cab8262b..6cf1130b 100644 --- a/compass_vue/components/nav-menu.vue +++ b/compass_vue/components/nav-menu.vue @@ -29,7 +29,36 @@
    • - +
  • ENROLLMENT SERVICES
    • +
  • + Reports +
    • +
  • + Affiliations +
    • + +
  • ADVISOR RESOURCES
  • Date: Wed, 15 May 2024 09:42:16 -0700 Subject: [PATCH 03/36] Adds affiliations routes. --- compass/urls.py | 2 +- compass_vue/components/nav-menu.vue | 17 +++++++++----- compass_vue/layout.vue | 2 +- compass_vue/pages/affiliations.vue | 36 +++++++++++++++++++++++++++++ compass_vue/router/index.js | 8 +++++++ 5 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 compass_vue/pages/affiliations.vue diff --git a/compass/urls.py b/compass/urls.py index 23cc33e5..2d8f7aff 100644 --- a/compass/urls.py +++ b/compass/urls.py @@ -170,7 +170,7 @@ name="visit_omad" ), # vue-router paths - re_path(r"^(checkins|student|caseload|reports).*$", LandingView.as_view()), + re_path(r"^(checkins|student|caseload|reports|affiliations).*$", LandingView.as_view()), # default landing re_path(r"^$", LandingView.as_view(), name="index"), ] diff --git a/compass_vue/components/nav-menu.vue b/compass_vue/components/nav-menu.vue index 6cf1130b..2d241a0a 100644 --- a/compass_vue/components/nav-menu.vue +++ b/compass_vue/components/nav-menu.vue @@ -21,15 +21,17 @@ >Caseload
    • + -
  • ENROLLMENT SERVICES
    • +
  • Affiliations
    • + + -
  • ADVISOR RESOURCES
    • +
  • diff --git a/compass_vue/pages/affiliations.vue b/compass_vue/pages/affiliations.vue new file mode 100644 index 00000000..a48ea4f1 --- /dev/null +++ b/compass_vue/pages/affiliations.vue @@ -0,0 +1,36 @@ +// reports.vue + + + + diff --git a/compass_vue/router/index.js b/compass_vue/router/index.js index 1ff0dfde..19749e4f 100644 --- a/compass_vue/router/index.js +++ b/compass_vue/router/index.js @@ -9,6 +9,7 @@ import CheckIn from "@/pages/check-ins.vue"; import Caseload from "@/pages/caseload.vue"; import Student from "@/pages/student.vue"; import Reports from "@/pages/reports.vue"; +import Affiliations from "@/pages/affiliations.vue"; const routes = [ { @@ -40,6 +41,13 @@ const routes = [ pathToRegexpOptions: { strict: true }, props: true, }, + { + path: "/affiliations", + component: Affiliations, + meta: { authorize: [Role.Manager] }, + pathToRegexpOptions: { strict: true }, + props: true, + }, ]; const router = createRouter({ From 24ff75db1d4bc1a0b2151f6862ab0167f51dcd98 Mon Sep 17 00:00:00 2001 From: Charlon Palacay Date: Wed, 15 May 2024 14:02:50 -0700 Subject: [PATCH 04/36] Updates dropdowns --- compass_vue/pages/affiliations.vue | 53 +++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/compass_vue/pages/affiliations.vue b/compass_vue/pages/affiliations.vue index a48ea4f1..719d70cf 100644 --- a/compass_vue/pages/affiliations.vue +++ b/compass_vue/pages/affiliations.vue @@ -5,16 +5,53 @@ From edccd26e869c2fbe0dae1bbc72d9ee2f12316eb8 Mon Sep 17 00:00:00 2001 From: Charlon Palacay Date: Wed, 15 May 2024 17:37:05 -0700 Subject: [PATCH 05/36] Add processUpload function --- compass_vue/pages/affiliations.vue | 57 +++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/compass_vue/pages/affiliations.vue b/compass_vue/pages/affiliations.vue index 719d70cf..fd040914 100644 --- a/compass_vue/pages/affiliations.vue +++ b/compass_vue/pages/affiliations.vue @@ -18,7 +18,7 @@

    Select Affliations

    - @@ -27,7 +27,7 @@

    Select Cohort

    - @@ -39,7 +39,7 @@ @@ -49,9 +49,51 @@
    -
    asdfasdf
    +
    +

    Upload resulsts...

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Student NumberFirstLastNetIDStatus
    2310335LouisKinglkingUploaded
    3776803NellieWoodsnwoodsUploaded
    6640182JefferyBridgesjeffbridgeserror
    1776982CeceliaGlovercecegloveUploaded
    +
    - @@ -69,5 +111,10 @@ export default { isLoading: true, }; }, + methods: { + processUpload: function () { + alert("jfsdalkj"); + }, + }, }; From bf111817f7cee0258c8e1bbc392b1b13ee9bc29d Mon Sep 17 00:00:00 2001 From: Jim Laney Date: Fri, 17 May 2024 11:28:59 -0700 Subject: [PATCH 06/36] add view and backend for affiliation upload file --- compass/dao/csv.py | 102 +++++++++++++++++++ compass/exceptions.py | 15 +-- compass/fixtures/uw_person/student.json | 4 +- compass/resources/csv/insensitive.csv | 2 + compass/resources/csv/missing_header.csv | 6 ++ compass/resources/csv/missing_student_id.csv | 7 ++ compass/resources/csv/valid.csv | 6 ++ compass/tests/dao/test_csv.py | 65 ++++++++++++ compass/tests/test_utils.py | 20 +++- compass/urls.py | 5 + compass/utils.py | 19 ++++ compass/views/api/student.py | 52 +++++++++- setup.py | 1 + 13 files changed, 294 insertions(+), 10 deletions(-) create mode 100644 compass/dao/csv.py create mode 100644 compass/resources/csv/insensitive.csv create mode 100644 compass/resources/csv/missing_header.csv create mode 100644 compass/resources/csv/missing_student_id.csv create mode 100644 compass/resources/csv/valid.csv create mode 100644 compass/tests/dao/test_csv.py diff --git a/compass/dao/csv.py b/compass/dao/csv.py new file mode 100644 index 00000000..9236e29b --- /dev/null +++ b/compass/dao/csv.py @@ -0,0 +1,102 @@ +# Copyright 2024 UW-IT, University of Washington +# SPDX-License-Identifier: Apache-2.0 + +from compass.dao.person import ( + get_person_by_student_number, PersonNotFoundException) +from compass.utils import format_student_number +from compass.exceptions import InvalidCSV +from logging import getLogger +import chardet +import csv +import os + +logger = getLogger(__name__) + + +def normalize(s): + return s.replace(' ', '').replace('_', '').lower() + + +class InsensitiveDict(dict): + """ + Override get() to strip() and lower() the input key, and strip() the + returned value. + """ + def get(self, *k, default=None): + for i in k: + if normalize(i) in self: + try: + return super().get(normalize(i)).strip() + except AttributeError: + break + return default + + +class InsensitiveDictReader(csv.DictReader): + """ + Override the csv.fieldnames property to strip() and lower() the fieldnames. + """ + @property + def fieldnames(self): + return [normalize(field) for field in super().fieldnames] + + def __next__(self): + return InsensitiveDict(super().__next__()) + + +class StudentCSV(): + def __init__(self): + self.encoding = None + self.student_identifiers = [ + 'studentid', 'studentno', 'studentnum', 'studentnumber'] + + def decode_file(self, csvfile): + if not self.encoding: + result = chardet.detect(csvfile) + self.encoding = result['encoding'] + return csvfile.decode(self.encoding) + + def validate(self, fileobj): + # Read the first line of the file to validate the header + decoded_file = self.decode_file(fileobj.readline()) + self.has_header = csv.Sniffer().has_header(decoded_file) + self.dialect = csv.Sniffer().sniff(decoded_file) + + reader = InsensitiveDictReader(decoded_file.splitlines(), + dialect=self.dialect) + + if not (any(s in reader.fieldnames for s in self.student_identifiers)): + raise InvalidCSV('Missing student identifier') + + fileobj.seek(0, 0) + + def persons_from_file(self, fileobj): + """ + Convert CSV file object into normalized JSON + + Supported column names are contained in self.student_identifiers + + All other field names are ignored. + """ + self.validate(fileobj) + decoded_file = self.decode_file(fileobj.read()).splitlines() + + persons = [] + for row in InsensitiveDictReader(decoded_file, dialect=self.dialect): + student_number = format_student_number( + row.get(*self.student_identifiers)) + + if student_number is None: + persons.append({'error': 'Missing student number'}) + continue + + try: + person = get_person_by_student_number(student_number) + person_dict = person.to_dict() + person_dict['student_number'] = student_number + persons.append(person_dict) + except PersonNotFoundException: + persons.append({ + 'error': f'Student number {student_number} not found'}) + + return persons diff --git a/compass/exceptions.py b/compass/exceptions.py index bc4f3794..a3d63f6c 100644 --- a/compass/exceptions.py +++ b/compass/exceptions.py @@ -2,11 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 -""" -Custom exceptions used by Compass. -""" - - class OverrideNotPermitted(Exception): def __str__(self): return "Action not permitted while using admin override" @@ -14,4 +9,12 @@ def __str__(self): class InvalidSystemKey(Exception): def __str__(self): - return "system_ey is invalid" + return "system_key is invalid" + + +class InvalidCSV(Exception): + def __init__(self, error="Invalid CSV file"): + self.error = error + + def __str__(self): + return self.error diff --git a/compass/fixtures/uw_person/student.json b/compass/fixtures/uw_person/student.json index 9c2b0fb7..48dbfae0 100644 --- a/compass/fixtures/uw_person/student.json +++ b/compass/fixtures/uw_person/student.json @@ -47,7 +47,7 @@ "ethnic_group_desc": "White", "exemption_code": "0", "exemption_desc": "NONE", - "external_email": "javerage@gmail.com", + "external_email": "lisa@test.com", "first_generation_4yr_ind": false, "first_generation_ind": false, "gender": "F", @@ -111,7 +111,7 @@ "spp_category": "1", "spp_category_dt": "2022-07-01 04:00:09.613-08:00", "sr_col_gpa": "0.00", - "student_email": "javerage@uw.edu", + "student_email": "lisa@uw.edu", "student_number": "1233338", "system_key": "888777333", "total_credits": "79.00", diff --git a/compass/resources/csv/insensitive.csv b/compass/resources/csv/insensitive.csv new file mode 100644 index 00000000..89471e7f --- /dev/null +++ b/compass/resources/csv/insensitive.csv @@ -0,0 +1,2 @@ +"field1"," field2","Field3",FIELD4,"field5","field 6","field7",Field_8 +"ök1",øk2,ok3,ok4," ok5 "," ",,ok_8 diff --git a/compass/resources/csv/missing_header.csv b/compass/resources/csv/missing_header.csv new file mode 100644 index 00000000..941d6799 --- /dev/null +++ b/compass/resources/csv/missing_header.csv @@ -0,0 +1,6 @@ +"800000","javerage","Average","J","AB","2.0","GRADUATE","Biology","javerage@uw.edu" +"40000","jjulius","Julius","J","AB","2.0","GRADUATE","Oceanography","jjulius@uw.edu" +"1000000","baverage","Average","Bill P","AC","2.0","GRADUATE","Oceanography","baverage@uw.edu" +"7000000","student1","Student 1","","AA","2.0","UNDERGRADUATE","Health Services","student1@uw.edu" +"9000000","student2","Student","2","AA","2.0","NON_MATRIC","Non Matriculated","student2@uw.edu" +"2000000","student3","Student","3","AC","2.0","NON_MATRIC","Non Matriculated","student3@uw.edu" diff --git a/compass/resources/csv/missing_student_id.csv b/compass/resources/csv/missing_student_id.csv new file mode 100644 index 00000000..59f30c77 --- /dev/null +++ b/compass/resources/csv/missing_student_id.csv @@ -0,0 +1,7 @@ +UWNetID,LastName,FirstName,Section,Credits,Class,Major,Email +"javerage","Average","J","AB","2.0","GRADUATE","Biology","javerage@uw.edu" +"jjulius","Julius","J","AB","2.0","GRADUATE","Oceanography","jjulius@uw.edu" +"baverage","Average","Bill P","AC","2.0","GRADUATE","Oceanography","baverage@uw.edu" +"student1","Student 1","","AA","2.0","UNDERGRADUATE","Health Services","student1@uw.edu" +"student2","Student","2","AA","2.0","NON_MATRIC","Non Matriculated","student2@uw.edu" +"student3","Student","3","AC","2.0","NON_MATRIC","Non Matriculated","student3@uw.edu" diff --git a/compass/resources/csv/valid.csv b/compass/resources/csv/valid.csv new file mode 100644 index 00000000..d5360194 --- /dev/null +++ b/compass/resources/csv/valid.csv @@ -0,0 +1,6 @@ +Student_No,UWNetID,LastName,FirstName,Section,Credits,Class,Major,Email +"1033334","javerage","Average","J","AB","2.0","GRADUATE","Biology","javerage@uw.edu" +"1233338","lisa","Simpson","Lisa","AB","2.0","GRADUATE","Oceanography","lisa@uw.edu" +"1233334","jbothell","Bothell","J","AC","2.0","GRADUATE","Oceanography","jbothell@uw.edu" +"7000000","student1","Student 1","","AA","2.0","UNDERGRADUATE","Health Services","student1@uw.edu" +"","student3","Student","3","AC","2.0","NON_MATRIC","Non Matriculated","student3@uw.edu" diff --git a/compass/tests/dao/test_csv.py b/compass/tests/dao/test_csv.py new file mode 100644 index 00000000..274667b2 --- /dev/null +++ b/compass/tests/dao/test_csv.py @@ -0,0 +1,65 @@ +# Copyright 2024 UW-IT, University of Washington +# SPDX-License-Identifier: Apache-2.0 + +from compass.dao.csv import InsensitiveDictReader, StudentCSV +from compass.exceptions import InvalidCSV +from compass.tests import CompassTestCase +import mock +import os + + +class StudentCSVTest(CompassTestCase): + def setUp(self): + self.resource_path = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', '..', 'resources', 'csv')) + + def test_validate(self): + csv_import = StudentCSV() + + with open(os.path.join(self.resource_path, 'valid.csv'), 'rb') as fh: + r = csv_import.validate(fh) + self.assertEqual(csv_import.has_header, True) + self.assertEqual(csv_import.dialect.delimiter, ',') + + with open(os.path.join( + self.resource_path, 'missing_header.csv'), 'rb') as fh: + self.assertRaisesRegex(InvalidCSV, 'Missing student identifier', + csv_import.validate, fh) + + with open(os.path.join( + self.resource_path, 'missing_student_id.csv'), 'rb') as fh: + self.assertRaisesRegex(InvalidCSV, 'Missing student identifier', + csv_import.validate, fh) + + def test_persons_from_file(self): + csv_import = StudentCSV() + + with open(os.path.join(self.resource_path, 'valid.csv'), 'rb') as fh: + result = csv_import.persons_from_file(fh) + + self.assertEqual(len(result), 5) + self.assertEqual(result[0]['student_number'], '1033334') + self.assertEqual(result[0]['uwnetid'], 'javerage') + self.assertEqual(result[1]['student_number'], '1233338') + self.assertEqual(result[1]['uwnetid'], 'lisa') + self.assertEqual(result[2]['student_number'], '1233334') + self.assertEqual(result[2]['uwnetid'], 'jbothell') + self.assertEqual(result[3]['error'], + 'Student number 7000000 not found') + self.assertEqual(result[4]['error'], 'Missing student number') + + +class InsensitiveDictReaderTest(StudentCSVTest): + def test_insensitive_dict_reader(self): + with open(os.path.join(self.resource_path, 'insensitive.csv')) as fh: + reader = InsensitiveDictReader(fh) + + row = next(reader) + self.assertEqual(row.get('Field1'), 'ök1') + self.assertEqual(row.get('Field2'), 'øk2') + self.assertEqual(row.get('Field3'), 'ok3') + self.assertEqual(row.get('Field4'), 'ok4') + self.assertEqual(row.get('Field 5', 'Field5'), 'ok5') + self.assertEqual(row.get('Field6', 'field 6'), '') + self.assertEqual(row.get('Field7'), '') + self.assertEqual(row.get('Field8'), 'ok_8') diff --git a/compass/tests/test_utils.py b/compass/tests/test_utils.py index 61fc3382..ed229be9 100644 --- a/compass/tests/test_utils.py +++ b/compass/tests/test_utils.py @@ -2,11 +2,29 @@ # SPDX-License-Identifier: Apache-2.0 from django.test import TestCase -from compass.utils import weekdays_before +from compass.utils import weekdays_before, zfill_or_none from datetime import datetime, date class TestUtils(TestCase): + def test_zfill_or_none(self): + return_value = zfill_or_none("12345", 7) + self.assertEqual(return_value, "0012345") + return_value = zfill_or_none("1234567", 7) + self.assertEqual(return_value, "1234567") + return_value = zfill_or_none("12345678", 7) + self.assertEqual(return_value, "12345678") + return_value = zfill_or_none("123", 5) + self.assertEqual(return_value, "00123") + return_value = zfill_or_none("0", 7) + self.assertEqual(return_value, None) + return_value = zfill_or_none(1234, 7) + self.assertEqual(return_value, "0001234") + return_value = zfill_or_none(None, 7) + self.assertEqual(return_value, None) + return_value = zfill_or_none("", 7) + self.assertEqual(return_value, None) + def test_weekdays_before(self): # date val = weekdays_before(date(2024, 4, 19), offset_days=3) diff --git a/compass/urls.py b/compass/urls.py index 2d8f7aff..779a2ef9 100644 --- a/compass/urls.py +++ b/compass/urls.py @@ -13,6 +13,7 @@ StudentSchedulesView, StudentTranscriptsView, StudentAffiliationsView, + StudentAffiliationsImportView, StudentVisitsView, StudentEligibilityView, StudentView, @@ -87,6 +88,10 @@ r"^api/internal/student/(?P[\w]+)/affiliations/$", StudentAffiliationsView.as_view(), ), + re_path( + r"^api/internal/import/affiliations/(?P[\w]+)/$", + StudentAffiliationsImportView.as_view(), + ), re_path( r"^api/internal/student/(?P[\w]+)" r"/affiliations/(?P[\w]+)/$", diff --git a/compass/utils.py b/compass/utils.py index e58c1fb8..bf20bb49 100644 --- a/compass/utils.py +++ b/compass/utils.py @@ -3,6 +3,25 @@ from datetime import timedelta +STUDENT_NUM_LEN = 7 +STUDENT_SYSKEY_LEN = 9 + + +def zfill_or_none(value, length): + value = str(value).zfill(length) if ( + value and len(str(value)) > 0) else None + if value == "0" * length: + return None + return value + + +def format_system_key(value): + return zfill_or_none(int(value), STUDENT_SYSKEY_LEN) + + +def format_student_number(value): + return zfill_or_none(value, STUDENT_NUM_LEN) + def weekdays_before(end_date, offset_days=3): """ diff --git a/compass/views/api/student.py b/compass/views/api/student.py index d1f342d0..a7187e3f 100644 --- a/compass/views/api/student.py +++ b/compass/views/api/student.py @@ -9,6 +9,7 @@ valid_uwnetid, valid_uwregid, valid_student_number, valid_system_key, get_person_by_uwnetid, get_person_by_student_number, get_transcripts_by_uwregid, PersonNotFoundException) +from compass.dao.csv import StudentCSV from compass.dao import current_datetime_utc from compass.models import ( AccessGroup, Student, AppUser, Contact, StudentAffiliation, Affiliation, @@ -18,7 +19,7 @@ VisitReadSerializer, StudentWriteSerializer, StudentEligibilitySerializer, EligibilityTypeSerializer, SpecialProgramSerializer) -from compass.exceptions import OverrideNotPermitted +from compass.exceptions import OverrideNotPermitted, InvalidCSV from restclients_core.exceptions import DataFailureException from uw_sws.term import ( get_current_term, get_next_term, get_term_after, @@ -250,6 +251,55 @@ def delete(self, request, systemkey, affiliation_id): return self.response_unauthorized(err) +class StudentAffiliationsImportView(BaseAPIView): + ''' + API endpoint for assigning the passed affiliation to multiple students + + /api/internal/import/affiliations/[affiliation_id] + ''' + def post(self, request, *args, **kwargs): + try: + access_group = self.get_access_group(request, require_manager=True) + except AccessGroup.DoesNotExist: + return self.response_unauthorized() + + affiliation_id = kwargs.get("affiliation_id") + cohort_id = request.data.get("cohort_id") + try: + affiliation = Affiliation.objects.get( + id=affiliation_id, access_group=access_group) + cohort = Cohort.objects.get(id=cohort_id) + except Affiliation.DoesNotExist: + return self.response_badrequest( + content="Unknown or unpermitted affiliation") + except Cohort.DoesNotExist: + return self.response_badrequest('Unknown cohort') + + uploaded_file = request.FILES.get("file") + if uploaded_file is None: + return self.response_badrequest(content="Missing file") + + try: + person_data = StudentCSV().persons_from_file(uploaded_file) + except InvalidCSV as err: + logger.error("POST upload failed for {}: {}".format( + uploaded_file.name, err)) + return self.response_badrequest(content=err) + + for p in person_data: + if hasattr(p, "system_key"): + student, _ = Student.objects.get_or_create( + system_key=p["system_key"]) + sa, _ = StudentAffiliation.objects.get_or_create( + student=student, affiliation=affiliation) + sa.date = current_datetime_utc().date() + sa.cohorts.clear() + sa.cohorts.add(cohort) + sa.save() + + return self.response_ok(person_data) + + class StudentVisitsView(BaseAPIView): ''' API endpoint returning a list of visits for a student diff --git a/setup.py b/setup.py index 36ae8595..82711bfd 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ 'urllib3~=1.25', 'django-person-client', 'python-dateutil~=2.8', + 'chardet~=5.0', 'lxml==4.9.4', 'xmlsec==1.3.13' ], From 6668118b1da67ae6d916ede098c8ee601119596c Mon Sep 17 00:00:00 2001 From: Jim Laney Date: Mon, 20 May 2024 12:14:23 -0700 Subject: [PATCH 07/36] populate affiliation and cohort select elements --- compass_vue/pages/affiliations.vue | 53 ++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/compass_vue/pages/affiliations.vue b/compass_vue/pages/affiliations.vue index fd040914..a657aee0 100644 --- a/compass_vue/pages/affiliations.vue +++ b/compass_vue/pages/affiliations.vue @@ -18,20 +18,34 @@

    Select Affliations

    - + +

    Select Cohort

    - + +
    @@ -100,18 +114,37 @@