Merge pull request #136 from RockefellerArchiveCenter/v1.1

v1.1

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+ignore = E501
+exclude = .git,.github,__pycache__,*/migrations,fixtures

.github/ISSUE_TEMPLATE/bug_report.md

@@ -24,7 +24,7 @@ A clear and concise description of what you expected to happen.
 If applicable, copy the error message or add screenshots to help explain your problem.
 
 ## Impact on your work
-How is this bug affecting your work? Are there alternatives you can use to getting your job done, or does this bug block your work completely? What deadlines are you up against? 
+How is this bug affecting your work? Are there alternatives you can use to getting your job done, or does this bug block your work completely? What deadlines are you up against?
 
 ## Additional context
 Add any other context about the problem here.

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+- repo: https://github.com/pre-commit/mirrors-autopep8
+  rev: v1.5
+  hooks:
+  - id: autopep8
+    args:
+      - --in-place
+      - --aggressive
+- repo: https://gitlab.com/pycqa/flake8
+  rev: 3.7.9
+  hooks:
+  - id: flake8
+- repo: git://github.com/doublify/pre-commit-isort
+  rev: v4.3.0
+  hooks:
+  - id: isort

.travis.yml

@@ -13,13 +13,15 @@ before_install:
   - cp aquarius/config.py.example aquarius/config.py
   - docker-compose up -d
   - sleep 20s
-  - docker-compose exec aquarius-web pip install coverage
+  - docker-compose exec aquarius-web pip install coverage pre-commit
+  - docker-compose exec aquarius-web pre-commit install
 install: true
 before_script:
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
   - chmod +x ./cc-test-reporter
   - ./cc-test-reporter before-build
 script:
+  - docker-compose exec aquarius-web pre-commit run --all-files --show-diff-on-failure
   - docker-compose exec aquarius-web coverage run manage.py test
 after_script:
   - docker-compose exec aquarius-web coverage xml

README.md

@@ -35,17 +35,15 @@ You will need to edit configuration values in `aquarius/config.py` to point to y
 
 ## Services
 
-aquarius has six services, all of which are exposed via HTTP endpoints (see [Routes](#routes) section below):
+aquarius has seven services, all of which are exposed via HTTP endpoints (see [Routes](#routes) section below):
 
 * Store Packages - saves package data received via a POST request. For an example of the package data aquarius expects, see `fixtures/data/`.
 * Transform Accessions - transforms accession data and delivers it to ArchivesSpace as an accession.
 * Transform Grouping Components - transforms accession data into a grouping component and delivers it to ArchivesSpace as an archival object.
 * Transform Transfer Component - transforms transfer data and delivers it to ArchivesSpace as an archival object.
 * Transform Digital Component - transforms transfer data and delivers it to ArchivesSpace as a digital object.
-* Update Status - sends updated data to Aurora.
-
-![TransferRoutine diagram](aquarius-services.png)
-
+* Update Accession Status - sends updated accession data to Aurora.
+* Update Transfer Status - sends updated transfer data to Aurora.
 
 ### Routes
 
@@ -58,7 +56,8 @@ aquarius has six services, all of which are exposed via HTTP endpoints (see [Rou
 |POST|/grouping-components| |200|Runs the GroupingComponentRoutine process|
 |POST|/transfer-components| |200|Runs the TransferComponentRoutine process|
 |POST|/digital-objects| |200|Runs the DigitalObjectRoutine process|
-|POST|/send-update| |200|Sends updated data to Aurora|
+|POST|/send-update| |200|Sends updated transfer data to Aurora|
+|POST|/send-accession-update| |200|Sends updated accession data to Aurora|
 |GET|/status||200|Return the status of the microservice|
 |GET|/schema.json||200|Returns the OpenAPI schema for this application|
 

aquarius/settings.py

@@ -11,6 +11,7 @@
 """
 
 import os
+
 import aquarius.config as CF
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -40,6 +41,7 @@
     'rest_framework',
     'health_check',
     'transformer',
+    'asterism',
 ]
 
 MIDDLEWARE = [

aquarius/urls.py

@@ -13,20 +13,24 @@
     1. Import the include() function: from django.urls import include, path
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
-from django.contrib import admin
 from django.conf.urls import url
-from django.urls import include, re_path
-from transformer.models import Package
-from transformer.views import *
+from django.contrib import admin
+from django.urls import include
 from rest_framework import routers
 from rest_framework.schemas import get_schema_view
+from transformer.views import (AccessionUpdateRequestView, PackageViewSet,
+                               ProcessAccessionsView,
+                               ProcessDigitalObjectsView,
+                               ProcessGroupingComponentsView,
+                               ProcessTransferComponentsView,
+                               TransferUpdateRequestView)
 
 router = routers.DefaultRouter()
 router.register(r'packages', PackageViewSet, 'package')
 
 schema_view = get_schema_view(
-      title="Aquarius API",
-      description="Endpoints for Aquarius microservice application."
+    title="Aquarius API",
+    description="Endpoints for Aquarius microservice application."
 )
 
 urlpatterns = [

fixtures/data/c33ff643-0f1b-4fa8-9105-08d2e6a9e0d5.json

@@ -1,6 +1,6 @@
 {
-    "uri": "http://dev.fedora.marist.edu:8000/fedora/rest/c33ff643-0f1b-4fa8-9105-08d2e6a9e0d5",
-    "identifier": "286e977d-ff33-4345-a532-4d6d85c25b3a",
+    "uri": "http://fedora.dev.rockarch.org:8080/fedora/rest/fdec6c9b-f1c7-4d04-a5a3-68e018e4d66a",
+    "identifier": "69ef97fe-c837-418a-9c3c-7246a4a36e5b",
     "package_type": "dip",
     "origin": "aurora"
 }

fixtures/data/d33ff643-0f1b-4fa8-9105-08d2e6a9e0d5.json

@@ -1,6 +1,6 @@
 {
     "uri": "http://dev.fedora.marist.edu:8000/fedora/rest/d33ff643-0f1b-4fa8-9105-08d2e6a9e0d5",
-    "identifier": "286e977d-ff33-4345-a532-4d6d85c25b3a",
+    "identifier": "69ef97fe-c837-418a-9c3c-7246a4a36e5b",
     "package_type": "aip",
     "origin": "digitization",
     "archivesspace_uri": "/repositories/2/archival_objects/170"

fixtures/data/e33ff643-0f1b-4fa8-9105-08d2e6a9e0d5.json

@@ -1,6 +1,6 @@
 {
     "uri": "http://dev.fedora.marist.edu:8000/fedora/rest/e33ff643-0f1b-4fa8-9105-08d2e6a9e0d5",
-    "identifier": "abcbcf60-d197-4c54-bf67-42f80b10f190",
+    "identifier": "69ef97fe-c837-418a-9c3c-7246a4a36e5b",
     "package_type": "aip",
     "origin": "legacy_digital",
     "archivesspace_uri": "/repositories/2/archival_objects/170"

fixtures/data/f33ff643-0f1b-4fa8-9105-08d2e6a9e0d5.json

@@ -1,5 +1,5 @@
 {
     "uri": "http://dev.fedora.marist.edu:8000/fedora/rest/f33ff643-0f1b-4fa8-9105-08d2e6a9e0d5",
-    "identifier": "ea245504-ea90-404e-b515-86cc53c4957c",
+    "identifier": "69ef97fe-c837-418a-9c3c-7246a4a36e5b",
     "package_type": "dip"
 }

requirements.txt

@@ -1,15 +1,15 @@
 ArchivesSnake==0.8.2
+asterism==0.5.2
 attrs==19.3.0
 boltons==20.0.0
 certifi==2019.9.11
 chardet==3.0.4
 click==7.0
 clinner==1.12.3
 colorlog==4.0.2
-Django==2.2.8
+Django==2.2.10
 djangorestframework==3.10.3
 ElectronBonder==0.5
-git+https://github.com/RockefellerArchiveCenter/[email protected]#egg=asterism
 gitdb2==2.0.6
 GitPython==3.0.4
 health-check==3.4.1
@@ -21,6 +21,7 @@ jsonschema==3.2.0
 MarkupSafe==1.1.1
 more-itertools==7.2.0
 multidict==4.5.2
+odin==1.5.1
 psycopg2-binary==2.8.4
 pytz==2019.3
 PyYAML==5.1.2

transformer/admin.py

@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.

transformer/clients.py

@@ -1,13 +1,20 @@
+import json
+from datetime import date
+
 from asnake.client import ASnakeClient
 from electronbonder.client import ElectronBond
-from datetime import date
-import json
 
 
-class ArchivesSpaceClientError(Exception): pass
-class ArchivesSpaceClientAccessionNumberError(Exception): pass
-class UrsaMajorClientError(Exception): pass
-class AuroraClientError(Exception): pass
+class ArchivesSpaceClientError(Exception):
+    pass
+
+
+class UrsaMajorClientError(Exception):
+    pass
+
+
+class AuroraClientError(Exception):
+    pass
 
 
 class ArchivesSpaceClient(object):
@@ -19,12 +26,12 @@ def __init__(self, baseurl, username, password, repo_id):
         if not self.client.authorize():
             raise ArchivesSpaceClientError("Couldn't authenticate user credentials for ArchivesSpace")
         self.TYPE_LIST = {
-            'family': ['agent_family', 'agents/families'],
-            'organization': ['agent_corporate_entity', 'agents/corporate_entities'],
-            'person': ['agent_person', 'agents/people'],
-            'component': ['archival_object', 'repositories/{repo_id}/archival_objects'.format(repo_id=self.repo_id)],
-            'accession': ['accession', 'repositories/{repo_id}/accessions'.format(repo_id=self.repo_id)],
-            'digital object': ['digital_objects', 'repositories/{repo_id}/digital_objects'.format(repo_id=self.repo_id)]
+            "family": ["agent_family", "agents/families"],
+            "organization": ["agent_corporate_entity", "agents/corporate_entities"],
+            "person": ["agent_person", "agents/people"],
+            "component": ["archival_object", "repositories/{repo_id}/archival_objects".format(repo_id=self.repo_id)],
+            "accession": ["accession", "repositories/{repo_id}/accessions".format(repo_id=self.repo_id)],
+            "digital object": ["digital_objects", "repositories/{repo_id}/digital_objects".format(repo_id=self.repo_id)]
         }
 
     def send_request(self, method, url, data=None, **kwargs):
@@ -33,18 +40,22 @@ def send_request(self, method, url, data=None, **kwargs):
         if r.status_code == 200:
             return r.json()
         else:
-            if r.json()['error'].get('id_0'):
-                raise ArchivesSpaceClientAccessionNumberError(r.json()['error'])
-            raise ArchivesSpaceClientError('Error sending {} request to {}: {}'.format(method, url, r.json()['error']))
+            if r.json()["error"].get("id_0"):
+                """Account for indexing delays by bumping up to the next accession number."""
+                id_1 = int(data["id_1"])
+                id_1 += 1
+                data["id_1"] = str(id_1).zfill(3)
+                return self.create(data, "accession")
+            raise ArchivesSpaceClientError("Error sending {} request to {}: {}".format(method, url, r.json()["error"]))
 
     def retrieve(self, url, **kwargs):
-        return self.send_request('get', url, **kwargs)
+        return self.send_request("get", url, **kwargs)
 
     def create(self, data, type, **kwargs):
-        return self.send_request('post', self.TYPE_LIST[type][1], data, **kwargs)
+        return self.send_request("post", self.TYPE_LIST[type][1], data, **kwargs)
 
     def update(self, uri, data, **kwargs):
-        return self.send_request('post', uri, data, **kwargs)
+        return self.send_request("post", uri, data, **kwargs)
 
     def get_or_create(self, type, field, value, last_updated, consumer_data):
         """
@@ -55,17 +66,17 @@ def get_or_create(self, type, field, value, last_updated, consumer_data):
         endpoint = self.TYPE_LIST[type][1]
         query = json.dumps({"query": {"field": field, "value": value, "jsonmodel_type": "field_query"}})
         try:
-            r = self.client.get('search', params={"page": 1, "type[]": model_type, "aq": query}).json()
-            if len(r['results']) == 0:
-                r = self.client.get(endpoint, params={"all_ids": True, "modified_since": last_updated-120}).json()
+            r = self.client.get("search", params={"page": 1, "type[]": model_type, "aq": query}).json()
+            if len(r["results"]) == 0:
+                r = self.client.get(endpoint, params={"all_ids": True, "modified_since": last_updated - 120}).json()
                 for ref in r:
-                    r = self.client.get('{}/{}'.format(endpoint, ref)).json()
+                    r = self.client.get("{}/{}".format(endpoint, ref)).json()
                     if r[field] == str(value):
-                        return r['uri']
-                return self.create(consumer_data, type)
-            return r['results'][0]['uri']
+                        return r["uri"]
+                return self.create(consumer_data, type).get("uri")
+            return r["results"][0]["uri"]
         except Exception as e:
-            raise ArchivesSpaceClientError('Error finding or creating object in ArchivesSpace: {}'.format(e))
+            raise ArchivesSpaceClientError("Error finding or creating object in ArchivesSpace: {}".format(e))
 
     def next_accession_number(self):
         """
@@ -78,17 +89,16 @@ def next_accession_number(self):
         current_year = str(date.today().year)
         try:
             query = json.dumps({"query": {"field": "four_part_id", "value": current_year, "jsonmodel_type": "field_query"}})
-            r = self.client.get('search', params={"page": 1, "type[]": "accession", "sort": "identifier desc", "aq": query}).json()
-            number = '001'
-            if r.get('total_hits') >= 1:
-                if r['results'][0]['identifier'].split("-")[0] == current_year:
-                    id_1 = int(r['results'][0]['identifier'].split("-")[1])
+            r = self.client.get("search", params={"page": 1, "type[]": "accession", "sort": "identifier desc", "aq": query}).json()
+            number = "1"
+            if r.get("total_hits") >= 1:
+                if r["results"][0]["identifier"].split("-")[0] == current_year:
+                    id_1 = int(r["results"][0]["identifier"].split("-")[1])
                     id_1 += 1
-                    updated = str(id_1).zfill(3)
-                    number = updated
-            return [current_year, number]
+                    number = str(id_1).zfill(3)
+            return ":".join([current_year, number.zfill(3)])
         except Exception as e:
-            raise ArchivesSpaceClientError('Error retrieving next accession number from ArchivesSpace: {}'.format(e))
+            raise ArchivesSpaceClientError("Error retrieving next accession number from ArchivesSpace: {}".format(e))
 
 
 class UrsaMajorClient(object):
@@ -105,10 +115,10 @@ def send_request(self, method, url, data=None, **kwargs):
             raise UrsaMajorClientError("Error sending {} request to {}: {}".format(method, url, e))
 
     def retrieve(self, url, *args, **kwargs):
-        return self.send_request('get', url, **kwargs)
+        return self.send_request("get", url, **kwargs)
 
     def update(self, url, data, **kwargs):
-        return self.send_request('put', url, data, headers={"Content-Type":"application/json"}, **kwargs)
+        return self.send_request("put", url, data, headers={"Content-Type": "application/json"}, **kwargs)
 
     def retrieve_paged(self, url, **kwargs):
         try:
@@ -121,24 +131,25 @@ def find_bag_by_id(self, identifier, **kwargs):
         """Finds a bag by its id."""
         try:
             bag_resp = self.client.get("bags/", params={"id": identifier}).json()
-            count = bag_resp.get('count')
+            count = bag_resp.get("count")
             if count != 1:
                 raise UrsaMajorClientError("Found {} bags matching id {}, expected 1".format(count, identifier))
-            bag_url = bag_resp.get('results')[0]['url']
-            return self.send_request('get', bag_url)
+            bag_url = bag_resp.get("results")[0]["url"]
+            return self.send_request("get", bag_url)
         except Exception as e:
             raise UrsaMajorClientError("Error finding bag by id: {}".format(e))
 
 
 class AuroraClient:
+    """Client to update data in Aurora."""
 
     def __init__(self, baseurl, username, password):
         self.client = ElectronBond(baseurl=baseurl, username=username, password=password)
         if not self.client.authorize():
             raise AuroraClientError("Could not authorize {} in Aurora".format(username))
 
     def update(self, url, data, **kwargs):
-        resp = self.client.put(url, data=json.dumps(data), headers={"Content-Type":"application/json"}, **kwargs)
+        resp = self.client.put(url, data=json.dumps(data), headers={"Content-Type": "application/json"}, **kwargs)
         if resp.status_code == 200:
             return resp.json()
         else: