Merge pull request #11130 from Vlad0n20/feature/ENG-7839

[ENG-7839] [ENG-7766] Add celery task to update Datacite metadata + Use new GV API endpoint to retrieve node's verified links + Update unit tests

Author: cslzchen

admin_tests/nodes/test_views.py

@@ -505,6 +505,7 @@ def setUp(self):
         self.user = AuthUserFactory()
         self.node = ProjectFactory(creator=self.user)
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_request_approval_is_approved(self):
         now = timezone.now()
         self.approval = RegistrationApprovalFactory(

admin_tests/registrations/test_registrations.py

@@ -92,12 +92,14 @@ def test_embargoed_registration_from_changed_to_private_project_spam_ham(self, e
         embargoed_registration_from_changed_to_private_project.confirm_ham(save=True)
         assert not embargoed_registration_from_changed_to_private_project.is_public
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_public_registration_from_public_project_spam_ham(self, superuser, public_registration_from_public_project):
         public_registration_from_public_project.confirm_spam(save=True)
         assert not public_registration_from_public_project.is_public
         public_registration_from_public_project.confirm_ham(save=True)
         assert public_registration_from_public_project.is_public
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_public_registration_from_private_project_spam_ham(self, superuser, public_registration_from_private_project):
         public_registration_from_private_project.confirm_spam(save=True)
         assert not public_registration_from_private_project.is_public
@@ -110,18 +112,21 @@ def test_private_registration_from_private_project_spam_ham(self, superuser, pri
         private_registration_from_public_project.confirm_ham(save=True)
         assert not private_registration_from_public_project.is_public
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_public_registration_from_changed_to_public_project_spam_ham(self, superuser, public_registration_from_changed_to_public_project):
         public_registration_from_changed_to_public_project.confirm_spam(save=True)
         assert not public_registration_from_changed_to_public_project.is_public
         public_registration_from_changed_to_public_project.confirm_ham(save=True)
         assert public_registration_from_changed_to_public_project.is_public
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_public_registration_from_changed_to_private_project_spam_ham(self, superuser, public_registration_from_changed_to_private_project):
         public_registration_from_changed_to_private_project.confirm_spam(save=True)
         assert not public_registration_from_changed_to_private_project.is_public
         public_registration_from_changed_to_private_project.confirm_ham(save=True)
         assert public_registration_from_changed_to_private_project.is_public
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_unapproved_registration_task(self, embargoed_registration_from_changed_to_public_project):
         embargoed_registration_from_changed_to_public_project.registration_approval.state = 'unapproved'
         embargoed_registration_from_changed_to_public_project.registration_approval.initiation_date -= timedelta(3)
@@ -131,6 +136,7 @@ def test_unapproved_registration_task(self, embargoed_registration_from_changed_
         embargoed_registration_from_changed_to_public_project.registration_approval.refresh_from_db()
         assert embargoed_registration_from_changed_to_public_project.registration_approval.state == 'approved'
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_unapproved_registration_task_after_spam(self, embargoed_registration_from_changed_to_public_project):
         embargoed_registration_from_changed_to_public_project.registration_approval.state = 'unapproved'
         embargoed_registration_from_changed_to_public_project.registration_approval.initiation_date -= timedelta(3)
@@ -141,6 +147,7 @@ def test_unapproved_registration_task_after_spam(self, embargoed_registration_fr
         embargoed_registration_from_changed_to_public_project.registration_approval.refresh_from_db()
         assert embargoed_registration_from_changed_to_public_project.registration_approval.state == 'unapproved'
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_unapproved_registration_task_after_spam_ham(self, embargoed_registration_from_changed_to_public_project):
         embargoed_registration_from_changed_to_public_project.registration_approval.state = 'unapproved'
         embargoed_registration_from_changed_to_public_project.registration_approval.initiation_date -= timedelta(3)

api_tests/identifiers/managment_commands/test_sync_dois.py

@@ -53,6 +53,7 @@ def preprint_identifier(self, preprint):
         identifier.save(update_modified=False)
         return identifier
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     @pytest.mark.enable_enqueue_task
     def test_doi_synced_datacite(self, app, registration, registration_identifier, mock_datacite):
         assert registration_identifier.modified.date() < datetime.datetime.now().date()
@@ -66,6 +67,7 @@ def test_doi_synced_datacite(self, app, registration, registration_identifier, m
         registration_identifier.reload()
         assert registration_identifier.modified.date() == datetime.datetime.now().date()
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     @pytest.mark.enable_enqueue_task
     def test_doi_synced_crossref(self, app, preprint_identifier, mock_crossref):
         assert preprint_identifier.modified.date() < datetime.datetime.now().date()
@@ -77,6 +79,7 @@ def test_doi_synced_crossref(self, app, preprint_identifier, mock_crossref):
         preprint_identifier.reload()
         assert preprint_identifier.modified.date() == datetime.datetime.now().date()
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     @pytest.mark.enable_enqueue_task
     def test_doi_sync_private(self, app, registration_private, registration_identifier, mock_datacite):
 
@@ -91,6 +94,7 @@ def test_doi_sync_private(self, app, registration_private, registration_identifi
         assert registration_identifier.modified.date() < datetime.datetime.now().date()
         assert registration_identifier.modified.date() < datetime.datetime.now().date()
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     @pytest.mark.enable_enqueue_task
     def test_doi_sync_public_only(self, app, registration_private, registration_identifier, mock_datacite):
         call_command('sync_doi_metadata', f'-m={datetime.datetime.now()}')

api_tests/identifiers/views/test_identifier_list.py

@@ -475,6 +475,7 @@ def ark_payload(self):
     def client(self, resource):
         return DataCiteClient(resource)
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     @responses.activate
     def test_create_identifier(self, app, resource, client, identifier_url, identifier_payload, user,
             write_contributor, read_contributor, ark_payload):

api_tests/registrations/views/test_registration_detail.py

@@ -335,6 +335,8 @@ def license_cc0(self):
 @pytest.mark.django_db
 @pytest.mark.enable_implicit_clean
 class TestRegistrationUpdate(TestRegistrationUpdateTestCase):
+
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_update_registration(
             self, app, user, read_only_contributor,
             read_write_contributor, public_registration,
@@ -415,6 +417,7 @@ def test_update_registration(
             expect_errors=True)
         assert res.status_code == 403
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_fields(
             self, app, user, public_registration,
             private_registration, public_url, institution_one,
@@ -561,6 +564,7 @@ def test_fields(
             expect_errors=True)
         assert res.status_code == 400
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_turning_private_registrations_public(
             self, app, user, make_payload):
         private_project = ProjectFactory(creator=user, is_public=False)
@@ -776,6 +780,7 @@ def test_initiate_withdrawal_success(self, mock_send_mail, app, user, public_reg
         assert public_registration.registered_from.logs.first().action == 'retraction_initiated'
         assert mock_send_mail.called
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_initiate_withdrawal_with_embargo_ends_embargo(
             self, app, user, public_project, public_registration, public_url, public_payload):
         public_registration.embargo_registration(
@@ -934,6 +939,7 @@ def new_tag_payload_withdrawn(self, registration_withdrawn):
             }
         }
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_registration_tags(
             self, app, registration_public, registration_private,
             url_registration_public, url_registration_private,

api_tests/registrations/views/test_registrations_childrens_list.py

@@ -68,7 +68,15 @@ def test_registrations_children_list(self, user, app, registration_with_children
         assert component_one._id in ids
         assert component_two._id in ids
 
-    def test_return_registrations_list_no_auth_approved(self, user, app, registration_with_children_approved, registration_with_children_approved_url):
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
+    def test_return_registrations_list_no_auth_approved(
+        self,
+        user,
+        app,
+        registration_with_children_approved,
+        registration_with_children_approved_url
+    ):
+
         component_one, component_two, component_three, component_four = registration_with_children_approved.nodes
 
         res = app.get(registration_with_children_approved_url)

api_tests/registrations/views/test_withdrawn_registrations.py

@@ -224,6 +224,7 @@ def test_field_specific_related_counts_retrieved_if_visible_field_on_withdrawn_r
         assert res.status_code == 200
         assert res.json['data']['relationships']['contributors']['links']['related']['meta']['count'] == 1
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_child_inherits_withdrawal_justification_and_date_withdrawn(
             self, app, user, withdrawn_registration_with_child, registration_with_child):
 

api_tests/registries_moderation/test_submissions.py

@@ -312,6 +312,7 @@ def test_registries_moderation_permission_log(self, app, registration_log_url, r
         resp = app.get(registration_log_url, auth=moderator.auth)
         assert resp.status_code == 200
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_registries_moderation_post_accept(self, app, registration, moderator, registration_actions_url, actions_payload_base, reg_creator):
         registration.require_approval(user=registration.creator)
         registration.registration_approval.accept()
@@ -378,6 +379,7 @@ def test_registries_moderation_post_embargo_reject(self, app, embargo_registrati
         embargo_registration.refresh_from_db()
         assert embargo_registration.moderation_state == RegistrationModerationStates.REJECTED.db_name
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_registries_moderation_post_withdraw_accept(self, app, retract_registration, moderator, retract_registration_actions_url, actions_payload_base, provider):
         retract_registration.sanction.accept()
         retract_registration.refresh_from_db()
@@ -408,6 +410,7 @@ def test_registries_moderation_post_withdraw_reject(self, app, retract_registrat
         retract_registration.refresh_from_db()
         assert retract_registration.moderation_state == RegistrationModerationStates.ACCEPTED.db_name
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_registries_moderation_post_force_withdraw(self, app, registration, moderator, registration_actions_url, actions_payload_base, provider, reg_creator):
         registration.require_approval(user=registration.creator)
         registration.registration_approval.accept()
@@ -477,6 +480,7 @@ def test_registries_moderation_post_embargo_admin_cant_accept(self, app, embargo
         resp = app.post_json_api(embargo_registration_actions_url, actions_payload_base, auth=reg_creator.auth, expect_errors=True)
         assert resp.status_code == 403
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_registries_moderation_post_admin_cant_force_withdraw(self, app, registration, moderator, registration_actions_url, actions_payload_base, provider, reg_creator):
         registration.require_approval(user=registration.creator)
 
@@ -509,6 +513,7 @@ def test_registries_moderation_post_admin_cant_force_withdraw(self, app, registr
             RegistrationModerationTriggers.REJECT_SUBMISSION
         ]
     )
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_post_submission_action_persists_comment(self, app, registration, moderator, registration_actions_url, actions_payload_base, moderator_trigger):
         assert registration.actions.count() == 0
         registration.require_approval(user=registration.creator)
@@ -692,6 +697,7 @@ def test_public_project_with_embargo_is_private_after_spam_and_ham_and_moderatio
         assert registration.moderation_state == RegistrationModerationStates.EMBARGO.db_name
         assert registration.is_public is False
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_private_project_without_embargo_is_public_after_spam_and_ham_and_moderation_approval(self, app, registration, moderator, registration_actions_url, actions_payload_base, provider):
         user = AuthUserFactory()
         user.is_staff = True
@@ -728,6 +734,7 @@ def test_private_project_without_embargo_is_public_after_spam_and_ham_and_modera
         assert registration.moderation_state == RegistrationModerationStates.ACCEPTED.db_name
         assert registration.is_public is True
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_public_project_without_embargo_is_public_after_spam_and_ham_and_moderation_approval(self, app, registration, moderator, registration_actions_url, actions_payload_base, provider):
         user = AuthUserFactory()
         user.is_staff = True

api_tests/search/serializers/test_serializers.py

@@ -16,6 +16,7 @@
 @pytest.mark.django_db
 class TestSearchSerializer:
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_search_serializer_mixed_model(self):
 
         user = AuthUserFactory()

api_tests/search/views/test_views.py

@@ -566,6 +566,7 @@ def registration_private(self, project_private, schema):
             registration_private.update_search()
             return registration_private
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_search_registrations(
             self, app, url_registration_search, user, user_one, user_two,
             registration, registration_public, registration_private):

conftest.py

@@ -357,3 +357,17 @@ def helpful_thing(self):
     ```
     """
     yield from rolledback_transaction('function_transaction')
+
+@pytest.fixture
+def mock_gravy_valet_get_verified_links():
+    """This fixture is used to mock a GV request which is made during node's identifier update. More specifically, when
+    the tree walker in datacite metadata building process asks GV for verified links. As a result, this request must be
+    mocked in many tests. The following decoration can be applied to either a test class or individual test methods.
+
+    ```
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
+    ```
+    """
+    with mock.patch('osf.external.gravy_valet.translations.get_verified_links') as mock_get_verified_links:
+        mock_get_verified_links.return_value = []
+        yield mock_get_verified_links

osf/external/gravy_valet/request_helpers.py

@@ -247,10 +247,9 @@ def iterate_gv_results(
         auth=auth
     )
 
-    if not response and raise_on_error:
-        raise GVException
-
     if not response:
+        if raise_on_error:
+            raise GVException
         return
 
     response_json = response.json()

osf/metadata/serializers/datacite/datacite_tree_walker.py

@@ -6,6 +6,7 @@
 
 import rdflib
 
+from framework import sentry
 from osf.exceptions import MetadataSerializationError
 from osf.metadata import gather
 from osf.metadata.rdfutils import (
@@ -113,7 +114,7 @@ def walk(self, doi_override=None):
         self._visit_rights(self.root)
         self._visit_descriptions(self.root, self.basket.focus.iri)
         self._visit_funding_references(self.root)
-        self._visit_related(self.root)
+        self._visit_related_and_verified_links(self.root)
 
     def _visit_identifier(self, parent_el, *, doi_override=None):
         if doi_override is None:
@@ -373,13 +374,16 @@ def _visit_related_identifier_and_item(self, identifier_parent_el, item_parent_e
         self._visit_publication_year(related_item_el, related_iri)
         self._visit_publisher(related_item_el, related_iri)
 
-    def _visit_related(self, parent_el):
+    def _visit_related_and_verified_links(self, parent_el):
         relation_pairs = set()
         for relation_iri, datacite_relation in RELATED_IDENTIFIER_TYPE_MAP.items():
             for related_iri in self.basket[relation_iri]:
                 relation_pairs.add((datacite_relation, related_iri))
+
         related_identifiers_el = self.visit(parent_el, 'relatedIdentifiers', is_list=True)
         related_items_el = self.visit(parent_el, 'relatedItems', is_list=True)
+
+        # First add regular related identifiers
         for datacite_relation, related_iri in sorted(relation_pairs):
             self._visit_related_identifier_and_item(
                 related_identifiers_el,
@@ -388,6 +392,35 @@ def _visit_related(self, parent_el):
                 datacite_relation,
             )
 
+        # Then add verified links to same relatedIdentifiers element
+        osf_item = self.basket.focus.dbmodel
+        from osf.models import AbstractNode
+
+        if isinstance(osf_item, AbstractNode):
+            gv_verified_link_list = osf_item.get_verified_links()
+            skipped_items = []
+            for item in gv_verified_link_list:
+                verified_link, resource_type = item.get('target_url', None), item.get('resource_type', None)
+                if not verified_link or not resource_type:
+                    logger.error(f'Must have both verified_link and resource_type: [item={item}]')
+                    skipped_items.append(f'Missing data: [link={verified_link}, type={resource_type}]')
+                    continue
+                if not smells_like_iri(verified_link):
+                    skipped_items.append(f'Invalid link: [link={verified_link}, type={resource_type}]')
+                    continue
+                self.visit(
+                    related_identifiers_el,
+                    'relatedIdentifier',
+                    text=verified_link,
+                    attrib={
+                        'relatedIdentifierType': 'URL',
+                        'relationType': 'IsReferencedBy',
+                        'resourceTypeGeneral': resource_type.title()
+                    }
+                )
+            if skipped_items:
+                sentry.log_message(f'Skipped items for node [{osf_item._id}]: {'; '.join(skipped_items)}. ')
+
     def _visit_name_identifiers(self, parent_el, agent_iri):
         for identifier in sorted(self.basket[agent_iri:DCTERMS.identifier]):
             identifier_type, identifier_value = self._identifier_type_and_value(identifier)

osf_tests/embargoes/test_embargoes.py

@@ -29,6 +29,7 @@ def registration(self, user):
         embargo.save()
         return embargo.registrations.last()
 
+    @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_request_early_termination_too_late(self, registration, user):
         """
         This is for an edge case test for where embargos are frozen and never expire when the user requests they be

osf_tests/metadata/expected_metadata_files/preprint_supplement.turtle

@@ -1,5 +1,6 @@
 @prefix osf: <https://osf.io/vocab/2022/> .
 @prefix skos: <http://www.w3.org/2004/02/skos/core#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
 
 <http://localhost:5000/w4ibb> osf:storageByteCount 1337 ;
     osf:storageRegion <http://localhost:8000/v2/regions/us/> .

osf_tests/metadata/expected_metadata_files/project_basic.datacite.json

@@ -63,6 +63,12 @@
       "relatedIdentifier": "11.pp/FK2osf.io/w4ibb_v1",
       "relatedIdentifierType": "DOI",
       "relationType": "IsSupplementTo"
+    },
+    {
+      "relatedIdentifier": "https://foo.bar",
+      "relatedIdentifierType": "URL",
+      "relationType": "IsReferencedBy",
+      "resourceTypeGeneral": "Other"
     }
   ],
   "relatedItems": [