diff --git a/admin_tests/nodes/test_views.py b/admin_tests/nodes/test_views.py index 9f978e75268..25904a611fb 100644 --- a/admin_tests/nodes/test_views.py +++ b/admin_tests/nodes/test_views.py @@ -546,6 +546,7 @@ def setUp(self): self.user = AuthUserFactory() self.node = ProjectFactory(creator=self.user) + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_request_approval_is_approved(self): now = timezone.now() self.approval = RegistrationApprovalFactory( diff --git a/admin_tests/registrations/test_registrations.py b/admin_tests/registrations/test_registrations.py index 5a7431c5977..708cb333098 100644 --- a/admin_tests/registrations/test_registrations.py +++ b/admin_tests/registrations/test_registrations.py @@ -92,12 +92,14 @@ def test_embargoed_registration_from_changed_to_private_project_spam_ham(self, e embargoed_registration_from_changed_to_private_project.confirm_ham(save=True) assert not embargoed_registration_from_changed_to_private_project.is_public + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_public_registration_from_public_project_spam_ham(self, superuser, public_registration_from_public_project): public_registration_from_public_project.confirm_spam(save=True) assert not public_registration_from_public_project.is_public public_registration_from_public_project.confirm_ham(save=True) assert public_registration_from_public_project.is_public + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_public_registration_from_private_project_spam_ham(self, superuser, public_registration_from_private_project): public_registration_from_private_project.confirm_spam(save=True) assert not public_registration_from_private_project.is_public @@ -110,18 +112,21 @@ def test_private_registration_from_private_project_spam_ham(self, superuser, pri private_registration_from_public_project.confirm_ham(save=True) assert not private_registration_from_public_project.is_public + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_public_registration_from_changed_to_public_project_spam_ham(self, superuser, public_registration_from_changed_to_public_project): public_registration_from_changed_to_public_project.confirm_spam(save=True) assert not public_registration_from_changed_to_public_project.is_public public_registration_from_changed_to_public_project.confirm_ham(save=True) assert public_registration_from_changed_to_public_project.is_public + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_public_registration_from_changed_to_private_project_spam_ham(self, superuser, public_registration_from_changed_to_private_project): public_registration_from_changed_to_private_project.confirm_spam(save=True) assert not public_registration_from_changed_to_private_project.is_public public_registration_from_changed_to_private_project.confirm_ham(save=True) assert public_registration_from_changed_to_private_project.is_public + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_unapproved_registration_task(self, embargoed_registration_from_changed_to_public_project): embargoed_registration_from_changed_to_public_project.registration_approval.state = 'unapproved' embargoed_registration_from_changed_to_public_project.registration_approval.initiation_date -= timedelta(3) @@ -131,6 +136,7 @@ def test_unapproved_registration_task(self, embargoed_registration_from_changed_ embargoed_registration_from_changed_to_public_project.registration_approval.refresh_from_db() assert embargoed_registration_from_changed_to_public_project.registration_approval.state == 'approved' + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_unapproved_registration_task_after_spam(self, embargoed_registration_from_changed_to_public_project): embargoed_registration_from_changed_to_public_project.registration_approval.state = 'unapproved' embargoed_registration_from_changed_to_public_project.registration_approval.initiation_date -= timedelta(3) @@ -141,6 +147,7 @@ def test_unapproved_registration_task_after_spam(self, embargoed_registration_fr embargoed_registration_from_changed_to_public_project.registration_approval.refresh_from_db() assert embargoed_registration_from_changed_to_public_project.registration_approval.state == 'unapproved' + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_unapproved_registration_task_after_spam_ham(self, embargoed_registration_from_changed_to_public_project): embargoed_registration_from_changed_to_public_project.registration_approval.state = 'unapproved' embargoed_registration_from_changed_to_public_project.registration_approval.initiation_date -= timedelta(3) diff --git a/api/share/utils.py b/api/share/utils.py index 5083220ce7f..438f2c738a6 100644 --- a/api/share/utils.py +++ b/api/share/utils.py @@ -12,6 +12,7 @@ from framework.celery_tasks.handlers import enqueue_task from framework.encryption import ensure_bytes from framework.sentry import log_exception +from osf.external.gravy_valet.exceptions import GVException from osf.metadata.osf_gathering import ( OsfmapPartition, pls_get_magic_metadata_basket, @@ -78,15 +79,20 @@ def task__update_share(self, guid: str, is_backfill=False, osfmap_partition_name raise ValueError(f'unknown osfguid "{guid}"') _resource = _osfid_instance.referent _is_deletion = _should_delete_indexcard(_resource) - _response = ( - pls_delete_trove_record(_resource, osfmap_partition=_osfmap_partition) - if _is_deletion - else pls_send_trove_record( - _resource, - is_backfill=is_backfill, - osfmap_partition=_osfmap_partition, + try: + _response = ( + pls_delete_trove_record(_resource, osfmap_partition=_osfmap_partition) + if _is_deletion + else pls_send_trove_record( + _resource, + is_backfill=is_backfill, + osfmap_partition=_osfmap_partition, + ) ) - ) + except GVException as e: + log_exception(e) + raise self.retry(exc=e) + try: _response.raise_for_status() except Exception as e: diff --git a/api_tests/identifiers/managment_commands/test_sync_dois.py b/api_tests/identifiers/managment_commands/test_sync_dois.py index e72893dc7dd..07b9c2b0ee8 100644 --- a/api_tests/identifiers/managment_commands/test_sync_dois.py +++ b/api_tests/identifiers/managment_commands/test_sync_dois.py @@ -53,6 +53,7 @@ def preprint_identifier(self, preprint): identifier.save(update_modified=False) return identifier + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') @pytest.mark.enable_enqueue_task def test_doi_synced_datacite(self, app, registration, registration_identifier, mock_datacite): assert registration_identifier.modified.date() < datetime.datetime.now().date() @@ -66,6 +67,7 @@ def test_doi_synced_datacite(self, app, registration, registration_identifier, m registration_identifier.reload() assert registration_identifier.modified.date() == datetime.datetime.now().date() + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') @pytest.mark.enable_enqueue_task def test_doi_synced_crossref(self, app, preprint_identifier, mock_crossref): assert preprint_identifier.modified.date() < datetime.datetime.now().date() @@ -77,6 +79,7 @@ def test_doi_synced_crossref(self, app, preprint_identifier, mock_crossref): preprint_identifier.reload() assert preprint_identifier.modified.date() == datetime.datetime.now().date() + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') @pytest.mark.enable_enqueue_task def test_doi_sync_private(self, app, registration_private, registration_identifier, mock_datacite): @@ -91,6 +94,7 @@ def test_doi_sync_private(self, app, registration_private, registration_identifi assert registration_identifier.modified.date() < datetime.datetime.now().date() assert registration_identifier.modified.date() < datetime.datetime.now().date() + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') @pytest.mark.enable_enqueue_task def test_doi_sync_public_only(self, app, registration_private, registration_identifier, mock_datacite): call_command('sync_doi_metadata', f'-m={datetime.datetime.now()}') diff --git a/api_tests/identifiers/views/test_identifier_list.py b/api_tests/identifiers/views/test_identifier_list.py index 88d98fa27ce..29252ab414a 100644 --- a/api_tests/identifiers/views/test_identifier_list.py +++ b/api_tests/identifiers/views/test_identifier_list.py @@ -475,6 +475,7 @@ def ark_payload(self): def client(self, resource): return DataCiteClient(resource) + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') @responses.activate def test_create_identifier(self, app, resource, client, identifier_url, identifier_payload, user, write_contributor, read_contributor, ark_payload): diff --git a/api_tests/registrations/views/test_registration_detail.py b/api_tests/registrations/views/test_registration_detail.py index 808ddc6b98d..471ab87ac31 100644 --- a/api_tests/registrations/views/test_registration_detail.py +++ b/api_tests/registrations/views/test_registration_detail.py @@ -335,6 +335,8 @@ def license_cc0(self): @pytest.mark.django_db @pytest.mark.enable_implicit_clean class TestRegistrationUpdate(TestRegistrationUpdateTestCase): + + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_update_registration( self, app, user, read_only_contributor, read_write_contributor, public_registration, @@ -415,6 +417,7 @@ def test_update_registration( expect_errors=True) assert res.status_code == 403 + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_fields( self, app, user, public_registration, private_registration, public_url, institution_one, @@ -561,6 +564,7 @@ def test_fields( expect_errors=True) assert res.status_code == 400 + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_turning_private_registrations_public( self, app, user, make_payload): private_project = ProjectFactory(creator=user, is_public=False) @@ -776,6 +780,7 @@ def test_initiate_withdrawal_success(self, mock_send_mail, app, user, public_reg assert public_registration.registered_from.logs.first().action == 'retraction_initiated' assert mock_send_mail.called + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_initiate_withdrawal_with_embargo_ends_embargo( self, app, user, public_project, public_registration, public_url, public_payload): public_registration.embargo_registration( @@ -934,6 +939,7 @@ def new_tag_payload_withdrawn(self, registration_withdrawn): } } + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_registration_tags( self, app, registration_public, registration_private, url_registration_public, url_registration_private, diff --git a/api_tests/registrations/views/test_registrations_childrens_list.py b/api_tests/registrations/views/test_registrations_childrens_list.py index 67ff993fa2a..37bda001d49 100644 --- a/api_tests/registrations/views/test_registrations_childrens_list.py +++ b/api_tests/registrations/views/test_registrations_childrens_list.py @@ -68,7 +68,15 @@ def test_registrations_children_list(self, user, app, registration_with_children assert component_one._id in ids assert component_two._id in ids - def test_return_registrations_list_no_auth_approved(self, user, app, registration_with_children_approved, registration_with_children_approved_url): + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') + def test_return_registrations_list_no_auth_approved( + self, + user, + app, + registration_with_children_approved, + registration_with_children_approved_url + ): + component_one, component_two, component_three, component_four = registration_with_children_approved.nodes res = app.get(registration_with_children_approved_url) diff --git a/api_tests/registrations/views/test_withdrawn_registrations.py b/api_tests/registrations/views/test_withdrawn_registrations.py index 9cf582e7889..f78cdfb99d2 100644 --- a/api_tests/registrations/views/test_withdrawn_registrations.py +++ b/api_tests/registrations/views/test_withdrawn_registrations.py @@ -224,6 +224,7 @@ def test_field_specific_related_counts_retrieved_if_visible_field_on_withdrawn_r assert res.status_code == 200 assert res.json['data']['relationships']['contributors']['links']['related']['meta']['count'] == 1 + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_child_inherits_withdrawal_justification_and_date_withdrawn( self, app, user, withdrawn_registration_with_child, registration_with_child): diff --git a/api_tests/registries_moderation/test_submissions.py b/api_tests/registries_moderation/test_submissions.py index feaf3141768..caec5fe1daf 100644 --- a/api_tests/registries_moderation/test_submissions.py +++ b/api_tests/registries_moderation/test_submissions.py @@ -312,6 +312,7 @@ def test_registries_moderation_permission_log(self, app, registration_log_url, r resp = app.get(registration_log_url, auth=moderator.auth) assert resp.status_code == 200 + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_registries_moderation_post_accept(self, app, registration, moderator, registration_actions_url, actions_payload_base, reg_creator): registration.require_approval(user=registration.creator) registration.registration_approval.accept() @@ -378,6 +379,7 @@ def test_registries_moderation_post_embargo_reject(self, app, embargo_registrati embargo_registration.refresh_from_db() assert embargo_registration.moderation_state == RegistrationModerationStates.REJECTED.db_name + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_registries_moderation_post_withdraw_accept(self, app, retract_registration, moderator, retract_registration_actions_url, actions_payload_base, provider): retract_registration.sanction.accept() retract_registration.refresh_from_db() @@ -408,6 +410,7 @@ def test_registries_moderation_post_withdraw_reject(self, app, retract_registrat retract_registration.refresh_from_db() assert retract_registration.moderation_state == RegistrationModerationStates.ACCEPTED.db_name + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_registries_moderation_post_force_withdraw(self, app, registration, moderator, registration_actions_url, actions_payload_base, provider, reg_creator): registration.require_approval(user=registration.creator) registration.registration_approval.accept() @@ -477,6 +480,7 @@ def test_registries_moderation_post_embargo_admin_cant_accept(self, app, embargo resp = app.post_json_api(embargo_registration_actions_url, actions_payload_base, auth=reg_creator.auth, expect_errors=True) assert resp.status_code == 403 + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_registries_moderation_post_admin_cant_force_withdraw(self, app, registration, moderator, registration_actions_url, actions_payload_base, provider, reg_creator): registration.require_approval(user=registration.creator) @@ -509,6 +513,7 @@ def test_registries_moderation_post_admin_cant_force_withdraw(self, app, registr RegistrationModerationTriggers.REJECT_SUBMISSION ] ) + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_post_submission_action_persists_comment(self, app, registration, moderator, registration_actions_url, actions_payload_base, moderator_trigger): assert registration.actions.count() == 0 registration.require_approval(user=registration.creator) @@ -692,6 +697,7 @@ def test_public_project_with_embargo_is_private_after_spam_and_ham_and_moderatio assert registration.moderation_state == RegistrationModerationStates.EMBARGO.db_name assert registration.is_public is False + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_private_project_without_embargo_is_public_after_spam_and_ham_and_moderation_approval(self, app, registration, moderator, registration_actions_url, actions_payload_base, provider): user = AuthUserFactory() user.is_staff = True @@ -728,6 +734,7 @@ def test_private_project_without_embargo_is_public_after_spam_and_ham_and_modera assert registration.moderation_state == RegistrationModerationStates.ACCEPTED.db_name assert registration.is_public is True + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_public_project_without_embargo_is_public_after_spam_and_ham_and_moderation_approval(self, app, registration, moderator, registration_actions_url, actions_payload_base, provider): user = AuthUserFactory() user.is_staff = True diff --git a/api_tests/search/serializers/test_serializers.py b/api_tests/search/serializers/test_serializers.py index 6fb9d0b8adf..92f267f77f9 100644 --- a/api_tests/search/serializers/test_serializers.py +++ b/api_tests/search/serializers/test_serializers.py @@ -16,6 +16,7 @@ @pytest.mark.django_db class TestSearchSerializer: + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_search_serializer_mixed_model(self): user = AuthUserFactory() diff --git a/api_tests/search/views/test_views.py b/api_tests/search/views/test_views.py index 675381668dc..5fff8411e61 100644 --- a/api_tests/search/views/test_views.py +++ b/api_tests/search/views/test_views.py @@ -566,6 +566,7 @@ def registration_private(self, project_private, schema): registration_private.update_search() return registration_private + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_search_registrations( self, app, url_registration_search, user, user_one, user_two, registration, registration_public, registration_private): diff --git a/api_tests/share/_utils.py b/api_tests/share/_utils.py index 4fde322fccc..9290f083f84 100644 --- a/api_tests/share/_utils.py +++ b/api_tests/share/_utils.py @@ -1,4 +1,5 @@ import contextlib +import re from urllib.parse import urlsplit from unittest import mock @@ -10,39 +11,48 @@ postcommit_celery_queue, postcommit_queue, ) -from website import settings as website_settings +from osf.models import Node, Preprint +from website import settings from api.share.utils import shtrove_ingest_url from osf.metadata.osf_gathering import OsfmapPartition +def gv_url(): + return fr'^{settings.GRAVYVALET_URL}/v1/configured-link-addons/\w*/verified-links' + + @contextlib.contextmanager def mock_share_responses(): ''' enable sending requests to shtrove with metadata updates, catch those requests in a yielded responses.RequestsMock ''' - with mock.patch.object(website_settings, 'SHARE_ENABLED', True): - with mock.patch.object(website_settings, 'SHARE_API_TOKEN', 'mock-api-token'): - with mock.patch.object(website_settings, 'USE_CELERY', False): # run tasks synchronously + with mock.patch.object(settings, 'SHARE_ENABLED', True): + with mock.patch.object(settings, 'SHARE_API_TOKEN', 'mock-api-token'): + with mock.patch.object(settings, 'USE_CELERY', False): # run tasks synchronously with responses.RequestsMock(assert_all_requests_are_fired=False) as _rsps: _ingest_url = shtrove_ingest_url() _rsps.add(responses.POST, _ingest_url, status=200) _rsps.add(responses.DELETE, _ingest_url, status=200) + _rsps.add(responses.GET, re.compile(gv_url()), status=200, body='{}') yield _rsps @contextlib.contextmanager def mock_update_share(): - with mock.patch.object(website_settings, 'SHARE_ENABLED', True): + with mock.patch.object(settings, 'SHARE_ENABLED', True): with mock.patch('api.share.utils._enqueue_update_share') as _mock_update_share: yield _mock_update_share @contextlib.contextmanager -def expect_ingest_request(mock_share_responses, osfguid, *, token=None, delete=False, count=1, error_response=False): +def expect_ingest_request(mock_share_responses, item, *, token=None, delete=False, count=1, error_response=False): + osfguid = item.get_guid()._id if isinstance(item, Preprint) else item._id mock_share_responses._calls.reset() yield _trove_main_count_per_item = 1 + expect_gv_call = isinstance(item, Node) and not delete + _gv_calls_number = 1 if expect_gv_call else 0 _trove_supplementary_count_per_item = ( 0 if (error_response or delete) @@ -51,20 +61,25 @@ def expect_ingest_request(mock_share_responses, osfguid, *, token=None, delete=F _total_count = count * ( _trove_main_count_per_item + _trove_supplementary_count_per_item + + _gv_calls_number ) assert len(mock_share_responses.calls) == _total_count, ( f'expected {_total_count} call(s), got {len(mock_share_responses.calls)}: {list(mock_share_responses.calls)}' ) _trove_ingest_calls = [] _trove_supp_ingest_calls = [] + _gv_links_calls = [] for _call in mock_share_responses.calls: if _call.request.url.startswith(shtrove_ingest_url()): if 'is_supplementary' in _call.request.url: _trove_supp_ingest_calls.append(_call) else: _trove_ingest_calls.append(_call) + elif _call.request.url.startswith(settings.GRAVYVALET_URL): + _gv_links_calls.append(_call) assert len(_trove_ingest_calls) == count assert len(_trove_supp_ingest_calls) == count * _trove_supplementary_count_per_item + assert len(_gv_links_calls) == (count if expect_gv_call else 0) for _call in _trove_ingest_calls: assert_ingest_request(_call.request, osfguid, token=token, delete=delete) for _call in _trove_supp_ingest_calls: @@ -83,10 +98,10 @@ def assert_ingest_request(request, expected_osfguid, *, token=None, delete=False else: assert request.method == 'POST' _focus_iri = _querydict['focus_iri'] - assert _focus_iri == f'{website_settings.DOMAIN}{expected_osfguid}' + assert _focus_iri == f'{settings.DOMAIN}{expected_osfguid}' _request_body = request.body.decode('utf-8') assert (_focus_iri in _request_body) or (supp and not _request_body.strip()) - _token = token or website_settings.SHARE_API_TOKEN + _token = token or settings.SHARE_API_TOKEN assert request.headers['Authorization'] == f'Bearer {_token}' @@ -96,7 +111,7 @@ def expect_preprint_ingest_request(mock_share_responses, preprint, *, delete=Fal # and postcommit-task handling (so on_preprint_updated actually runs) with expect_ingest_request( mock_share_responses, - preprint.get_guid()._id, + preprint, token=preprint.provider.access_token, delete=delete, count=count, diff --git a/api_tests/share/test_share_node.py b/api_tests/share/test_share_node.py index 791e7d0099a..f398c2247e7 100644 --- a/api_tests/share/test_share_node.py +++ b/api_tests/share/test_share_node.py @@ -99,11 +99,11 @@ def registration_outcome(self, registration): return o def test_update_node_share(self, mock_share_responses, node, user): - with expect_ingest_request(mock_share_responses, node._id): + with expect_ingest_request(mock_share_responses, node): on_node_updated(node._id, user._id, False, {'is_public'}) def test_update_registration_share(self, mock_share_responses, registration, user): - with expect_ingest_request(mock_share_responses, registration._id): + with expect_ingest_request(mock_share_responses, registration): on_node_updated(registration._id, user._id, False, {'is_public'}) def test_update_share_correctly_for_projects(self, mock_share_responses, node, user): @@ -125,7 +125,7 @@ def test_update_share_correctly_for_projects(self, mock_share_responses, node, u for i, case in enumerate(cases): for attr, value in case['attrs'].items(): setattr(node, attr, value) - with expect_ingest_request(mock_share_responses, node._id, delete=case['is_deleted']): + with expect_ingest_request(mock_share_responses, node, delete=case['is_deleted']): node.save() def test_update_share_correctly_for_registrations(self, mock_share_responses, registration, user): @@ -144,38 +144,38 @@ def test_update_share_correctly_for_registrations(self, mock_share_responses, re for i, case in enumerate(cases): for attr, value in case['attrs'].items(): setattr(registration, attr, value) - with expect_ingest_request(mock_share_responses, registration._id, delete=case['is_deleted']): + with expect_ingest_request(mock_share_responses, registration, delete=case['is_deleted']): registration.save() assert registration.is_registration def test_update_share_correctly_for_projects_with_qa_tags(self, mock_share_responses, node, user): - with expect_ingest_request(mock_share_responses, node._id, delete=True): + with expect_ingest_request(mock_share_responses, node, delete=True): node.add_tag(settings.DO_NOT_INDEX_LIST['tags'][0], auth=Auth(user)) - with expect_ingest_request(mock_share_responses, node._id, delete=False): + with expect_ingest_request(mock_share_responses, node, delete=False): node.remove_tag(settings.DO_NOT_INDEX_LIST['tags'][0], auth=Auth(user), save=True) def test_update_share_correctly_for_registrations_with_qa_tags(self, mock_share_responses, registration, user): - with expect_ingest_request(mock_share_responses, registration._id, delete=True): + with expect_ingest_request(mock_share_responses, registration, delete=True): registration.add_tag(settings.DO_NOT_INDEX_LIST['tags'][0], auth=Auth(user)) - with expect_ingest_request(mock_share_responses, registration._id): + with expect_ingest_request(mock_share_responses, registration): registration.remove_tag(settings.DO_NOT_INDEX_LIST['tags'][0], auth=Auth(user), save=True) def test_update_share_correctly_for_projects_with_qa_titles(self, mock_share_responses, node, user): node.title = settings.DO_NOT_INDEX_LIST['titles'][0] + ' arbitary text for test title.' node.save() - with expect_ingest_request(mock_share_responses, node._id, delete=True): + with expect_ingest_request(mock_share_responses, node, delete=True): on_node_updated(node._id, user._id, False, {'is_public'}) node.title = 'Not a qa title' - with expect_ingest_request(mock_share_responses, node._id): + with expect_ingest_request(mock_share_responses, node): node.save() assert node.title not in settings.DO_NOT_INDEX_LIST['titles'] def test_update_share_correctly_for_registrations_with_qa_titles(self, mock_share_responses, registration, user): registration.title = settings.DO_NOT_INDEX_LIST['titles'][0] + ' arbitary text for test title.' - with expect_ingest_request(mock_share_responses, registration._id, delete=True): + with expect_ingest_request(mock_share_responses, registration, delete=True): registration.save() registration.title = 'Not a qa title' - with expect_ingest_request(mock_share_responses, registration._id): + with expect_ingest_request(mock_share_responses, registration): registration.save() assert registration.title not in settings.DO_NOT_INDEX_LIST['titles'] @@ -189,18 +189,18 @@ def test_call_async_update_on_500_retry(self, mock_share_responses, node, user): """This is meant to simulate a temporary outage, so the retry mechanism should kick in and complete it.""" mock_share_responses.replace(responses.POST, shtrove_ingest_url(), status=500) mock_share_responses.add(responses.POST, shtrove_ingest_url(), status=200) - with expect_ingest_request(mock_share_responses, node._id, count=2): + with expect_ingest_request(mock_share_responses, node, count=2): on_node_updated(node._id, user._id, False, {'is_public'}) @mark.skip('Synchronous retries not supported if celery >=5.0') def test_call_async_update_on_500_failure(self, mock_share_responses, node, user): """This is meant to simulate a total outage, so the retry mechanism should try X number of times and quit.""" mock_share_responses.replace(responses.POST, shtrove_ingest_url(), status=500) - with expect_ingest_request(mock_share_responses, node._id, count=5): # tries five times + with expect_ingest_request(mock_share_responses, node, count=5): # tries five times on_node_updated(node._id, user._id, False, {'is_public'}) @mark.skip('Synchronous retries not supported if celery >=5.0') def test_no_call_async_update_on_400_failure(self, mock_share_responses, node, user): mock_share_responses.replace(responses.POST, shtrove_ingest_url(), status=400) - with expect_ingest_request(mock_share_responses, node._id): + with expect_ingest_request(mock_share_responses, node): on_node_updated(node._id, user._id, False, {'is_public'}) diff --git a/conftest.py b/conftest.py index 6f870093ed4..4ef8080ff85 100644 --- a/conftest.py +++ b/conftest.py @@ -357,3 +357,17 @@ def helpful_thing(self): ``` """ yield from rolledback_transaction('function_transaction') + +@pytest.fixture +def mock_gravy_valet_get_verified_links(): + """This fixture is used to mock a GV request which is made during node's identifier update. More specifically, when + the tree walker in datacite metadata building process asks GV for verified links. As a result, this request must be + mocked in many tests. The following decoration can be applied to either a test class or individual test methods. + + ``` + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') + ``` + """ + with mock.patch('osf.external.gravy_valet.translations.get_verified_links') as mock_get_verified_links: + mock_get_verified_links.return_value = [] + yield mock_get_verified_links diff --git a/framework/auth/__init__.py b/framework/auth/__init__.py index 73ca8dd06a8..9df43069936 100644 --- a/framework/auth/__init__.py +++ b/framework/auth/__init__.py @@ -35,6 +35,7 @@ def authenticate(user, response, user_updates=None): 'auth_user_username': user.username, 'auth_user_id': user._primary_key, 'auth_user_fullname': user.fullname, + 'user_reference_uri': user.get_semantic_iri(), } print_cas_log(f'Finalizing authentication - data updated: user=[{user._id}]', LogLevel.INFO) enqueue_task(update_user_from_activity.s(user._id, timezone.now().timestamp(), cas_login=True, updates=user_updates)) diff --git a/osf/__init__.py b/osf/__init__.py index e69de29bb2d..ca04eadc2c2 100644 --- a/osf/__init__.py +++ b/osf/__init__.py @@ -0,0 +1,2 @@ +from .tasks import log_gv_addon +__all__ = ['log_gv_addon'] diff --git a/osf/external/gravy_valet/exceptions.py b/osf/external/gravy_valet/exceptions.py new file mode 100644 index 00000000000..a1b11533084 --- /dev/null +++ b/osf/external/gravy_valet/exceptions.py @@ -0,0 +1,2 @@ +class GVException(Exception): + pass diff --git a/osf/external/gravy_valet/request_helpers.py b/osf/external/gravy_valet/request_helpers.py index 76fef67b6a5..3da2f81e4f8 100644 --- a/osf/external/gravy_valet/request_helpers.py +++ b/osf/external/gravy_valet/request_helpers.py @@ -9,6 +9,7 @@ from website import settings from . import auth_helpers +from .exceptions import GVException logger = logging.getLogger(__name__) @@ -17,7 +18,7 @@ # {{placeholder}} format allows f-string to return a formatable string ACCOUNT_ENDPOINT = f'{API_BASE}authorized-storage-accounts/{{pk}}' -ADDONS_ENDPOINT = f'{API_BASE}configured-storage-addons' +ADDONS_ENDPOINT = f'{API_BASE}configured-{{addon_type}}-addons' GENERIC_ADDONS_ENDPOINT = f'{API_BASE}{{addon_type}}' ADDON_ENDPOINT = f'{GENERIC_ADDONS_ENDPOINT}/{{pk}}' WB_CONFIG_ENDPOINT = f'{ADDON_ENDPOINT}/waterbutler-credentials' @@ -41,6 +42,7 @@ class AddonType(enum.StrEnum): STORAGE = enum.auto() CITATION = enum.auto() COMPUTING = enum.auto() + LINK = enum.auto() CITATION_ITEM_TYPE_ALIASES = { 'COLLECTION': 'folder', @@ -56,6 +58,13 @@ def get_account(gv_account_pk, requesting_user): # -> JSONAPIResultEntry params={'include': ACCOUNT_EXTERNAL_STORAGE_SERVICE_PATH}, ) +def get_verified_links(node_guid, requesting_user=None): + return iterate_gv_results( + ADDONS_ENDPOINT.format(addon_type=AddonType.LINK) + f'/{node_guid}/verified-links', + requesting_user=requesting_user, + raise_on_error=True + ) + def create_addon(requested_resource, requesting_user, attributes: dict, relationships: dict, addon_type: str): # -> JSONAPIResultEntry '''Return a JSONAPIResultEntry representing a known ConfiguredStorageAddon.''' @@ -225,7 +234,8 @@ def iterate_gv_results( requested_resource=None, request_method='GET', params: dict = None, - auth=None + auth=None, + raise_on_error: bool = False ): # -> typing.Iterator[JSONAPIResultEntry] '''Processes the result of a request to GravyValet list endpoint into a generator of JSONAPIResultEntires.''' response = _make_gv_request( @@ -236,7 +246,10 @@ def iterate_gv_results( params=params, auth=auth ) + if not response: + if raise_on_error: + raise GVException return response_json = response.json() @@ -423,6 +436,10 @@ def __init__(self, result_entry: dict, included_entities_lookup: dict = None): def json(self): return self._result_entry + @property + def attributes(self): + return self._attributes + def get_attribute(self, attribute_name): return self._attributes.get(attribute_name) diff --git a/osf/external/gravy_valet/translations.py b/osf/external/gravy_valet/translations.py index d53bc0ad065..2c284fd6b15 100644 --- a/osf/external/gravy_valet/translations.py +++ b/osf/external/gravy_valet/translations.py @@ -23,6 +23,12 @@ def make_ephemeral_user_settings(gv_account_data, requesting_user): active_user=requesting_user, ) +def get_verified_links(node_guid, requesting_user=None): + links = gv_requests.get_verified_links(node_guid, requesting_user) + return [ + link.attributes for link in links + ] + def make_ephemeral_node_settings(gv_addon_data: gv_requests.JSONAPIResultEntry, requested_resource, requesting_user): addon_type = gv_addon_data.resource_type.split('-')[1] diff --git a/osf/metadata/osf_gathering.py b/osf/metadata/osf_gathering.py index 728e18d3587..0463de03dc1 100644 --- a/osf/metadata/osf_gathering.py +++ b/osf/metadata/osf_gathering.py @@ -142,6 +142,7 @@ def pls_get_magic_metadata_basket(osf_item) -> gather.Basket: **OSF_OBJECT, OSF.supplements: OSF_OBJECT_REFERENCE, OSF.hasCedarTemplate: None, + OSF.verifiedLink: None, }, OSF.ProjectComponent: { **OSF_OBJECT, @@ -921,6 +922,14 @@ def gather_qualified_attributions(focus): yield (_attribution_ref, DCAT.hadRole, _osfrole_ref) yield (_attribution_ref, OSF.order, index) +@gather.er(OSF.verifiedLink) +def gather_verified_link(focus): + links = focus.dbmodel.get_verified_links() + for link in links: + ref = rdflib.BNode() + yield (OSF.verifiedLink, ref) + yield (ref, DCAT.accessURL, link['target_url']) + yield (ref, DCTERMS.type, DATACITE[link['resource_type']]) @gather.er(OSF.affiliation) def gather_affiliated_institutions(focus): diff --git a/osf/metadata/serializers/datacite/datacite_tree_walker.py b/osf/metadata/serializers/datacite/datacite_tree_walker.py index 640e00e76f0..86081d27369 100644 --- a/osf/metadata/serializers/datacite/datacite_tree_walker.py +++ b/osf/metadata/serializers/datacite/datacite_tree_walker.py @@ -6,6 +6,7 @@ import rdflib +from framework import sentry from osf.exceptions import MetadataSerializationError from osf.metadata import gather from osf.metadata.rdfutils import ( @@ -113,7 +114,7 @@ def walk(self, doi_override=None): self._visit_rights(self.root) self._visit_descriptions(self.root, self.basket.focus.iri) self._visit_funding_references(self.root) - self._visit_related(self.root) + self._visit_related_and_verified_links(self.root) def _visit_identifier(self, parent_el, *, doi_override=None): if doi_override is None: @@ -373,13 +374,16 @@ def _visit_related_identifier_and_item(self, identifier_parent_el, item_parent_e self._visit_publication_year(related_item_el, related_iri) self._visit_publisher(related_item_el, related_iri) - def _visit_related(self, parent_el): + def _visit_related_and_verified_links(self, parent_el): relation_pairs = set() for relation_iri, datacite_relation in RELATED_IDENTIFIER_TYPE_MAP.items(): for related_iri in self.basket[relation_iri]: relation_pairs.add((datacite_relation, related_iri)) + related_identifiers_el = self.visit(parent_el, 'relatedIdentifiers', is_list=True) related_items_el = self.visit(parent_el, 'relatedItems', is_list=True) + + # First add regular related identifiers for datacite_relation, related_iri in sorted(relation_pairs): self._visit_related_identifier_and_item( related_identifiers_el, @@ -388,6 +392,35 @@ def _visit_related(self, parent_el): datacite_relation, ) + # Then add verified links to same relatedIdentifiers element + osf_item = self.basket.focus.dbmodel + from osf.models import AbstractNode + + if isinstance(osf_item, AbstractNode): + gv_verified_link_list = osf_item.get_verified_links() + skipped_items = [] + for item in gv_verified_link_list: + verified_link, resource_type = item.get('target_url', None), item.get('resource_type', None) + if not verified_link or not resource_type: + logger.error(f'Must have both verified_link and resource_type: [item={item}]') + skipped_items.append(f'Missing data: [link={verified_link}, type={resource_type}]') + continue + if not smells_like_iri(verified_link): + skipped_items.append(f'Invalid link: [link={verified_link}, type={resource_type}]') + continue + self.visit( + related_identifiers_el, + 'relatedIdentifier', + text=verified_link, + attrib={ + 'relatedIdentifierType': 'URL', + 'relationType': 'IsReferencedBy', + 'resourceTypeGeneral': resource_type.title() + } + ) + if skipped_items: + sentry.log_message(f'Skipped items for node [{osf_item._id}]: {'; '.join(skipped_items)}. ') + def _visit_name_identifiers(self, parent_el, agent_iri): for identifier in sorted(self.basket[agent_iri:DCTERMS.identifier]): identifier_type, identifier_value = self._identifier_type_and_value(identifier) diff --git a/osf/models/node.py b/osf/models/node.py index 83d646cc717..5c60e3f0a3c 100644 --- a/osf/models/node.py +++ b/osf/models/node.py @@ -2342,6 +2342,9 @@ def is_fork_of(self, other): def is_registration_of(self, other): return self.is_derived_from(other, 'registered_from') + def get_verified_links(self): + return gv_translations.get_verified_links(self._id) + def serialize_waterbutler_credentials(self, provider_name): return self.get_addon(provider_name).serialize_waterbutler_credentials() diff --git a/osf/tasks.py b/osf/tasks.py new file mode 100644 index 00000000000..8070832a886 --- /dev/null +++ b/osf/tasks.py @@ -0,0 +1,47 @@ +import logging +import re + +from django.db.models import Model + +from framework.celery_tasks import app +from framework.sentry import log_message + +logger = logging.getLogger(__name__) +iri_regex = re.compile(r'http://[^/]+/(?P\w{5})/?') + +def get_object_by_url[T: Model](url: str, model: type[T]) -> T | None: + if not (match := iri_regex.match(url)): + log_message(f"received invalid {model.__name__} {url=}. ", skip_session=True) + return None + try: + return model.objects.get(guids___id=match['id']) + except model.DoesNotExist: + log_message(f"Could not find {model.__name__} with id={match['id']}", skip_session=True) + return None + +@app.task(max_retries=5, name='osf.tasks.log_gv_addon', default_retry_delay=10) +def log_gv_addon(node_url: str, action: str, user_url: str, addon: str): + from osf.models import NodeLog, OSFUser, Node + + PERMITTED_GV_ACTIONS = frozenset({ + NodeLog.ADDON_ADDED, + NodeLog.ADDON_REMOVED + }) + if action not in PERMITTED_GV_ACTIONS: + log_message(f"{action} is not permitted to be logged from GV", skip_session=True) + return + + node = get_object_by_url(node_url, Node) + user = get_object_by_url(user_url, OSFUser) + if not node or not user: + return + + node.add_log( + action=action, + auth=user, + params={ + 'node': node._id, + 'project': node.parent_id, + 'addon': addon + } + ) diff --git a/osf_tests/embargoes/test_embargoes.py b/osf_tests/embargoes/test_embargoes.py index 18df621d15f..468f5b03aaf 100644 --- a/osf_tests/embargoes/test_embargoes.py +++ b/osf_tests/embargoes/test_embargoes.py @@ -29,6 +29,7 @@ def registration(self, user): embargo.save() return embargo.registrations.last() + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_request_early_termination_too_late(self, registration, user): """ This is for an edge case test for where embargos are frozen and never expire when the user requests they be diff --git a/osf_tests/metadata/expected_metadata_files/preprint_supplement.turtle b/osf_tests/metadata/expected_metadata_files/preprint_supplement.turtle index 9ff0732a509..ca3e023d5dd 100644 --- a/osf_tests/metadata/expected_metadata_files/preprint_supplement.turtle +++ b/osf_tests/metadata/expected_metadata_files/preprint_supplement.turtle @@ -1,5 +1,6 @@ @prefix osf: . @prefix skos: . +@prefix xsd: . osf:storageByteCount 1337 ; osf:storageRegion . diff --git a/osf_tests/metadata/expected_metadata_files/project_basic.datacite.json b/osf_tests/metadata/expected_metadata_files/project_basic.datacite.json index 1f85e773f2f..d866a786a89 100644 --- a/osf_tests/metadata/expected_metadata_files/project_basic.datacite.json +++ b/osf_tests/metadata/expected_metadata_files/project_basic.datacite.json @@ -63,6 +63,12 @@ "relatedIdentifier": "11.pp/FK2osf.io/w4ibb_v1", "relatedIdentifierType": "DOI", "relationType": "IsSupplementTo" + }, + { + "relatedIdentifier": "https://foo.bar", + "relatedIdentifierType": "URL", + "relationType": "IsReferencedBy", + "resourceTypeGeneral": "Other" } ], "relatedItems": [ diff --git a/osf_tests/metadata/expected_metadata_files/project_basic.datacite.xml b/osf_tests/metadata/expected_metadata_files/project_basic.datacite.xml index 8b4abfb5d87..d395415a708 100644 --- a/osf_tests/metadata/expected_metadata_files/project_basic.datacite.xml +++ b/osf_tests/metadata/expected_metadata_files/project_basic.datacite.xml @@ -38,6 +38,7 @@ http://localhost:5000/w5ibb 11.pp/FK2osf.io/w4ibb_v1 + https://foo.bar diff --git a/osf_tests/metadata/expected_metadata_files/project_basic.turtle b/osf_tests/metadata/expected_metadata_files/project_basic.turtle index 5a259408730..c5208ec295e 100644 --- a/osf_tests/metadata/expected_metadata_files/project_basic.turtle +++ b/osf_tests/metadata/expected_metadata_files/project_basic.turtle @@ -29,7 +29,9 @@ osf:supplements ; prov:qualifiedAttribution [ dcat:hadRole osf:admin-contributor ; prov:agent ; - osf:order 0 ] . + osf:order 0 ] ; + osf:verifiedLink [ dcterms:type ; + dcat:accessURL "https://foo.bar" ] . a osf:Preprint ; dcterms:created "2123-05-04" ; diff --git a/osf_tests/metadata/expected_metadata_files/project_full.datacite.json b/osf_tests/metadata/expected_metadata_files/project_full.datacite.json index 4ead1090105..f4a43d07bd6 100644 --- a/osf_tests/metadata/expected_metadata_files/project_full.datacite.json +++ b/osf_tests/metadata/expected_metadata_files/project_full.datacite.json @@ -97,6 +97,12 @@ "relatedIdentifier": "11.pp/FK2osf.io/w4ibb_v1", "relatedIdentifierType": "DOI", "relationType": "IsSupplementTo" + }, + { + "relatedIdentifier": "https://foo.bar", + "relatedIdentifierType": "URL", + "relationType": "IsReferencedBy", + "resourceTypeGeneral": "Other" } ], "relatedItems": [ diff --git a/osf_tests/metadata/expected_metadata_files/project_full.datacite.xml b/osf_tests/metadata/expected_metadata_files/project_full.datacite.xml index e8649704d02..f707bb2e077 100644 --- a/osf_tests/metadata/expected_metadata_files/project_full.datacite.xml +++ b/osf_tests/metadata/expected_metadata_files/project_full.datacite.xml @@ -56,6 +56,7 @@ http://localhost:5000/w5ibb 11.pp/FK2osf.io/w4ibb_v1 + https://foo.bar diff --git a/osf_tests/metadata/expected_metadata_files/project_full.turtle b/osf_tests/metadata/expected_metadata_files/project_full.turtle index c93e286db01..6856faa651f 100644 --- a/osf_tests/metadata/expected_metadata_files/project_full.turtle +++ b/osf_tests/metadata/expected_metadata_files/project_full.turtle @@ -35,7 +35,9 @@ osf:supplements ; prov:qualifiedAttribution [ dcat:hadRole osf:admin-contributor ; prov:agent ; - osf:order 0 ] . + osf:order 0 ] ; + osf:verifiedLink [ dcterms:type ; + dcat:accessURL "https://foo.bar" ] . a osf:Preprint ; dcterms:created "2123-05-04" ; diff --git a/osf_tests/metadata/expected_metadata_files/project_supplement.turtle b/osf_tests/metadata/expected_metadata_files/project_supplement.turtle index d055e97554f..b1fc08a015d 100644 --- a/osf_tests/metadata/expected_metadata_files/project_supplement.turtle +++ b/osf_tests/metadata/expected_metadata_files/project_supplement.turtle @@ -1,6 +1,7 @@ @prefix dcterms: . @prefix osf: . @prefix skos: . +@prefix xsd: . osf:hasOsfAddon ; osf:storageByteCount 7 ; diff --git a/osf_tests/metadata/expected_metadata_files/registration_supplement.turtle b/osf_tests/metadata/expected_metadata_files/registration_supplement.turtle index 9e8201b7915..e73d437696b 100644 --- a/osf_tests/metadata/expected_metadata_files/registration_supplement.turtle +++ b/osf_tests/metadata/expected_metadata_files/registration_supplement.turtle @@ -1,5 +1,6 @@ @prefix osf: . @prefix skos: . +@prefix xsd: . osf:storageByteCount 17 ; osf:storageRegion . diff --git a/osf_tests/metadata/test_osf_gathering.py b/osf_tests/metadata/test_osf_gathering.py index 98de761a836..33be346e2df 100644 --- a/osf_tests/metadata/test_osf_gathering.py +++ b/osf_tests/metadata/test_osf_gathering.py @@ -1,6 +1,7 @@ import datetime from unittest import mock +import pytest from django.test import TestCase import rdflib from rdflib import Literal, URIRef @@ -703,6 +704,7 @@ def test_gather_collection_membership(self): (_collection_ref, DCTERMS.title, Literal(_collection_provider.name)), }) + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_gather_registration_withdrawal(self): # focus: registration assert_triples(osf_gathering.gather_registration_withdrawal(self.registrationfocus), set()) diff --git a/osf_tests/metadata/test_serialized_metadata.py b/osf_tests/metadata/test_serialized_metadata.py index 7df56f14894..bcbb18a52ae 100644 --- a/osf_tests/metadata/test_serialized_metadata.py +++ b/osf_tests/metadata/test_serialized_metadata.py @@ -2,6 +2,7 @@ import pathlib from unittest import mock +import pytest import rdflib from osf import models as osfdb @@ -176,6 +177,9 @@ def setUp(self): mock.patch('osf.models.base.Guid.objects.get_or_create', new=osfguid_sequence.get_or_create), mock.patch('django.utils.timezone.now', new=forever_now), mock.patch('osf.models.metaschema.RegistrationSchema.absolute_api_v2_url', new='http://fake.example/schema/for/test'), + mock.patch('osf.models.node.Node.get_verified_links', return_value=[ + {'target_url': 'https://foo.bar', 'resource_type': 'Other'} + ]) ): self.enterContext(patcher) # build test objects @@ -326,6 +330,7 @@ def _setUp_full(self): self.project.node_license.year = '2250-2254' self.project.node_license.save() + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_serialized_metadata(self): self._assert_scenario(BASIC_METADATA_SCENARIO) self._setUp_full() @@ -335,7 +340,8 @@ def _assert_scenario(self, scenario_dict): for focus_type, by_partition in scenario_dict.items(): for osfmap_partition, expected_files in by_partition.items(): for format_key, filename in expected_files.items(): - self._assert_scenario_file(focus_type, osfmap_partition, format_key, filename) + with self.subTest(msg=f'{focus_type=}, {format_key=}. {filename=}'): + self._assert_scenario_file(focus_type, osfmap_partition, format_key, filename) def _assert_scenario_file( self, diff --git a/osf_tests/test_archiver.py b/osf_tests/test_archiver.py index 59c178b839d..8a1643cc83f 100644 --- a/osf_tests/test_archiver.py +++ b/osf_tests/test_archiver.py @@ -533,6 +533,7 @@ def test_archive_addon(self, mock_make_copy_request): } ) + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_archive_success(self): node = factories.NodeFactory(creator=self.user) file_trees, selected_files, node_index = generate_file_tree([node]) @@ -565,6 +566,7 @@ def test_archive_success(self): assert registration_files == set(selected_files.keys()) + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_archive_success_escaped_file_names(self): file_tree = file_tree_factory(0, 0, 0) fake_file = file_factory(name='>and&and<') @@ -593,6 +595,7 @@ def test_archive_success_escaped_file_names(self): updated_response = registration.schema_responses.get().all_responses[qid] assert updated_response[0]['file_name'] == fake_file_name + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_archive_success_with_components(self): node = factories.NodeFactory(creator=self.user) comp1 = factories.NodeFactory(parent=node, creator=self.user) @@ -632,6 +635,7 @@ def mock_get_file_tree(self, *args, **kwargs): assert parent_registration._id in file_response['file_urls']['html'] registration_files.add(file_sha) + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_archive_success_different_name_same_sha(self): file_tree = file_tree_factory(0, 0, 0) fake_file = file_factory() @@ -658,6 +662,7 @@ def test_archive_success_different_name_same_sha(self): for key, question in registration.registered_meta[schema._id].items(): assert question['extra'][0]['selectedFileName'] == fake_file['name'] + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_archive_failure_different_name_same_sha(self): file_tree = file_tree_factory(0, 0, 0) fake_file = file_factory() @@ -683,6 +688,7 @@ def test_archive_failure_different_name_same_sha(self): with pytest.raises(ArchivedFileNotFound): archive_success(registration._id, job._id) + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_archive_success_same_file_in_component(self): file_tree = file_tree_factory(3, 3, 3) selected = list(select_files_from_tree(file_tree).values())[0] diff --git a/osf_tests/test_node.py b/osf_tests/test_node.py index 4fcd6e542cf..05b5fc1290d 100644 --- a/osf_tests/test_node.py +++ b/osf_tests/test_node.py @@ -3707,6 +3707,7 @@ def test_update_is_public(self, node, user, auth): last_log = node.logs.latest() assert last_log.action == NodeLog.MADE_PRIVATE + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_update_can_make_registration_public(self): reg = RegistrationFactory(is_public=False) reg.update({'is_public': True}) diff --git a/osf_tests/test_notable_domains.py b/osf_tests/test_notable_domains.py index 68e39912a65..96e7c28d704 100644 --- a/osf_tests/test_notable_domains.py +++ b/osf_tests/test_notable_domains.py @@ -190,6 +190,7 @@ def test_check_resource_for_domains_spam(self, spam_domain, marked_as_spam_domai @pytest.mark.enable_enqueue_task @pytest.mark.parametrize('factory', [NodeFactory, RegistrationFactory, PreprintFactory]) + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_spam_check(self, app, factory, spam_domain, marked_as_spam_domain, request_context): obj = factory() obj.is_public = True diff --git a/osf_tests/test_pigeon.py b/osf_tests/test_pigeon.py index a1905c41ba8..561b26b2eb4 100644 --- a/osf_tests/test_pigeon.py +++ b/osf_tests/test_pigeon.py @@ -61,6 +61,7 @@ def test_pigeon_archive_immediately(self, registration, mock_pigeon, mock_celery @pytest.mark.enable_enqueue_task @pytest.mark.enable_implicit_clean + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_pigeon_archive_embargo(self, embargo, mock_pigeon, mock_celery): embargo._get_registration().terminate_embargo() guid = embargo._get_registration()._id diff --git a/osf_tests/test_registrations.py b/osf_tests/test_registrations.py index 970b3670a3e..a93ffba6264 100644 --- a/osf_tests/test_registrations.py +++ b/osf_tests/test_registrations.py @@ -51,7 +51,6 @@ def project(user, auth, fake): def auth(user): return Auth(user) - # copied from tests/test_models.py def test_factory(user, project): # Create a registration with kwargs @@ -119,6 +118,7 @@ def test_update_article_doi(self, auth): # copied from tests/test_models.py +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class TestRegisterNode: @pytest.fixture() @@ -404,6 +404,7 @@ def registration(self, project_two, component, contributor_unregistered, contrib with mock_archive(project_two, autoapprove=True) as registration: return registration + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_unregistered_contributors_unclaimed_records_get_copied(self, user, project, component, registration, contributor_unregistered, contributor_unregistered_no_email): contributor_unregistered.refresh_from_db() contributor_unregistered_no_email.refresh_from_db() @@ -417,6 +418,7 @@ def test_unregistered_contributors_unclaimed_records_get_copied(self, user, proj # copied from tests/test_registrations +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class TestNodeApprovalStates: def test_sanction_none(self): @@ -633,7 +635,7 @@ def test_expand_registration_responses_veer(self, draft_veer): assert registration_metadata == veer_condensed - +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class TestRegistationModerationStates: @pytest.fixture diff --git a/osf_tests/test_sanctions.py b/osf_tests/test_sanctions.py index 244ce7f7628..de4161ced4a 100644 --- a/osf_tests/test_sanctions.py +++ b/osf_tests/test_sanctions.py @@ -14,6 +14,7 @@ @pytest.mark.django_db +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class TestRegistrationApprovalHooks: # Regression test for https://openscience.atlassian.net/browse/OSF-4940 @@ -29,6 +30,7 @@ def test_unmoderated_accept_sets_state_to_approved(self, mock_update_search): @pytest.mark.django_db +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class TestNodeEmbargoTerminations: @pytest.fixture() @@ -164,6 +166,7 @@ def test_render_non_admin_emails( @pytest.mark.django_db +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class TestDOICreation: def make_test_registration(self, embargoed=False, moderated=False): diff --git a/tests/identifiers/test_datacite.py b/tests/identifiers/test_datacite.py index bff0542297a..5e5563f8917 100644 --- a/tests/identifiers/test_datacite.py +++ b/tests/identifiers/test_datacite.py @@ -1,6 +1,7 @@ import lxml import pytest import responses +from unittest import mock from datacite import schema40 from django.utils import timezone @@ -27,6 +28,7 @@ def _assert_unordered_list_of_dicts_equal(actual_list_of_dicts, expected_list_of @pytest.mark.django_db +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class TestDataCiteClient: @pytest.fixture() @@ -260,6 +262,7 @@ def setUp(self): self.client = DataCiteClient(self.node) @responses.activate + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_datacite_create_identifiers_not_exists(self): responses.add( responses.Response( diff --git a/tests/test_registrations/test_embargoes.py b/tests/test_registrations/test_embargoes.py index 8b87cf4e252..9ae7f84529c 100644 --- a/tests/test_registrations/test_embargoes.py +++ b/tests/test_registrations/test_embargoes.py @@ -150,6 +150,7 @@ def test_embargo_with_valid_end_date_starts_pending_embargo(self): self.registration.save() assert self.registration.is_pending_embargo + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_embargo_public_project_makes_private_pending_embargo(self): self.registration.is_public = True assert self.registration.is_public @@ -427,6 +428,7 @@ def test_on_complete_raises_error_if_registration_is_spam(self): assert mock_notify.call_count == 0 # Regression for OSF-8840 + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_public_embargo_cannot_be_deleted_with_initial_token(self): embargo_termination_approval = EmbargoTerminationApprovalFactory() registration = Registration.objects.get(embargo_termination_approval=embargo_termination_approval) @@ -697,6 +699,7 @@ def test_GET_disapprove_for_existing_registration_returns_200(self): assert res.status_code == 200 assert res.request.path == self.registration.web_url_for('view_project') + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_GET_from_unauthorized_user_with_registration_token(self): unauthorized_user = AuthUserFactory() @@ -747,6 +750,7 @@ def test_GET_from_unauthorized_user_with_registration_token(self): ) assert res.status_code == 200 + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_GET_from_authorized_user_with_registration_app_token(self): self.registration.require_approval(self.user) self.registration.save() @@ -960,6 +964,7 @@ def test_GET_disapprove_for_existing_registration_returns_200(self): assert res.status_code == 302 assert res.request.path == self.registration.web_url_for('token_action') + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_GET_from_unauthorized_user_with_registration_token(self): unauthorized_user = AuthUserFactory() @@ -1010,6 +1015,7 @@ def test_GET_from_unauthorized_user_with_registration_token(self): ) assert res.status_code == 302 + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_GET_from_authorized_user_with_registration_app_token(self): self.registration.require_approval(self.user) self.registration.save() diff --git a/tests/test_registrations/test_registration_approvals.py b/tests/test_registrations/test_registration_approvals.py index 965b48c4e96..2802e0e5a30 100644 --- a/tests/test_registrations/test_registration_approvals.py +++ b/tests/test_registrations/test_registration_approvals.py @@ -114,6 +114,7 @@ def test_non_admin_approval_token_raises_PermissionsError(self): self.registration.registration_approval.approve(user=non_admin, token=approval_token) assert self.registration.is_pending_registration + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_approval_adds_to_parent_projects_log(self): initial_project_logs = self.registration.registered_from.logs.count() self.registration.require_approval( @@ -126,6 +127,7 @@ def test_approval_adds_to_parent_projects_log(self): # adds initiated, approved, and registered logs assert self.registration.registered_from.logs.count() == initial_project_logs + 3 + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_one_approval_with_two_admins_stays_pending(self): admin2 = UserFactory() Contributor.objects.create(node=self.registration, user=admin2) @@ -251,6 +253,7 @@ def test_new_registration_is_pending_registration(self): self.registration.save() assert self.registration.is_pending_registration + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_should_suppress_emails(self): self.registration = RegistrationFactory(project=self.project) self.registration.external_registration = True @@ -280,6 +283,7 @@ def test_should_suppress_emails(self): self.registration.sanction.ask(contributors) assert mock_notify_non_authorizer.call_count == 0 + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_on_complete_notify_initiator(self): self.registration.require_approval( self.user, diff --git a/tests/test_registrations/test_retractions.py b/tests/test_registrations/test_retractions.py index 67f0b0fb497..a29993698ac 100644 --- a/tests/test_registrations/test_retractions.py +++ b/tests/test_registrations/test_retractions.py @@ -26,7 +26,9 @@ from osf.utils import permissions + @pytest.mark.enable_bookmark_creation +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class RegistrationRetractionModelsTestCase(OsfTestCase): def setUp(self): super().setUp() @@ -400,6 +402,7 @@ def test_new_retraction_is_pending_retraction(self): @pytest.mark.enable_bookmark_creation +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class RegistrationWithChildNodesRetractionModelTestCase(OsfTestCase): def setUp(self): super().setUp() @@ -757,6 +760,7 @@ def test_POST_retraction_to_subproject_component_returns_HTTPError_BAD_REQUEST(s assert res.status_code == http_status.HTTP_400_BAD_REQUEST @pytest.mark.enable_bookmark_creation +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class RegistrationRetractionViewsTestCase(OsfTestCase): def setUp(self): super().setUp() diff --git a/tests/test_registrations/test_review_flows.py b/tests/test_registrations/test_review_flows.py index 7aeaa241c13..a8653d32a0f 100644 --- a/tests/test_registrations/test_review_flows.py +++ b/tests/test_registrations/test_review_flows.py @@ -1,3 +1,5 @@ +from unittest import mock + import pytest from api.providers.workflows import Workflows @@ -61,6 +63,7 @@ def retraction(provider=None): @pytest.mark.enable_bookmark_creation @pytest.mark.django_db +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class TestUnmoderatedFlows(): @pytest.mark.parametrize( @@ -194,6 +197,7 @@ def test_approve_after_reject_is_noop(self, sanction_fixture): @pytest.mark.enable_bookmark_creation @pytest.mark.django_db +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class TestModeratedFlows(): @pytest.fixture @@ -530,6 +534,7 @@ def test_provider_admin_can_reject_as_moderator( assert sanction_object.approval_stage is ApprovalStates.MODERATOR_REJECTED @pytest.mark.enable_bookmark_creation +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class TestEmbargoTerminationFlows(OsfTestCase): def setUp(self): @@ -640,6 +645,7 @@ def test_accept_after_reject_raises_machine_error(self): @pytest.mark.enable_bookmark_creation @pytest.mark.django_db +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class TestModerationActions: @pytest.fixture @@ -754,6 +760,7 @@ def test_no_actions_written_on_unmoderated_rejection(self, sanction_object, prov @pytest.mark.django_db +@pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') class TestNestedFlows(): @pytest.fixture(params=[registration_approval, embargo, retraction]) diff --git a/tests/test_tokens.py b/tests/test_tokens.py index 3afafa2363a..d64dffd37cc 100644 --- a/tests/test_tokens.py +++ b/tests/test_tokens.py @@ -88,6 +88,7 @@ def setUp(self, *args, **kwargs): self.reg = AbstractNode.objects.get(Q(**{self.Model.SHORT_NAME: self.sanction})) self.user = self.reg.creator + @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') def test_sanction_handler(self): if not self.kind: return diff --git a/website/identifiers/tasks.py b/website/identifiers/tasks.py index f940956d54b..29afa9645ee 100644 --- a/website/identifiers/tasks.py +++ b/website/identifiers/tasks.py @@ -1,10 +1,13 @@ import logging from django.apps import apps +from osf.external.gravy_valet.exceptions import GVException from framework.celery_tasks import app as celery_app from framework.celery_tasks.handlers import queued_task from framework import sentry +logger = logging.getLogger(__name__) + @celery_app.task(bind=True, max_retries=5, acks_late=True) def task__update_doi_metadata_on_change(self, target_guid): sentry.log_message('Updating DOI for guid', extra_data={'guid': target_guid}, level=logging.INFO) @@ -17,3 +20,30 @@ def task__update_doi_metadata_on_change(self, target_guid): @celery_app.task(ignore_results=True) def update_doi_metadata_on_change(target_guid): task__update_doi_metadata_on_change(target_guid) + +@celery_app.task(bind=True, max_retries=5, acks_late=True) +def task__update_verified_links(self, target_guid): + logger.debug(f'Updating DOI metadata for guid due to verified links configuration change in Gravy Valet: ' + f'[guid={target_guid}]') + + Guid = apps.get_model('osf.Guid') + target_object = Guid.load(target_guid).referent + try: + target_object.request_identifier_update(category='doi') + logger.debug(f'DOI metadata for guid with verified links updated: [guid={target_guid}]') + except GVException as e: + logger.error(f'DOI metadata for guid with verified links failed to update: [guid={target_guid}]') + raise self.retry(exc=e) + + try: + target_object.update_search() + except GVException as e: + logger.error(f'Share update for guid with verified links failed to update: [guid={target_guid}]') + raise self.retry(exc=e) + + +@queued_task +@celery_app.task(ignore_results=True) +def update_verified_links(target_guid): + # TODO: log sentry if fails after max retry + task__update_verified_links(target_guid) diff --git a/website/settings/defaults.py b/website/settings/defaults.py index d891e886873..377f16f83e7 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -407,6 +407,8 @@ class CeleryConfig: task_high_queue = 'high' task_remote_computing_queue = 'remote' task_account_status_changes_queue = 'account_status_changes' + task_external_high_queue = 'external_high' + task_external_low_queue = 'external_low' remote_computing_modules = { 'addons.boa.tasks.submit_to_boa', @@ -473,17 +475,53 @@ class CeleryConfig: pass else: task_queues = ( - Queue(task_remote_computing_queue, Exchange(task_remote_computing_queue), - routing_key=task_remote_computing_queue, consumer_arguments={'x-priority': -10}), - Queue(task_low_queue, Exchange(task_low_queue), - routing_key=task_low_queue, consumer_arguments={'x-priority': -1}), - Queue(task_default_queue, Exchange(task_default_queue), - routing_key=task_default_queue, consumer_arguments={'x-priority': 0}), - Queue(task_med_queue, Exchange(task_med_queue), - routing_key=task_med_queue, consumer_arguments={'x-priority': 1}), - Queue(task_high_queue, Exchange(task_high_queue), - routing_key=task_high_queue, consumer_arguments={'x-priority': 10}), - Queue(task_account_status_changes_queue, Exchange(task_account_status_changes_queue), routing_key=task_account_status_changes_queue) + Queue( + task_remote_computing_queue, + Exchange(task_remote_computing_queue), + routing_key=task_remote_computing_queue, + consumer_arguments={'x-priority': -10}, + ), + Queue( + task_low_queue, + Exchange(task_low_queue), + routing_key=task_low_queue, + consumer_arguments={'x-priority': -1}, + ), + Queue( + task_default_queue, + Exchange(task_default_queue), + routing_key=task_default_queue, + consumer_arguments={'x-priority': 0}, + ), + Queue( + task_med_queue, + Exchange(task_med_queue), + routing_key=task_med_queue, + consumer_arguments={'x-priority': 1}, + ), + Queue( + task_high_queue, + Exchange(task_high_queue), + routing_key=task_high_queue, + consumer_arguments={'x-priority': 10}, + ), + Queue( + task_account_status_changes_queue, + Exchange(task_account_status_changes_queue), + routing_key=task_account_status_changes_queue, + ), + Queue( + task_external_high_queue, + Exchange(task_external_high_queue), + routing_key=task_external_high_queue, + consumer_arguments={'x-priority': 9}, + ), + Queue( + task_external_low_queue, + Exchange(task_external_low_queue), + routing_key=task_external_low_queue, + consumer_arguments={'x-priority': -2}, + ), ) task_default_exchange_type = 'direct' diff --git a/website/templates/include/profile/settings_navpanel.mako b/website/templates/include/profile/settings_navpanel.mako index 44cd57e65e5..f898047d809 100644 --- a/website/templates/include/profile/settings_navpanel.mako +++ b/website/templates/include/profile/settings_navpanel.mako @@ -9,7 +9,7 @@ Account settings
  • - Configure add-on accounts
    • + Configure add-on & link service accounts
  • Notifications
    • diff --git a/website/templates/profile/addons.mako b/website/templates/profile/addons.mako index dff81c81e8f..4f65659031a 100644 --- a/website/templates/profile/addons.mako +++ b/website/templates/profile/addons.mako @@ -1,5 +1,5 @@ <%inherit file="base.mako"/> -<%def name="title()">Configure Add-on Accounts +<%def name="title()">Configure add-on & link service accounts <%def name="stylesheets()"> ${parent.stylesheets()} @@ -25,7 +25,7 @@
    -

    Configure Add-on Accounts

    +

    Configure add-on & link service accounts

    % for addon in addon_settings: ${ render_user_settings(addon) } diff --git a/website/templates/project/project_header.mako b/website/templates/project/project_header.mako index 3a8944c7602..8aa26140ee4 100644 --- a/website/templates/project/project_header.mako +++ b/website/templates/project/project_header.mako @@ -87,7 +87,7 @@ % if permissions.WRITE in user['permissions'] and not node['is_registration'] and not node['link']:
  • Add-ons
    • % endif - +
  • Linked services
    • % if not node['link'] and (user['has_read_permissions'] and not node['is_registration'] or (node['is_registration'] and permissions.WRITE in user['permissions'])):
  • Settings
    • % endif diff --git a/website/views.py b/website/views.py index aa523f80fd1..72a69f0bd26 100644 --- a/website/views.py +++ b/website/views.py @@ -339,7 +339,7 @@ def resolve_guid(guid, suffix=None): elif isinstance(resource, Node) and clean_suffix and clean_suffix.startswith('files') and flag_is_active(request, features.EMBER_PROJECT_FILES): return use_ember_app() - elif isinstance(resource, Node) and clean_suffix and (clean_suffix.startswith('metadata') or clean_suffix.startswith('components')): + elif isinstance(resource, Node) and clean_suffix and (clean_suffix.startswith('metadata') or clean_suffix.startswith('components') or clean_suffix.startswith('links')): return use_ember_app() elif isinstance(resource, BaseFileNode) and resource.is_file and not isinstance(resource.target, Preprint):