Ensure we retain/can crawl remotely retrieved resources.

Closes: #25

Author: Julian

referencing/_core.py

@@ -1,7 +1,16 @@
 from __future__ import annotations
 
 from collections.abc import Iterable, Iterator, Sequence
-from typing import Any, Callable, ClassVar, Generic, Protocol, Tuple, cast
+from typing import (
+    Any,
+    Callable,
+    ClassVar,
+    Generic,
+    Protocol,
+    Tuple,
+    TypeVar,
+    cast,
+)
 from urllib.parse import unquote, urldefrag, urljoin
 
 from attrs import evolve, field
@@ -252,16 +261,8 @@ def __getitem__(self, uri: URI) -> Resource[D]:
         """
         try:
             return self._resources[uri]
-        except LookupError:
-            try:
-                return self._retrieve(uri)
-            except (
-                exceptions.CannotDetermineSpecification,
-                exceptions.NoSuchResource,
-            ):
-                raise
-            except Exception:
-                raise exceptions.Unretrievable(ref=uri)
+        except KeyError:
+            raise exceptions.NoSuchResource(ref=uri)
 
     def __iter__(self) -> Iterator[URI]:
         """
@@ -288,6 +289,32 @@ def __repr__(self) -> str:
             summary = f"{pluralized}"
         return f"<Registry ({size} {summary})>"
 
+    def get_or_retrieve(self, uri: URI):
+        """
+        Get a resource from the registry, crawling or retrieving if necessary.
+        """
+        resource = self._resources.get(uri)
+        if resource is not None:
+            return Retrieved(registry=self, value=resource)
+
+        registry = self.crawl()
+        resource = registry._resources.get(uri)
+        if resource is not None:
+            return Retrieved(registry=registry, value=resource)
+
+        try:
+            resource = registry._retrieve(uri)
+        except (
+            exceptions.CannotDetermineSpecification,
+            exceptions.NoSuchResource,
+        ):
+            raise
+        except Exception:
+            raise exceptions.Unretrievable(ref=uri)
+        else:
+            registry = registry.with_resource(uri, resource)
+            return Retrieved(registry=registry, value=resource)
+
     def remove(self, uri: URI):
         """
         Return a registry with the resource identified by a given URI removed.
@@ -308,7 +335,15 @@ def anchor(self, uri: URI, name: str):
         """
         Retrieve the given anchor, which must already have been found.
         """
-        return self._anchors[uri, name]
+        value = self._anchors.get((uri, name))
+        if value is not None:
+            return Retrieved(value=value, registry=self)
+
+        registry = self.crawl()
+        value = registry._anchors.get((uri, name))
+        if value is not None:
+            return Retrieved(value=value, registry=registry)
+        raise exceptions.NoSuchAnchor(ref=uri, resource=self[uri], anchor=name)
 
     def contents(self, uri: URI) -> D:
         """
@@ -424,10 +459,23 @@ def resolver_with_root(self, resource: Resource[D]) -> Resolver[D]:
         )
 
 
+T = TypeVar("T", AnchorType[Any], Resource[Any])
+
+
+@frozen
+class Retrieved(Generic[D, T]):
+    """
+    A value retrieved from a `Registry`.
+    """
+
+    value: T
+    registry: Registry[D]
+
+
 @frozen
 class Resolved(Generic[D]):
     """
-    A resolved reference.
+    A reference resolved to its contents by a `Resolver`.
     """
 
     contents: D
@@ -486,44 +534,24 @@ def lookup(self, ref: URI) -> Resolved[D]:
             uri, fragment = self._base_uri, ref[1:]
         else:
             uri, fragment = urldefrag(urljoin(self._base_uri, ref))
-        registry = self._registry
-        resource = registry.get(uri)
-        if resource is None:
-            registry = registry.crawl()
-            try:
-                resource = registry[uri]
-            except exceptions.NoSuchResource:
-                raise exceptions.Unresolvable(ref=ref) from None
-            except exceptions.Unretrievable:
-                raise exceptions.Unresolvable(ref=ref)
+        try:
+            retrieved = self._registry.get_or_retrieve(uri)
+        except exceptions.NoSuchResource:
+            raise exceptions.Unresolvable(ref=ref) from None
+        except exceptions.Unretrievable:
+            raise exceptions.Unresolvable(ref=ref)
 
         if fragment.startswith("/"):
-            return resource.pointer(
-                pointer=fragment,
-                resolver=self._evolve(registry=registry, base_uri=uri),
-            )
+            resolver = self._evolve(registry=retrieved.registry, base_uri=uri)
+            return retrieved.value.pointer(pointer=fragment, resolver=resolver)
 
         if fragment:
-            try:
-                anchor = registry.anchor(uri, fragment)
-            except LookupError:
-                registry = registry.crawl()
-                try:
-                    anchor = registry.anchor(uri, fragment)
-                except LookupError:
-                    raise exceptions.NoSuchAnchor(
-                        ref=ref,
-                        resource=resource,
-                        anchor=fragment,
-                    )
-            return anchor.resolve(
-                resolver=self._evolve(registry=registry, base_uri=uri),
-            )
+            retrieved = retrieved.registry.anchor(uri, fragment)
+            resolver = self._evolve(registry=retrieved.registry, base_uri=uri)
+            return retrieved.value.resolve(resolver=resolver)
 
-        return Resolved(
-            contents=resource.contents,
-            resolver=self._evolve(registry=registry, base_uri=uri),
-        )
+        resolver = self._evolve(registry=retrieved.registry, base_uri=uri)
+        return Resolved(contents=retrieved.value.contents, resolver=resolver)
 
     def in_subresource(self, subresource: Resource[D]) -> Resolver[D]:
         """

referencing/jsonschema.py

@@ -7,7 +7,7 @@
 from collections.abc import Sequence, Set
 from typing import Any, Iterable, Union
 
-from referencing import Anchor, Registry, Resource, Specification
+from referencing import Anchor, Registry, Resource, Specification, exceptions
 from referencing._attrs import frozen
 from referencing._core import Resolved as _Resolved, Resolver as _Resolver
 from referencing.typing import URI, Anchor as AnchorType, Mapping
@@ -544,8 +544,8 @@ def resolve(self, resolver: _Resolver[Schema]):
         last = self.resource
         for uri, registry in resolver.dynamic_scope():
             try:
-                anchor = registry.anchor(uri, self.name)
-            except LookupError:
+                anchor = registry.anchor(uri, self.name).value
+            except exceptions.NoSuchAnchor:
                 continue
             if isinstance(anchor, DynamicAnchor):
                 last = anchor.resource

referencing/tests/test_core.py

@@ -110,21 +110,17 @@ def test_crawl_finds_anchors_with_id(self):
             {"ID": "urn:bar", "anchors": {"foo": 12}},
         )
         registry = Registry().with_resource(resource.id(), resource)
-        with pytest.raises(LookupError):
-            registry.anchor(resource.id(), "foo")
 
-        assert registry.crawl().anchor(resource.id(), "foo") == Anchor(
+        assert registry.crawl().anchor(resource.id(), "foo").value == Anchor(
             name="foo",
             resource=ID_AND_CHILDREN.create_resource(12),
         )
 
     def test_crawl_finds_anchors_no_id(self):
         resource = ID_AND_CHILDREN.create_resource({"anchors": {"foo": 12}})
         registry = Registry().with_resource("urn:root", resource)
-        with pytest.raises(LookupError):
-            registry.anchor("urn:root", "foo")
 
-        assert registry.crawl().anchor("urn:root", "foo") == Anchor(
+        assert registry.crawl().anchor("urn:root", "foo").value == Anchor(
             name="foo",
             resource=ID_AND_CHILDREN.create_resource(12),
         )
@@ -135,6 +131,16 @@ def test_contents(self):
         registry = Registry().with_resource(uri, resource)
         assert registry.contents(uri) == {"foo": "bar"}
 
+    def test_crawled_anchor(self):
+        resource = ID_AND_CHILDREN.create_resource({"anchors": {"foo": "bar"}})
+        registry = Registry().with_resource("urn:example", resource)
+        retrieved = registry.anchor("urn:example", "foo")
+        assert retrieved.value == Anchor(
+            name="foo",
+            resource=ID_AND_CHILDREN.create_resource("bar"),
+        )
+        assert retrieved.registry == registry.crawl()
+
     def test_init(self):
         one = Resource.opaque(contents={})
         two = ID_AND_CHILDREN.create_resource({"foo": "bar"})
@@ -401,37 +407,41 @@ def test_remove_nonexistent_uri(self):
     def test_retrieve(self):
         foo = Resource.opaque({"foo": "bar"})
         registry = Registry(retrieve=lambda uri: foo)
-        assert registry["urn:example"] == foo
+        assert registry.get_or_retrieve("urn:example").value == foo
 
     def test_retrieve_arbitrary_exception(self):
+        foo = Resource.opaque({"foo": "bar"})
+
         def retrieve(uri):
             if uri == "urn:succeed":
-                return {}
+                return foo
             raise Exception("Oh no!")
 
         registry = Registry(retrieve=retrieve)
-        assert registry["urn:succeed"] == {}
+        assert registry.get_or_retrieve("urn:succeed").value == foo
         with pytest.raises(exceptions.Unretrievable):
-            registry["urn:uhoh"]
+            registry.get_or_retrieve("urn:uhoh")
 
     def test_retrieve_no_such_resource(self):
+        foo = Resource.opaque({"foo": "bar"})
+
         def retrieve(uri):
             if uri == "urn:succeed":
-                return {}
+                return foo
             raise exceptions.NoSuchResource(ref=uri)
 
         registry = Registry(retrieve=retrieve)
-        assert registry["urn:succeed"] == {}
+        assert registry.get_or_retrieve("urn:succeed").value == foo
         with pytest.raises(exceptions.NoSuchResource):
-            registry["urn:uhoh"]
+            registry.get_or_retrieve("urn:uhoh")
 
     def test_retrieve_cannot_determine_specification(self):
         def retrieve(uri):
             return Resource.from_contents({})
 
         registry = Registry(retrieve=retrieve)
         with pytest.raises(exceptions.CannotDetermineSpecification):
-            registry["urn:uhoh"]
+            registry.get_or_retrieve("urn:uhoh")
 
     def test_retrieve_already_available_resource(self):
         def retrieve(uri):
@@ -440,6 +450,7 @@ def retrieve(uri):
         foo = Resource.opaque({"foo": "bar"})
         registry = Registry({"urn:example": foo})
         assert registry["urn:example"] == foo
+        assert registry.get_or_retrieve("urn:example").value == foo
 
     def test_retrieve_first_checks_crawlable_resource(self):
         def retrieve(uri):
@@ -687,7 +698,7 @@ def test_lookup_non_existent_anchor(self):
         with pytest.raises(exceptions.Unresolvable) as e:
             resolver.lookup(ref)
         assert e.value == exceptions.NoSuchAnchor(
-            ref=ref,
+            ref="urn:example",
             resource=root,
             anchor="noSuchAnchor",
         )
@@ -698,6 +709,38 @@ def test_lookup_retrieved_resource(self):
         resolved = resolver.lookup("http://example.com/")
         assert resolved.contents == resource.contents
 
+    def test_repeated_lookup_from_retrieved_resource(self):
+        """
+        A (custom-)retrieved resource is added to the registry returned by
+        looking it up.
+        """
+        resource = Resource.opaque(contents={"foo": "baz"})
+        once = [resource]
+
+        def retrieve(uri: str):
+            return once.pop()
+
+        resolver = Registry(retrieve=retrieve).resolver()
+        resolved = resolver.lookup("http://example.com/")
+        assert resolved.contents == resource.contents
+
+        resolved = resolved.resolver.lookup("http://example.com/")
+        assert resolved.contents == resource.contents
+
+    def test_repeated_anchor_lookup_from_retrieved_resource(self):
+        resource = Resource.opaque(contents={"foo": "baz"})
+        once = [resource]
+
+        def retrieve(uri: str):
+            return once.pop()
+
+        resolver = Registry(retrieve=retrieve).resolver()
+        resolved = resolver.lookup("http://example.com/")
+        assert resolved.contents == resource.contents
+
+        resolved = resolved.resolver.lookup("#")
+        assert resolved.contents == resource.contents
+
     # FIXME: The tests below aren't really representable in the current
     #        suite, though we should probably think of ways to do so.