From e3dd3de8d9e82e21bcc10ab2757091e4efa6b65d Mon Sep 17 00:00:00 2001 From: danwusbu Date: Tue, 21 May 2024 18:37:20 -0400 Subject: [PATCH] Refactor PackageLoader to use importlib_resources --- CHANGES.rst | 1 + requirements/dev.txt | 138 ++++++++++++++++++++++++--------------- requirements/docs.in | 1 + requirements/docs.txt | 7 +- requirements/tests.in | 1 + requirements/tests.txt | 4 +- requirements/tests37.in | 1 + requirements/tests37.txt | 19 +----- requirements/typing.in | 1 + requirements/typing.txt | 19 +++++- src/jinja2/loaders.py | 26 ++++---- tests/test_loader.py | 5 -- 12 files changed, 132 insertions(+), 91 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f23b6c96f..14c06d1de 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,6 +9,7 @@ Unreleased - Use modern packaging metadata with ``pyproject.toml`` instead of ``setup.cfg``. :pr:`1793` - Use ``flit_core`` instead of ``setuptools`` as build backend. +- Refactor ``PackageLoader`` to use ``importlib_resources`` to fix ``PackageLoader.list_templates()`` for Python 3.13. Version 3.1.5 diff --git a/requirements/dev.txt b/requirements/dev.txt index 076912b0b..b03872bdb 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,39 +1,44 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # pip-compile dev.in # alabaster==0.7.16 - # via sphinx + # via + # -r docs.txt + # sphinx attrs==23.2.0 # via + # -r tests.txt # outcome # trio babel==2.15.0 - # via sphinx -build==1.2.1 - # via pip-tools + # via + # -r docs.txt + # sphinx cachetools==5.3.3 # via tox certifi==2024.2.2 - # via requests + # via + # -r docs.txt + # requests cfgv==3.4.0 # via pre-commit chardet==5.2.0 # via tox charset-normalizer==3.3.2 - # via requests -click==8.1.7 # via - # pip-compile-multi - # pip-tools + # -r docs.txt + # requests colorama==0.4.6 # via tox distlib==0.3.8 # via virtualenv docutils==0.21.2 - # via sphinx + # via + # -r docs.txt + # sphinx filelock==3.14.0 # via # tox @@ -42,107 +47,138 @@ identify==2.5.36 # via pre-commit idna==3.7 # via + # -r docs.txt + # -r tests.txt # requests # trio imagesize==1.4.1 - # via sphinx + # via + # -r docs.txt + # sphinx +importlib-resources==6.4.0 + # via -r tests.txt iniconfig==2.0.0 - # via pytest + # via + # -r tests.txt + # pytest jinja2==3.1.4 - # via sphinx + # via + # -r docs.txt + # sphinx markupsafe==2.1.5 - # via jinja2 + # via + # -r docs.txt + # jinja2 mypy==1.10.0 - # via -r typing.in + # via -r typing.txt mypy-extensions==1.0.0 - # via mypy + # via + # -r typing.txt + # mypy nodeenv==1.8.0 # via pre-commit outcome==1.3.0.post0 - # via trio + # via + # -r tests.txt + # trio packaging==24.0 # via - # build + # -r docs.txt + # -r tests.txt # pallets-sphinx-themes # pyproject-api # pytest # sphinx # tox pallets-sphinx-themes==2.1.3 - # via -r docs.in -pip-compile-multi==2.6.3 - # via -r dev.in -pip-tools==7.4.1 - # via pip-compile-multi + # via -r docs.txt platformdirs==4.2.1 # via # tox # virtualenv pluggy==1.5.0 # via + # -r tests.txt # pytest # tox pre-commit==3.7.1 # via -r dev.in pygments==2.18.0 - # via sphinx + # via + # -r docs.txt + # sphinx pyproject-api==1.6.1 # via tox -pyproject-hooks==1.1.0 - # via - # build - # pip-tools pytest==8.2.0 - # via -r tests.in + # via -r tests.txt pyyaml==6.0.1 # via pre-commit requests==2.31.0 - # via sphinx + # via + # -r docs.txt + # sphinx sniffio==1.3.1 - # via trio + # via + # -r tests.txt + # trio snowballstemmer==2.2.0 - # via sphinx + # via + # -r docs.txt + # sphinx sortedcontainers==2.4.0 - # via trio + # via + # -r tests.txt + # trio sphinx==7.3.7 # via - # -r docs.in + # -r docs.txt # pallets-sphinx-themes # sphinx-issues # sphinxcontrib-log-cabinet sphinx-issues==4.1.0 - # via -r docs.in + # via -r docs.txt sphinxcontrib-applehelp==1.0.8 - # via sphinx + # via + # -r docs.txt + # sphinx sphinxcontrib-devhelp==1.0.6 - # via sphinx + # via + # -r docs.txt + # sphinx sphinxcontrib-htmlhelp==2.0.5 - # via sphinx + # via + # -r docs.txt + # sphinx sphinxcontrib-jsmath==1.0.1 - # via sphinx + # via + # -r docs.txt + # sphinx sphinxcontrib-log-cabinet==1.0.1 - # via -r docs.in + # via -r docs.txt sphinxcontrib-qthelp==1.0.7 - # via sphinx + # via + # -r docs.txt + # sphinx sphinxcontrib-serializinghtml==1.1.10 - # via sphinx -toposort==1.10 - # via pip-compile-multi + # via + # -r docs.txt + # sphinx tox==4.15.0 # via -r dev.in trio==0.25.0 - # via -r tests.in + # via -r tests.txt typing-extensions==4.11.0 - # via mypy + # via + # -r typing.txt + # mypy urllib3==2.2.1 - # via requests + # via + # -r docs.txt + # requests virtualenv==20.26.1 # via # pre-commit # tox -wheel==0.43.0 - # via pip-tools # The following packages are considered to be unsafe in a requirements file: -# pip # setuptools diff --git a/requirements/docs.in b/requirements/docs.in index ba3fd7774..5cecc47b2 100644 --- a/requirements/docs.in +++ b/requirements/docs.in @@ -1,3 +1,4 @@ pallets-sphinx-themes sphinx sphinxcontrib-log-cabinet +importlib-resources diff --git a/requirements/docs.txt b/requirements/docs.txt index 2cbd73fa8..11f06a5f2 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # pip-compile docs.in @@ -18,6 +18,8 @@ idna==3.7 # via requests imagesize==1.4.1 # via sphinx +importlib-resources==6.4.0 + # via -r docs.in jinja2==3.1.4 # via sphinx markupsafe==2.1.5 @@ -38,10 +40,7 @@ sphinx==7.3.7 # via # -r docs.in # pallets-sphinx-themes - # sphinx-issues # sphinxcontrib-log-cabinet -sphinx-issues==4.1.0 - # via -r docs.in sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 diff --git a/requirements/tests.in b/requirements/tests.in index 5669c6ecd..201bbdf01 100644 --- a/requirements/tests.in +++ b/requirements/tests.in @@ -1,2 +1,3 @@ pytest trio +importlib-resources diff --git a/requirements/tests.txt b/requirements/tests.txt index de18d477d..01ebbb948 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # pip-compile tests.in @@ -10,6 +10,8 @@ attrs==23.2.0 # trio idna==3.7 # via trio +importlib-resources==6.4.0 + # via -r tests.in iniconfig==2.0.0 # via pytest outcome==1.3.0.post0 diff --git a/requirements/tests37.in b/requirements/tests37.in index 9c2bed180..86c8c9025 100644 --- a/requirements/tests37.in +++ b/requirements/tests37.in @@ -1,2 +1,3 @@ pytest trio==0.22.2 +importlib-resources diff --git a/requirements/tests37.txt b/requirements/tests37.txt index 578789e7a..254bdea76 100644 --- a/requirements/tests37.txt +++ b/requirements/tests37.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.7 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # pip-compile tests37.in @@ -8,17 +8,10 @@ attrs==23.2.0 # via # outcome # trio -exceptiongroup==1.2.1 - # via - # pytest - # trio idna==3.7 # via trio -importlib-metadata==6.7.0 - # via - # attrs - # pluggy - # pytest +importlib-resources==6.4.0 + # via -r tests37.in iniconfig==2.0.0 # via pytest outcome==1.3.0.post0 @@ -33,11 +26,5 @@ sniffio==1.3.1 # via trio sortedcontainers==2.4.0 # via trio -tomli==2.0.1 - # via pytest trio==0.22.2 # via -r tests37.in -typing-extensions==4.7.1 - # via importlib-metadata -zipp==3.15.0 - # via importlib-metadata diff --git a/requirements/typing.in b/requirements/typing.in index 8be59c5dc..2e162b078 100644 --- a/requirements/typing.in +++ b/requirements/typing.in @@ -1,3 +1,4 @@ mypy pyright pytest +importlib-resources diff --git a/requirements/typing.txt b/requirements/typing.txt index c08a53767..272ca0e1b 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -1,12 +1,29 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # pip-compile typing.in # +importlib-resources==6.4.0 + # via -r typing.in +iniconfig==2.0.0 + # via pytest mypy==1.10.0 # via -r typing.in mypy-extensions==1.0.0 # via mypy +nodeenv==1.8.0 + # via pyright +packaging==24.0 + # via pytest +pluggy==1.5.0 + # via pytest +pyright==1.1.363 + # via -r typing.in +pytest==8.2.1 + # via -r typing.in typing-extensions==4.11.0 # via mypy + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index 5cb49271e..a68136bc3 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -14,6 +14,8 @@ from importlib import import_module from types import ModuleType +import importlib_resources + from .exceptions import TemplateNotFound from .utils import internalcode @@ -371,6 +373,15 @@ def up_to_date() -> bool: def list_templates(self) -> t.List[str]: results: t.List[str] = [] + def _find_files_in_dir(path: importlib_resources.abc.Traversable) -> None: + if path.is_dir(): + for sub_path in path.iterdir(): + _find_files_in_dir(sub_path) + else: + print(str(path), self._template_root) + if str(path).startswith(self._template_root): + results.append(str(path)[len(self._template_root) + len(os.sep) :]) + if self._archive is None: # Package is a directory. offset = len(self._template_root) @@ -382,20 +393,9 @@ def list_templates(self) -> t.List[str]: for name in filenames ) else: - if not hasattr(self._loader, "_files"): - raise TypeError( - "This zip import does not have the required" - " metadata to list templates." - ) - # Package is a zip file. - prefix = self._template_root[len(self._archive) :].lstrip(os.sep) + os.sep - offset = len(prefix) - - for name in self._loader._files.keys(): - # Find names under the templates directory that aren't directories. - if name.startswith(prefix) and name[-1] != os.sep: - results.append(name[offset:].replace(os.sep, "/")) + for path in importlib_resources.files(self.package_name).iterdir(): + _find_files_in_dir(path) results.sort() return results diff --git a/tests/test_loader.py b/tests/test_loader.py index 3e64f6237..20449a66e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -362,11 +362,6 @@ def test_package_zip_source(package_zip_loader, template, expect): assert up_to_date is None -@pytest.mark.xfail( - sys.implementation.name == "pypy" or sys.version_info > (3, 13), - reason="zipimporter doesn't have a '_files' attribute", - raises=TypeError, -) def test_package_zip_list(package_zip_loader): assert package_zip_loader.list_templates() == ["foo/test.html", "test.html"]