Merge pull request #11143 from itamarst/2984-new-cache-lower-memory

New HTTP cache with lower memory usage

Author: pfmoore

docs/html/topics/caching.md

@@ -27,6 +27,13 @@ While this cache attempts to minimize network activity, it does not prevent
 network access altogether. If you want a local install solution that
 circumvents accessing PyPI, see {ref}`Installing from local packages`.
 
+```{versionchanged} 23.3
+A new cache format is now used, stored in a directory called `http-v2` (see
+below for this directory's location). Previously this cache was stored in a
+directory called `http` in the main cache directory. If you have completely
+switched to newer versions of `pip`, you may wish to delete the old directory.
+```
+
 (wheel-caching)=
 
 ### Locally built wheels
@@ -124,11 +131,11 @@ The {ref}`pip cache` command can be used to manage pip's cache.
 
 ### Removing a single package
 
-`pip cache remove setuptools` removes all wheel files related to setuptools from pip's cache.
+`pip cache remove setuptools` removes all wheel files related to setuptools from pip's cache. HTTP cache files are not removed at this time.
 
 ### Removing the cache
 
-`pip cache purge` will clear all wheel files from pip's cache.
+`pip cache purge` will clear all files from pip's wheel and HTTP caches.
 
 ### Listing cached files
 

news/2984.bugfix.rst

@@ -0,0 +1 @@
+pip uses less memory when caching large packages. As a result, there is a new on-disk cache format stored in a new directory ($PIP_CACHE_DIR/http-v2).

src/pip/_internal/cli/req_command.py

@@ -120,7 +120,7 @@ def _build_session(
             ssl_context = None
 
         session = PipSession(
-            cache=os.path.join(cache_dir, "http") if cache_dir else None,
+            cache=os.path.join(cache_dir, "http-v2") if cache_dir else None,
             retries=retries if retries is not None else options.retries,
             trusted_hosts=options.trusted_hosts,
             index_urls=self._get_index_urls(options),

src/pip/_internal/commands/cache.py

@@ -93,24 +93,30 @@ def get_cache_info(self, options: Values, args: List[Any]) -> None:
         num_http_files = len(self._find_http_files(options))
         num_packages = len(self._find_wheels(options, "*"))
 
-        http_cache_location = self._cache_dir(options, "http")
+        http_cache_location = self._cache_dir(options, "http-v2")
+        old_http_cache_location = self._cache_dir(options, "http")
         wheels_cache_location = self._cache_dir(options, "wheels")
-        http_cache_size = filesystem.format_directory_size(http_cache_location)
+        http_cache_size = filesystem.format_size(
+            filesystem.directory_size(http_cache_location)
+            + filesystem.directory_size(old_http_cache_location)
+        )
         wheels_cache_size = filesystem.format_directory_size(wheels_cache_location)
 
         message = (
             textwrap.dedent(
                 """
-                    Package index page cache location: {http_cache_location}
+                    Package index page cache location (pip v23.3+): {http_cache_location}
+                    Package index page cache location (older pips): {old_http_cache_location}
                     Package index page cache size: {http_cache_size}
                     Number of HTTP files: {num_http_files}
                     Locally built wheels location: {wheels_cache_location}
                     Locally built wheels size: {wheels_cache_size}
                     Number of locally built wheels: {package_count}
-                """
+                """  # noqa: E501
             )
             .format(
                 http_cache_location=http_cache_location,
+                old_http_cache_location=old_http_cache_location,
                 http_cache_size=http_cache_size,
                 num_http_files=num_http_files,
                 wheels_cache_location=wheels_cache_location,
@@ -189,8 +195,11 @@ def _cache_dir(self, options: Values, subdir: str) -> str:
         return os.path.join(options.cache_dir, subdir)
 
     def _find_http_files(self, options: Values) -> List[str]:
-        http_dir = self._cache_dir(options, "http")
-        return filesystem.find_files(http_dir, "*")
+        old_http_dir = self._cache_dir(options, "http")
+        new_http_dir = self._cache_dir(options, "http-v2")
+        return filesystem.find_files(old_http_dir, "*") + filesystem.find_files(
+            new_http_dir, "*"
+        )
 
     def _find_wheels(self, options: Values, pattern: str) -> List[str]:
         wheel_dir = self._cache_dir(options, "wheels")

src/pip/_internal/network/cache.py

@@ -3,10 +3,11 @@
 
 import os
 from contextlib import contextmanager
-from typing import Generator, Optional
+from datetime import datetime
+from typing import BinaryIO, Generator, Optional, Union
 
-from pip._vendor.cachecontrol.cache import BaseCache
-from pip._vendor.cachecontrol.caches import FileCache
+from pip._vendor.cachecontrol.cache import SeparateBodyBaseCache
+from pip._vendor.cachecontrol.caches import SeparateBodyFileCache
 from pip._vendor.requests.models import Response
 
 from pip._internal.utils.filesystem import adjacent_tmp_file, replace
@@ -28,7 +29,7 @@ def suppressed_cache_errors() -> Generator[None, None, None]:
         pass
 
 
-class SafeFileCache(BaseCache):
+class SafeFileCache(SeparateBodyBaseCache):
     """
     A file based cache which is safe to use even when the target directory may
     not be accessible or writable.
@@ -43,7 +44,7 @@ def _get_cache_path(self, name: str) -> str:
         # From cachecontrol.caches.file_cache.FileCache._fn, brought into our
         # class for backwards-compatibility and to avoid using a non-public
         # method.
-        hashed = FileCache.encode(name)
+        hashed = SeparateBodyFileCache.encode(name)
         parts = list(hashed[:5]) + [hashed]
         return os.path.join(self.directory, *parts)
 
@@ -53,17 +54,33 @@ def get(self, key: str) -> Optional[bytes]:
             with open(path, "rb") as f:
                 return f.read()
 
-    def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None:
-        path = self._get_cache_path(key)
+    def _write(self, path: str, data: bytes) -> None:
         with suppressed_cache_errors():
             ensure_dir(os.path.dirname(path))
 
             with adjacent_tmp_file(path) as f:
-                f.write(value)
+                f.write(data)
 
             replace(f.name, path)
 
+    def set(
+        self, key: str, value: bytes, expires: Union[int, datetime, None] = None
+    ) -> None:
+        path = self._get_cache_path(key)
+        self._write(path, value)
+
     def delete(self, key: str) -> None:
         path = self._get_cache_path(key)
         with suppressed_cache_errors():
             os.remove(path)
+        with suppressed_cache_errors():
+            os.remove(path + ".body")
+
+    def get_body(self, key: str) -> Optional[BinaryIO]:
+        path = self._get_cache_path(key) + ".body"
+        with suppressed_cache_errors():
+            return open(path, "rb")
+
+    def set_body(self, key: str, body: bytes) -> None:
+        path = self._get_cache_path(key) + ".body"
+        self._write(path, body)

src/pip/_vendor/cachecontrol.pyi

@@ -8,11 +8,21 @@
 """
 __author__ = "Eric Larson"
 __email__ = "[email protected]"
-__version__ = "0.12.11"
+__version__ = "0.13.1"
 
-from .wrapper import CacheControl
-from .adapter import CacheControlAdapter
-from .controller import CacheController
+from pip._vendor.cachecontrol.adapter import CacheControlAdapter
+from pip._vendor.cachecontrol.controller import CacheController
+from pip._vendor.cachecontrol.wrapper import CacheControl
+
+__all__ = [
+    "__author__",
+    "__email__",
+    "__version__",
+    "CacheControlAdapter",
+    "CacheController",
+    "CacheControl",
+]
 
 import logging
+
 logging.getLogger(__name__).addHandler(logging.NullHandler())

src/pip/_vendor/cachecontrol/__init__.py

@@ -1,43 +1,49 @@
 # SPDX-FileCopyrightText: 2015 Eric Larson
 #
 # SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
 
 import logging
+from argparse import ArgumentParser
+from typing import TYPE_CHECKING
 
 from pip._vendor import requests
 
 from pip._vendor.cachecontrol.adapter import CacheControlAdapter
 from pip._vendor.cachecontrol.cache import DictCache
 from pip._vendor.cachecontrol.controller import logger
 
-from argparse import ArgumentParser
+if TYPE_CHECKING:
+    from argparse import Namespace
 
+    from pip._vendor.cachecontrol.controller import CacheController
 
-def setup_logging():
+
+def setup_logging() -> None:
     logger.setLevel(logging.DEBUG)
     handler = logging.StreamHandler()
     logger.addHandler(handler)
 
 
-def get_session():
+def get_session() -> requests.Session:
     adapter = CacheControlAdapter(
         DictCache(), cache_etags=True, serializer=None, heuristic=None
     )
     sess = requests.Session()
     sess.mount("http://", adapter)
     sess.mount("https://", adapter)
 
-    sess.cache_controller = adapter.controller
+    sess.cache_controller = adapter.controller  # type: ignore[attr-defined]
     return sess
 
 
-def get_args():
+def get_args() -> Namespace:
     parser = ArgumentParser()
     parser.add_argument("url", help="The URL to try and cache")
     return parser.parse_args()
 
 
-def main(args=None):
+def main() -> None:
     args = get_args()
     sess = get_session()
 
@@ -48,10 +54,13 @@ def main(args=None):
     setup_logging()
 
     # try setting the cache
-    sess.cache_controller.cache_response(resp.request, resp.raw)
+    cache_controller: CacheController = (
+        sess.cache_controller  # type: ignore[attr-defined]
+    )
+    cache_controller.cache_response(resp.request, resp.raw)
 
     # Now try to get it
-    if sess.cache_controller.cached_request(resp.request):
+    if cache_controller.cached_request(resp.request):
         print("Cached!")
     else:
         print("Not cached :(")