Fix client for HTTPS endpoints with Python 3.12 (#1454)

* Fix client for HTTPS endpoints for python 3.12

* Use only `DEFAULT_SSL_CONTEXT_OPTIONS` to prevent `CRIME` attacks

* lint fix

* install certifi types

* pre commit

* Avoid using dep options for >=3.10

* More tests

* Fix lint issues

* Use `3.12` by default everywhere

* Skip `grout -h` test on windows

* Add http only test

* Rollback to python versions, 3.12 causes doc/lint failures

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Keep proxy.py benchmarking on so that users dont run into surprises

* Install certifi

* No need of certifi

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: abhinavsingh

README.md

@@ -2563,7 +2563,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT]
           [--filtered-client-ips FILTERED_CLIENT_IPS]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
 
-proxy.py v2.4.6.dev27+g975b6b68.d20240811
+proxy.py v2.4.6.dev25+g2754b928.d20240812
 
 options:
   -h, --help            show this help message and exit
@@ -2692,8 +2692,8 @@ options:
                         Default: None. Signing certificate to use for signing
                         dynamically generated HTTPS certificates. If used,
                         must also pass --ca-key-file and --ca-signing-key-file
-  --ca-file CA_FILE     Default: /Users/abhinavsingh/Dev/proxy.py/.venv3118/li
-                        b/python3.11/site-packages/certifi/cacert.pem. Provide
+  --ca-file CA_FILE     Default: /Users/abhinavsingh/Dev/proxy.py/.venv3122/li
+                        b/python3.12/site-packages/certifi/cacert.pem. Provide
                         path to custom CA bundle for peer certificate
                         verification
   --ca-signing-key-file CA_SIGNING_KEY_FILE

benchmark/compare.sh

@@ -93,10 +93,10 @@ benchmark_asgi() {
   fi
 }
 
-# echo "============================="
-# echo "Benchmarking Proxy.Py"
-# PYTHONPATH=. benchmark_lib proxy $PROXYPY_PORT
-# echo "============================="
+echo "============================="
+echo "Benchmarking Proxy.Py"
+PYTHONPATH=. benchmark_lib proxy $PROXYPY_PORT
+echo "============================="
 
 # echo "============================="
 # echo "Benchmarking Blacksheep"

benchmark/requirements.txt

@@ -1,4 +1,4 @@
-aiohttp==3.8.2
+aiohttp==3.10.3
 # Blacksheep depends upon essentials_openapi which is pinned to pyyaml==5.4.1
 # and pyyaml>5.3.1 is broken for cython 3
 # See https://github.com/yaml/pyyaml/issues/724#issuecomment-1638587228

proxy/common/constants.py

@@ -161,7 +161,9 @@ def _env_threadless_compliant() -> bool:
 DEFAULT_WAIT_FOR_TASKS_TIMEOUT = 1 / 1000
 DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT = 1   # in seconds
 DEFAULT_SSL_CONTEXT_OPTIONS = (
-    ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1
+    ssl.OP_NO_COMPRESSION
+    if sys.version_info >= (3, 10)
+    else (ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1)
 )
 
 DEFAULT_DEVTOOLS_DOC_URL = 'http://proxy'

proxy/http/client.py

@@ -9,11 +9,18 @@
     :license: BSD, see LICENSE for more details.
 """
 import ssl
+import logging
 from typing import Optional
 
 from .parser import HttpParser, httpParserTypes
+from ..common.types import TcpOrTlsSocket
 from ..common.utils import build_http_request, new_socket_connection
-from ..common.constants import HTTPS_PROTO, DEFAULT_TIMEOUT
+from ..common.constants import (
+    HTTPS_PROTO, DEFAULT_TIMEOUT, DEFAULT_SSL_CONTEXT_OPTIONS,
+)
+
+
+logger = logging.getLogger(__name__)
 
 
 def client(
@@ -25,34 +32,37 @@ def client(
     conn_close: bool = True,
     scheme: bytes = HTTPS_PROTO,
     timeout: float = DEFAULT_TIMEOUT,
+    content_type: bytes = b'application/x-www-form-urlencoded',
 ) -> Optional[HttpParser]:
     """Makes a request to remote registry endpoint"""
     request = build_http_request(
         method=method,
         url=path,
         headers={
             b'Host': host,
-            b'Content-Type': b'application/x-www-form-urlencoded',
+            b'Content-Type': content_type,
         },
         body=body,
         conn_close=conn_close,
     )
     try:
         conn = new_socket_connection((host.decode(), port))
-    except ConnectionRefusedError:
+    except Exception as exc:
+        logger.exception('Cannot establish connection', exc_info=exc)
         return None
-    try:
-        sock = (
-            ssl.wrap_socket(sock=conn, ssl_version=ssl.PROTOCOL_TLSv1_2)
-            if scheme == HTTPS_PROTO
-            else conn
-        )
-    except Exception:
-        conn.close()
-        return None
-    parser = HttpParser(
-        httpParserTypes.RESPONSE_PARSER,
-    )
+    sock: TcpOrTlsSocket = conn
+    if scheme == HTTPS_PROTO:
+        try:
+            ctx = ssl.SSLContext(protocol=(ssl.PROTOCOL_TLS_CLIENT))
+            ctx.options |= DEFAULT_SSL_CONTEXT_OPTIONS
+            ctx.verify_mode = ssl.CERT_REQUIRED
+            ctx.load_default_certs()
+            sock = ctx.wrap_socket(conn, server_hostname=host.decode())
+        except Exception as exc:
+            logger.exception('Unable to wrap', exc_info=exc)
+            conn.close()
+            return None
+    parser = HttpParser(httpParserTypes.RESPONSE_PARSER)
     sock.settimeout(timeout)
     try:
         sock.sendall(request)

tests/http/test_client.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import unittest
+
+from proxy.http.client import client
+
+
+class TestClient(unittest.TestCase):
+
+    def test_http(self) -> None:
+        response = client(
+            host=b'google.com',
+            port=80,
+            scheme=b'http',
+            path=b'/',
+            method=b'GET',
+            content_type=b'text/html',
+        )
+        assert response is not None
+        self.assertEqual(response.code, b'301')
+
+    def test_client(self) -> None:
+        response = client(
+            host=b'google.com',
+            port=443,
+            scheme=b'https',
+            path=b'/',
+            method=b'GET',
+            content_type=b'text/html',
+        )
+        assert response is not None
+        self.assertEqual(response.code, b'301')
+
+    def test_client_connection_refused(self) -> None:
+        response = client(
+            host=b'cannot-establish-connection.com',
+            port=443,
+            scheme=b'https',
+            path=b'/',
+            method=b'GET',
+            content_type=b'text/html',
+        )
+        assert response is None
+
+    def test_cannot_ssl_wrap(self) -> None:
+        response = client(
+            host=b'example.com',
+            port=80,
+            scheme=b'https',
+            path=b'/',
+            method=b'GET',
+            content_type=b'text/html',
+        )
+        assert response is None

tests/test_grout.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import sys
+
+import pytest
+import unittest
+
+from proxy import grout
+from proxy.common.constants import IS_WINDOWS
+
+
+@pytest.mark.skipif(
+    IS_WINDOWS,
+    reason="sys.argv replacement don't really work on windows",
+)
+class TestGrout(unittest.TestCase):
+
+    def test_grout(self) -> None:
+        with self.assertRaises(SystemExit):
+            original = sys.argv
+            sys.argv = ['grout', '-h']
+            grout()
+            sys.argv = original