From a9a94ef277acb9365900549f8fc599a9d9a1b834 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 26 Dec 2023 02:32:25 -0700 Subject: [PATCH 01/19] Initial check-in of type specification conformance suite. --- conformance/.gitignore | 47 ++++ conformance/README.md | 61 +++++ conformance/requirements.txt | 6 + .../results/mypy/annotations_typeexpr.toml | 20 ++ .../results/mypy/generics_self_advanced.toml | 15 ++ .../mypy/generics_self_attributes.toml | 5 + .../results/mypy/generics_self_basic.toml | 10 + .../results/mypy/generics_self_protocols.toml | 15 ++ .../results/mypy/generics_self_usage.toml | 19 ++ .../results/mypy/literals_interactions.toml | 12 + .../results/mypy/literals_literalstring.toml | 15 ++ .../mypy/literals_parameterizations.toml | 22 ++ .../results/mypy/literals_semantics.toml | 7 + .../results/mypy/narrowing_typeguard.toml | 5 + .../results/mypy/typeddicts_alt_syntax.toml | 13 + .../results/mypy/typeddicts_class_syntax.toml | 8 + .../results/mypy/typeddicts_final.toml | 3 + .../results/mypy/typeddicts_inheritance.toml | 6 + .../results/mypy/typeddicts_operations.toml | 14 + .../results/mypy/typeddicts_required.toml | 12 + .../mypy/typeddicts_type_consistency.toml | 14 + .../results/mypy/typeddicts_usage.toml | 10 + conformance/results/mypy/version.toml | 1 + .../results/pyre/annotations_typeexpr.toml | 19 ++ .../results/pyre/generics_self_advanced.toml | 15 ++ .../pyre/generics_self_attributes.toml | 10 + .../results/pyre/generics_self_basic.toml | 15 ++ .../results/pyre/generics_self_protocols.toml | 7 + .../results/pyre/generics_self_usage.toml | 10 + .../results/pyre/literals_interactions.toml | 11 + .../results/pyre/literals_literalstring.toml | 13 + .../pyre/literals_parameterizations.toml | 30 +++ .../results/pyre/literals_semantics.toml | 9 + .../results/pyre/narrowing_typeguard.toml | 9 + .../results/pyre/typeddicts_alt_syntax.toml | 10 + .../results/pyre/typeddicts_class_syntax.toml | 11 + .../results/pyre/typeddicts_final.toml | 7 + .../results/pyre/typeddicts_inheritance.toml | 8 + .../results/pyre/typeddicts_operations.toml | 14 + .../results/pyre/typeddicts_required.toml | 11 + .../pyre/typeddicts_type_consistency.toml | 19 ++ .../results/pyre/typeddicts_usage.toml | 10 + conformance/results/pyre/version.toml | 1 + .../results/pyright/annotations_typeexpr.toml | 31 +++ .../pyright/generics_self_advanced.toml | 3 + .../pyright/generics_self_attributes.toml | 14 + .../results/pyright/generics_self_basic.toml | 8 + .../pyright/generics_self_protocols.toml | 15 ++ .../results/pyright/generics_self_usage.toml | 20 ++ .../pyright/literals_interactions.toml | 7 + .../pyright/literals_literalstring.toml | 25 ++ .../pyright/literals_parameterizations.toml | 21 ++ .../results/pyright/literals_semantics.toml | 10 + .../results/pyright/narrowing_typeguard.toml | 5 + .../pyright/typeddicts_alt_syntax.toml | 11 + .../pyright/typeddicts_class_syntax.toml | 10 + .../results/pyright/typeddicts_final.toml | 3 + .../pyright/typeddicts_inheritance.toml | 9 + .../pyright/typeddicts_operations.toml | 24 ++ .../results/pyright/typeddicts_required.toml | 7 + .../pyright/typeddicts_type_consistency.toml | 25 ++ .../results/pyright/typeddicts_usage.toml | 13 + conformance/results/pyright/version.toml | 1 + conformance/results/results.html | 240 ++++++++++++++++++ conformance/src/__init__.py | 0 conformance/src/main.py | 122 +++++++++ conformance/src/reporting.py | 99 ++++++++ conformance/src/results_template.html | 129 ++++++++++ conformance/src/test_groups.py | 46 ++++ conformance/src/test_groups.toml | 66 +++++ conformance/src/type_checker.py | 150 +++++++++++ conformance/tests/annotations_typeexpr.py | 99 ++++++++ conformance/tests/generics_self_advanced.py | 46 ++++ conformance/tests/generics_self_attributes.py | 34 +++ conformance/tests/generics_self_basic.py | 82 ++++++ conformance/tests/generics_self_protocols.py | 64 +++++ conformance/tests/generics_self_usage.py | 126 +++++++++ conformance/tests/literals_interactions.py | 116 +++++++++ conformance/tests/literals_literalstring.py | 170 +++++++++++++ .../tests/literals_parameterizations.py | 67 +++++ conformance/tests/literals_semantics.py | 42 +++ conformance/tests/narrowing_typeguard.py | 108 ++++++++ conformance/tests/typeddicts_alt_syntax.py | 45 ++++ conformance/tests/typeddicts_class_syntax.py | 79 ++++++ conformance/tests/typeddicts_final.py | 26 ++ conformance/tests/typeddicts_inheritance.py | 66 +++++ conformance/tests/typeddicts_operations.py | 65 +++++ conformance/tests/typeddicts_required.py | 76 ++++++ .../tests/typeddicts_type_consistency.py | 150 +++++++++++ conformance/tests/typeddicts_usage.py | 42 +++ 90 files changed, 3196 insertions(+) create mode 100644 conformance/.gitignore create mode 100644 conformance/README.md create mode 100644 conformance/requirements.txt create mode 100644 conformance/results/mypy/annotations_typeexpr.toml create mode 100644 conformance/results/mypy/generics_self_advanced.toml create mode 100644 conformance/results/mypy/generics_self_attributes.toml create mode 100644 conformance/results/mypy/generics_self_basic.toml create mode 100644 conformance/results/mypy/generics_self_protocols.toml create mode 100644 conformance/results/mypy/generics_self_usage.toml create mode 100644 conformance/results/mypy/literals_interactions.toml create mode 100644 conformance/results/mypy/literals_literalstring.toml create mode 100644 conformance/results/mypy/literals_parameterizations.toml create mode 100644 conformance/results/mypy/literals_semantics.toml create mode 100644 conformance/results/mypy/narrowing_typeguard.toml create mode 100644 conformance/results/mypy/typeddicts_alt_syntax.toml create mode 100644 conformance/results/mypy/typeddicts_class_syntax.toml create mode 100644 conformance/results/mypy/typeddicts_final.toml create mode 100644 conformance/results/mypy/typeddicts_inheritance.toml create mode 100644 conformance/results/mypy/typeddicts_operations.toml create mode 100644 conformance/results/mypy/typeddicts_required.toml create mode 100644 conformance/results/mypy/typeddicts_type_consistency.toml create mode 100644 conformance/results/mypy/typeddicts_usage.toml create mode 100644 conformance/results/mypy/version.toml create mode 100644 conformance/results/pyre/annotations_typeexpr.toml create mode 100644 conformance/results/pyre/generics_self_advanced.toml create mode 100644 conformance/results/pyre/generics_self_attributes.toml create mode 100644 conformance/results/pyre/generics_self_basic.toml create mode 100644 conformance/results/pyre/generics_self_protocols.toml create mode 100644 conformance/results/pyre/generics_self_usage.toml create mode 100644 conformance/results/pyre/literals_interactions.toml create mode 100644 conformance/results/pyre/literals_literalstring.toml create mode 100644 conformance/results/pyre/literals_parameterizations.toml create mode 100644 conformance/results/pyre/literals_semantics.toml create mode 100644 conformance/results/pyre/narrowing_typeguard.toml create mode 100644 conformance/results/pyre/typeddicts_alt_syntax.toml create mode 100644 conformance/results/pyre/typeddicts_class_syntax.toml create mode 100644 conformance/results/pyre/typeddicts_final.toml create mode 100644 conformance/results/pyre/typeddicts_inheritance.toml create mode 100644 conformance/results/pyre/typeddicts_operations.toml create mode 100644 conformance/results/pyre/typeddicts_required.toml create mode 100644 conformance/results/pyre/typeddicts_type_consistency.toml create mode 100644 conformance/results/pyre/typeddicts_usage.toml create mode 100644 conformance/results/pyre/version.toml create mode 100644 conformance/results/pyright/annotations_typeexpr.toml create mode 100644 conformance/results/pyright/generics_self_advanced.toml create mode 100644 conformance/results/pyright/generics_self_attributes.toml create mode 100644 conformance/results/pyright/generics_self_basic.toml create mode 100644 conformance/results/pyright/generics_self_protocols.toml create mode 100644 conformance/results/pyright/generics_self_usage.toml create mode 100644 conformance/results/pyright/literals_interactions.toml create mode 100644 conformance/results/pyright/literals_literalstring.toml create mode 100644 conformance/results/pyright/literals_parameterizations.toml create mode 100644 conformance/results/pyright/literals_semantics.toml create mode 100644 conformance/results/pyright/narrowing_typeguard.toml create mode 100644 conformance/results/pyright/typeddicts_alt_syntax.toml create mode 100644 conformance/results/pyright/typeddicts_class_syntax.toml create mode 100644 conformance/results/pyright/typeddicts_final.toml create mode 100644 conformance/results/pyright/typeddicts_inheritance.toml create mode 100644 conformance/results/pyright/typeddicts_operations.toml create mode 100644 conformance/results/pyright/typeddicts_required.toml create mode 100644 conformance/results/pyright/typeddicts_type_consistency.toml create mode 100644 conformance/results/pyright/typeddicts_usage.toml create mode 100644 conformance/results/pyright/version.toml create mode 100644 conformance/results/results.html create mode 100644 conformance/src/__init__.py create mode 100644 conformance/src/main.py create mode 100644 conformance/src/reporting.py create mode 100644 conformance/src/results_template.html create mode 100644 conformance/src/test_groups.py create mode 100644 conformance/src/test_groups.toml create mode 100644 conformance/src/type_checker.py create mode 100644 conformance/tests/annotations_typeexpr.py create mode 100644 conformance/tests/generics_self_advanced.py create mode 100644 conformance/tests/generics_self_attributes.py create mode 100644 conformance/tests/generics_self_basic.py create mode 100644 conformance/tests/generics_self_protocols.py create mode 100644 conformance/tests/generics_self_usage.py create mode 100644 conformance/tests/literals_interactions.py create mode 100644 conformance/tests/literals_literalstring.py create mode 100644 conformance/tests/literals_parameterizations.py create mode 100644 conformance/tests/literals_semantics.py create mode 100644 conformance/tests/narrowing_typeguard.py create mode 100644 conformance/tests/typeddicts_alt_syntax.py create mode 100644 conformance/tests/typeddicts_class_syntax.py create mode 100644 conformance/tests/typeddicts_final.py create mode 100644 conformance/tests/typeddicts_inheritance.py create mode 100644 conformance/tests/typeddicts_operations.py create mode 100644 conformance/tests/typeddicts_required.py create mode 100644 conformance/tests/typeddicts_type_consistency.py create mode 100644 conformance/tests/typeddicts_usage.py diff --git a/conformance/.gitignore b/conformance/.gitignore new file mode 100644 index 000000000..5eae7a458 --- /dev/null +++ b/conformance/.gitignore @@ -0,0 +1,47 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Environments +.env +.venv + +# Tools +.mypy_cache +.pyre_configuration +.pyre +.coverage +htmlcov + +# General +.DS_Store + +# Editor temp files +.*.swp + +# Workspace configurations +.vscode diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 000000000..f3f8633ac --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,61 @@ +# Type System Conformance + +## Motivation + +[PEP 729](https://peps.python.org/pep-0729/) provides a structured and documented way to specify and evolve the Python type system. In support of this effort, an official [Python typing spec](https://github.com/python/typing/tree/main/docs/spec) has been drafted. This spec consolidates details from various historical typing-related PEPs. The spec will be modified over time to clarify unspecified and under-specified parts of the type system. It will also be extended to cover new features of the type system. + +Accompanying the typing specification is this conformance test suite which validates the behavior of static type checkers against the specification. + +## Structure & Name + +This project contains test cases for behaviors defined in the Python typing spec. Tests are structured and grouped in accordance with the specification's chapter headings. + +* [concepts](https://typing.readthedocs.io/en/latest/spec/concepts.html) +* [annotations](https://typing.readthedocs.io/en/latest/spec/annotations.html) +* [specialtypes](https://typing.readthedocs.io/en/latest/spec/special-types.html) +* [generics](https://typing.readthedocs.io/en/latest/spec/generics.html) +* [qualifiers](https://typing.readthedocs.io/en/latest/spec/qualifiers.html) +* [classes](https://typing.readthedocs.io/en/latest/spec/class-compat.html) +* [aliases](https://typing.readthedocs.io/en/latest/spec/aliases.html) +* [literals](https://typing.readthedocs.io/en/latest/spec/literal.html) +* [protocols](https://typing.readthedocs.io/en/latest/spec/protocol.html) +* [callables](https://typing.readthedocs.io/en/latest/spec/callables.html) +* [overloads](https://typing.readthedocs.io/en/latest/spec/overload.html) +* [dataclasses](https://typing.readthedocs.io/en/latest/spec/dataclasses.html) +* [typeddicts](https://typing.readthedocs.io/en/latest/spec/typeddict.html) +* [narrowing](https://typing.readthedocs.io/en/latest/spec/narrowing.html) +* [directives](https://typing.readthedocs.io/en/latest/spec/directives.html) +* [distribution](https://typing.readthedocs.io/en/latest/spec/distributing.html) +* [historical](https://typing.readthedocs.io/en/latest/spec/historical.html) + +A test file is a ".py" file. The file name should start with one of the above names followed by a description of the test (with words separated by underscores). For example, `generics_paramspec_basic_usage.py` would contain the basic usage tests for `ParamSpec`. Each test file can contain multiple individual unit tests, but these tests should be related to each other. If the number of unit tests in a single test file exceeds ten, it may be desirable to split it into separate test files. This will help maintain a consistent level of granularity across tests. + +## Notes About Tests + +Tests are designed to run on all current and future static type checkers. They must therefore be type-checker agnostic and should not rely on functionality or behaviors that are specific to one type checker or another. + +Test cases are meant to be human readable. They should include comments that help explain their purpose (what is being tested, whether an error should be generated, etc.). They should also contain links to the typing spec, discussions, and issue trackers. + +The test suite focuses on static type checking not general Python semantics. Tests should therefore focus on static analysis behaviors, not runtime behaviors. + +## Reporting Conformance Results + +Different type checkers report errors in different ways (with different wording in error messages and different line numbers or character ranges for errors). This variation makes it difficult to fully automate test validation given that tests will want to check for both false positive and false negative type errors. Some level of manual inspection will therefore be needed to determine whether a type checker is fully conformant with all tests in any given test file. This "scoring" process is required only when the output of a test changes — e.g. when a new version of that type checker is released and the tests are rerun. We assume that the output of a type checker will be the same from one run to the next unless/until a new version is released that fixes or introduces a bug. In this case, the output will need to be manually inspected and the conformance results re-scored for those tests whose output has changed. + +Conformance results are reported and summarized for each supported type checker. Initially, results will be reported for mypy and pyright. It is the goal and desire to add additional type checkers over time. + +## Adding a New Test Case + +To add a new test, create a new ".py" file in the `tests` directory. Its name must begin with one of the above test group names followed by an underscore. Write the contents of the test including a module docstring describing the purpose of the test. Next, run the tool. This will generate a new `.toml` file corresponding to the new test for each supported type checker. Manually review the output from each type checker and determine whether it conforms to the specification. If so, add `conformant = "Yes"` to the `.toml` file. If it does not fully comply, add `conformant = "Partial"` and a `notes` section detailing where it is not compliant. If the type checker doesn't support the feature in the test add `conformant = "Unsupported"`. Once the conformance status has been updated, re-run the test tool again to regenerate the summary report. + +## Updating A Test Case + +If a test is updated (augmented or fixed), the process is similar to when adding a new test. Run the tool to generate new results and manually examine the output of each supported type checker. Then update the conformance status accordingly. Once the conformance status has been updated, re-run the test tool again to regenerate the summary report. + +## Updating a Type Checker + +If a new version of a type checker is released, re-run the test tool with the new version. If the type checker output has changed for any test cases, the tool will supply the old and new outputs. Examine these to determine whether the conformance status has changed. Once the conformance status has been updated, re-run the test tool again to regenerate the summary report. + +## Contributing + +Contributions are welcome! diff --git a/conformance/requirements.txt b/conformance/requirements.txt new file mode 100644 index 000000000..b60d217d6 --- /dev/null +++ b/conformance/requirements.txt @@ -0,0 +1,6 @@ +tomli +tomlkit +pyright +mypy +pyre-check +pytype diff --git a/conformance/results/mypy/annotations_typeexpr.toml b/conformance/results/mypy/annotations_typeexpr.toml new file mode 100644 index 000000000..47799bf5c --- /dev/null +++ b/conformance/results/mypy/annotations_typeexpr.toml @@ -0,0 +1,20 @@ +conformant = "Yes" +output = """ +annotations_typeexpr.py:77: error: Invalid type comment or annotation [valid-type] +annotations_typeexpr.py:77: note: Suggestion: use eval[...] instead of eval(...) +annotations_typeexpr.py:78: error: Bracketed expression "[...]" is not valid as a type [valid-type] +annotations_typeexpr.py:79: error: Syntax error in type annotation [syntax] +annotations_typeexpr.py:79: note: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) +annotations_typeexpr.py:80: error: Invalid type comment or annotation [valid-type] +annotations_typeexpr.py:81: error: Invalid type comment or annotation [valid-type] +annotations_typeexpr.py:82: error: Invalid type comment or annotation [valid-type] +annotations_typeexpr.py:83: error: Invalid type comment or annotation [valid-type] +annotations_typeexpr.py:84: error: Invalid type comment or annotation [valid-type] +annotations_typeexpr.py:85: error: Variable "annotations_typeexpr.var1" is not valid as a type [valid-type] +annotations_typeexpr.py:85: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +annotations_typeexpr.py:86: error: Invalid type: try using Literal[True] instead? [valid-type] +annotations_typeexpr.py:87: error: Invalid type: try using Literal[1] instead? [valid-type] +annotations_typeexpr.py:88: error: Invalid type: try using Literal[-1] instead? [valid-type] +annotations_typeexpr.py:89: error: Invalid type comment or annotation [valid-type] +annotations_typeexpr.py:90: error: Invalid type comment or annotation [valid-type] +""" diff --git a/conformance/results/mypy/generics_self_advanced.toml b/conformance/results/mypy/generics_self_advanced.toml new file mode 100644 index 000000000..bfee9ad9e --- /dev/null +++ b/conformance/results/mypy/generics_self_advanced.toml @@ -0,0 +1,15 @@ +conformant = "Partial" +notes = """ +Does not infer the type of an unannotated `self` parameter to be type `Self`. +Does not retain `Self` when calling method that returns `Self`. +Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`. +Does not retain `Self` when accessing attribute through `type[Self]`. +""" +output = """ +generics_self_advanced.py:35: error: Expression is of type "ChildB", not "Self" [assert-type] +generics_self_advanced.py:38: error: Expression is of type "ChildB", not "Self" [assert-type] +generics_self_advanced.py:42: error: Expression is of type "type[ChildB]", not "type[Self]" [assert-type] +generics_self_advanced.py:43: error: Expression is of type "list[ChildB]", not "list[Self]" [assert-type] +generics_self_advanced.py:44: error: Expression is of type "ChildB", not "Self" [assert-type] +generics_self_advanced.py:45: error: Expression is of type "ChildB", not "Self" [assert-type] +""" diff --git a/conformance/results/mypy/generics_self_attributes.toml b/conformance/results/mypy/generics_self_attributes.toml new file mode 100644 index 000000000..560756a55 --- /dev/null +++ b/conformance/results/mypy/generics_self_attributes.toml @@ -0,0 +1,5 @@ +conformant = "Yes" +output = """ +generics_self_attributes.py:26: error: Argument "next" to "OrdinalLinkedList" has incompatible type "LinkedList[int]"; expected "OrdinalLinkedList | None" [arg-type] +generics_self_attributes.py:32: error: Incompatible types in assignment (expression has type "LinkedList[int]", variable has type "OrdinalLinkedList | None") [assignment] +""" diff --git a/conformance/results/mypy/generics_self_basic.toml b/conformance/results/mypy/generics_self_basic.toml new file mode 100644 index 000000000..317f9ac13 --- /dev/null +++ b/conformance/results/mypy/generics_self_basic.toml @@ -0,0 +1,10 @@ +conformant = "Partial" +notes = """ +Does not properly handle constructor call through `cls` parameter. +""" +output = """ +generics_self_basic.py:19: error: Incompatible return value type (got "Shape", expected "Self") [return-value] +generics_self_basic.py:27: error: Too many arguments for "Shape" [call-arg] +generics_self_basic.py:32: error: Incompatible return value type (got "Shape", expected "Self") [return-value] +generics_self_basic.py:64: error: Self type cannot have type arguments [misc] +""" diff --git a/conformance/results/mypy/generics_self_protocols.toml b/conformance/results/mypy/generics_self_protocols.toml new file mode 100644 index 000000000..432b4860f --- /dev/null +++ b/conformance/results/mypy/generics_self_protocols.toml @@ -0,0 +1,15 @@ +conformant = "Yes" +output = """ +generics_self_protocols.py:61: error: Argument 1 to "accepts_shape" has incompatible type "BadReturnType"; expected "ShapeProtocol" [arg-type] +generics_self_protocols.py:61: note: Following member(s) of "BadReturnType" have conflicts: +generics_self_protocols.py:61: note: Expected: +generics_self_protocols.py:61: note: def set_scale(self, scale: float) -> BadReturnType +generics_self_protocols.py:61: note: Got: +generics_self_protocols.py:61: note: def set_scale(self, scale: float) -> int +generics_self_protocols.py:64: error: Argument 1 to "accepts_shape" has incompatible type "ReturnDifferentClass"; expected "ShapeProtocol" [arg-type] +generics_self_protocols.py:64: note: Following member(s) of "ReturnDifferentClass" have conflicts: +generics_self_protocols.py:64: note: Expected: +generics_self_protocols.py:64: note: def set_scale(self, scale: float) -> ReturnDifferentClass +generics_self_protocols.py:64: note: Got: +generics_self_protocols.py:64: note: def set_scale(self, scale: float) -> ReturnConcreteShape +""" diff --git a/conformance/results/mypy/generics_self_usage.toml b/conformance/results/mypy/generics_self_usage.toml new file mode 100644 index 000000000..e9642785d --- /dev/null +++ b/conformance/results/mypy/generics_self_usage.toml @@ -0,0 +1,19 @@ +conformant = "Yes" +output = """ +generics_self_usage.py:73: error: Self type is only allowed in annotations within class definition [misc] +generics_self_usage.py:76: error: Self type is only allowed in annotations within class definition [misc] +generics_self_usage.py:82: error: Method cannot have explicit self annotation and Self type [misc] +generics_self_usage.py:82: error: A function returning TypeVar should receive at least one argument containing the same TypeVar [type-var] +generics_self_usage.py:82: note: Consider using the upper bound "Foo2" instead +generics_self_usage.py:86: error: Incompatible return value type (got "Foo3", expected "Self") [return-value] +generics_self_usage.py:101: error: Self type is only allowed in annotations within class definition [misc] +generics_self_usage.py:103: error: Self type is only allowed in annotations within class definition [misc] +generics_self_usage.py:106: error: Self type is only allowed in annotations within class definition [misc] +generics_self_usage.py:106: error: Self type cannot be used in type alias target [misc] +generics_self_usage.py:111: error: Static methods cannot use Self type [misc] +generics_self_usage.py:111: error: A function returning TypeVar should receive at least one argument containing the same TypeVar [type-var] +generics_self_usage.py:111: note: Consider using the upper bound "Base" instead +generics_self_usage.py:116: error: Static methods cannot use Self type [misc] +generics_self_usage.py:121: error: Self type cannot be used in a metaclass [misc] +generics_self_usage.py:125: error: Self type cannot be used in a metaclass [misc] +""" diff --git a/conformance/results/mypy/literals_interactions.toml b/conformance/results/mypy/literals_interactions.toml new file mode 100644 index 000000000..b2b4ddbd0 --- /dev/null +++ b/conformance/results/mypy/literals_interactions.toml @@ -0,0 +1,12 @@ +conformant = "Partial" +notes = """ +Does not narrow type of `x` with `x in Literal` type guard pattern. +""" +output = """ +literals_interactions.py:15: error: Tuple index out of range [misc] +literals_interactions.py:16: error: Tuple index out of range [misc] +literals_interactions.py:17: error: Tuple index out of range [misc] +literals_interactions.py:18: error: Tuple index out of range [misc] +literals_interactions.py:106: error: Argument 1 to "expects_bad_status" has incompatible type "str"; expected "Literal['MALFORMED', 'ABORTED']" [arg-type] +literals_interactions.py:109: error: Argument 1 to "expects_pending_status" has incompatible type "str"; expected "Literal['PENDING']" [arg-type] +""" diff --git a/conformance/results/mypy/literals_literalstring.toml b/conformance/results/mypy/literals_literalstring.toml new file mode 100644 index 000000000..c3574fccf --- /dev/null +++ b/conformance/results/mypy/literals_literalstring.toml @@ -0,0 +1,15 @@ +conformant = "Unsupported" +notes = """ +Support for `LiteralString` has not been implemented in mypy. +""" +output = """ +literals_literalstring.py:36: error: Parameter 2 of Literal[...] is invalid [valid-type] +literals_literalstring.py:37: error: Parameter 1 of Literal[...] is invalid [valid-type] +literals_literalstring.py:43: error: Incompatible types in assignment (expression has type "Literal['two']", variable has type "Literal['']") [assignment] +literals_literalstring.py:74: error: Incompatible types in assignment (expression has type "int", variable has type "str") [assignment] +literals_literalstring.py:75: error: Incompatible types in assignment (expression has type "bytes", variable has type "str") [assignment] +literals_literalstring.py:142: error: Overloaded function signatures 1 and 2 overlap with incompatible return types [overload-overlap] +literals_literalstring.py:142: error: Overloaded function signatures 1 and 3 overlap with incompatible return types [overload-overlap] +literals_literalstring.py:152: error: Overloaded function signature 3 will never be matched: signature 2's parameter type(s) are the same or broader [misc] +literals_literalstring.py:162: error: Expression is of type "bool", not "str" [assert-type] +""" diff --git a/conformance/results/mypy/literals_parameterizations.toml b/conformance/results/mypy/literals_parameterizations.toml new file mode 100644 index 000000000..2b099f547 --- /dev/null +++ b/conformance/results/mypy/literals_parameterizations.toml @@ -0,0 +1,22 @@ +conformant = "Partial" +notes = """ +Does not reject tuple within Literal. +""" +output = """ +literals_parameterizations.py:40: error: Invalid type: Literal[...] cannot contain arbitrary expressions [valid-type] +literals_parameterizations.py:41: error: Invalid type: Literal[...] cannot contain arbitrary expressions [valid-type] +literals_parameterizations.py:42: error: Invalid type: Literal[...] cannot contain arbitrary expressions [valid-type] +literals_parameterizations.py:43: error: Invalid type: Literal[...] cannot contain arbitrary expressions [valid-type] +literals_parameterizations.py:44: error: Invalid type: Literal[...] cannot contain arbitrary expressions [valid-type] +literals_parameterizations.py:46: error: Invalid type: Literal[...] cannot contain arbitrary expressions [valid-type] +literals_parameterizations.py:47: error: Parameter 1 of Literal[...] is invalid [valid-type] +literals_parameterizations.py:48: error: Parameter 1 of Literal[...] is invalid [valid-type] +literals_parameterizations.py:49: error: Parameter 1 of Literal[...] is invalid [valid-type] +literals_parameterizations.py:50: error: Parameter 1 of Literal[...] cannot be of type "float" [valid-type] +literals_parameterizations.py:51: error: Parameter 1 of Literal[...] cannot be of type "Any" [valid-type] +literals_parameterizations.py:52: error: Parameter 1 of Literal[...] is invalid [valid-type] +literals_parameterizations.py:55: error: Invalid type: Literal[...] cannot contain arbitrary expressions [valid-type] +literals_parameterizations.py:58: error: Literal[...] must have at least one parameter [valid-type] +literals_parameterizations.py:59: error: Parameter 1 of Literal[...] is invalid [valid-type] +literals_parameterizations.py:63: error: Incompatible types in assignment (expression has type "Literal[Color.RED]", variable has type "Literal['Color.RED']") [assignment] +""" diff --git a/conformance/results/mypy/literals_semantics.toml b/conformance/results/mypy/literals_semantics.toml new file mode 100644 index 000000000..384e68cff --- /dev/null +++ b/conformance/results/mypy/literals_semantics.toml @@ -0,0 +1,7 @@ +conformant = "Yes" +output = """ +literals_semantics.py:10: error: Incompatible types in assignment (expression has type "Literal[4]", variable has type "Literal[3]") [assignment] +literals_semantics.py:24: error: Incompatible types in assignment (expression has type "Literal[0]", variable has type "Literal[False]") [assignment] +literals_semantics.py:25: error: Incompatible types in assignment (expression has type "Literal[False]", variable has type "Literal[0]") [assignment] +literals_semantics.py:33: error: Incompatible types in assignment (expression has type "int", variable has type "Literal[3, 4, 5]") [assignment] +""" diff --git a/conformance/results/mypy/narrowing_typeguard.toml b/conformance/results/mypy/narrowing_typeguard.toml new file mode 100644 index 000000000..f92bc925a --- /dev/null +++ b/conformance/results/mypy/narrowing_typeguard.toml @@ -0,0 +1,5 @@ +conformant = "Yes" +output = """ +narrowing_typeguard.py:102: error: TypeGuard functions must have a positional argument [valid-type] +narrowing_typeguard.py:107: error: TypeGuard functions must have a positional argument [valid-type] +""" diff --git a/conformance/results/mypy/typeddicts_alt_syntax.toml b/conformance/results/mypy/typeddicts_alt_syntax.toml new file mode 100644 index 000000000..bb1da440f --- /dev/null +++ b/conformance/results/mypy/typeddicts_alt_syntax.toml @@ -0,0 +1,13 @@ +conformant = "Partial" +notes = """ +Does not support keyword-argument form of alternative syntax (deprecated in 3.11). +""" +output = """ +typeddicts_alt_syntax.py:23: error: TypedDict() expects a dictionary literal as the second argument [misc] +typeddicts_alt_syntax.py:27: error: Invalid TypedDict() field name [misc] +typeddicts_alt_syntax.py:31: error: First argument "WrongName" to TypedDict() does not match variable name "BadTypedDict3" [name-match] +typeddicts_alt_syntax.py:35: error: Too many arguments for TypedDict() [misc] +typeddicts_alt_syntax.py:41: error: Unexpected arguments to TypedDict() [misc] +typeddicts_alt_syntax.py:44: error: Extra keys ("name", "year") for TypedDict "Movie2" [typeddict-unknown-key] +typeddicts_alt_syntax.py:45: error: Extra keys ("name", "year") for TypedDict "Movie2" [typeddict-unknown-key] +""" diff --git a/conformance/results/mypy/typeddicts_class_syntax.toml b/conformance/results/mypy/typeddicts_class_syntax.toml new file mode 100644 index 000000000..edbcfb888 --- /dev/null +++ b/conformance/results/mypy/typeddicts_class_syntax.toml @@ -0,0 +1,8 @@ +conformant = "Yes" +output = """ +typeddicts_class_syntax.py:29: error: Invalid statement in TypedDict definition; expected "field_name: field_type" [misc] +typeddicts_class_syntax.py:33: error: Invalid statement in TypedDict definition; expected "field_name: field_type" [misc] +typeddicts_class_syntax.py:38: error: Invalid statement in TypedDict definition; expected "field_name: field_type" [misc] +typeddicts_class_syntax.py:44: error: Unexpected keyword argument "metaclass" for "__init_subclass__" of "TypedDict" [call-arg] +typeddicts_class_syntax.py:49: error: Unexpected keyword argument "other" for "__init_subclass__" of "TypedDict" [call-arg] +""" diff --git a/conformance/results/mypy/typeddicts_final.toml b/conformance/results/mypy/typeddicts_final.toml new file mode 100644 index 000000000..74f99ddf8 --- /dev/null +++ b/conformance/results/mypy/typeddicts_final.toml @@ -0,0 +1,3 @@ +conformant = "Yes" +output = """ +""" diff --git a/conformance/results/mypy/typeddicts_inheritance.toml b/conformance/results/mypy/typeddicts_inheritance.toml new file mode 100644 index 000000000..3d12cac25 --- /dev/null +++ b/conformance/results/mypy/typeddicts_inheritance.toml @@ -0,0 +1,6 @@ +conformant = "Yes" +output = """ +typeddicts_inheritance.py:44: error: All bases of a new TypedDict must be TypedDict types [misc] +typeddicts_inheritance.py:55: error: Overwriting TypedDict field "x" while extending [misc] +typeddicts_inheritance.py:65: error: Overwriting TypedDict field "x" while merging [misc] +""" diff --git a/conformance/results/mypy/typeddicts_operations.toml b/conformance/results/mypy/typeddicts_operations.toml new file mode 100644 index 000000000..87d4ffbee --- /dev/null +++ b/conformance/results/mypy/typeddicts_operations.toml @@ -0,0 +1,14 @@ +conformant = "Yes" +output = """ +typeddicts_operations.py:22: error: Value of "name" has incompatible type "int"; expected "str" [typeddict-item] +typeddicts_operations.py:23: error: Value of "year" has incompatible type "str"; expected "int" [typeddict-item] +typeddicts_operations.py:24: error: TypedDict "Movie" has no key "other" [typeddict-unknown-key] +typeddicts_operations.py:26: error: TypedDict "Movie" has no key "other" [typeddict-item] +typeddicts_operations.py:28: error: Missing key "year" for TypedDict "Movie" [typeddict-item] +typeddicts_operations.py:29: error: Incompatible types (expression has type "float", TypedDict item "year" has type "int") [typeddict-item] +typeddicts_operations.py:32: error: Extra key "other" for TypedDict "Movie" [typeddict-unknown-key] +typeddicts_operations.py:37: error: Expected TypedDict key to be string literal [misc] +typeddicts_operations.py:47: error: "Movie" has no attribute "clear" [attr-defined] +typeddicts_operations.py:49: error: Key "name" of TypedDict "Movie" cannot be deleted [misc] +typeddicts_operations.py:62: error: "MovieOptional" has no attribute "clear" [attr-defined] +""" diff --git a/conformance/results/mypy/typeddicts_required.toml b/conformance/results/mypy/typeddicts_required.toml new file mode 100644 index 000000000..4e11a4ad6 --- /dev/null +++ b/conformance/results/mypy/typeddicts_required.toml @@ -0,0 +1,12 @@ +conformant = "Partial" +notes = """ +Does not support nesting of `Annotated` and `Required` or `NotRequired`. +""" +output = """ +typeddicts_required.py:12: error: Required[] can be only used in a TypedDict definition [valid-type] +typeddicts_required.py:16: error: NotRequired[] can be only used in a TypedDict definition [valid-type] +typeddicts_required.py:59: error: Required[] can be only used in a TypedDict definition [valid-type] +typeddicts_required.py:60: error: NotRequired[] can be only used in a TypedDict definition [valid-type] +typeddicts_required.py:65: error: Required[] can be only used in a TypedDict definition [valid-type] +typeddicts_required.py:67: error: Required[] can be only used in a TypedDict definition [valid-type] +""" diff --git a/conformance/results/mypy/typeddicts_type_consistency.toml b/conformance/results/mypy/typeddicts_type_consistency.toml new file mode 100644 index 000000000..40b7fba9d --- /dev/null +++ b/conformance/results/mypy/typeddicts_type_consistency.toml @@ -0,0 +1,14 @@ +conformant = "Yes" +output = """ +typeddicts_type_consistency.py:21: error: Incompatible types in assignment (expression has type "B1", variable has type "A1") [assignment] +typeddicts_type_consistency.py:38: error: Incompatible types in assignment (expression has type "B2", variable has type "A2") [assignment] +typeddicts_type_consistency.py:65: error: Incompatible types in assignment (expression has type "A3", variable has type "B3") [assignment] +typeddicts_type_consistency.py:69: error: Extra key "y" for TypedDict "A3" [typeddict-unknown-key] +typeddicts_type_consistency.py:76: error: Incompatible types in assignment (expression has type "B3", variable has type "dict[str, int]") [assignment] +typeddicts_type_consistency.py:77: error: Incompatible types in assignment (expression has type "B3", variable has type "dict[str, object]") [assignment] +typeddicts_type_consistency.py:78: error: Incompatible types in assignment (expression has type "B3", variable has type "dict[Any, Any]") [assignment] +typeddicts_type_consistency.py:82: error: Incompatible types in assignment (expression has type "B3", variable has type "Mapping[str, int]") [assignment] +typeddicts_type_consistency.py:99: error: Incompatible types in assignment (expression has type "str | None", variable has type "str") [assignment] +typeddicts_type_consistency.py:105: error: Incompatible types in assignment (expression has type "int | str", variable has type "int") [assignment] +typeddicts_type_consistency.py:124: error: Incompatible types (expression has type "int", TypedDict item "inner_key" has type "str") [typeddict-item] +""" diff --git a/conformance/results/mypy/typeddicts_usage.toml b/conformance/results/mypy/typeddicts_usage.toml new file mode 100644 index 000000000..00081b5ea --- /dev/null +++ b/conformance/results/mypy/typeddicts_usage.toml @@ -0,0 +1,10 @@ +conformant = "Yes" +output = """ +typeddicts_usage.py:23: error: TypedDict "Movie" has no key "director" [typeddict-unknown-key] +typeddicts_usage.py:24: error: Value of "year" has incompatible type "str"; expected "int" [typeddict-item] +typeddicts_usage.py:28: error: Missing key "name" for TypedDict "Movie" [typeddict-item] +typeddicts_usage.py:28: error: Extra key "title" for TypedDict "Movie" [typeddict-unknown-key] +typeddicts_usage.py:35: error: Cannot use isinstance() with TypedDict type [misc] +typeddicts_usage.py:40: error: Variable "typing.TypedDict" is not valid as a type [valid-type] +typeddicts_usage.py:40: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +""" diff --git a/conformance/results/mypy/version.toml b/conformance/results/mypy/version.toml new file mode 100644 index 000000000..f0d528a44 --- /dev/null +++ b/conformance/results/mypy/version.toml @@ -0,0 +1 @@ +version = "mypy 1.8.0" diff --git a/conformance/results/pyre/annotations_typeexpr.toml b/conformance/results/pyre/annotations_typeexpr.toml new file mode 100644 index 000000000..6c7949f8c --- /dev/null +++ b/conformance/results/pyre/annotations_typeexpr.toml @@ -0,0 +1,19 @@ +conformant = "Yes" +output = """ +annotations_typeexpr.py:65:26 Incompatible parameter type [6]: In call `typing.GenericMeta.__getitem__`, for 1st positional argument, expected `Type[Variable[_T]]` but got `object`. +annotations_typeexpr.py:67:27 Incompatible parameter type [6]: In call `typing.GenericMeta.__getitem__`, for 1st positional argument, expected `Type[Variable[_T_co](covariant)]` but got `Tuple[object, typing.Any]`. +annotations_typeexpr.py:77:8 Invalid type [31]: Expression `eval("".join(map(chr, [105, 110, 116])))` is not a valid type. +annotations_typeexpr.py:78:8 Invalid type [31]: Expression `[int, str]` is not a valid type. +annotations_typeexpr.py:79:8 Invalid type [31]: Expression `(int, str)` is not a valid type. +annotations_typeexpr.py:80:8 Invalid type [31]: Expression `comprehension(int for generators(generator($target$i in range(1) if )))` is not a valid type. +annotations_typeexpr.py:81:8 Invalid type [31]: Expression `{ }` is not a valid type. +annotations_typeexpr.py:82:8 Invalid type [31]: Expression `lambda () (int)()` is not a valid type. +annotations_typeexpr.py:83:8 Invalid type [31]: Expression `[int][0]` is not a valid type. +annotations_typeexpr.py:84:8 Invalid type [31]: Expression `int if 1 < 3 else str` is not a valid type. +annotations_typeexpr.py:85:8 Undefined or invalid type [11]: Annotation `var1` is not defined as a type. +annotations_typeexpr.py:86:9 Invalid type [31]: Expression `True` is not a valid type. +annotations_typeexpr.py:87:9 Invalid type [31]: Expression `1` is not a valid type. +annotations_typeexpr.py:88:9 Invalid type [31]: Expression `-1` is not a valid type. +annotations_typeexpr.py:89:9 Invalid type [31]: Expression `int or str` is not a valid type. +annotations_typeexpr.py:90:9 Invalid type [31]: Expression `"int"` is not a valid type. +""" diff --git a/conformance/results/pyre/generics_self_advanced.toml b/conformance/results/pyre/generics_self_advanced.toml new file mode 100644 index 000000000..2930afeb0 --- /dev/null +++ b/conformance/results/pyre/generics_self_advanced.toml @@ -0,0 +1,15 @@ +conformant = "Unsupported" +notes = """ +Does not understand `Self` type. +""" +output = """ +generics_self_advanced.py:25:7 Undefined or invalid type [11]: Annotation `Self` is not defined as a type. +generics_self_advanced.py:35:26 Undefined attribute [16]: Module `typing` has no attribute `Self`. +generics_self_advanced.py:36:33 Undefined attribute [16]: Module `typing` has no attribute `Self`. +generics_self_advanced.py:37:31 Undefined attribute [16]: Module `typing` has no attribute `Self`. +generics_self_advanced.py:38:36 Undefined attribute [16]: Module `typing` has no attribute `Self`. +generics_self_advanced.py:42:30 Undefined attribute [16]: Module `typing` has no attribute `Self`. +generics_self_advanced.py:43:32 Undefined attribute [16]: Module `typing` has no attribute `Self`. +generics_self_advanced.py:44:30 Undefined attribute [16]: Module `typing` has no attribute `Self`. +generics_self_advanced.py:45:35 Undefined attribute [16]: Module `typing` has no attribute `Self`. +""" diff --git a/conformance/results/pyre/generics_self_attributes.toml b/conformance/results/pyre/generics_self_attributes.toml new file mode 100644 index 000000000..f772cd636 --- /dev/null +++ b/conformance/results/pyre/generics_self_attributes.toml @@ -0,0 +1,10 @@ +conformant = "Unsupported" +notes = """ +Does not understand `Self` type. +""" +output = """ +generics_self_attributes.py:16:10 Undefined or invalid type [11]: Annotation `Self` is not defined as a type. +generics_self_attributes.py:26:5 Unexpected keyword [28]: Unexpected keyword argument `next` to call `OrdinalLinkedList.__init__`. +generics_self_attributes.py:29:14 Unexpected keyword [28]: Unexpected keyword argument `next` to call `OrdinalLinkedList.__init__`. +generics_self_attributes.py:32:14 Unexpected keyword [28]: Unexpected keyword argument `next` to call `LinkedList.__init__`. +""" diff --git a/conformance/results/pyre/generics_self_basic.toml b/conformance/results/pyre/generics_self_basic.toml new file mode 100644 index 000000000..66191b50c --- /dev/null +++ b/conformance/results/pyre/generics_self_basic.toml @@ -0,0 +1,15 @@ +conformant = "Unsupported" +notes = """ +Does not understand `Self` type. +""" +output = """ +generics_self_basic.py:13:26 Undefined attribute [16]: Module `typing` has no attribute `Self`. +generics_self_basic.py:19:8 Incompatible return type [7]: Expected `Variable[_Self_generics_self_basic_Shape__ (bound to Shape)]` but got `Shape`. +generics_self_basic.py:26:30 Undefined attribute [16]: Module `typing` has no attribute `Self`. +generics_self_basic.py:27:15 Too many arguments [19]: Call `object.__init__` expects 0 positional arguments, 1 was provided. +generics_self_basic.py:32:8 Incompatible return type [7]: Expected `Variable[_Self_generics_self_basic_Shape__ (bound to Shape)]` but got `Shape`. +generics_self_basic.py:39:27 Undefined attribute [16]: Module `typing` has no attribute `Self`. +generics_self_basic.py:57:0 Uninitialized attribute [13]: Attribute `value` is declared in class `Container` to have type `Variable[T]` but is never initialized. +generics_self_basic.py:64:25 Undefined or invalid type [11]: Annotation `Self` is not defined as a type. +generics_self_basic.py:80:31 Incompatible parameter type [6]: In call `typing.GenericMeta.__getitem__`, for 1st positional argument, expected `Type[Variable[T]]` but got `TypeVar`. +""" diff --git a/conformance/results/pyre/generics_self_protocols.toml b/conformance/results/pyre/generics_self_protocols.toml new file mode 100644 index 000000000..3943248a8 --- /dev/null +++ b/conformance/results/pyre/generics_self_protocols.toml @@ -0,0 +1,7 @@ +conformant = "Partial" +notes = """ +Does not reject protocol compatibility due to method `Self` return type. +""" +output = """ +generics_self_protocols.py:61:18 Incompatible parameter type [6]: In call `accepts_shape`, for 1st positional argument, expected `ShapeProtocol` but got `BadReturnType`. +""" diff --git a/conformance/results/pyre/generics_self_usage.toml b/conformance/results/pyre/generics_self_usage.toml new file mode 100644 index 000000000..6349aa8a7 --- /dev/null +++ b/conformance/results/pyre/generics_self_usage.toml @@ -0,0 +1,10 @@ +conformant = "Unsupported" +notes = """ +Does not understand `Self` type. +""" +output = """ +generics_self_usage.py:20:34 Undefined or invalid type [11]: Annotation `Self` is not defined as a type. +generics_self_usage.py:86:8 Incompatible return type [7]: Expected `Variable[_Self_generics_self_usage_Foo3__ (bound to Foo3)]` but got `Foo3`. +generics_self_usage.py:106:0 Incompatible variable type [9]: TupleSelf is declared to have type `TypeAlias` but is used as type `Type[tuple[Variable[_T_co](covariant)]]`. +generics_self_usage.py:106:29 Undefined attribute [16]: Module `typing` has no attribute `Self`. +""" diff --git a/conformance/results/pyre/literals_interactions.toml b/conformance/results/pyre/literals_interactions.toml new file mode 100644 index 000000000..4f88bb35a --- /dev/null +++ b/conformance/results/pyre/literals_interactions.toml @@ -0,0 +1,11 @@ +conformant = "Partial" +notes = """ +Does not detect out-of-bound tuple literal index. +Does not narrow type of `x` with `x in Literal` type guard pattern. +Does not narrow type of `x` with `x == Literal` type guard pattern. +""" +output = """ +literals_interactions.py:51:38 Incompatible parameter type [6]: In call `typing.GenericMeta.__getitem__`, for 1st positional argument, expected `Type[Variable[AnyStr <: [str, bytes]]]` but got `object`. +literals_interactions.py:106:34 Incompatible parameter type [6]: In call `expects_bad_status`, for 1st positional argument, expected `Union[typing_extensions.Literal['ABORTED'], typing_extensions.Literal['MALFORMED']]` but got `str`. +literals_interactions.py:109:31 Non-literal string [62]: In call `expects_pending_status`, for 1st positional argument, expected `LiteralString` but got `str`. Ensure only a string literal or a `LiteralString` is used. +""" diff --git a/conformance/results/pyre/literals_literalstring.toml b/conformance/results/pyre/literals_literalstring.toml new file mode 100644 index 000000000..c482077b1 --- /dev/null +++ b/conformance/results/pyre/literals_literalstring.toml @@ -0,0 +1,13 @@ +conformant = "Yes" +output = """ +literals_literalstring.py:36:11 Invalid type [31]: Expression `LiteralString` is not a literal value. +literals_literalstring.py:36:11 Undefined or invalid type [11]: Annotation `typing` is not defined as a type. +literals_literalstring.py:37:13 Invalid type [31]: Expression `LiteralString` is not a literal value. +literals_literalstring.py:43:4 Incompatible variable type [9]: x2 is declared to have type `typing_extensions.Literal['']` but is used as type `typing_extensions.Literal['two']`. +literals_literalstring.py:66:4 Incompatible variable type [9]: x1 is declared to have type `typing_extensions.LiteralString` but is used as type `str`. +literals_literalstring.py:74:4 Incompatible variable type [9]: x3 is declared to have type `typing_extensions.LiteralString` but is used as type `typing_extensions.Literal[3]`. +literals_literalstring.py:75:4 Incompatible variable type [9]: x4 is declared to have type `typing_extensions.LiteralString` but is used as type `typing_extensions.Literal[b'test']`. +literals_literalstring.py:120:21 Incompatible parameter type [6]: In call `literal_identity`, for 1st positional argument, expected `Variable[TLiteral (bound to typing_extensions.LiteralString)]` but got `str`. +literals_literalstring.py:134:50 Incompatible parameter type [6]: In call `Container.__init__`, for 1st positional argument, expected `Variable[T (bound to typing_extensions.LiteralString)]` but got `str`. +literals_literalstring.py:166:4 Incompatible variable type [9]: x1 is declared to have type `List[str]` but is used as type `List[typing_extensions.LiteralString]`. +""" diff --git a/conformance/results/pyre/literals_parameterizations.toml b/conformance/results/pyre/literals_parameterizations.toml new file mode 100644 index 000000000..89d38c177 --- /dev/null +++ b/conformance/results/pyre/literals_parameterizations.toml @@ -0,0 +1,30 @@ +conformant = "Partial" +notes = """ +Does not support type aliases in Literal type expression. +Does not support nested Literal type expression. +Does not reject unary + operator in Literal type expression. +Does not reject tuple in Literal type expression. +Does not reject "bare" Literal in type expression. +""" +output = """ +literals_parameterizations.py:32:0 Invalid type [31]: Expression `AppendMode` is not a literal value. +literals_parameterizations.py:32:0 Invalid type [31]: Expression `ReadOnlyMode` is not a literal value. +literals_parameterizations.py:32:0 Invalid type [31]: Expression `WriteAndTruncateMode` is not a literal value. +literals_parameterizations.py:32:0 Invalid type [31]: Expression `WriteNoTruncateMode` is not a literal value. +literals_parameterizations.py:32:0 Undefined or invalid type [11]: Annotation `` is not defined as a type. +literals_parameterizations.py:34:8 Invalid type [31]: Expression `typing.Literal[(typing.Literal[(typing.Literal[(1, 2, 3)], "foo")], 5, None)]` is not a valid type. +literals_parameterizations.py:40:6 Invalid type [31]: Expression `typing.Literal[3.__add__(4)]` is not a valid type. +literals_parameterizations.py:41:6 Invalid type [31]: Expression `typing.Literal["foo".replace("o", "b")]` is not a valid type. +literals_parameterizations.py:42:6 Invalid type [31]: Expression `typing.Literal[4.__add__(3.000000j)]` is not a valid type. +literals_parameterizations.py:44:6 Invalid type [31]: Expression `typing.Literal[not False]` is not a valid type. +literals_parameterizations.py:46:6 Invalid type [31]: Expression `typing.Literal[{ "a":"b","c":"d" }]` is not a valid type. +literals_parameterizations.py:47:6 Invalid type [31]: Expression `typing.Literal[int]` is not a valid type. +literals_parameterizations.py:48:6 Invalid type [31]: Expression `variable` is not a literal value. +literals_parameterizations.py:49:7 Invalid type [31]: Expression `T` is not a literal value. +literals_parameterizations.py:50:7 Invalid type [31]: Expression `typing.Literal[3.140000]` is not a valid type. +literals_parameterizations.py:51:7 Invalid type [31]: Expression `Any` is not a literal value. +literals_parameterizations.py:52:7 Invalid type [31]: Expression `typing.Literal[...]` is not a valid type. +literals_parameterizations.py:55:19 Invalid type [31]: Expression `typing.Literal[1.__add__(2)]` is not a valid type. +literals_parameterizations.py:59:3 Invalid type [31]: Expression `my_function` is not a literal value. +literals_parameterizations.py:63:4 Incompatible variable type [9]: x1 is declared to have type `typing_extensions.Literal['Color.RED']` but is used as type `typing_extensions.Literal[Color.RED]`. +""" diff --git a/conformance/results/pyre/literals_semantics.toml b/conformance/results/pyre/literals_semantics.toml new file mode 100644 index 000000000..c7f8f8c9f --- /dev/null +++ b/conformance/results/pyre/literals_semantics.toml @@ -0,0 +1,9 @@ +conformant = "Partial" +notes = """ +Does not reject augmented operation that modifies literal value. +""" +output = """ +literals_semantics.py:10:0 Incompatible variable type [9]: v2 is declared to have type `typing_extensions.Literal[3]` but is used as type `typing_extensions.Literal[4]`. +literals_semantics.py:24:4 Incompatible variable type [9]: x1 is declared to have type `typing_extensions.Literal[False]` but is used as type `typing_extensions.Literal[0]`. +literals_semantics.py:25:4 Incompatible variable type [9]: x2 is declared to have type `typing_extensions.Literal[0]` but is used as type `typing_extensions.Literal[False]`. +""" diff --git a/conformance/results/pyre/narrowing_typeguard.toml b/conformance/results/pyre/narrowing_typeguard.toml new file mode 100644 index 000000000..b14ed9e43 --- /dev/null +++ b/conformance/results/pyre/narrowing_typeguard.toml @@ -0,0 +1,9 @@ +conformant = "Partial" +notes = """ +Does not support `tuple` in `assert_type` call. +Does not reject TypeGuard method with too few parameters. +""" +output = """ +narrowing_typeguard.py:17:33 Incompatible parameter type [6]: In call `typing.GenericMeta.__getitem__`, for 1st positional argument, expected `Type[Variable[_T_co](covariant)]` but got `Tuple[Type[str], Type[str]]`. +narrowing_typeguard.py:19:33 Incompatible parameter type [6]: In call `typing.GenericMeta.__getitem__`, for 1st positional argument, expected `Type[Variable[_T_co](covariant)]` but got `Tuple[Type[str], typing.Any]`. +""" diff --git a/conformance/results/pyre/typeddicts_alt_syntax.toml b/conformance/results/pyre/typeddicts_alt_syntax.toml new file mode 100644 index 000000000..1435a730a --- /dev/null +++ b/conformance/results/pyre/typeddicts_alt_syntax.toml @@ -0,0 +1,10 @@ +conformant = "Partial" +notes = """ +Does not report when name of TypedDict doesn't match assigned identifier name. +Does not support keyword-argument form of alternative syntax (deprecated in 3.11). +""" +output = """ +typeddicts_alt_syntax.py:23:16 Call error [29]: `object` is not a function. +typeddicts_alt_syntax.py:41:9 Call error [29]: `object` is not a function. +typeddicts_alt_syntax.py:43:8 Undefined or invalid type [11]: Annotation `Movie2` is not defined as a type. +""" diff --git a/conformance/results/pyre/typeddicts_class_syntax.toml b/conformance/results/pyre/typeddicts_class_syntax.toml new file mode 100644 index 000000000..50ffee139 --- /dev/null +++ b/conformance/results/pyre/typeddicts_class_syntax.toml @@ -0,0 +1,11 @@ +conformant = "Partial" +notes = """ +Does not reject methods within TypedDict class. +Does not report when metaclass is provided. +Does not report when other keyword argument is provided. +Does not support generic TypedDict class. +""" +output = """ +typeddicts_class_syntax.py:57:0 Uninitialized attribute [13]: Attribute `name` is declared in class `GenericTypedDict` to have type `str` but is never initialized. +typeddicts_class_syntax.py:57:0 Uninitialized attribute [13]: Attribute `value` is declared in class `GenericTypedDict` to have type `Variable[T]` but is never initialized. +""" diff --git a/conformance/results/pyre/typeddicts_final.toml b/conformance/results/pyre/typeddicts_final.toml new file mode 100644 index 000000000..c55712229 --- /dev/null +++ b/conformance/results/pyre/typeddicts_final.toml @@ -0,0 +1,7 @@ +conformant = "Partial" +notes = """ +Does not handle value with literal type as index to TypedDict object. +""" +output = """ +typeddicts_final.py:26:17 Incompatible parameter type [6]: In call `TypedDictionary.__getitem__`, for 1st positional argument, expected `typing_extensions.Literal['name']` but got `Union[typing_extensions.Literal['name'], typing_extensions.Literal['year']]`. +""" diff --git a/conformance/results/pyre/typeddicts_inheritance.toml b/conformance/results/pyre/typeddicts_inheritance.toml new file mode 100644 index 000000000..d15915ca3 --- /dev/null +++ b/conformance/results/pyre/typeddicts_inheritance.toml @@ -0,0 +1,8 @@ +conformant = "Partial" +notes = """ +Does not reject TypedDict class that inherits from non-TypedDict class. +""" +output = """ +typeddicts_inheritance.py:54:0 Inconsistent override [15]: `x` overrides attribute defined in `X1` inconsistently. Type `int` is not a subtype of the overridden attribute `str`. +typeddicts_inheritance.py:65:0 Invalid inheritance [39]: Field `x` has type `int` in base class `X2` and type `str` in base class `Y2`. +""" diff --git a/conformance/results/pyre/typeddicts_operations.toml b/conformance/results/pyre/typeddicts_operations.toml new file mode 100644 index 000000000..3c18b76a0 --- /dev/null +++ b/conformance/results/pyre/typeddicts_operations.toml @@ -0,0 +1,14 @@ +conformant = "Yes" +output = """ +typeddicts_operations.py:22:16 Invalid TypedDict operation [54]: Expected `str` to be assigned to `Movie` field `name` but got `int`. +typeddicts_operations.py:23:16 Invalid TypedDict operation [54]: Expected `int` to be assigned to `Movie` field `year` but got `str`. +typeddicts_operations.py:24:6 TypedDict accessed with a missing key [27]: TypedDict `Movie` has no key `other`. +typeddicts_operations.py:26:12 TypedDict accessed with a missing key [27]: TypedDict `Movie` has no key `other`. +typeddicts_operations.py:28:8 TypedDict initialization error [55]: Missing required field `year` for TypedDict `Movie`. +typeddicts_operations.py:29:8 TypedDict initialization error [55]: Expected type `int` for `Movie` field `year` but got `float`. +typeddicts_operations.py:32:8 TypedDict initialization error [55]: TypedDict `Movie` has no field `other`. +typeddicts_operations.py:37:4 Incompatible variable type [9]: movie is declared to have type `Movie` but is used as type `Dict[str, Union[int, str]]`. +typeddicts_operations.py:44:10 TypedDict accessed with a missing key [27]: TypedDict `Movie` has no key `other`. +typeddicts_operations.py:47:0 Undefined attribute [16]: `Movie` has no attribute `clear`. +typeddicts_operations.py:62:0 Undefined attribute [16]: `MovieOptional` has no attribute `clear`. +""" diff --git a/conformance/results/pyre/typeddicts_required.toml b/conformance/results/pyre/typeddicts_required.toml new file mode 100644 index 000000000..9a654d815 --- /dev/null +++ b/conformance/results/pyre/typeddicts_required.toml @@ -0,0 +1,11 @@ +conformant = "Partial" +notes = """ +Does not reject use of `Required` in function parameter annotation. +Does not reject nested use of `Required` in type annotation. +Does not support recursive TypedDict definitions. +""" +output = """ +typeddicts_required.py:11:0 Uninitialized attribute [13]: Attribute `x` is declared in class `NotTypedDict` to have type `Required[int]` but is never initialized. +typeddicts_required.py:71:62 Undefined or invalid type [11]: Annotation `RecursiveMovie` is not defined as a type. +typeddicts_required.py:74:24 TypedDict initialization error [55]: Expected type `unknown` for `RecursiveMovie` field `predecessor` but got `typing.Dict[str, str]`. +""" diff --git a/conformance/results/pyre/typeddicts_type_consistency.toml b/conformance/results/pyre/typeddicts_type_consistency.toml new file mode 100644 index 000000000..28b9043e4 --- /dev/null +++ b/conformance/results/pyre/typeddicts_type_consistency.toml @@ -0,0 +1,19 @@ +conformant = "Partial" +notes = """ +Does not reject assignment of TypedDict with missing key. +Does not return non-Optional value from `get` method for required key. +Does not properly handle nested TypedDicts. +""" +output = """ +typeddicts_type_consistency.py:21:0 Incompatible variable type [9]: a1 is declared to have type `A1` but is used as type `B1`. +typeddicts_type_consistency.py:38:0 Incompatible variable type [9]: a2 is declared to have type `A2` but is used as type `B2`. +typeddicts_type_consistency.py:69:11 TypedDict initialization error [55]: TypedDict `A3` has no field `y`. +typeddicts_type_consistency.py:76:0 Incompatible variable type [9]: d1 is declared to have type `Dict[str, int]` but is used as type `B3`. +typeddicts_type_consistency.py:77:0 Incompatible variable type [9]: d2 is declared to have type `Dict[str, object]` but is used as type `B3`. +typeddicts_type_consistency.py:78:0 Incompatible variable type [9]: d3 is declared to have type `Dict[typing.Any, typing.Any]` but is used as type `B3`. +typeddicts_type_consistency.py:82:0 Incompatible variable type [9]: m1 is declared to have type `Mapping[str, int]` but is used as type `B3`. +typeddicts_type_consistency.py:99:0 Incompatible variable type [9]: name3 is declared to have type `str` but is used as type `Optional[str]`. +typeddicts_type_consistency.py:105:0 Incompatible variable type [9]: age4 is declared to have type `int` but is used as type `Union[str, int]`. +typeddicts_type_consistency.py:124:41 TypedDict initialization error [55]: Expected type `str` for `Inner1` field `inner_key` but got `int`. +typeddicts_type_consistency.py:150:0 Incompatible variable type [9]: o4 is declared to have type `Outer3` but is used as type `Outer2`. +""" diff --git a/conformance/results/pyre/typeddicts_usage.toml b/conformance/results/pyre/typeddicts_usage.toml new file mode 100644 index 000000000..739dbf331 --- /dev/null +++ b/conformance/results/pyre/typeddicts_usage.toml @@ -0,0 +1,10 @@ +conformant = "Partial" +notes = """ +Does not report errant use of TypedDict in `isinstance` call. +Does not reject use of TypedDict as TypeVar bound. +""" +output = """ +typeddicts_usage.py:23:6 TypedDict accessed with a missing key [27]: TypedDict `Movie` has no key `director`. +typeddicts_usage.py:24:16 Invalid TypedDict operation [54]: Expected `int` to be assigned to `Movie` field `year` but got `str`. +typeddicts_usage.py:28:16 TypedDict initialization error [55]: Missing required field `name` for TypedDict `Movie`. +""" diff --git a/conformance/results/pyre/version.toml b/conformance/results/pyre/version.toml new file mode 100644 index 000000000..abda16e02 --- /dev/null +++ b/conformance/results/pyre/version.toml @@ -0,0 +1 @@ +version = "pyre 0.9.19" diff --git a/conformance/results/pyright/annotations_typeexpr.toml b/conformance/results/pyright/annotations_typeexpr.toml new file mode 100644 index 000000000..80d2f522e --- /dev/null +++ b/conformance/results/pyright/annotations_typeexpr.toml @@ -0,0 +1,31 @@ +conformant = "Partial" +notes = """ +Does not reject ternary expression in annotation. +Does not reject binary expression in annotation. +""" +output = """ +annotations_typeexpr.py:77:9 - error: Call expression not allowed in type expression (reportGeneralTypeIssues) +annotations_typeexpr.py:78:9 - error: List expression not allowed in type annotation +  Use List[T] to indicate a list type or Union[T1, T2] to indicate a union type (reportGeneralTypeIssues) +annotations_typeexpr.py:78:9 - error: Expected type expression but received "list[Unknown]" (reportGeneralTypeIssues) +annotations_typeexpr.py:79:9 - error: Tuple expression not allowed in type annotation +  Use Tuple[T1, ..., Tn] to indicate a tuple type or Union[T1, T2] to indicate a union type (reportGeneralTypeIssues) +annotations_typeexpr.py:79:9 - error: Expected type expression but received "tuple[type[int], type[str]]" (reportGeneralTypeIssues) +annotations_typeexpr.py:80:9 - error: List expression not allowed in type annotation +  Use List[T] to indicate a list type or Union[T1, T2] to indicate a union type (reportGeneralTypeIssues) +annotations_typeexpr.py:80:9 - error: Expected type expression but received "list[type[int]]" (reportGeneralTypeIssues) +annotations_typeexpr.py:81:9 - error: Dictionary expression not allowed in type annotation +  Use Dict[T1, T2] to indicate a dictionary type (reportGeneralTypeIssues) +annotations_typeexpr.py:81:9 - error: Expected type expression but received "dict[Unknown, Unknown]" (reportGeneralTypeIssues) +annotations_typeexpr.py:82:9 - error: Call expression not allowed in type expression (reportGeneralTypeIssues) +annotations_typeexpr.py:83:9 - error: List expression not allowed in type annotation +  Use List[T] to indicate a list type or Union[T1, T2] to indicate a union type (reportGeneralTypeIssues) +annotations_typeexpr.py:83:9 - error: Expected type expression but received "list[type[int]]" (reportGeneralTypeIssues) +annotations_typeexpr.py:85:9 - error: Variable not allowed in type expression (reportGeneralTypeIssues) +annotations_typeexpr.py:86:10 - error: Expected type expression but received "Literal[True]" (reportGeneralTypeIssues) +annotations_typeexpr.py:87:10 - error: Expected type expression but received "Literal[1]" (reportGeneralTypeIssues) +annotations_typeexpr.py:88:10 - error: Expected type expression but received "Literal[-1]" (reportGeneralTypeIssues) +annotations_typeexpr.py:90:10 - error: Expected expression +annotations_typeexpr.py:90:10 - error: Tuple expression not allowed in type annotation +  Use Tuple[T1, ..., Tn] to indicate a tuple type or Union[T1, T2] to indicate a union type (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/generics_self_advanced.toml b/conformance/results/pyright/generics_self_advanced.toml new file mode 100644 index 000000000..74f99ddf8 --- /dev/null +++ b/conformance/results/pyright/generics_self_advanced.toml @@ -0,0 +1,3 @@ +conformant = "Yes" +output = """ +""" diff --git a/conformance/results/pyright/generics_self_attributes.toml b/conformance/results/pyright/generics_self_attributes.toml new file mode 100644 index 000000000..030b5f4b2 --- /dev/null +++ b/conformance/results/pyright/generics_self_attributes.toml @@ -0,0 +1,14 @@ +conformant = "Yes" +output = """ +generics_self_attributes.py:26:38 - error: Argument of type "LinkedList[int]" cannot be assigned to parameter "next" of type "OrdinalLinkedList | None" in function "__init__" +  Type "LinkedList[int]" cannot be assigned to type "OrdinalLinkedList | None" +    "LinkedList[int]" is incompatible with "OrdinalLinkedList" +    "LinkedList[int]" is incompatible with "None" (reportGeneralTypeIssues) +generics_self_attributes.py:32:8 - error: Cannot assign member "next" for type "OrdinalLinkedList" +  Expression of type "LinkedList[int]" cannot be assigned to member "next" of class "OrdinalLinkedList" +    Member "__set__" is unknown +    Member "__set__" is unknown +    Type "LinkedList[int]" cannot be assigned to type "OrdinalLinkedList | None" +      "LinkedList[int]" is incompatible with "OrdinalLinkedList" +      "LinkedList[int]" is incompatible with "None" (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/generics_self_basic.toml b/conformance/results/pyright/generics_self_basic.toml new file mode 100644 index 000000000..5c024f0bb --- /dev/null +++ b/conformance/results/pyright/generics_self_basic.toml @@ -0,0 +1,8 @@ +conformant = "Yes" +output = """ +generics_self_basic.py:19:16 - error: Expression of type "Shape" cannot be assigned to return type "Self@Shape" +  Type "Shape" cannot be assigned to type "Self@Shape" (reportGeneralTypeIssues) +generics_self_basic.py:32:16 - error: Expression of type "Shape" cannot be assigned to return type "Self@Shape" +  Type "Shape" cannot be assigned to type "Self@Shape" (reportGeneralTypeIssues) +generics_self_basic.py:64:31 - error: Expected no type arguments for class "Self" (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/generics_self_protocols.toml b/conformance/results/pyright/generics_self_protocols.toml new file mode 100644 index 000000000..14f1ace93 --- /dev/null +++ b/conformance/results/pyright/generics_self_protocols.toml @@ -0,0 +1,15 @@ +conformant = "Yes" +output = """ +generics_self_protocols.py:61:19 - error: Argument of type "BadReturnType" cannot be assigned to parameter "shape" of type "ShapeProtocol" in function "accepts_shape" +  "BadReturnType" is incompatible with protocol "ShapeProtocol" +    "set_scale" is an incompatible type +      Type "(scale: float) -> int" cannot be assigned to type "(scale: float) -> BadReturnType" +        Function return type "int" is incompatible with type "BadReturnType" +          "int" is incompatible with "BadReturnType" (reportGeneralTypeIssues) +generics_self_protocols.py:64:19 - error: Argument of type "ReturnDifferentClass" cannot be assigned to parameter "shape" of type "ShapeProtocol" in function "accepts_shape" +  "ReturnDifferentClass" is incompatible with protocol "ShapeProtocol" +    "set_scale" is an incompatible type +      Type "(scale: float) -> ReturnConcreteShape" cannot be assigned to type "(scale: float) -> ReturnDifferentClass" +        Function return type "ReturnConcreteShape" is incompatible with type "ReturnDifferentClass" +          "ReturnConcreteShape" is incompatible with "ReturnDifferentClass" (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/generics_self_usage.toml b/conformance/results/pyright/generics_self_usage.toml new file mode 100644 index 000000000..f5507f65a --- /dev/null +++ b/conformance/results/pyright/generics_self_usage.toml @@ -0,0 +1,20 @@ +conformant = "Partial" +notes = """ +Does not reject invalid use of `Self` in class definition. +Does not reject invalid use of `Self` in metaclass. +""" +output = """ +generics_self_usage.py:73:14 - error: "Self" is not valid in this context (reportGeneralTypeIssues) +generics_self_usage.py:73:23 - error: "Self" is not valid in this context (reportGeneralTypeIssues) +generics_self_usage.py:76:6 - error: "Self" is not valid in this context (reportGeneralTypeIssues) +generics_self_usage.py:82:54 - error: "Self" cannot be used in a function with a `self` or `cls` parameter that has a type annotation other than "Self" (reportGeneralTypeIssues) +generics_self_usage.py:82:44 - warning: TypeVar "TFoo2" appears only once in generic function signature +  Use "Foo2" instead (reportInvalidTypeVarUse) +generics_self_usage.py:86:16 - error: Expression of type "Foo3" cannot be assigned to return type "Self@Foo3" +  Type "Foo3" cannot be assigned to type "Self@Foo3" (reportGeneralTypeIssues) +generics_self_usage.py:103:12 - error: Class cannot derive from itself +generics_self_usage.py:106:30 - error: "Self" is not valid in this context (reportGeneralTypeIssues) +generics_self_usage.py:111:19 - error: "Self" is not valid in this context (reportGeneralTypeIssues) +generics_self_usage.py:116:31 - error: "Self" is not valid in this context (reportGeneralTypeIssues) +generics_self_usage.py:116:40 - error: "Self" is not valid in this context (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/literals_interactions.toml b/conformance/results/pyright/literals_interactions.toml new file mode 100644 index 000000000..edd762e5e --- /dev/null +++ b/conformance/results/pyright/literals_interactions.toml @@ -0,0 +1,7 @@ +conformant = "Yes" +output = """ +literals_interactions.py:15:5 - error: Index 5 is out of range for type tuple[int, str, list[bool]] (reportGeneralTypeIssues) +literals_interactions.py:16:5 - error: Index -5 is out of range for type tuple[int, str, list[bool]] (reportGeneralTypeIssues) +literals_interactions.py:17:5 - error: Index 4 is out of range for type tuple[int, str, list[bool]] (reportGeneralTypeIssues) +literals_interactions.py:18:5 - error: Index -4 is out of range for type tuple[int, str, list[bool]] (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/literals_literalstring.toml b/conformance/results/pyright/literals_literalstring.toml new file mode 100644 index 000000000..5bcdc78b2 --- /dev/null +++ b/conformance/results/pyright/literals_literalstring.toml @@ -0,0 +1,25 @@ +conformant = "Yes" +output = """ +literals_literalstring.py:36:29 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_literalstring.py:37:22 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_literalstring.py:43:23 - error: Expression of type "Literal['two']" cannot be assigned to declared type "Literal['']" +  "Literal['two']" cannot be assigned to type "Literal['']" (reportGeneralTypeIssues) +literals_literalstring.py:66:25 - error: Expression of type "str" cannot be assigned to declared type "LiteralString" +  "str" is incompatible with "LiteralString" (reportGeneralTypeIssues) +literals_literalstring.py:74:25 - error: Expression of type "Literal[3]" cannot be assigned to declared type "LiteralString" +  "Literal[3]" is incompatible with "LiteralString" (reportGeneralTypeIssues) +literals_literalstring.py:75:25 - error: Expression of type "Literal[b"test"]" cannot be assigned to declared type "LiteralString" +  "Literal[b"test"]" is incompatible with "LiteralString" (reportGeneralTypeIssues) +literals_literalstring.py:120:22 - error: Argument of type "str" cannot be assigned to parameter "s" of type "TLiteral@literal_identity" in function "literal_identity" +  Type "str" cannot be assigned to type "LiteralString" +    "str" is incompatible with "LiteralString" (reportGeneralTypeIssues) +literals_literalstring.py:134:51 - error: Argument of type "str" cannot be assigned to parameter "value" of type "T@Container" in function "__init__" +  Type "str" cannot be assigned to type "LiteralString" +    "str" is incompatible with "LiteralString" (reportGeneralTypeIssues) +literals_literalstring.py:142:5 - error: Overload 1 for "func8" overlaps overload 2 and returns an incompatible type (reportOverlappingOverload) +literals_literalstring.py:142:5 - error: Overload 1 for "func8" overlaps overload 3 and returns an incompatible type (reportOverlappingOverload) +literals_literalstring.py:166:21 - error: Expression of type "list[LiteralString]" cannot be assigned to declared type "list[str]" +  "list[LiteralString]" is incompatible with "list[str]" +    Type parameter "_T@list" is invariant, but "LiteralString" is not the same as "str" +    Consider switching from "list" to "Sequence" which is covariant (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/literals_parameterizations.toml b/conformance/results/pyright/literals_parameterizations.toml new file mode 100644 index 000000000..e8c639f17 --- /dev/null +++ b/conformance/results/pyright/literals_parameterizations.toml @@ -0,0 +1,21 @@ +conformant = "Yes" +output = """ +literals_parameterizations.py:40:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:41:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:42:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:43:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:44:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:45:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:46:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:47:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:48:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:49:16 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:50:16 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:51:16 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:52:16 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:55:28 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:58:4 - error: "Literal" cannot be used in this context without a type argument +literals_parameterizations.py:59:12 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value +literals_parameterizations.py:63:32 - error: Expression of type "Literal[Color.RED]" cannot be assigned to declared type "Literal['Color.RED']" +  "Literal[Color.RED]" cannot be assigned to type "Literal['Color.RED']" (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/literals_semantics.toml b/conformance/results/pyright/literals_semantics.toml new file mode 100644 index 000000000..d0c9c5509 --- /dev/null +++ b/conformance/results/pyright/literals_semantics.toml @@ -0,0 +1,10 @@ +conformant = "Yes" +output = """ +literals_semantics.py:10:18 - error: Expression of type "Literal[4]" cannot be assigned to declared type "Literal[3]" +  "Literal[4]" cannot be assigned to type "Literal[3]" (reportGeneralTypeIssues) +literals_semantics.py:24:26 - error: Expression of type "Literal[0]" cannot be assigned to declared type "Literal[False]" +  "Literal[0]" cannot be assigned to type "Literal[False]" (reportGeneralTypeIssues) +literals_semantics.py:25:22 - error: Expression of type "Literal[False]" cannot be assigned to declared type "Literal[0]" +  "Literal[False]" cannot be assigned to type "Literal[0]" (reportGeneralTypeIssues) +literals_semantics.py:33:10 - error: Expression of type "Literal[6, 7, 8]" cannot be assigned to declared type "Literal[3, 4, 5]" (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/narrowing_typeguard.toml b/conformance/results/pyright/narrowing_typeguard.toml new file mode 100644 index 000000000..e9bd7c115 --- /dev/null +++ b/conformance/results/pyright/narrowing_typeguard.toml @@ -0,0 +1,5 @@ +conformant = "Yes" +output = """ +narrowing_typeguard.py:102:9 - error: User-defined type guard functions and methods must have at least one input parameter (reportGeneralTypeIssues) +narrowing_typeguard.py:107:9 - error: User-defined type guard functions and methods must have at least one input parameter (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/typeddicts_alt_syntax.toml b/conformance/results/pyright/typeddicts_alt_syntax.toml new file mode 100644 index 000000000..e7c416e9a --- /dev/null +++ b/conformance/results/pyright/typeddicts_alt_syntax.toml @@ -0,0 +1,11 @@ +conformant = "Partial" +notes = """ +Does not report when name of TypedDict doesn't match assigned identifier name. +""" +output = """ +typeddicts_alt_syntax.py:23:17 - error: Expected dict or keyword parameter as second parameter +typeddicts_alt_syntax.py:27:45 - error: Expected string literal for dictionary entry name +typeddicts_alt_syntax.py:35:78 - error: Extra TypedDict arguments not supported +typeddicts_alt_syntax.py:45:43 - error: Expression of type "dict[str, str]" cannot be assigned to declared type "Movie2" +  "Literal['']" is incompatible with "int" (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/typeddicts_class_syntax.toml b/conformance/results/pyright/typeddicts_class_syntax.toml new file mode 100644 index 000000000..1147a3487 --- /dev/null +++ b/conformance/results/pyright/typeddicts_class_syntax.toml @@ -0,0 +1,10 @@ +conformant = "Partial" +notes = """ +Does not report when metaclass is provided. +Does not report when other keyword argument is provided. +""" +output = """ +typeddicts_class_syntax.py:29:5 - error: TypedDict classes can contain only type annotations +typeddicts_class_syntax.py:33:5 - error: TypedDict classes can contain only type annotations +typeddicts_class_syntax.py:38:5 - error: TypedDict classes can contain only type annotations +""" diff --git a/conformance/results/pyright/typeddicts_final.toml b/conformance/results/pyright/typeddicts_final.toml new file mode 100644 index 000000000..74f99ddf8 --- /dev/null +++ b/conformance/results/pyright/typeddicts_final.toml @@ -0,0 +1,3 @@ +conformant = "Yes" +output = """ +""" diff --git a/conformance/results/pyright/typeddicts_inheritance.toml b/conformance/results/pyright/typeddicts_inheritance.toml new file mode 100644 index 000000000..e37cec165 --- /dev/null +++ b/conformance/results/pyright/typeddicts_inheritance.toml @@ -0,0 +1,9 @@ +conformant = "Yes" +output = """ +typeddicts_inheritance.py:44:7 - error: All base classes for TypedDict classes must also be TypedDict classes +  Class "NonTypedDict" is not a TypedDict +typeddicts_inheritance.py:55:4 - error: "x" overrides symbol of same name in class "X1" +  Variable is mutable so its type is invariant +    Override type "int" is not the same as base type "str" (reportIncompatibleVariableOverride) +typeddicts_inheritance.py:65:7 - error: Base classes for class "XYZ2" define variable "x" in incompatible way (reportIncompatibleVariableOverride) +""" diff --git a/conformance/results/pyright/typeddicts_operations.toml b/conformance/results/pyright/typeddicts_operations.toml new file mode 100644 index 000000000..c21ae97eb --- /dev/null +++ b/conformance/results/pyright/typeddicts_operations.toml @@ -0,0 +1,24 @@ +conformant = "Yes" +output = """ +typeddicts_operations.py:22:1 - error: Could not assign item in TypedDict +  "Literal[1982]" is incompatible with "str" (reportGeneralTypeIssues) +typeddicts_operations.py:23:1 - error: Could not assign item in TypedDict +  "Literal['']" is incompatible with "int" (reportGeneralTypeIssues) +typeddicts_operations.py:24:1 - error: Could not assign item in TypedDict +  "other" is not a defined key in "Movie" (reportGeneralTypeIssues) +typeddicts_operations.py:26:7 - error: Could not access item in TypedDict +  "other" is not a defined key in "Movie" (reportGeneralTypeIssues) +typeddicts_operations.py:28:9 - error: Expression of type "dict[str, str]" cannot be assigned to declared type "Movie" +  "year" is required in "Movie" (reportGeneralTypeIssues) +typeddicts_operations.py:29:42 - error: Expression of type "dict[str, str | float]" cannot be assigned to declared type "Movie" +  "float" is incompatible with "int" (reportGeneralTypeIssues) +typeddicts_operations.py:32:36 - error: Expression of type "dict[str, str | int]" cannot be assigned to declared type "Movie" +  "other" is an undefined field in type "Movie" (reportGeneralTypeIssues) +typeddicts_operations.py:37:20 - error: Expression of type "dict[str, str | int]" cannot be assigned to declared type "Movie" (reportGeneralTypeIssues) +typeddicts_operations.py:47:7 - error: Cannot access member "clear" for type "Movie" +  Member "clear" is unknown (reportGeneralTypeIssues) +typeddicts_operations.py:49:5 - error: Could not delete item in TypedDict +  "name" is a required key and cannot be deleted (reportGeneralTypeIssues) +typeddicts_operations.py:62:16 - error: Cannot access member "clear" for type "MovieOptional" +  Member "clear" is unknown (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/typeddicts_required.toml b/conformance/results/pyright/typeddicts_required.toml new file mode 100644 index 000000000..ccc5ef2de --- /dev/null +++ b/conformance/results/pyright/typeddicts_required.toml @@ -0,0 +1,7 @@ +conformant = "Yes" +output = """ +typeddicts_required.py:12:8 - error: "Required" is not allowed in this context +typeddicts_required.py:16:8 - error: "NotRequired" is not allowed in this context +typeddicts_required.py:59:8 - error: "Required" is not allowed in this context +typeddicts_required.py:60:8 - error: "Required" is not allowed in this context +""" diff --git a/conformance/results/pyright/typeddicts_type_consistency.toml b/conformance/results/pyright/typeddicts_type_consistency.toml new file mode 100644 index 000000000..0785a39eb --- /dev/null +++ b/conformance/results/pyright/typeddicts_type_consistency.toml @@ -0,0 +1,25 @@ +conformant = "Yes" +output = """ +typeddicts_type_consistency.py:21:10 - error: Expression of type "B1" cannot be assigned to declared type "A1" +  "x" is an incompatible type +    Type "int" cannot be assigned to type "int | None" +      "int" is incompatible with "None" (reportGeneralTypeIssues) +typeddicts_type_consistency.py:38:10 - error: Expression of type "B2" cannot be assigned to declared type "A2" +  "x" is not required in "type[A2]" (reportGeneralTypeIssues) +typeddicts_type_consistency.py:65:6 - error: Expression of type "A3" cannot be assigned to declared type "B3" +  "y" is missing from "type[A3]" (reportGeneralTypeIssues) +typeddicts_type_consistency.py:69:21 - error: Expression of type "dict[str, int]" cannot be assigned to declared type "A3" +  "y" is an undefined field in type "A3" (reportGeneralTypeIssues) +typeddicts_type_consistency.py:76:22 - error: Expression of type "B3" cannot be assigned to declared type "dict[str, int]" +  "B3" is incompatible with "dict[str, int]" (reportGeneralTypeIssues) +typeddicts_type_consistency.py:77:25 - error: Expression of type "B3" cannot be assigned to declared type "dict[str, object]" +  "B3" is incompatible with "dict[str, object]" (reportGeneralTypeIssues) +typeddicts_type_consistency.py:78:22 - error: Expression of type "B3" cannot be assigned to declared type "dict[Any, Any]" +  "B3" is incompatible with "dict[Any, Any]" (reportGeneralTypeIssues) +typeddicts_type_consistency.py:82:25 - error: Expression of type "B3" cannot be assigned to declared type "Mapping[str, int]" +  "B3" is incompatible with "Mapping[str, int]" +    Type parameter "_VT_co@Mapping" is covariant, but "object" is not a subtype of "int" +      "object" is incompatible with "int" (reportGeneralTypeIssues) +typeddicts_type_consistency.py:124:56 - error: Expression of type "dict[str, dict[str, dict[str, int]]]" cannot be assigned to declared type "Outer1" +  "Literal[1]" is incompatible with "str" (reportGeneralTypeIssues) +""" diff --git a/conformance/results/pyright/typeddicts_usage.toml b/conformance/results/pyright/typeddicts_usage.toml new file mode 100644 index 000000000..10f072bcd --- /dev/null +++ b/conformance/results/pyright/typeddicts_usage.toml @@ -0,0 +1,13 @@ +conformant = "Partial" +notes = """ +Does not report errant use of TypedDict in `isinstance` call. +""" +output = """ +typeddicts_usage.py:23:1 - error: Could not assign item in TypedDict +  "director" is not a defined key in "Movie" (reportGeneralTypeIssues) +typeddicts_usage.py:24:1 - error: Could not assign item in TypedDict +  "Literal['1982']" is incompatible with "int" (reportGeneralTypeIssues) +typeddicts_usage.py:28:18 - error: Expression of type "dict[str, str | int]" cannot be assigned to declared type "Movie" +  "title" is an undefined field in type "Movie" (reportGeneralTypeIssues) +typeddicts_usage.py:40:24 - error: "TypedDict" cannot be used in this context +""" diff --git a/conformance/results/pyright/version.toml b/conformance/results/pyright/version.toml new file mode 100644 index 000000000..e28ed20e8 --- /dev/null +++ b/conformance/results/pyright/version.toml @@ -0,0 +1 @@ +version = "pyright 1.1.342" diff --git a/conformance/results/results.html b/conformance/results/results.html new file mode 100644 index 000000000..741237752 --- /dev/null +++ b/conformance/results/results.html @@ -0,0 +1,240 @@ + + + + + + + Type System Test Results + + + + +
+
+

Python Type System Conformance Test Results

+
+

mypy 1.8.0

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+Type annotations
     annotations_typeexprYes
+Generics
     generics_self_advancedPartialDoes not infer the type of an unannotated `self` parameter to be type `Self`.
Does not retain `Self` when calling method that returns `Self`.
Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.
Does not retain `Self` when accessing attribute through `type[Self]`.
     generics_self_attributesYes
     generics_self_basicPartialDoes not properly handle constructor call through `cls` parameter.
     generics_self_protocolsYes
     generics_self_usageYes
+Literals
     literals_interactionsPartialDoes not narrow type of `x` with `x in Literal` type guard pattern.
     literals_literalstringUnsupportedSupport for `LiteralString` has not been implemented in mypy.
     literals_parameterizationsPartialDoes not reject tuple within Literal.
     literals_semanticsYes
+Typed dictionaries
     typeddicts_alt_syntaxPartialDoes not support keyword-argument form of alternative syntax (deprecated in 3.11).
     typeddicts_class_syntaxYes
     typeddicts_finalYes
     typeddicts_inheritanceYes
     typeddicts_operationsYes
     typeddicts_requiredPartialDoes not support nesting of `Annotated` and `Required` or `NotRequired`.
     typeddicts_type_consistencyYes
     typeddicts_usageYes
+Type narrowing
     narrowing_typeguardYes
+

pyright 1.1.342

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+Type annotations
     annotations_typeexprPartialDoes not reject ternary expression in annotation.
Does not reject binary expression in annotation.
+Generics
     generics_self_advancedYes
     generics_self_attributesYes
     generics_self_basicYes
     generics_self_protocolsYes
     generics_self_usagePartialDoes not reject invalid use of `Self` in class definition.
Does not reject invalid use of `Self` in metaclass.
+Literals
     literals_interactionsYes
     literals_literalstringYes
     literals_parameterizationsYes
     literals_semanticsYes
+Typed dictionaries
     typeddicts_alt_syntaxPartialDoes not report when name of TypedDict doesn't match assigned identifier name.
     typeddicts_class_syntaxPartialDoes not report when metaclass is provided.
Does not report when other keyword argument is provided.
     typeddicts_finalYes
     typeddicts_inheritanceYes
     typeddicts_operationsYes
     typeddicts_requiredYes
     typeddicts_type_consistencyYes
     typeddicts_usagePartialDoes not report errant use of TypedDict in `isinstance` call.
+Type narrowing
     narrowing_typeguardYes
+

pyre 0.9.19

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+Type annotations
     annotations_typeexprYes
+Generics
     generics_self_advancedUnsupportedDoes not understand `Self` type.
     generics_self_attributesUnsupportedDoes not understand `Self` type.
     generics_self_basicUnsupportedDoes not understand `Self` type.
     generics_self_protocolsPartialDoes not reject protocol compatibility due to method `Self` return type.
     generics_self_usageUnsupportedDoes not understand `Self` type.
+Literals
     literals_interactionsPartialDoes not detect out-of-bound tuple literal index.
Does not narrow type of `x` with `x in Literal` type guard pattern.
Does not narrow type of `x` with `x == Literal` type guard pattern.
     literals_literalstringYes
     literals_parameterizationsPartialDoes not support type aliases in Literal type expression.
Does not support nested Literal type expression.
Does not reject unary + operator in Literal type expression.
Does not reject tuple in Literal type expression.
Does not reject "bare" Literal in type expression.
     literals_semanticsPartialDoes not reject augmented operation that modifies literal value.
+Typed dictionaries
     typeddicts_alt_syntaxPartialDoes not report when name of TypedDict doesn't match assigned identifier name.
Does not support keyword-argument form of alternative syntax (deprecated in 3.11).
     typeddicts_class_syntaxPartialDoes not reject methods within TypedDict class.
Does not report when metaclass is provided.
Does not report when other keyword argument is provided.
Does not support generic TypedDict class.
     typeddicts_finalPartialDoes not handle value with literal type as index to TypedDict object.
     typeddicts_inheritancePartialDoes not reject TypedDict class that inherits from non-TypedDict class.
     typeddicts_operationsYes
     typeddicts_requiredPartialDoes not reject use of `Required` in function parameter annotation.
Does not reject nested use of `Required` in type annotation.
Does not support recursive TypedDict definitions.
     typeddicts_type_consistencyPartialDoes not reject assignment of TypedDict with missing key.
Does not return non-Optional value from `get` method for required key.
Does not properly handle nested TypedDicts.
     typeddicts_usagePartialDoes not report errant use of TypedDict in `isinstance` call.
Does not reject use of TypedDict as TypeVar bound.
+Type narrowing
     narrowing_typeguardPartialDoes not support `tuple` in `assert_type` call.
Does not reject TypeGuard method with too few parameters.
+ + +
+ + + \ No newline at end of file diff --git a/conformance/src/__init__.py b/conformance/src/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/conformance/src/main.py b/conformance/src/main.py new file mode 100644 index 000000000..7a9f76718 --- /dev/null +++ b/conformance/src/main.py @@ -0,0 +1,122 @@ +""" +Type system conformance test for static type checkers. +""" + +import os +from pathlib import Path +import re +import sys +from typing import Sequence + +import tomli +import tomlkit + +from reporting import generate_summary +from test_groups import get_test_cases, get_test_groups +from type_checker import TYPE_CHECKERS, TypeChecker + + +def run_tests( + root_dir: Path, + type_checker: TypeChecker, + test_cases: Sequence[Path], +): + print(f"Running tests for {type_checker.name}") + tests_output = type_checker.run_tests([file.name for file in test_cases]) + results_dir = root_dir / "results" / type_checker.name + + for test_case in test_cases: + update_output_for_test( + type_checker, results_dir, test_case, tests_output.get(test_case.name, "") + ) + + update_type_checker_version(type_checker, root_dir) + + +def update_output_for_test( + type_checker: TypeChecker, + results_dir: Path, + test_case: Path, + output: str, +): + test_name = test_case.stem + output = f"\n{output}" + + results_file = results_dir / f"{test_name}.toml" + results_file.parent.mkdir(parents=True, exist_ok=True) + + # Read the existing results file if present. + try: + with open(results_file, "rb") as f: + existing_results = tomli.load(f) + except FileNotFoundError: + existing_results = {} + + old_output = existing_results.get("output", "") + old_output = f"\n{old_output}" + + # Did the type checker output change since last time the + # test was run? + if old_output != output: + print(f"Output changed for {test_name} when running {type_checker.name}") + print(f"Old output: {old_output}") + print(f"New output: {output}") + print("") + + # Use multiline formatting for any strings that contain newlines. + for key, value in existing_results.items(): + if isinstance(value, str) and "\n" in value: + existing_results[key] = tomlkit.string(f"\n{value}", multiline=True) + + existing_results["output"] = tomlkit.string(output, multiline=True) + + results_file.parent.mkdir(parents=True, exist_ok=True) + with open(results_file, "w") as f: + tomlkit.dump(existing_results, f) + + +def update_type_checker_version(type_checker: TypeChecker, root_dir: Path): + # Record the version of the type checker used for the latest run. + version_file = root_dir / "results" / type_checker.name / "version.toml" + + # Read the existing version file if present. + try: + with open(version_file, "rb") as f: + existing_info = tomli.load(f) + except FileNotFoundError: + existing_info = {} + + existing_info["version"] = type_checker.get_version() + + version_file.parent.mkdir(parents=True, exist_ok=True) + with open(version_file, "w") as f: + tomlkit.dump(existing_info, f) + + +def main(): + # Some tests cover features that are available only in the + # latest version of Python (3.12), so we need this version. + assert sys.version_info >= (3, 12) + + root_dir = Path(__file__).resolve().parent.parent + + tests_dir = root_dir / "tests" + assert tests_dir.is_dir() + + test_groups = get_test_groups(root_dir) + test_cases = get_test_cases(test_groups, tests_dir) + + # Switch to the tests directory. + os.chdir(tests_dir) + + # Run each test case with each type checker. + for type_checker in TYPE_CHECKERS: + type_checker.install() + run_tests(root_dir, type_checker, test_cases) + + # Generate a summary report. + generate_summary(root_dir) + + +if __name__ == "__main__": + main() diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py new file mode 100644 index 000000000..0a253c46d --- /dev/null +++ b/conformance/src/reporting.py @@ -0,0 +1,99 @@ +""" +Generates a summary of the type checker conformant tests. +""" + +from pathlib import Path + +import tomli + +from test_groups import get_test_cases, get_test_groups +from type_checker import TYPE_CHECKERS + + +def generate_summary(root_dir: Path): + print('Generating summary report') + template_file = root_dir / "src" / "results_template.html" + with open(template_file, "r") as f: + template = f.read() + + summary = template.replace("{{summary}}", generate_summary_html(root_dir)) + + results_file = root_dir / "results" / "results.html" + + with open(results_file, "w") as f: + f.write(summary) + + +def generate_summary_html(root_dir: Path): + test_groups = get_test_groups(root_dir) + test_cases = get_test_cases(test_groups, root_dir / "tests") + + summary_html = "" + + for type_checker in TYPE_CHECKERS: + # Load the version file for the type checker. + version_file = root_dir / "results" / type_checker.name / "version.toml" + + try: + with open(version_file, "rb") as f: + existing_info = tomli.load(f) + except FileNotFoundError: + existing_info = {} + + version = existing_info["version"] or "Unknown version" + + summary_html += f"

{version}

\n" + summary_html += '
\n' + + for test_group_name, test_group in test_groups.items(): + tests_in_group = [ + case + for case in test_cases + if case.name.startswith(f"{test_group_name}_") + ] + + tests_in_group.sort(key=lambda x: x.name) + + # Are there any test cases in this group? + if len(tests_in_group) > 0: + summary_html += '\n" + + for test_case in tests_in_group: + test_case_name = test_case.stem + + try: + results_file = ( + root_dir + / "results" + / type_checker.name + / f"{test_case_name}.toml" + ) + with open(results_file, "rb") as f: + results = tomli.load(f) + except FileNotFoundError: + results = {} + + conformance = results.get("conformant", "Unknown") + notes = results.get("notes", "").replace("\n", "
") + + conformance_class = ( + "conformant" + if conformance == "Yes" + else "partially-conformant" + if conformance == "Partial" + else "not-conformant" + ) + + summary_html += f"" + summary_html += f'' + summary_html += f'' + summary_html += f'\n' + + # Add a spacer row after this group to help with readability. + summary_html += '\n' + + summary_html += "
\n' + summary_html += f'{test_group.name}' + summary_html += "
     {test_case_name}{conformance}{notes}
\n" + + return summary_html diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html new file mode 100644 index 000000000..c7c91fc8f --- /dev/null +++ b/conformance/src/results_template.html @@ -0,0 +1,129 @@ + + + + + + + Type System Test Results + + + + +
+
+

Python Type System Conformance Test Results

+
+ {{summary}} + +
+ + + \ No newline at end of file diff --git a/conformance/src/test_groups.py b/conformance/src/test_groups.py new file mode 100644 index 000000000..f0145ff11 --- /dev/null +++ b/conformance/src/test_groups.py @@ -0,0 +1,46 @@ +""" +Reads a template file that describes groups of tests in the +conformance test suite. +""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping, Sequence + +import tomli + + +@dataclass +class TestGroup: + name: str + href: str + + +def get_test_groups(root_dir: Path) -> Mapping[str, TestGroup]: + # Read the TOML file that defines the test groups. Each test + # group has a name that associated test cases must start with. + test_group_file = root_dir / "src" / "test_groups.toml" + with open(test_group_file, "rb") as f: + test_groups = tomli.load(f) + + return { + k: TestGroup(v.get("name", "unknown"), v.get("href", "")) + for k, v in test_groups.items() + } + + +def get_test_cases( + test_groups: Mapping[str, TestGroup], tests_dir: Path +) -> Sequence[Path]: + test_group_names = test_groups.keys() + + # Filter test cases based on test group names. Files that do + # not begin with a known test group name are assumed to be + # files that support one or more tests. + test_cases = [ + p + for p in Path(tests_dir).glob("*.py") + if p.name.split("_")[0] in test_group_names + ] + + return test_cases diff --git a/conformance/src/test_groups.toml b/conformance/src/test_groups.toml new file mode 100644 index 000000000..b5d26e546 --- /dev/null +++ b/conformance/src/test_groups.toml @@ -0,0 +1,66 @@ + +[concepts] +name = "Type system concepts" +href = "https://typing.readthedocs.io/en/latest/spec/concepts.html" + +[annotations] +name = "Type annotations" +href = "https://typing.readthedocs.io/en/latest/spec/annotations.html" + +[specialtypes] +name = "Special types in annotations" +href = "https://typing.readthedocs.io/en/latest/spec/special-types.html" + +[generics] +name = "Generics" +href = "https://typing.readthedocs.io/en/latest/spec/generics.html" + +[qualifiers] +name = "Type qualifiers" +href = "https://typing.readthedocs.io/en/latest/spec/qualifiers.html" + +[classes] +name = "Class type compatibility" +href = "https://typing.readthedocs.io/en/latest/spec/class-compat.html" + +[aliases] +name = "Type aliases" +href = "https://typing.readthedocs.io/en/latest/spec/aliases.html" + +[literals] +name = "Literals" +href = "https://typing.readthedocs.io/en/latest/spec/literal.html" + +[protocols] +href = "https://typing.readthedocs.io/en/latest/spec/protocol.html" + +[callables] +name = "Callables" +href = "https://typing.readthedocs.io/en/latest/spec/callables.html" + +[overloads] +name = "Overloads" +href = "https://typing.readthedocs.io/en/latest/spec/overload.html" + +[dataclasses] +href = "https://typing.readthedocs.io/en/latest/spec/dataclasses.html" + +[typeddicts] +name = "Typed dictionaries" +href = "https://typing.readthedocs.io/en/latest/spec/typeddict.html" + +[narrowing] +name = "Type narrowing" +href = "https://typing.readthedocs.io/en/latest/spec/narrowing.html" + +[directives] +name = "Type checker directives" +href = "https://typing.readthedocs.io/en/latest/spec/directives.html" + +[distribution] +name = "Distributing type information" +href = "https://typing.readthedocs.io/en/latest/spec/distributing.html" + +[historical] +name = "Historical and deprecated features" +href = "https://typing.readthedocs.io/en/latest/spec/historical.html" diff --git a/conformance/src/type_checker.py b/conformance/src/type_checker.py new file mode 100644 index 000000000..b42148e1a --- /dev/null +++ b/conformance/src/type_checker.py @@ -0,0 +1,150 @@ +""" +Classes that abstract differences between type checkers. +""" + +from abc import ABC, abstractmethod +import json +from pathlib import Path +from subprocess import PIPE, run +from typing import Sequence + + +class TypeChecker(ABC): + @property + @abstractmethod + def name(self) -> str: + """ + Returns the name of the type checker. + """ + raise NotImplementedError + + @abstractmethod + def install(self) -> None: + """ + Ensures that the latest version of the type checker is installed. + """ + raise NotImplementedError + + @abstractmethod + def get_version(self) -> str: + """ + Returns the current version string for the type checker. + """ + raise NotImplementedError + + @abstractmethod + def run_tests(self, test_files: Sequence[str]) -> dict[str, str]: + """ + Runs the type checker on the specified test file and + returns the output. + """ + raise NotImplementedError + + +class MypyTypeChecker(TypeChecker): + @property + def name(self) -> str: + return "mypy" + + def install(self) -> None: + run("pip install mypy --upgrade", shell=True) + + def get_version(self) -> str: + proc = run("mypy --version", stdout=PIPE, text=True, shell=True) + version = proc.stdout.strip() + + # Remove the " (compiled)" if it's present. + version = version.split(" (")[0] + return version + + def run_tests(self, test_files: Sequence[str]) -> dict[str, str]: + command = f"mypy . --disable-error-code empty-body" + proc = run(command, stdout=PIPE, text=True, shell=True) + lines = proc.stdout.split('\n') + + # Add results to a dictionary keyed by the file name. + results_dict: dict[str, str] = {} + for line in lines: + file_name = line.split(':')[0].strip() + results_dict[file_name] = results_dict.get(file_name, '') + line + '\n' + + return results_dict + + +class PyrightTypeChecker(TypeChecker): + @property + def name(self) -> str: + return "pyright" + + def install(self) -> None: + # Install the Python wrapper if it's not installed. + run("pip install pyright", shell=True) + + # Force the Python wrapper to install node if needed + # and download the latest version of pyright. + self.get_version() + + def get_version(self) -> str: + proc = run("pyright --version", stdout=PIPE, text=True, shell=True) + return proc.stdout.strip() + + def run_tests(self, test_files: Sequence[str]) -> dict[str, str]: + command = f"pyright . --outputjson" + proc = run(command, stdout=PIPE, text=True, shell=True) + output_json = json.loads(proc.stdout) + diagnostics = output_json['generalDiagnostics'] + + # Add results to a dictionary keyed by the file name. + results_dict: dict[str, str] = {} + for diagnostic in diagnostics: + file_path = Path(diagnostic.get('file', '')) + file_name = file_path.name + line_number = diagnostic['range']['start']['line'] + 1 + col_number = diagnostic['range']['start']['character'] + 1 + severity = diagnostic['severity'] + message = diagnostic['message'] + rule = f' ({diagnostic['rule']})' if 'rule' in diagnostic else '' + + line_text = f'{file_name}:{line_number}:{col_number} - {severity}: {message}{rule}\n' + results_dict[file_name] = results_dict.get(file_name, '') + line_text + + return results_dict + +class PyreTypeChecker(TypeChecker): + @property + def name(self) -> str: + return "pyre" + + def install(self) -> None: + run("pip install pyre-check --upgrade", shell=True) + + # Generate a default config file. + pyre_config = '{"site_package_search_strategy": "pep561", "source_directories": ["."]}\n' + with open('.pyre_configuration', 'w') as f: + f.write(pyre_config) + + def get_version(self) -> str: + proc = run("pyre --version", stdout=PIPE, text=True, shell=True) + version = proc.stdout.strip() + version = version.replace('Client version:', 'pyre') + return version + + def run_tests(self, test_files: Sequence[str]) -> dict[str, str]: + command = f"pyre check" + proc = run(command, stdout=PIPE, text=True, shell=True) + lines = proc.stdout.split('\n') + + # Add results to a dictionary keyed by the file name. + results_dict: dict[str, str] = {} + for line in lines: + file_name = line.split(':')[0].strip() + results_dict[file_name] = results_dict.get(file_name, '') + line + '\n' + + return results_dict + + +TYPE_CHECKERS: Sequence[TypeChecker] = ( + MypyTypeChecker(), + PyrightTypeChecker(), + PyreTypeChecker(), +) diff --git a/conformance/tests/annotations_typeexpr.py b/conformance/tests/annotations_typeexpr.py new file mode 100644 index 000000000..064cac65c --- /dev/null +++ b/conformance/tests/annotations_typeexpr.py @@ -0,0 +1,99 @@ +""" +Test for type expressions used in annotations. +""" + +import abc +import abc +import types +import types +from typing import Any, Callable, Tuple, Union, assert_type + +# https://typing.readthedocs.io/en/latest/spec/annotations.html#valid-type-expression-forms + +def greeting(name: str) -> str: + return 'Hello ' + name + +assert_type(greeting('Monty'), str) + + +# > Expressions whose type is a subtype of a specific argument type are also accepted for that argument. +class StrSub(str): ... +assert_type(greeting(StrSub('Monty')), str) + + +# > Type hints may be built-in classes (including those defined in standard library or third-party +# > extension modules), abstract base classes, types available in the types module, and user-defined +# > classes (including those defined in the standard library or third-party modules). + + +class UserDefinedClass: ... +class AbstractBaseClass(abc.ABC): + @abc.abstractmethod + def abstract_method(self): ... + +# The following parameter annotations should all be considered +# valid and not generate errors. +def valid_annotations( + p1: int, + p2: str, + p3: bytes, + p4: bytearray, + p5: memoryview, + p6: complex, + p7: float, + p8: bool, + p9: object, + p10: type, + p11: types.ModuleType, + p12: types.FunctionType, + p13: types.BuiltinFunctionType, + p14: UserDefinedClass, + p15: AbstractBaseClass, + p16: int, + p17: Union[int, str], + p18: None, + p19: list, + p20: list[int], + p21: tuple, + p22: Tuple[int, ...], + p23: Tuple[int, int, str], + p24: Callable[..., int], + p25: Callable[[int, str], None], + p26: Any, +): + assert_type(p17, int | str) + assert_type(p19, list[Any]) + assert_type(p20, list[int]) + assert_type(p21, tuple[Any, ...]) + + +# > Annotations should be kept simple or static analysis tools may not be able to interpret the values. + +var1 = 3 + +# The following parameter annotations should all be considered +# invalid and generate errors. +def invalid_annotations( + p1: eval("".join(map(chr, [105, 110, 116]))), + p2: [int, str], + p3: (int, str), + p4: [int for i in range(1)], + p5: {}, + p6: (lambda : int)(), + p7: [int][0], + p8: int if 1 < 3 else str, + p9: var1, + p10: True, + p11: 1, + p12: -1, + p13: int or str, + p14: f"int", +): + pass + + +# > When used in a type hint, the expression None is considered equivalent to type(None). + +def takes_None(x: None) -> None: ... +assert_type(takes_None(None), None) + diff --git a/conformance/tests/generics_self_advanced.py b/conformance/tests/generics_self_advanced.py new file mode 100644 index 000000000..9a837e40f --- /dev/null +++ b/conformance/tests/generics_self_advanced.py @@ -0,0 +1,46 @@ +""" +Tests for advanced or special-case usage of the typing.Self type. +""" + +from typing import assert_type, Self + + +class ParentA: + # Test for property that returns Self. + @property + def prop1(self) -> Self: + ... + +class ChildA(ParentA): + ... + + +assert_type(ParentA().prop1, ParentA) +assert_type(ChildA().prop1, ChildA) + + +# Test for a child that accesses an attribute within a parent +# whose type is annotated using Self. +class ParentB: + a: list[Self] + + @classmethod + def method1(cls) -> Self: + ... + +class ChildB(ParentB): + b: int = 0 + + def method2(self) -> None: + assert_type(self, Self) + assert_type(self.a, list[Self]) + assert_type(self.a[0], Self) + assert_type(self.method1(), Self) + + @classmethod + def method3(cls) -> None: + assert_type(cls, type[Self]) + assert_type(cls.a, list[Self]) + assert_type(cls.a[0], Self) + assert_type(cls.method1(), Self) + diff --git a/conformance/tests/generics_self_attributes.py b/conformance/tests/generics_self_attributes.py new file mode 100644 index 000000000..2fad31cb9 --- /dev/null +++ b/conformance/tests/generics_self_attributes.py @@ -0,0 +1,34 @@ +""" +Tests for usage of the typing.Self type with attributes. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/generics.html#use-in-attribute-annotations + +from typing import TypeVar, Generic, Self +from dataclasses import dataclass + + +T = TypeVar("T") + +@dataclass +class LinkedList(Generic[T]): + value: T + next: Self | None = None + + +@dataclass +class OrdinalLinkedList(LinkedList[int]): + def ordinal_value(self) -> str: + return str(self.value) + + +# This should result in a type error. +xs = OrdinalLinkedList(value=1, next=LinkedList[int](value=2)) + +if xs.next is not None: + xs.next = OrdinalLinkedList(value=3, next=None) # OK + + # This should result in a type error. + xs.next = LinkedList[int](value=3, next=None) + + diff --git a/conformance/tests/generics_self_basic.py b/conformance/tests/generics_self_basic.py new file mode 100644 index 000000000..aea671078 --- /dev/null +++ b/conformance/tests/generics_self_basic.py @@ -0,0 +1,82 @@ +""" +Tests for basic usage of the typing.Self type. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/generics.html#self. + +from typing import Callable, Generic, Self, TypeVar, assert_type + +T = TypeVar("T") + +class Shape: + def set_scale(self, scale: float) -> Self: + assert_type(self, Self) + self.scale = scale + return self + + def method2(self) -> Self: + # This should result in a type error. + return Shape() + + def method3(self) -> "Shape": + return self + + @classmethod + def from_config(cls, config: dict[str, float]) -> Self: + assert_type(cls, type[Self]) + return cls(config["scale"]) + + @classmethod + def cls_method2(cls) -> Self: + # This should result in a type error. + return Shape() + + @classmethod + def cls_method3(cls) -> "Shape": + return cls() + + def difference(self, other: Self) -> float: + assert_type(other, Self) + return 0.0 + + def apply(self, f: Callable[[Self], None]) -> None: + return f(self) + + +class Circle(Shape): + pass + + +assert_type(Shape().set_scale(1.0), Shape) +assert_type(Circle().set_scale(1.0), Circle) + +assert_type(Shape.from_config({}), Shape) +assert_type(Circle.from_config({}), Circle) + + +class Container(Generic[T]): + value: T + + def set_value(self, value: T) -> Self: + ... + + # This should generate an error because Self isn't subscriptable. + def foo(self, other: Self[int]) -> None: + pass + + +def object_with_concrete_type( + int_container: Container[int], str_container: Container[str] +) -> None: + assert_type(int_container.set_value(42), Container[int]) + assert_type(str_container.set_value("hello"), Container[str]) + + +def object_with_generic_type( + container: Container[T], + value: T, +) -> Container[T]: + val = container.set_value(value) + assert_type(val, Container[T]) + return val + diff --git a/conformance/tests/generics_self_protocols.py b/conformance/tests/generics_self_protocols.py new file mode 100644 index 000000000..6f27bf0b9 --- /dev/null +++ b/conformance/tests/generics_self_protocols.py @@ -0,0 +1,64 @@ +""" +Tests for usage of the typing.Self type with protocols. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/generics.html#use-in-protocols + +from typing import Protocol, Self, assert_type + + +class ShapeProtocol(Protocol): + def set_scale(self, scale: float) -> Self: + ... + + +class ReturnSelf: + scale: float = 1.0 + + def set_scale(self, scale: float) -> Self: + self.scale = scale + return self + + +class ReturnConcreteShape: + scale: float = 1.0 + + def set_scale(self, scale: float) -> "ReturnConcreteShape": + self.scale = scale + return self + + +class BadReturnType: + scale: float = 1.0 + + def set_scale(self, scale: float) -> int: + self.scale = scale + return 42 + + +class ReturnDifferentClass: + scale: float = 1.0 + + def set_scale(self, scale: float) -> ReturnConcreteShape: + return ReturnConcreteShape() + + +def accepts_shape(shape: ShapeProtocol) -> None: + y = shape.set_scale(0.5) + assert_type(y, ShapeProtocol) + + +def main( + return_self_shape: ReturnSelf, + return_concrete_shape: ReturnConcreteShape, + bad_return_type: BadReturnType, + return_different_class: ReturnDifferentClass, +) -> None: + accepts_shape(return_self_shape) # OK + accepts_shape(return_concrete_shape) # OK + + # This should generate a type error. + accepts_shape(bad_return_type) + + # Not OK because it returns a non-subclass. + accepts_shape(return_different_class) diff --git a/conformance/tests/generics_self_usage.py b/conformance/tests/generics_self_usage.py new file mode 100644 index 000000000..657849b24 --- /dev/null +++ b/conformance/tests/generics_self_usage.py @@ -0,0 +1,126 @@ +""" +Tests for valid and invalid usage of the typing.Self type. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/generics.html#valid-locations-for-self + +from typing import Any, Callable, Generic, Self, TypeAlias, TypeVar + +class ReturnsSelf: + def foo(self) -> Self: # Accepted + return self + + @classmethod + def bar(cls) -> Self: # Accepted + return cls(1) + + def __new__(cls, value: int) -> Self: # Accepted + return cls(1) + + def explicitly_use_self(self: Self) -> Self: # Accepted + return self + + # Accepted (Self can be nested within other types) + def returns_list(self) -> list[Self]: + return [] + + # Accepted (Self can be nested within other types) + @classmethod + def return_cls(cls) -> type[Self]: + return cls + +class Child(ReturnsSelf): + # Accepted (we can override a method that uses Self annotations) + def foo(self) -> Self: + return self + +class TakesSelf: + def foo(self, other: Self) -> bool: # Accepted + return True + +class Recursive: + # Accepted (treated as an @property returning ``Self | None``) + next: Self | None + +class CallableAttribute: + def foo(self) -> int: + return 0 + + # Accepted (treated as an @property returning the Callable type) + bar: Callable[[Self], int] = foo + +class HasNestedFunction: + x: int = 42 + + def foo(self) -> None: + + # Accepted (Self is bound to HasNestedFunction). + def nested(z: int, inner_self: Self) -> Self: + print(z) + print(inner_self.x) + return inner_self + + nested(42, self) # OK + + +class Outer: + class Inner: + def foo(self) -> Self: # Accepted (Self is bound to Inner) + return self + + +# This should generate an error. +def foo(bar: Self) -> Self: ... # Rejected (not within a class) + +# This should generate an error. +bar: Self # Rejected (not within a class) + +TFoo2 = TypeVar("TFoo2", bound="Foo2") + +class Foo2: + # Rejected (Self is treated as unknown). + def has_existing_self_annotation(self: TFoo2) -> Self: ... + +class Foo3: + def return_concrete_type(self) -> Self: + return Foo3() # Rejected (see FooChild below for rationale) + +class Foo3Child(Foo3): + child_value: int = 42 + + def child_method(self) -> None: + y = self.return_concrete_type() + y.child_value + +T = TypeVar("T") + +class Bar(Generic[T]): + def bar(self) -> T: ... + +# This should generate an error. +class Baz(Bar[Self]): ... # Rejected + +class Baz2(Self): ... # Rejected + +# This should generate an error. +TupleSelf: TypeAlias = tuple[Self] # Rejected + +class Base: + @staticmethod + # This should generate an error. + def make() -> Self: # Rejected + ... + + @staticmethod + # This should generate an error. + def return_parameter(foo: Self) -> Self: # Rejected + ... + +class MyMetaclass(type): + # This should generate an error. + def __new__(cls, *args: Any) -> Self: # Rejected + ... + + # This should generate an error. + def __mul__(cls, count: int) -> list[Self]: # Rejected + ... diff --git a/conformance/tests/literals_interactions.py b/conformance/tests/literals_interactions.py new file mode 100644 index 000000000..f9782c4f1 --- /dev/null +++ b/conformance/tests/literals_interactions.py @@ -0,0 +1,116 @@ +""" +Tests interactions between Literal types and other typing features. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/literal.html#interactions-with-other-types-and-features + +from enum import Enum +from typing import IO, Any, Final, Generic, Literal, TypeVar, assert_type, overload + + +def func1(v: tuple[int, str, list[bool]], a: Literal[0], b: Literal[5], c: Literal[-5]): + assert_type(v[a], int) + assert_type(v[2], list[bool]) + + v[b] # Type error: index out of range + v[c] # Type error: index out of range + v[4] # Type error: index out of range + v[-4] # Type error: index out of range + + +_PathType = str | bytes | int + + +@overload +def open( + path: _PathType, + mode: Literal["r", "w", "a", "x", "r+", "w+", "a+", "x+"], +) -> IO[str]: + ... + + +@overload +def open( + path: _PathType, + mode: Literal["rb", "wb", "ab", "xb", "r+b", "w+b", "a+b", "x+b"], +) -> IO[bytes]: + ... + + +@overload +def open(path: _PathType, mode: str) -> IO[Any]: + ... + + +def open(path: _PathType, mode: Any) -> Any: + pass + + +assert_type(open("path", "r"), IO[str]) +assert_type(open("path", "wb"), IO[bytes]) +assert_type(open("path", "other"), IO[Any]) + + +A = TypeVar("A", bound=int) +B = TypeVar("B", bound=int) +C = TypeVar("C", bound=int) + + +class Matrix(Generic[A, B]): + def __add__(self, other: "Matrix[A, B]") -> "Matrix[A, B]": + ... + + def __matmul__(self, other: "Matrix[B, C]") -> "Matrix[A, C]": + ... + + def transpose(self) -> "Matrix[B, A]": + ... + + +def func2(a: Matrix[Literal[2], Literal[3]], b: Matrix[Literal[3], Literal[7]]): + c = a @ b + assert_type(c, Matrix[Literal[2], Literal[7]]) + + +T = TypeVar("T", Literal["a"], Literal["b"], Literal["c"]) +S = TypeVar("S", bound=Literal["foo"]) + + +class Status(Enum): + SUCCESS = 0 + INVALID_DATA = 1 + FATAL_ERROR = 2 + + +def parse_status1(s: str | Status) -> None: + if s is Status.SUCCESS: + assert_type(s, Literal[Status.SUCCESS]) + elif s is Status.INVALID_DATA: + assert_type(s, Literal[Status.INVALID_DATA]) + elif s is Status.FATAL_ERROR: + assert_type(s, Literal[Status.FATAL_ERROR]) + else: + assert_type(s, str) + + +def expects_bad_status(status: Literal["MALFORMED", "ABORTED"]): + ... + + +def expects_pending_status(status: Literal["PENDING"]): + ... + + +def parse_status2(status: str) -> None: + if status in ("MALFORMED", "ABORTED"): + return expects_bad_status(status) + + if status == "PENDING": + expects_pending_status(status) + + +final_val1: Final = 3 +assert_type(final_val1, Literal[3]) + +final_val2: Final = True +assert_type(final_val2, Literal[True]) diff --git a/conformance/tests/literals_literalstring.py b/conformance/tests/literals_literalstring.py new file mode 100644 index 000000000..10b50e080 --- /dev/null +++ b/conformance/tests/literals_literalstring.py @@ -0,0 +1,170 @@ +""" +Tests handling of the LiteralString special form. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/literal.html#literalstring + + +from typing import ( + Any, + Generic, + Literal, + LiteralString, + Sequence, + TypeVar, + assert_type, + overload, +) + + +variable_annotation: LiteralString + + +def my_function(literal_string: LiteralString) -> LiteralString: + ... + + +class Foo: + my_attribute: LiteralString = "" + + +type_argument: list[LiteralString] + +T = TypeVar("T", bound=LiteralString) + + +bad_union: Literal["hello", LiteralString] # Type error +bad_nesting: Literal[LiteralString] # Type error + + +def func1(a: Literal["one"], b: Literal["two"]): + x1: LiteralString = a + + x2: Literal[""] = b # Type error + + +def func2(a: LiteralString, b: LiteralString): + # > Addition: x + y is of type LiteralString if both x and y are compatible with LiteralString. + assert_type(a + b, LiteralString) + + # > Joining: sep.join(xs) is of type LiteralString if sep’s type is + # > compatible with LiteralString and xs’s type is compatible with Iterable[LiteralString]. + assert_type(",".join((a, b)), LiteralString) + assert_type(",".join((a, str(b))), str) + + # > In-place addition: If s has type LiteralString and x has type compatible with + # > LiteralString, then s += x preserves s’s type as LiteralString. + a += "hello" + b += a + + # > String formatting: An f-string has type LiteralString if and only if its constituent + # > expressions are literal strings. s.format(...) has type LiteralString if and only if + # > s and the arguments have types compatible with LiteralString. + assert_type(f"{a} {b}", LiteralString) + + variable = 3 + x1: LiteralString = f"{a} {str(variable)}" # Type error + + assert_type(a + str(1), str) + + # > LiteralString is compatible with the type str + x2: str = a + + # > Other literal types, such as literal integers, are not compatible with LiteralString. + x3: LiteralString = 3 # Type error + x4: LiteralString = b"test" # Type error + + +# > Conditional statements and expressions work as expected. +def condition1() -> bool: + ... + + +def return_literal_string() -> LiteralString: + return "foo" if condition1() else "bar" # OK + + +def return_literal_str2(literal_string: LiteralString) -> LiteralString: + return "foo" if condition1() else literal_string # OK + + +def return_literal_str3() -> LiteralString: + result: LiteralString + if condition1(): + result = "foo" + else: + result = "bar" + + return result # OK + + +# > TypeVars can be bound to LiteralString. +TLiteral = TypeVar("TLiteral", bound=LiteralString) + + +def literal_identity(s: TLiteral) -> TLiteral: + return s + + +def func3(s: Literal["hello"]): + y = literal_identity(s) + assert_type(y, Literal["hello"]) + + +def func4(s: LiteralString): + y2 = literal_identity(s) + assert_type(y2, LiteralString) + + +def func5(s: str): + literal_identity(s) # Type error + + +# > LiteralString can be used as a type argument for generic classes. +class Container(Generic[T]): + def __init__(self, value: T) -> None: + self.value = value + + +def func6(s: LiteralString): + x: Container[LiteralString] = Container(s) # OK + + +def func7(s: str): + x_error: Container[LiteralString] = Container(s) # Type error + + +# > Standard containers like List work as expected. +xs: list[LiteralString] = ["foo", "bar", "baz"] + + +@overload +def func8(x: Literal["foo"]) -> int: + ... + + +@overload +def func8(x: LiteralString) -> bool: + ... + + +@overload +def func8(x: str) -> str: + ... + + +def func8(x: Any) -> Any: + ... + + +assert_type(func8("foo"), int) # First overload +assert_type(func8("bar"), bool) # Second overload +assert_type(func8(str(1)), str) # Third overload + + +def func9(val: list[LiteralString]): + x1: list[str] = val # Type error + + +def func10(val: Sequence[LiteralString]): + x1: Sequence[str] = val # OK diff --git a/conformance/tests/literals_parameterizations.py b/conformance/tests/literals_parameterizations.py new file mode 100644 index 000000000..0edc1d4d8 --- /dev/null +++ b/conformance/tests/literals_parameterizations.py @@ -0,0 +1,67 @@ +""" +Tests legal and illegal parameterizations of Literal. +""" + +# > Literal must be parameterized with at least one type. + +from typing import Any, Literal, TypeVar +from enum import Enum + + +class Color(Enum): + RED = 0 + GREEN = 1 + BLUE = 2 + + +good1: Literal[26] +good2: Literal[0x1A] +good3: Literal[-4] +good4: Literal["hello world"] +good5: Literal[b"hello world"] +good6: Literal["hello world"] +good7: Literal[True] +good8: Literal[Color.RED] +good9: Literal[None] + +ReadOnlyMode = Literal["r", "r+"] +WriteAndTruncateMode = Literal["w", "w+", "wt", "w+t"] +WriteNoTruncateMode = Literal["r+", "r+t"] +AppendMode = Literal["a", "a+", "at", "a+t"] + +AllModes = Literal[ReadOnlyMode, WriteAndTruncateMode, WriteNoTruncateMode, AppendMode] + +good10: Literal[Literal[Literal[1, 2, 3], "foo"], 5, None] + +variable = 3 +T = TypeVar("T") + +# > Arbitrary expressions [are illegal] +bad1: Literal[3 + 4] # Type error +bad2: Literal["foo".replace("o", "b")] # Type error +bad3: Literal[4 + 3j] # Type error +bad4: Literal[+5] # Type error +bad5: Literal[not False] # Type error +bad6: Literal[(1, "foo", "bar")] # Type error +bad7: Literal[{"a": "b", "c": "d"}] # Type error +bad8: Literal[int] # Type error +bad9: Literal[variable] # Type error +bad10: Literal[T] # Type error +bad11: Literal[3.14] # Type error +bad12: Literal[Any] # Type error +bad13: Literal[...] # Type error + + +def my_function(x: Literal[1 + 2]) -> int: # Type error + return x * 3 + +x: Literal # Type error +y: Literal[my_function] = my_function # Type error + + +def func2(a: Literal[Color.RED]): + x1: Literal["Color.RED"] = a # Type error + + x2: "Literal[Color.RED]" = a # OK + + diff --git a/conformance/tests/literals_semantics.py b/conformance/tests/literals_semantics.py new file mode 100644 index 000000000..a75b48c5d --- /dev/null +++ b/conformance/tests/literals_semantics.py @@ -0,0 +1,42 @@ +""" +Tests the semantics of the Literal special form. +""" + +from typing import Literal +from typing import Literal as L + + +v1: Literal[3] = 3 +v2: Literal[3] = 4 # Type error + +v3: L[-3] = -3 + + +# > Literal[20] and Literal[0x14] are equivalent +def func1(a: Literal[20], b: Literal[0x14], c: Literal[0b10100]): + x1: Literal[0x14] = a + x2: Literal[0x14] = b + x3: Literal[0x14] = c + + +# > Literal[0] and Literal[False] are not equivalent +def func2(a: Literal[0], b: Literal[False]): + x1: Literal[False] = a # Type Error + x2: Literal[0] = b # Type Error + + +# > Given some value v that is a member of type T, the type Literal[v] shall +# > be treated as a subtype of T. For example, Literal[3] is a subtype of int. +def func3(a: L[3, 4, 5]): + b = a.__add__(3) + c = a + 3 + a += 3 # Type error + + +# > When a Literal is parameterized with more than one value, it’s treated +# > as exactly equivalent to the union of those types. +def func4(a: L[None, 3] | L[3, "foo", b"bar", True]): + x1: Literal[3, b"bar", True, "foo", None] = a + a = x1 + + diff --git a/conformance/tests/narrowing_typeguard.py b/conformance/tests/narrowing_typeguard.py new file mode 100644 index 000000000..fbdbca821 --- /dev/null +++ b/conformance/tests/narrowing_typeguard.py @@ -0,0 +1,108 @@ +""" +Tests TypeGuard functionality. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/narrowing.html#typeguard + +from typing import Any, Self, TypeGuard, TypeVar, assert_type + + +T = TypeVar("T") + +def is_two_element_tuple(val: tuple[T, ...]) -> TypeGuard[tuple[T, T]]: + return len(val) == 2 + +def func1(names: tuple[str, ...]): + if is_two_element_tuple(names): + assert_type(names, tuple[str, str]) + else: + assert_type(names, tuple[str, ...]) + + +def is_str_list(val: list[object], allow_empty: bool) -> TypeGuard[list[str]]: + if len(val) == 0: + return allow_empty + return all(isinstance(x, str) for x in val) + +def is_set_of(val: set[Any], type: type[T]) -> TypeGuard[set[T]]: + return all(isinstance(x, type) for x in val) + +def func2(val: set[object]): + if is_set_of(val, int): + assert_type(val, set[int]) + else: + assert_type(val, set[object]) + + +T_A = TypeVar("T_A", bound="A") + +class A: + def tg_1(self, val: object) -> TypeGuard[int]: + return isinstance(val, int) + + @classmethod + def tg_2(cls, val: object) -> TypeGuard[int]: + return isinstance(val, int) + + @staticmethod + def tg_3(val: object) -> TypeGuard[int]: + return isinstance(val, int) + + def tg4(self, val: object) -> TypeGuard[Self]: + return isinstance(val, type(self)) + + def tg5(self: T_A, val: object) -> TypeGuard[T_A]: + return isinstance(val, type(self)) + +class B(A): + pass + +# > Type checkers should assume that type narrowing should be applied to +# > the expression that is passed as the first positional argument to a +# > user-defined type guard. If the type guard function accepts more than +# > one argument, no type narrowing is applied to those additional argument +# > expressions. + +def func3() -> None: + val1 = object() + if A().tg_1(val1): + assert_type(val1, int) + + val2 = object() + if A().tg_2(val2): + assert_type(val2, int) + + val3 = object() + if A.tg_2(val3): + assert_type(val3, int) + + val4 = object() + if A().tg_3(val4): + assert_type(val4, int) + + val5 = object() + if A.tg_3(val5): + assert_type(val5, int) + + val6 = object() + if B().tg4(val6): + assert_type(val6, B) + + val7 = object() + if B().tg4(val7): + assert_type(val7, B) + + +# > If a type guard function is implemented as an instance method or class +# > method, the first positional argument maps to the second parameter +# > (after “self” or “cls”). + +class C: + # Type checker should emit error here. + def tg_1(self) -> TypeGuard[int]: + return False + + @classmethod + # Type checker should emit error here. + def tg_2(cls) -> TypeGuard[int]: + return False diff --git a/conformance/tests/typeddicts_alt_syntax.py b/conformance/tests/typeddicts_alt_syntax.py new file mode 100644 index 000000000..1baf016cb --- /dev/null +++ b/conformance/tests/typeddicts_alt_syntax.py @@ -0,0 +1,45 @@ +""" +Tests the "alternative" (non-class) syntax for defining a TypedDict. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/typeddict.html#alternative-syntax + +from typing import TypedDict + + +Movie = TypedDict("Movie", {"name": str, "year": int, "illegal key name": bool}) + +movie: Movie = {"name": "Blade Runner", "year": 1982, "illegal key name": True} + +MovieOptional = TypedDict("MovieOptional", {"name": str, "year": int}, total=False) + +movie_opt1: MovieOptional = {} +movie_opt2: MovieOptional = {"year": 1982} + +# > A type checker is only expected to accept a dictionary display expression as +# > the second argument to TypedDict. In particular, a variable that refers to a +# > dictionary object does not need to be supported, to simplify implementation. +my_dict = {"name": str} +BadTypedDict1 = TypedDict("BadTypedDict1", my_dict) + + +# This should generate an error because it uses a non-str key. +BadTypedDict2 = TypedDict("BadTypedDict2", {1: str}) + + +# This should generate an error because it uses a non-matching name. +BadTypedDict3 = TypedDict("WrongName", {"name": str}) + + +# This should generate an error because of the additional parameter. +BadTypedDict4 = TypedDict("BadTypedDict4", {"name": str}, total=False, other=False) + + +# > The keyword-argument syntax is deprecated in 3.11 and will be removed in 3.13. +# > It may also be unsupported by static type checkers. + +Movie2 = TypedDict("Movie2", name=str, year=int) + +movie2: Movie2 +movie2 = {"name": "Blade Runner", "year": 1982} +movie2 = {"name": "Blade Runner", "year": ""} # Type error: Incorrect type for year diff --git a/conformance/tests/typeddicts_class_syntax.py b/conformance/tests/typeddicts_class_syntax.py new file mode 100644 index 000000000..38ed09c0c --- /dev/null +++ b/conformance/tests/typeddicts_class_syntax.py @@ -0,0 +1,79 @@ +""" +Tests the class syntax for defining a TypedDict. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/typeddict.html#typeddict + +from typing import Generic, TypeVar, TypedDict + + +class Movie(TypedDict): + name: str + year: int + + # > String literal forward references are valid in the value types. + director: "Person" + + +class Person(TypedDict): + name: str + age: int + + +# > Methods are not allowed, since the runtime type of a TypedDict object +# > will always be just dict (it is never a subclass of dict). +class BadTypedDict1(TypedDict): + name: str + + # Methods are not allowed, so this should generate an error. + def method1(self): + pass + + # Methods are not allowed, so this should generate an error. + @classmethod + def method2(cls): + pass + + # Methods are not allowed, so this should generate an error. + @staticmethod + def method3(): + pass + + +# > Specifying a metaclass is not allowed. +class BadTypedDict2(TypedDict, metaclass=type): + name: str + + +# This should generate an error because "other" is not an allowed keyword argument. +class BadTypedDict3(TypedDict, other=True): + name: str + + +# > TypedDicts may be made generic by adding Generic[T] among the bases. +T = TypeVar("T") + + +class GenericTypedDict(Generic[T]): + name: str + value: T + + +# > An empty TypedDict can be created by only including pass in the +# > body (if there is a docstring, pass can be omitted): +class EmptyDict1(TypedDict): + pass + + +class EmptyDict2(TypedDict): + """Docstring""" + + +class MovieTotal(TypedDict, total=True): + name: str + + +class MovieOptional(TypedDict, total=False): + name: str + + diff --git a/conformance/tests/typeddicts_final.py b/conformance/tests/typeddicts_final.py new file mode 100644 index 000000000..1cd08f2f9 --- /dev/null +++ b/conformance/tests/typeddicts_final.py @@ -0,0 +1,26 @@ +""" +Tests the use of Final values when used with TypedDicts. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/typeddict.html#use-of-final-values-and-literal-types + +from typing import Final, Literal, TypedDict + + +class Movie(TypedDict): + name: str + year: int + + +# > Type checkers should allow final names (PEP 591) with string values to be +# > used instead of string literals in operations on TypedDict objects. +YEAR: Final = "year" + +m: Movie = {"name": "Alien", "year": 1979} +years_since_epoch = m[YEAR] - 1970 + + +# > An expression with a suitable literal type (PEP 586) can be used instead of +# > a literal value. +def get_value(movie: Movie, key: Literal["year", "name"]) -> int | str: + return movie[key] diff --git a/conformance/tests/typeddicts_inheritance.py b/conformance/tests/typeddicts_inheritance.py new file mode 100644 index 000000000..38b72403a --- /dev/null +++ b/conformance/tests/typeddicts_inheritance.py @@ -0,0 +1,66 @@ +""" +Tests inheritance rules for TypedDict classes. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/typeddict.html#typeddict + +from typing import TypedDict + + +class Movie(TypedDict): + name: str + year: int + +class BookBasedMovie(Movie): + based_on: str + +class BookBasedMovieAlso(TypedDict): + name: str + year: int + based_on: str + +b1: BookBasedMovie = {"name": "Little Women", "year": 2019, "based_on": "Little Women"} + +b2: BookBasedMovieAlso = b1 + + +class X(TypedDict): + x: int + +class Y(TypedDict): + y: str + +class XYZ(X, Y): + z: bool + +x1 = XYZ(x=1, y="", z=True) + +# > A TypedDict cannot inherit from both a TypedDict type and a +# > non-TypedDict base class other than Generic. + +class NonTypedDict: + pass + +class BadTypedDict(TypedDict, NonTypedDict): + pass + + +# > Changing a field type of a parent TypedDict class in a subclass is +# > not allowed. + +class X1(TypedDict): + x: str + +class Y1(X1): + x: int # Type check error: cannot overwrite TypedDict field "x" + + +# > Multiple inheritance does not allow conflict types for the same name field: +class X2(TypedDict): + x: int + +class Y2(TypedDict): + x: str + +class XYZ2(X2, Y2): # Type check error: cannot overwrite TypedDict field "x" while merging + xyz: bool diff --git a/conformance/tests/typeddicts_operations.py b/conformance/tests/typeddicts_operations.py new file mode 100644 index 000000000..11c449043 --- /dev/null +++ b/conformance/tests/typeddicts_operations.py @@ -0,0 +1,65 @@ +""" +Tests operations provided by a TypedDict object. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/typeddict.html#supported-and-unsupported-operations + + +from typing import TypedDict, assert_type + + +class Movie(TypedDict): + name: str + year: int + + +movie: Movie + +movie = {"name": "Blade Runner", "year": 1982} +movie["name"] = "Other" +movie["year"] = 1981 + +movie["name"] = 1982 # Type error: wrong type +movie["year"] = "" # Type error: wrong type +movie["other"] = "" # Type error: unknown key added + +print(movie["other"]) # Type error: unknown key referenced + +movie = {"name": "Blade Runner"} # Type error: year is missing +movie = {"name": "Blade Runner", "year": 1982.1} # Type error: year is wrong type + +# > The use of a key that is not known to exist should be reported as an error. +movie = {"name": "", "year": 1900, "other": 2} # Type error: extra key + + +def func1(variable_key: str): + # > A key that is not a literal should generally be rejected. + movie: Movie = {variable_key: "", "year": 1900} # Type error: variable key + + +# It's not clear from the spec what type this should be. +movie.get("name") + +# It's not clear from the spec what type this should be. +movie.get("other") + + +movie.clear() # Type error: clear not allowed + +del movie["name"] # Type error: del not allowed for required key + + + +class MovieOptional(TypedDict, total=False): + name: str + year: int + + +movie_optional: MovieOptional = {} + +assert_type(movie_optional.get("name"), str | None) + +movie_optional.clear() # Type error: clear not allowed + +del movie_optional["name"] + diff --git a/conformance/tests/typeddicts_required.py b/conformance/tests/typeddicts_required.py new file mode 100644 index 000000000..22485f0f1 --- /dev/null +++ b/conformance/tests/typeddicts_required.py @@ -0,0 +1,76 @@ +""" +Tests the Required and NotRequired special forms. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/typeddict.html#required-and-notrequired + +from typing import Annotated, NotRequired, Required, TypedDict + + +# Required and NotRequired are valid only within a TypedDict. +class NotTypedDict: + x: Required[int] # Type error: Required not allowed in this context + + +def func1( + x: NotRequired[int], +) -> None: # Type error: Required not allowed in this context + pass + + +class TD1(TypedDict, total=False): + a: int + + +class TD2(TD1, total=True): + b: int + + +class TD3(TypedDict): + a: NotRequired[int] + b: Required[int] + + +class TD4(TypedDict, total=False): + a: int + b: Required[int] + + +class TD5(TypedDict, total=True): + a: NotRequired[int] + b: int + + +td3: TD3 = {"b": 0} +td4: TD4 = {"b": 0} +td5: TD5 = {"b": 0} + +# These are all equivalent types, so they should be +# bidirectionally type compatible. +td3 = td4 +td3 = td5 +td4 = td3 +td4 = td5 +td5 = td3 +td5 = td4 + + +class TD6(TypedDict): + a: Required[Required[int]] # Type error: Nesting not allowed + b: Required[NotRequired[int]] # Type error: Nesting not allowed + + +class TD7(TypedDict): + # > Required[] and NotRequired[] can be used with Annotated[], in any nesting order. + x: Annotated[Required[int], ""] + y: Required[Annotated[int, ""]] + z: Annotated[Required[Annotated[int, ""]], ""] + + +RecursiveMovie = TypedDict( + "RecursiveMovie", {"title": Required[str], "predecessor": NotRequired["RecursiveMovie"]} +) + +movie: RecursiveMovie = {"title": "Beethoven 3", "predecessor": {"title": "Beethoven 2"}} + + diff --git a/conformance/tests/typeddicts_type_consistency.py b/conformance/tests/typeddicts_type_consistency.py new file mode 100644 index 000000000..c782b2cc3 --- /dev/null +++ b/conformance/tests/typeddicts_type_consistency.py @@ -0,0 +1,150 @@ +""" +Tests the type consistency rules for TypedDict objects. +""" + +# Specification: https://typing.readthedocs.io/en/latest/spec/typeddict.html#type-consistency + +from typing import Any, Literal, Mapping, TypedDict + + +class A1(TypedDict): + x: int | None + + +class B1(TypedDict): + x: int + + +b1: B1 = {"x": 0} + +# > Value types behave invariantly. +a1: A1 = b1 # Type check error: 'B1' not compatible with 'A1' + +# > any TypedDict type is consistent with Mapping[str, object] +v1: Mapping[str, object] = b1 + +# > A TypedDict type with a required key is not consistent with a TypedDict +# > type where the same key is a non-required key. + +class A2(TypedDict, total=False): + x: int + + +class B2(TypedDict): + x: int + + +b2: B2 = {"x": 0} +a2: A2 = b2 # Type check error: 'B2' not compatible with 'A2' + + +# > A TypedDict type A is consistent with TypedDict B if A is structurally +# > compatible with B. This is true if and only if both of these conditions +# > are satisfied: +# > 1. For each key in B, A has the corresponding key and the corresponding +# > value type in A is consistent with the value type in B. For each key in B, +# > the value type in B is also consistent with the corresponding value type +# > in A. +# > 2. For each required key in B, the corresponding key is required in A. For +# > each non-required key in B, the corresponding key is not required in A. + + +class A3(TypedDict): + x: int + + +class B3(TypedDict): + x: int + y: int + + +b3: B3 = {"x": 0, "y": 0} +a3: A3 = b3 + +a3 = {"x": 0} +b3 = a3 # Type checker error + + +# This should generate an error because it's a literal assignment. +a3_1: A3 = {"x": 0, "y": 0} + +# This should not generate an error. +a3_2 = b3 + +# > A TypedDict isn’t consistent with any Dict[...] type. + +d1: dict[str, int] = b3 # Type checker error +d2: dict[str, object] = b3 # Type checker error +d3: dict[Any, Any] = b3 # Type checker error + +# > A TypedDict with all int values is not consistent with Mapping[str, int]. + +m1: Mapping[str, int] = b3 # Type checker error +m2: Mapping[str, object] = b3 # OK +m3: Mapping[str, Any] = b3 # OK + + +# Test "get" method. +UserType1 = TypedDict("UserType1", {"name": str, "age": int}, total=False) +user1: UserType1 = {"name": "Bob", "age": 40} + +name1: str = user1.get("name", "n/a") +age1: int = user1.get("age", 42) + +UserType2 = TypedDict("UserType2", {"name": str, "age": int}) +user2: UserType2 = {"name": "Bob", "age": 40} + +name2: str | None = user2.get("name") + +name3: str = user2.get("name") + +age2: int = user2.get("age", 42) + +age3: int | str = user2.get("age", "42") + +age4: int = user2.get("age", "42") + +# Test nested TypedDicts. +class Inner1(TypedDict): + inner_key: str + + +class Inner2(TypedDict): + inner_key: Inner1 + + +class Outer1(TypedDict): + outer_key: Inner2 + + +o1: Outer1 = {"outer_key": {"inner_key": {"inner_key": "hi"}}} + +# This should generate an error because the inner-most value +# should be a string. +o2: Outer1 = {"outer_key": {"inner_key": {"inner_key": 1}}} + + +class Inner3(TypedDict): + x: int + + +class Inner4(TypedDict): + x: int + + +class Outer2(TypedDict): + y: str + z: Literal[""] | Inner3 + + +class Outer3(TypedDict): + y: str + z: Literal[""] | Inner4 + + +def func1(td: Outer3): + ... + + +o3: Outer2 = {"y": "", "z": {"x": 0}} +o4: Outer3 = o3 diff --git a/conformance/tests/typeddicts_usage.py b/conformance/tests/typeddicts_usage.py new file mode 100644 index 000000000..99232872d --- /dev/null +++ b/conformance/tests/typeddicts_usage.py @@ -0,0 +1,42 @@ +""" +Tests for basic usage of TypedDict. +""" + +from typing import TypeVar, TypedDict + + +class Movie(TypedDict): + name: str + year: int + + +movie: Movie = {"name": "Blade Runner", "year": 1982} + + +def record_movie(movie: Movie) -> None: + ... + + +record_movie({"name": "Blade Runner", "year": 1982}) + + +movie["director"] = "Ridley Scott" # Error: invalid key 'director' +movie["year"] = "1982" # Error: invalid value type ("int" expected) + +# The code below should be rejected, since 'title' is not a valid key, +# and the 'name' key is missing: +movie2: Movie = {"title": "Blade Runner", "year": 1982} + +m = Movie(name='Blade Runner', year=1982) + + +# > TypedDict type objects cannot be used in isinstance() tests such as +# > isinstance(d, Movie). +if isinstance(movie, Movie): + pass + + +# TypedDict should not be allowed as a bound for a TypeVar. +T = TypeVar("T", bound=TypedDict) # Type checker error + + From 2f565e5b9af02a7139746bb1900a9ad0165de18e Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 26 Dec 2023 02:36:27 -0700 Subject: [PATCH 02/19] Fixed syntax error in f-string that affects 3.11 and older. --- conformance/src/type_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance/src/type_checker.py b/conformance/src/type_checker.py index b42148e1a..85ce56fd9 100644 --- a/conformance/src/type_checker.py +++ b/conformance/src/type_checker.py @@ -103,7 +103,7 @@ def run_tests(self, test_files: Sequence[str]) -> dict[str, str]: col_number = diagnostic['range']['start']['character'] + 1 severity = diagnostic['severity'] message = diagnostic['message'] - rule = f' ({diagnostic['rule']})' if 'rule' in diagnostic else '' + rule = f" ({diagnostic['rule']})" if 'rule' in diagnostic else '' line_text = f'{file_name}:{line_number}:{col_number} - {severity}: {message}{rule}\n' results_dict[file_name] = results_dict.get(file_name, '') + line_text From 815ec19869f1b8291a39b8c398a572c60cb6ed67 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 26 Dec 2023 10:29:19 -0700 Subject: [PATCH 03/19] Addressed PR feedback. Addressed linter issues. --- .flake8 | 1 + conformance/README.md | 14 +++++++-- conformance/src/main.py | 1 - conformance/src/type_checker.py | 53 +++++++++++++++++---------------- 4 files changed, 40 insertions(+), 29 deletions(-) diff --git a/.flake8 b/.flake8 index ff64f42c1..13a85deb9 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,7 @@ [flake8] max-line-length = 90 +exclude = conformance ignore = # irrelevant plugins B3, diff --git a/conformance/README.md b/conformance/README.md index f3f8633ac..61d76e10c 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -38,6 +38,14 @@ Test cases are meant to be human readable. They should include comments that hel The test suite focuses on static type checking not general Python semantics. Tests should therefore focus on static analysis behaviors, not runtime behaviors. +## Running the Conformance Test Tool + +To run the conformance test suite: +* Clone the https://github.com/python/typing repo. +* Create and activate a Python 3.12 virtual environment. +* Switch to the `conformance` subdirectory and install all dependencies (`pip install -r requirements.txt`). +* Switch to the `src` subdirectory and run `python main.py`. + ## Reporting Conformance Results Different type checkers report errors in different ways (with different wording in error messages and different line numbers or character ranges for errors). This variation makes it difficult to fully automate test validation given that tests will want to check for both false positive and false negative type errors. Some level of manual inspection will therefore be needed to determine whether a type checker is fully conformant with all tests in any given test file. This "scoring" process is required only when the output of a test changes — e.g. when a new version of that type checker is released and the tests are rerun. We assume that the output of a type checker will be the same from one run to the next unless/until a new version is released that fixes or introduces a bug. In this case, the output will need to be manually inspected and the conformance results re-scored for those tests whose output has changed. @@ -46,11 +54,11 @@ Conformance results are reported and summarized for each supported type checker. ## Adding a New Test Case -To add a new test, create a new ".py" file in the `tests` directory. Its name must begin with one of the above test group names followed by an underscore. Write the contents of the test including a module docstring describing the purpose of the test. Next, run the tool. This will generate a new `.toml` file corresponding to the new test for each supported type checker. Manually review the output from each type checker and determine whether it conforms to the specification. If so, add `conformant = "Yes"` to the `.toml` file. If it does not fully comply, add `conformant = "Partial"` and a `notes` section detailing where it is not compliant. If the type checker doesn't support the feature in the test add `conformant = "Unsupported"`. Once the conformance status has been updated, re-run the test tool again to regenerate the summary report. +To add a new test, create a new ".py" file in the `tests` directory. Its name must begin with one of the above test group names followed by an underscore. Write the contents of the test including a module docstring describing the purpose of the test. Next, run the conformance test tool. This will generate a new `.toml` file in the `results` subdirectory corresponding each type checker. Manually review the output from each type checker and determine whether it conforms to the specification. If so, add `conformant = "Yes"` to the `.toml` file. If it does not fully comply, add `conformant = "Partial"` and a `notes` section detailing where it is not compliant. If the type checker doesn't support the feature in the test add `conformant = "Unsupported"`. Once the conformance status has been updated, rerun the conformance test tool to regenerate the summary report. -## Updating A Test Case +## Updating a Test Case -If a test is updated (augmented or fixed), the process is similar to when adding a new test. Run the tool to generate new results and manually examine the output of each supported type checker. Then update the conformance status accordingly. Once the conformance status has been updated, re-run the test tool again to regenerate the summary report. +If a test is updated (augmented or fixed), the process is similar to when adding a new test. Run the conformance test tool to generate new results and manually examine the output of each supported type checker. Then update the conformance status accordingly. Once the conformance status has been updated, rerun the conformance test tool to regenerate the summary report. ## Updating a Type Checker diff --git a/conformance/src/main.py b/conformance/src/main.py index 7a9f76718..d657292fd 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -4,7 +4,6 @@ import os from pathlib import Path -import re import sys from typing import Sequence diff --git a/conformance/src/type_checker.py b/conformance/src/type_checker.py index 85ce56fd9..e87cad33f 100644 --- a/conformance/src/type_checker.py +++ b/conformance/src/type_checker.py @@ -58,16 +58,16 @@ def get_version(self) -> str: return version def run_tests(self, test_files: Sequence[str]) -> dict[str, str]: - command = f"mypy . --disable-error-code empty-body" + command = "mypy . --disable-error-code empty-body" proc = run(command, stdout=PIPE, text=True, shell=True) - lines = proc.stdout.split('\n') + lines = proc.stdout.split("\n") # Add results to a dictionary keyed by the file name. results_dict: dict[str, str] = {} for line in lines: - file_name = line.split(':')[0].strip() - results_dict[file_name] = results_dict.get(file_name, '') + line + '\n' - + file_name = line.split(":")[0].strip() + results_dict[file_name] = results_dict.get(file_name, "") + line + "\n" + return results_dict @@ -89,27 +89,28 @@ def get_version(self) -> str: return proc.stdout.strip() def run_tests(self, test_files: Sequence[str]) -> dict[str, str]: - command = f"pyright . --outputjson" + command = "pyright . --outputjson" proc = run(command, stdout=PIPE, text=True, shell=True) output_json = json.loads(proc.stdout) - diagnostics = output_json['generalDiagnostics'] + diagnostics = output_json["generalDiagnostics"] # Add results to a dictionary keyed by the file name. results_dict: dict[str, str] = {} for diagnostic in diagnostics: - file_path = Path(diagnostic.get('file', '')) + file_path = Path(diagnostic.get("file", "")) file_name = file_path.name - line_number = diagnostic['range']['start']['line'] + 1 - col_number = diagnostic['range']['start']['character'] + 1 - severity = diagnostic['severity'] - message = diagnostic['message'] - rule = f" ({diagnostic['rule']})" if 'rule' in diagnostic else '' - - line_text = f'{file_name}:{line_number}:{col_number} - {severity}: {message}{rule}\n' - results_dict[file_name] = results_dict.get(file_name, '') + line_text - + line_number = diagnostic["range"]["start"]["line"] + 1 + col_number = diagnostic["range"]["start"]["character"] + 1 + severity = diagnostic["severity"] + message = diagnostic["message"] + rule = f" ({diagnostic['rule']})" if "rule" in diagnostic else "" + + line_text = f"{file_name}:{line_number}:{col_number} - {severity}: {message}{rule}\n" + results_dict[file_name] = results_dict.get(file_name, "") + line_text + return results_dict + class PyreTypeChecker(TypeChecker): @property def name(self) -> str: @@ -119,27 +120,29 @@ def install(self) -> None: run("pip install pyre-check --upgrade", shell=True) # Generate a default config file. - pyre_config = '{"site_package_search_strategy": "pep561", "source_directories": ["."]}\n' - with open('.pyre_configuration', 'w') as f: + pyre_config = ( + '{"site_package_search_strategy": "pep561", "source_directories": ["."]}\n' + ) + with open(".pyre_configuration", "w") as f: f.write(pyre_config) def get_version(self) -> str: proc = run("pyre --version", stdout=PIPE, text=True, shell=True) version = proc.stdout.strip() - version = version.replace('Client version:', 'pyre') + version = version.replace("Client version:", "pyre") return version def run_tests(self, test_files: Sequence[str]) -> dict[str, str]: - command = f"pyre check" + command = "pyre check" proc = run(command, stdout=PIPE, text=True, shell=True) - lines = proc.stdout.split('\n') + lines = proc.stdout.split("\n") # Add results to a dictionary keyed by the file name. results_dict: dict[str, str] = {} for line in lines: - file_name = line.split(':')[0].strip() - results_dict[file_name] = results_dict.get(file_name, '') + line + '\n' - + file_name = line.split(":")[0].strip() + results_dict[file_name] = results_dict.get(file_name, "") + line + "\n" + return results_dict From b35bfa9d2e68b20dc02c073e071c179bede418a9 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 26 Dec 2023 10:47:45 -0700 Subject: [PATCH 04/19] Updated test results for pyright 1.1.343, which addresses all conformance issues currently covered by tests. --- .../results/pyright/annotations_typeexpr.toml | 10 ++++------ .../results/pyright/generics_self_usage.toml | 11 +++++------ .../results/pyright/typeddicts_alt_syntax.toml | 6 ++---- .../results/pyright/typeddicts_class_syntax.toml | 8 +++----- conformance/results/pyright/typeddicts_usage.toml | 7 +++---- conformance/results/pyright/version.toml | 2 +- conformance/results/results.html | 13 +++++++------ conformance/src/results_template.html | 1 + conformance/src/type_checker.py | 2 +- 9 files changed, 27 insertions(+), 33 deletions(-) diff --git a/conformance/results/pyright/annotations_typeexpr.toml b/conformance/results/pyright/annotations_typeexpr.toml index 80d2f522e..e2bbaf74c 100644 --- a/conformance/results/pyright/annotations_typeexpr.toml +++ b/conformance/results/pyright/annotations_typeexpr.toml @@ -1,8 +1,4 @@ -conformant = "Partial" -notes = """ -Does not reject ternary expression in annotation. -Does not reject binary expression in annotation. -""" +conformant = "Yes" output = """ annotations_typeexpr.py:77:9 - error: Call expression not allowed in type expression (reportGeneralTypeIssues) annotations_typeexpr.py:78:9 - error: List expression not allowed in type annotation @@ -21,10 +17,12 @@ annotations_typeexpr.py:82:9 - error: Call expression not allowed in type expres annotations_typeexpr.py:83:9 - error: List expression not allowed in type annotation   Use List[T] to indicate a list type or Union[T1, T2] to indicate a union type (reportGeneralTypeIssues) annotations_typeexpr.py:83:9 - error: Expected type expression but received "list[type[int]]" (reportGeneralTypeIssues) +annotations_typeexpr.py:84:9 - error: Ternary expression not allowed in type annotation annotations_typeexpr.py:85:9 - error: Variable not allowed in type expression (reportGeneralTypeIssues) annotations_typeexpr.py:86:10 - error: Expected type expression but received "Literal[True]" (reportGeneralTypeIssues) annotations_typeexpr.py:87:10 - error: Expected type expression but received "Literal[1]" (reportGeneralTypeIssues) -annotations_typeexpr.py:88:10 - error: Expected type expression but received "Literal[-1]" (reportGeneralTypeIssues) +annotations_typeexpr.py:88:10 - error: Unary operator not allowed in type annotation +annotations_typeexpr.py:89:10 - error: Binary operator not allowed in type annotation annotations_typeexpr.py:90:10 - error: Expected expression annotations_typeexpr.py:90:10 - error: Tuple expression not allowed in type annotation   Use Tuple[T1, ..., Tn] to indicate a tuple type or Union[T1, T2] to indicate a union type (reportGeneralTypeIssues) diff --git a/conformance/results/pyright/generics_self_usage.toml b/conformance/results/pyright/generics_self_usage.toml index f5507f65a..12031239e 100644 --- a/conformance/results/pyright/generics_self_usage.toml +++ b/conformance/results/pyright/generics_self_usage.toml @@ -1,8 +1,4 @@ -conformant = "Partial" -notes = """ -Does not reject invalid use of `Self` in class definition. -Does not reject invalid use of `Self` in metaclass. -""" +conformant = "Yes" output = """ generics_self_usage.py:73:14 - error: "Self" is not valid in this context (reportGeneralTypeIssues) generics_self_usage.py:73:23 - error: "Self" is not valid in this context (reportGeneralTypeIssues) @@ -12,9 +8,12 @@ generics_self_usage.py:82:44 - warning: TypeVar "TFoo2" appears only once in gen   Use "Foo2" instead (reportInvalidTypeVarUse) generics_self_usage.py:86:16 - error: Expression of type "Foo3" cannot be assigned to return type "Self@Foo3"   Type "Foo3" cannot be assigned to type "Self@Foo3" (reportGeneralTypeIssues) -generics_self_usage.py:103:12 - error: Class cannot derive from itself +generics_self_usage.py:101:15 - error: "Self" is not valid in this context (reportGeneralTypeIssues) +generics_self_usage.py:103:12 - error: "Self" is not valid in this context (reportGeneralTypeIssues) generics_self_usage.py:106:30 - error: "Self" is not valid in this context (reportGeneralTypeIssues) generics_self_usage.py:111:19 - error: "Self" is not valid in this context (reportGeneralTypeIssues) generics_self_usage.py:116:31 - error: "Self" is not valid in this context (reportGeneralTypeIssues) generics_self_usage.py:116:40 - error: "Self" is not valid in this context (reportGeneralTypeIssues) +generics_self_usage.py:121:37 - error: "Self" cannot be used within a metaclass (a subclass of "type") (reportGeneralTypeIssues) +generics_self_usage.py:125:42 - error: "Self" cannot be used within a metaclass (a subclass of "type") (reportGeneralTypeIssues) """ diff --git a/conformance/results/pyright/typeddicts_alt_syntax.toml b/conformance/results/pyright/typeddicts_alt_syntax.toml index e7c416e9a..e95787a8a 100644 --- a/conformance/results/pyright/typeddicts_alt_syntax.toml +++ b/conformance/results/pyright/typeddicts_alt_syntax.toml @@ -1,10 +1,8 @@ -conformant = "Partial" -notes = """ -Does not report when name of TypedDict doesn't match assigned identifier name. -""" +conformant = "Yes" output = """ typeddicts_alt_syntax.py:23:17 - error: Expected dict or keyword parameter as second parameter typeddicts_alt_syntax.py:27:45 - error: Expected string literal for dictionary entry name +typeddicts_alt_syntax.py:31:1 - error: TypedDict must be assigned to a variable named "WrongName" typeddicts_alt_syntax.py:35:78 - error: Extra TypedDict arguments not supported typeddicts_alt_syntax.py:45:43 - error: Expression of type "dict[str, str]" cannot be assigned to declared type "Movie2"   "Literal['']" is incompatible with "int" (reportGeneralTypeIssues) diff --git a/conformance/results/pyright/typeddicts_class_syntax.toml b/conformance/results/pyright/typeddicts_class_syntax.toml index 1147a3487..a01baa99d 100644 --- a/conformance/results/pyright/typeddicts_class_syntax.toml +++ b/conformance/results/pyright/typeddicts_class_syntax.toml @@ -1,10 +1,8 @@ -conformant = "Partial" -notes = """ -Does not report when metaclass is provided. -Does not report when other keyword argument is provided. -""" +conformant = "Yes" output = """ typeddicts_class_syntax.py:29:5 - error: TypedDict classes can contain only type annotations typeddicts_class_syntax.py:33:5 - error: TypedDict classes can contain only type annotations typeddicts_class_syntax.py:38:5 - error: TypedDict classes can contain only type annotations +typeddicts_class_syntax.py:44:32 - error: TypedDict does not support __init_subclass__ parameter "metaclass" +typeddicts_class_syntax.py:49:32 - error: TypedDict does not support __init_subclass__ parameter "other" """ diff --git a/conformance/results/pyright/typeddicts_usage.toml b/conformance/results/pyright/typeddicts_usage.toml index 10f072bcd..55660c6d4 100644 --- a/conformance/results/pyright/typeddicts_usage.toml +++ b/conformance/results/pyright/typeddicts_usage.toml @@ -1,7 +1,4 @@ -conformant = "Partial" -notes = """ -Does not report errant use of TypedDict in `isinstance` call. -""" +conformant = "Yes" output = """ typeddicts_usage.py:23:1 - error: Could not assign item in TypedDict   "director" is not a defined key in "Movie" (reportGeneralTypeIssues) @@ -9,5 +6,7 @@ typeddicts_usage.py:24:1 - error: Could not assign item in TypedDict   "Literal['1982']" is incompatible with "int" (reportGeneralTypeIssues) typeddicts_usage.py:28:18 - error: Expression of type "dict[str, str | int]" cannot be assigned to declared type "Movie"   "title" is an undefined field in type "Movie" (reportGeneralTypeIssues) +typeddicts_usage.py:35:22 - error: Second argument to "isinstance" must be a class or tuple of classes +  TypedDict class not allowed for instance or class checks (reportGeneralTypeIssues) typeddicts_usage.py:40:24 - error: "TypedDict" cannot be used in this context """ diff --git a/conformance/results/pyright/version.toml b/conformance/results/pyright/version.toml index e28ed20e8..493948174 100644 --- a/conformance/results/pyright/version.toml +++ b/conformance/results/pyright/version.toml @@ -1 +1 @@ -version = "pyright 1.1.342" +version = "pyright 1.1.343" diff --git a/conformance/results/results.html b/conformance/results/results.html index 741237752..e58727b9f 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -82,6 +82,7 @@ } .col3 { + width: 65%; vertical-align: top; } @@ -146,11 +147,11 @@

Python Type System Conformance Test Results

     narrowing_typeguardYes -

pyright 1.1.342

+

pyright 1.1.343

- + @@ -158,7 +159,7 @@

Python Type System Conformance Test Results

- + @@ -169,14 +170,14 @@

Python Type System Conformance Test Results

- - + + - + diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html index c7c91fc8f..ac7d20e4e 100644 --- a/conformance/src/results_template.html +++ b/conformance/src/results_template.html @@ -82,6 +82,7 @@ } .col3 { + width: 65%; vertical-align: top; } diff --git a/conformance/src/type_checker.py b/conformance/src/type_checker.py index e87cad33f..3ee4bc77d 100644 --- a/conformance/src/type_checker.py +++ b/conformance/src/type_checker.py @@ -78,7 +78,7 @@ def name(self) -> str: def install(self) -> None: # Install the Python wrapper if it's not installed. - run("pip install pyright", shell=True) + run("pip install pyright --upgrade", shell=True) # Force the Python wrapper to install node if needed # and download the latest version of pyright. From 89d41d333bcbc7e6e508af47a569bfc04ff7ca14 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 26 Dec 2023 12:59:44 -0700 Subject: [PATCH 05/19] Added support for pytest. --- .../results/pytype/annotations_typeexpr.toml | 26 ++++++++ .../pytype/generics_self_advanced.toml | 51 ++++++++++++++++ .../pytype/generics_self_attributes.toml | 9 +++ .../results/pytype/generics_self_basic.toml | 52 ++++++++++++++++ .../pytype/generics_self_protocols.toml | 2 + .../results/pytype/generics_self_usage.toml | 31 ++++++++++ .../results/pytype/literals_interactions.toml | 48 +++++++++++++++ .../pytype/literals_literalstring.toml | 6 ++ .../pytype/literals_parameterizations.toml | 41 +++++++++++++ .../results/pytype/literals_semantics.toml | 25 ++++++++ .../results/pytype/narrowing_typeguard.toml | 6 ++ .../results/pytype/typeddicts_alt_syntax.toml | 14 +++++ .../pytype/typeddicts_class_syntax.toml | 8 +++ .../results/pytype/typeddicts_final.toml | 3 + .../pytype/typeddicts_inheritance.toml | 9 +++ .../results/pytype/typeddicts_operations.toml | 24 ++++++++ .../results/pytype/typeddicts_required.toml | 8 +++ .../pytype/typeddicts_type_consistency.toml | 20 +++++++ .../results/pytype/typeddicts_usage.toml | 14 +++++ conformance/results/pytype/version.toml | 1 + conformance/results/results.html | 37 ++++++++++++ conformance/src/main.py | 2 +- conformance/src/type_checker.py | 60 +++++++++++++++++++ 23 files changed, 496 insertions(+), 1 deletion(-) create mode 100644 conformance/results/pytype/annotations_typeexpr.toml create mode 100644 conformance/results/pytype/generics_self_advanced.toml create mode 100644 conformance/results/pytype/generics_self_attributes.toml create mode 100644 conformance/results/pytype/generics_self_basic.toml create mode 100644 conformance/results/pytype/generics_self_protocols.toml create mode 100644 conformance/results/pytype/generics_self_usage.toml create mode 100644 conformance/results/pytype/literals_interactions.toml create mode 100644 conformance/results/pytype/literals_literalstring.toml create mode 100644 conformance/results/pytype/literals_parameterizations.toml create mode 100644 conformance/results/pytype/literals_semantics.toml create mode 100644 conformance/results/pytype/narrowing_typeguard.toml create mode 100644 conformance/results/pytype/typeddicts_alt_syntax.toml create mode 100644 conformance/results/pytype/typeddicts_class_syntax.toml create mode 100644 conformance/results/pytype/typeddicts_final.toml create mode 100644 conformance/results/pytype/typeddicts_inheritance.toml create mode 100644 conformance/results/pytype/typeddicts_operations.toml create mode 100644 conformance/results/pytype/typeddicts_required.toml create mode 100644 conformance/results/pytype/typeddicts_type_consistency.toml create mode 100644 conformance/results/pytype/typeddicts_usage.toml create mode 100644 conformance/results/pytype/version.toml diff --git a/conformance/results/pytype/annotations_typeexpr.toml b/conformance/results/pytype/annotations_typeexpr.toml new file mode 100644 index 000000000..6b91be0f3 --- /dev/null +++ b/conformance/results/pytype/annotations_typeexpr.toml @@ -0,0 +1,26 @@ +conformant = "Partial" +notes = """ +Does not reject call expressions in type annotation. +Does not reject call lambda expression in type annotation. +Does not reject list expression in type annotation. +Does not reject ternary expression in type annotation. +Does not reject f-string in type annotation. +""" +output = """ +File "annotations_typeexpr.py", line 76, in : Invalid type annotation '[int, str]' for p2 [invalid-annotation] + Not a type +File "annotations_typeexpr.py", line 76, in : Invalid type annotation '(int, str)' for p3 [invalid-annotation] + Not a type +File "annotations_typeexpr.py", line 76, in : Invalid type annotation '' for p4 [invalid-annotation] + Not a type +File "annotations_typeexpr.py", line 76, in : Invalid type annotation '{}' for p5 [invalid-annotation] + Not a type +File "annotations_typeexpr.py", line 76, in : Invalid type annotation '3' for p9 [invalid-annotation] + Not a type +File "annotations_typeexpr.py", line 76, in : Invalid type annotation 'True' for p10 [invalid-annotation] + Not a type +File "annotations_typeexpr.py", line 76, in : Invalid type annotation '1' for p11 [invalid-annotation] + Not a type +File "annotations_typeexpr.py", line 76, in : Invalid type annotation '-1' for p12 [invalid-annotation] + Not a type +""" diff --git a/conformance/results/pytype/generics_self_advanced.toml b/conformance/results/pytype/generics_self_advanced.toml new file mode 100644 index 000000000..d980d2f3d --- /dev/null +++ b/conformance/results/pytype/generics_self_advanced.toml @@ -0,0 +1,51 @@ +conformant = "Unsupported" +notes = """ +Does not understand `Self` type. +""" +output = """ +File "generics_self_advanced.py", line 12, in prop1: bad return type [bad-return-type] + Expected: ParentA + Actually returned: None +Called from (traceback): + line 18, in current file +File "generics_self_advanced.py", line 12, in prop1: bad return type [bad-return-type] + Expected: ParentA + Actually returned: None +Called from (traceback): + line 19, in current file +File "generics_self_advanced.py", line 19, in : ParentA [assert-type] + Expected: ChildA + Actual: ParentA +File "generics_self_advanced.py", line 29, in method1: bad return type [bad-return-type] + Expected: ChildB + Actually returned: None +Called from (traceback): + line 38, in method2 +File "generics_self_advanced.py", line 29, in method1: bad return type [bad-return-type] + Expected: ParentB + Actually returned: None +File "generics_self_advanced.py", line 35, in method2: ChildB [assert-type] + Expected: Any + Actual: ChildB +File "generics_self_advanced.py", line 36, in method2: List[ChildB] [assert-type] + Expected: list + Actual: List[ChildB] +File "generics_self_advanced.py", line 37, in method2: ChildB [assert-type] + Expected: Any + Actual: ChildB +File "generics_self_advanced.py", line 38, in method2: ChildB [assert-type] + Expected: Any + Actual: ChildB +File "generics_self_advanced.py", line 42, in method3: Type[ChildB] [assert-type] + Expected: Any + Actual: Type[ChildB] +File "generics_self_advanced.py", line 43, in method3: List[ChildB] [assert-type] + Expected: list + Actual: List[ChildB] +File "generics_self_advanced.py", line 44, in method3: ChildB [assert-type] + Expected: Any + Actual: ChildB +File "generics_self_advanced.py", line 45, in method3: ChildB [assert-type] + Expected: Any + Actual: ChildB +""" diff --git a/conformance/results/pytype/generics_self_attributes.toml b/conformance/results/pytype/generics_self_attributes.toml new file mode 100644 index 000000000..ec77829e1 --- /dev/null +++ b/conformance/results/pytype/generics_self_attributes.toml @@ -0,0 +1,9 @@ +conformant = "Unsupported" +notes = """ +Does not understand `Self` type. +""" +output = """ +File "generics_self_attributes.py", line 26, in : Function OrdinalLinkedList.__init__ was called with the wrong arguments [wrong-arg-types] + Expected: (self, value, next: Optional[OrdinalLinkedList] = ...) + Actually passed: (self, value, next: LinkedList[int]) +""" diff --git a/conformance/results/pytype/generics_self_basic.toml b/conformance/results/pytype/generics_self_basic.toml new file mode 100644 index 000000000..768d64c2a --- /dev/null +++ b/conformance/results/pytype/generics_self_basic.toml @@ -0,0 +1,52 @@ +conformant = "Unsupported" +notes = """ +Does not understand `Self` type. +""" +output = """ +File "generics_self_basic.py", line 13, in set_scale: Shape [assert-type] + Expected: Any + Actual: Shape +File "generics_self_basic.py", line 13, in set_scale: Circle [assert-type] + Expected: Any + Actual: Circle +Called from (traceback): + line 51, in current file +File "generics_self_basic.py", line 26, in from_config: Type[Shape] [assert-type] + Expected: Any + Actual: Type[Shape] +Called from (traceback): + line 53, in current file +File "generics_self_basic.py", line 26, in from_config: Type[Circle] [assert-type] + Expected: Any + Actual: Type[Circle] +Called from (traceback): + line 54, in current file +File "generics_self_basic.py", line 27, in from_config: Function Shape.__init__ expects 1 arg(s), got 2 [wrong-arg-count] + Expected: (self) + Actually passed: (self, _) +Called from (traceback): + line 53, in current file +File "generics_self_basic.py", line 27, in from_config: Function Circle.__init__ expects 1 arg(s), got 2 [wrong-arg-count] + Expected: (self) + Actually passed: (self, _) +Called from (traceback): + line 54, in current file +File "generics_self_basic.py", line 39, in difference: Shape [assert-type] + Expected: Any + Actual: Shape +File "generics_self_basic.py", line 61, in set_value: bad return type [bad-return-type] + Expected: Container + Actually returned: None +File "generics_self_basic.py", line 61, in set_value: bad return type [bad-return-type] + Expected: Container[int] + Actually returned: None +Called from (traceback): + line 71, in object_with_concrete_type +File "generics_self_basic.py", line 61, in set_value: bad return type [bad-return-type] + Expected: Container[str] + Actually returned: None +Called from (traceback): + line 72, in object_with_concrete_type +File "generics_self_basic.py", line 64, in Container: unsupported operand type(s) for item retrieval: 'Self: TypeVar' and 'int: Type[int]' [unsupported-operands] + No attribute '__getitem__' on 'Self: TypeVar' +""" diff --git a/conformance/results/pytype/generics_self_protocols.toml b/conformance/results/pytype/generics_self_protocols.toml new file mode 100644 index 000000000..8f9fb5f9b --- /dev/null +++ b/conformance/results/pytype/generics_self_protocols.toml @@ -0,0 +1,2 @@ +output = """ +""" diff --git a/conformance/results/pytype/generics_self_usage.toml b/conformance/results/pytype/generics_self_usage.toml new file mode 100644 index 000000000..0326533e3 --- /dev/null +++ b/conformance/results/pytype/generics_self_usage.toml @@ -0,0 +1,31 @@ +conformant = "Unsupported" +notes = """ +Does not understand `Self` type. +""" +output = """ +File "generics_self_usage.py", line 58, in foo: Invalid type annotation 'Self' [invalid-annotation] + Cannot use 'typing.Self' outside of a class +File "generics_self_usage.py", line 73, in : Invalid type annotation 'Self' [invalid-annotation] + Cannot use 'typing.Self' outside of a class +File "generics_self_usage.py", line 76, in : Invalid type annotation 'Self' for bar [invalid-annotation] + TypeVar(s) 'Self' not in scope +File "generics_self_usage.py", line 82, in has_existing_self_annotation: bad return type [bad-return-type] + Expected: Foo2 + Actually returned: None +File "generics_self_usage.py", line 86, in return_concrete_type: bad return type [bad-return-type] + Expected: Foo3Child + Actually returned: Foo3 +Called from (traceback): + line 92, in child_method +File "generics_self_usage.py", line 103, in : Invalid base class: Self [base-class-error] +File "generics_self_usage.py", line 111, in Base: Invalid type annotation 'Self' [invalid-annotation] + Cannot use 'typing.Self' outside of a class +File "generics_self_usage.py", line 116, in Base: Invalid type annotation 'Self' [invalid-annotation] + Cannot use 'typing.Self' outside of a class +File "generics_self_usage.py", line 122, in __new__: bad return type [bad-return-type] + Expected: MyMetaclass + Actually returned: None +File "generics_self_usage.py", line 126, in __mul__: bad return type [bad-return-type] + Expected: List[MyMetaclass] + Actually returned: None +""" diff --git a/conformance/results/pytype/literals_interactions.toml b/conformance/results/pytype/literals_interactions.toml new file mode 100644 index 000000000..6b3e93b19 --- /dev/null +++ b/conformance/results/pytype/literals_interactions.toml @@ -0,0 +1,48 @@ +conformant = "Partial" +notes = """ +Incorrectly rejects some legal Literal annotations. +Does not reject some illegal Literal annotations. +Does not use Literal to distinguish overloads. +Does not narrow based on `x is Literal` type guard pattern. +Does not narrow based on `x == Literal` type guard pattern. +""" +output = """ +File "literals_interactions.py", line 11, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'int' at index 0 +File "literals_interactions.py", line 46, in open: bad return type [bad-return-type] + Expected: IO[str] + Actually returned: None +Called from (traceback): + line 49, in current file +File "literals_interactions.py", line 46, in open: bad return type [bad-return-type] + Expected: IO[bytes] + Actually returned: None +Called from (traceback): + line 50, in current file +File "literals_interactions.py", line 46, in open: bad return type [bad-return-type] + Expected: IO + Actually returned: None +Called from (traceback): + line 51, in current file +File "literals_interactions.py", line 61, in __add__: bad return type [bad-return-type] + Expected: Matrix[int, int] + Actually returned: None +File "literals_interactions.py", line 64, in __matmul__: bad return type [bad-return-type] + Expected: Matrix[int, int] + Actually returned: None +File "literals_interactions.py", line 67, in transpose: bad return type [bad-return-type] + Expected: Matrix[int, int] + Actually returned: None +File "literals_interactions.py", line 72, in func2: Matrix[Any, int] [assert-type] + Expected: Matrix[int, int] + Actual: Matrix[Any, int] +File "literals_interactions.py", line 93, in parse_status1: Union[Status, str] [assert-type] + Expected: str + Actual: Union[Status, str] +File "literals_interactions.py", line 106, in parse_status2: Function expects_bad_status was called with the wrong arguments [wrong-arg-types] + Expected: (status: Literal['ABORTED', 'MALFORMED']) + Actually passed: (status: str) +File "literals_interactions.py", line 109, in parse_status2: Function expects_pending_status was called with the wrong arguments [wrong-arg-types] + Expected: (status: Literal['PENDING']) + Actually passed: (status: str) +""" diff --git a/conformance/results/pytype/literals_literalstring.toml b/conformance/results/pytype/literals_literalstring.toml new file mode 100644 index 000000000..0ffdbe488 --- /dev/null +++ b/conformance/results/pytype/literals_literalstring.toml @@ -0,0 +1,6 @@ +conformant = "Unsupported" +notes = """ +Does not understand `LiteralString` special form. +""" +output = """ +""" diff --git a/conformance/results/pytype/literals_parameterizations.toml b/conformance/results/pytype/literals_parameterizations.toml new file mode 100644 index 000000000..f24946a15 --- /dev/null +++ b/conformance/results/pytype/literals_parameterizations.toml @@ -0,0 +1,41 @@ +conformant = "Unsupported" +notes = """ +Does not understand `Literal` type annotation. +""" +output = """ +File "literals_parameterizations.py", line 17, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'int' at index 0 +File "literals_parameterizations.py", line 18, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'int' at index 0 +File "literals_parameterizations.py", line 19, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'int' at index 0 +File "literals_parameterizations.py", line 32, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'Union' at index 0 + Bad parameter 'Union' at index 1 + Bad parameter 'Union' at index 2 + Bad parameter 'Union' at index 3 +File "literals_parameterizations.py", line 34, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'Union' at index 0 +File "literals_parameterizations.py", line 41, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'str' at index 0 +File "literals_parameterizations.py", line 42, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'complex' at index 0 +File "literals_parameterizations.py", line 46, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'dict' at index 0 +File "literals_parameterizations.py", line 47, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'int' at index 0 +File "literals_parameterizations.py", line 49, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'T' at index 0 +File "literals_parameterizations.py", line 50, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'float' at index 0 +File "literals_parameterizations.py", line 52, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter '...' at index 0 +File "literals_parameterizations.py", line 59, in : Invalid type annotation 'Literal[my_function]' [invalid-annotation] + Invalid type annotation 'Literal' + Bad parameter 'my_function' at index 0 +File "literals_parameterizations.py", line 59, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'my_function' at index 0 +File "literals_parameterizations.py", line 63, in func2: Type annotation for x1 does not match type of assignment [annotation-type-mismatch] + Annotation: Literal['Color.RED'] + Assignment: Color +""" diff --git a/conformance/results/pytype/literals_semantics.toml b/conformance/results/pytype/literals_semantics.toml new file mode 100644 index 000000000..e33ebf37b --- /dev/null +++ b/conformance/results/pytype/literals_semantics.toml @@ -0,0 +1,25 @@ +conformant = "Unsupported" +notes = """ +Does not understand `Literal` type annotation. +""" +output = """ +File "literals_semantics.py", line 10, in : Type annotation for v2 does not match type of assignment [annotation-type-mismatch] + Annotation: Literal[3] + Assignment: Literal[4] +File "literals_semantics.py", line 12, in : Invalid type annotation 'L[-3]' [invalid-annotation] + Invalid type annotation 'Literal' + Bad parameter 'int' at index 0 +File "literals_semantics.py", line 12, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'int' at index 0 +File "literals_semantics.py", line 16, in : Invalid type annotation 'Literal' [invalid-annotation] + Bad parameter 'int' at index 0 +File "literals_semantics.py", line 17, in func1: Invalid type annotation 'Literal[20]' [invalid-annotation] + Invalid type annotation 'Literal' + Bad parameter 'int' at index 0 +File "literals_semantics.py", line 18, in func1: Invalid type annotation 'Literal[20]' [invalid-annotation] + Invalid type annotation 'Literal' + Bad parameter 'int' at index 0 +File "literals_semantics.py", line 19, in func1: Invalid type annotation 'Literal[20]' [invalid-annotation] + Invalid type annotation 'Literal' + Bad parameter 'int' at index 0 +""" diff --git a/conformance/results/pytype/narrowing_typeguard.toml b/conformance/results/pytype/narrowing_typeguard.toml new file mode 100644 index 000000000..699b0799f --- /dev/null +++ b/conformance/results/pytype/narrowing_typeguard.toml @@ -0,0 +1,6 @@ +conformant = "Partial" +notes = """ +Does not reject TypeGuard method with too few parameters. +""" +output = """ +""" diff --git a/conformance/results/pytype/typeddicts_alt_syntax.toml b/conformance/results/pytype/typeddicts_alt_syntax.toml new file mode 100644 index 000000000..5bc176026 --- /dev/null +++ b/conformance/results/pytype/typeddicts_alt_syntax.toml @@ -0,0 +1,14 @@ +conformant = "Partial" +notes = """ +Does not reject use of variable as second argument to `TypedDict` call. +Does not report when name of TypedDict doesn't match assigned identifier name. +Does not support keyword-argument form of alternative syntax (deprecated in 3.11). +""" +output = """ +File "typeddicts_alt_syntax.py", line 27, in : Function typing.TypedDict was called with the wrong arguments [wrong-arg-types] + Expected: (name, fields: dict, ...) + Actually passed: (name, fields: Dict[int, Type[str]]) +File "typeddicts_alt_syntax.py", line 41, in : Function typing.TypedDict expects 2 arg(s), got 3 [wrong-arg-count] + Expected: (name, fields, *, total) + Actually passed: (name, name, year) +""" diff --git a/conformance/results/pytype/typeddicts_class_syntax.toml b/conformance/results/pytype/typeddicts_class_syntax.toml new file mode 100644 index 000000000..b187f00c5 --- /dev/null +++ b/conformance/results/pytype/typeddicts_class_syntax.toml @@ -0,0 +1,8 @@ +conformant = "Partial" +notes = """ +Does not reject methods within TypedDict class. +Does not report when metaclass is provided. +Does not report when other keyword argument is provided. +""" +output = """ +""" diff --git a/conformance/results/pytype/typeddicts_final.toml b/conformance/results/pytype/typeddicts_final.toml new file mode 100644 index 000000000..74f99ddf8 --- /dev/null +++ b/conformance/results/pytype/typeddicts_final.toml @@ -0,0 +1,3 @@ +conformant = "Yes" +output = """ +""" diff --git a/conformance/results/pytype/typeddicts_inheritance.toml b/conformance/results/pytype/typeddicts_inheritance.toml new file mode 100644 index 000000000..c07c15419 --- /dev/null +++ b/conformance/results/pytype/typeddicts_inheritance.toml @@ -0,0 +1,9 @@ +conformant = "Yes" +output = """ +File "typeddicts_inheritance.py", line 44, in : Invalid base class: NonTypedDict [base-class-error] + TypedDict BadTypedDict cannot inherit from a non-TypedDict class. +File "typeddicts_inheritance.py", line 54, in : Invalid base class: X1 [base-class-error] + Duplicate TypedDict key x in classes X1 and Y1 +File "typeddicts_inheritance.py", line 65, in : Invalid base class: Y2 [base-class-error] + Duplicate TypedDict key x in classes Y2 and X2 +""" diff --git a/conformance/results/pytype/typeddicts_operations.toml b/conformance/results/pytype/typeddicts_operations.toml new file mode 100644 index 000000000..8f53a4408 --- /dev/null +++ b/conformance/results/pytype/typeddicts_operations.toml @@ -0,0 +1,24 @@ +conformant = "Partial" +notes = """ +Does not report type violation with TypedDict value assignment. +Does not report reference to unknown key in TypedDict. +Does not reject `clear` method on TypedDict with required keys. +Does not reject delete operation for required key in TypedDict. +""" +output = """ +File "typeddicts_operations.py", line 28, in : Type annotation for movie does not match type of assignment [annotation-type-mismatch] + Annotation: Movie(TypedDict) + Assignment: Dict[str, str] +File "typeddicts_operations.py", line 29, in : Type annotation for movie does not match type of assignment [annotation-type-mismatch] + Annotation: Movie(TypedDict) + Assignment: Dict[str, Union[float, str]] +File "typeddicts_operations.py", line 32, in : Type annotation for movie does not match type of assignment [annotation-type-mismatch] + Annotation: Movie(TypedDict) + Assignment: Dict[str, Union[int, str]] +File "typeddicts_operations.py", line 37, in func1: Type annotation for movie does not match type of assignment [annotation-type-mismatch] + Annotation: Movie(TypedDict) + Assignment: Dict[str, Union[int, str]] +File "typeddicts_operations.py", line 60, in : str [assert-type] + Expected: Optional[str] + Actual: str +""" diff --git a/conformance/results/pytype/typeddicts_required.toml b/conformance/results/pytype/typeddicts_required.toml new file mode 100644 index 000000000..bdedbc7d6 --- /dev/null +++ b/conformance/results/pytype/typeddicts_required.toml @@ -0,0 +1,8 @@ +conformant = "Partial" +notes = """ +Does not reject use of `Required` in non-TypedDict class. +Does not reject use of `Required` in function parameter annotation. +Does not reject nested use of `Required` in type annotation. +""" +output = """ +""" diff --git a/conformance/results/pytype/typeddicts_type_consistency.toml b/conformance/results/pytype/typeddicts_type_consistency.toml new file mode 100644 index 000000000..b80ed1d7a --- /dev/null +++ b/conformance/results/pytype/typeddicts_type_consistency.toml @@ -0,0 +1,20 @@ +conformant = "Partial" +notes = """ +Does not report some type violations for TypedDict type compatibility. +Incorrectly reports type violation in cases where there is none. +Does not report type incompatibility between TypedDict and `dict[str, Any]`. +""" +output = """ +File "typeddicts_type_consistency.py", line 62, in : Type annotation for a3 does not match type of assignment [annotation-type-mismatch] + Annotation: A3(TypedDict) + Assignment: B3 +File "typeddicts_type_consistency.py", line 65, in : Type annotation for b3 does not match type of assignment [annotation-type-mismatch] + Annotation: B3(TypedDict) + Assignment: Dict[str, int] +File "typeddicts_type_consistency.py", line 69, in : Type annotation for a3_1 does not match type of assignment [annotation-type-mismatch] + Annotation: A3(TypedDict) + Assignment: Dict[str, int] +File "typeddicts_type_consistency.py", line 124, in : Type annotation for o2 does not match type of assignment [annotation-type-mismatch] + Annotation: Outer1(TypedDict) + Assignment: Dict[str, Dict[str, Dict[str, int]]] +""" diff --git a/conformance/results/pytype/typeddicts_usage.toml b/conformance/results/pytype/typeddicts_usage.toml new file mode 100644 index 000000000..41139d698 --- /dev/null +++ b/conformance/results/pytype/typeddicts_usage.toml @@ -0,0 +1,14 @@ +conformant = "Partial" +notes = """ +Does not report errant use of TypedDict in `isinstance` call. +Does not reject use of TypedDict as TypeVar bound. +""" +output = """ +File "typeddicts_usage.py", line 23, in : TypedDict Movie does not contain key director [typed-dict-error] +File "typeddicts_usage.py", line 24, in : Type annotation for key year in TypedDict Movie does not match type of assignment [annotation-type-mismatch] + Annotation: int + Assignment: str +File "typeddicts_usage.py", line 28, in : Type annotation for movie2 does not match type of assignment [annotation-type-mismatch] + Annotation: Movie(TypedDict) + Assignment: Dict[str, Union[int, str]] +""" diff --git a/conformance/results/pytype/version.toml b/conformance/results/pytype/version.toml new file mode 100644 index 000000000..1c85d9c11 --- /dev/null +++ b/conformance/results/pytype/version.toml @@ -0,0 +1 @@ +version = "pytype 2023.12.18" diff --git a/conformance/results/results.html b/conformance/results/results.html index e58727b9f..b8b465954 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -220,6 +220,43 @@

Python Type System Conformance Test Results

Type narrowing
+
Type annotations
     annotations_typeexprPartialDoes not reject ternary expression in annotation.
Does not reject binary expression in annotation.
     annotations_typeexprYes
Generics
     generics_self_attributesYes
     generics_self_basicYes
     generics_self_protocolsYes
     generics_self_usagePartialDoes not reject invalid use of `Self` in class definition.
Does not reject invalid use of `Self` in metaclass.
     generics_self_usageYes
Literals
Typed dictionaries
     typeddicts_alt_syntaxPartialDoes not report when name of TypedDict doesn't match assigned identifier name.
     typeddicts_class_syntaxPartialDoes not report when metaclass is provided.
Does not report when other keyword argument is provided.
     typeddicts_alt_syntaxYes
     typeddicts_class_syntaxYes
     typeddicts_finalYes
     typeddicts_inheritanceYes
     typeddicts_operationsYes
     typeddicts_requiredYes
     typeddicts_type_consistencyYes
     typeddicts_usagePartialDoes not report errant use of TypedDict in `isinstance` call.
     typeddicts_usageYes
Type narrowing
     narrowing_typeguardPartialDoes not support `tuple` in `assert_type` call.
Does not reject TypeGuard method with too few parameters.
+

pytype 2023.12.18

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+Type annotations
     annotations_typeexprPartialDoes not reject call expressions in type annotation.
Does not reject call lambda expression in type annotation.
Does not reject list expression in type annotation.
Does not reject ternary expression in type annotation.
Does not reject f-string in type annotation.
+Generics
     generics_self_advancedUnsupportedDoes not understand `Self` type.
     generics_self_attributesUnsupportedDoes not understand `Self` type.
     generics_self_basicUnsupportedDoes not understand `Self` type.
     generics_self_protocolsUnknown
     generics_self_usageUnsupportedDoes not understand `Self` type.
+Literals
     literals_interactionsPartialIncorrectly rejects some legal Literal annotations.
Does not reject some illegal Literal annotations.
Does not use Literal to distinguish overloads.
Does not narrow based on `x is Literal` type guard pattern.
Does not narrow based on `x == Literal` type guard pattern.
     literals_literalstringUnsupportedDoes not understand `LiteralString` special form.
     literals_parameterizationsUnsupportedDoes not understand `Literal` type annotation.
     literals_semanticsUnsupportedDoes not understand `Literal` type annotation.
+Typed dictionaries
     typeddicts_alt_syntaxPartialDoes not reject use of variable as second argument to `TypedDict` call.
Does not report when name of TypedDict doesn't match assigned identifier name.
Does not support keyword-argument form of alternative syntax (deprecated in 3.11).
     typeddicts_class_syntaxPartialDoes not reject methods within TypedDict class.
Does not report when metaclass is provided.
Does not report when other keyword argument is provided.
     typeddicts_finalYes
     typeddicts_inheritanceYes
     typeddicts_operationsPartialDoes not report type violation with TypedDict value assignment.
Does not report reference to unknown key in TypedDict.
Does not reject `clear` method on TypedDict with required keys.
Does not reject delete operation for required key in TypedDict.
     typeddicts_requiredPartialDoes not reject use of `Required` in non-TypedDict class.
Does not reject use of `Required` in function parameter annotation.
Does not reject nested use of `Required` in type annotation.
     typeddicts_type_consistencyPartialDoes not report some type violations for TypedDict type compatibility.
Incorrectly reports type violation in cases where there is none.
Does not report type incompatibility between TypedDict and `dict[str, Any]`.
     typeddicts_usagePartialDoes not report errant use of TypedDict in `isinstance` call.
Does not reject use of TypedDict as TypeVar bound.
+Type narrowing
     narrowing_typeguardPartialDoes not reject TypeGuard method with too few parameters.