add tests for step-1 of overload evaluation

conformance/results/mypy/overloads_evaluation.toml

@@ -0,0 +1,25 @@
+conformant = "Partial"
+notes = """
+Does not pick a winning overload based on arity, prior to considering argument types.
+"""
+conformance_automated = "Fail"
+errors_diff = """
+Line 36: Unexpected errors ['overloads_evaluation.py:36: error: Expression is of type "Any", not "int"  [assert-type]']
+Line 42: Unexpected errors ['overloads_evaluation.py:42: error: Expression is of type "Any", not "str"  [assert-type]']
+"""
+output = """
+overloads_evaluation.py:26: error: All overload variants of "num_args" require at least one argument  [call-overload]
+overloads_evaluation.py:26: note: Possible overload variants:
+overloads_evaluation.py:26: note:     def num_args(x: int, y: str) -> int
+overloads_evaluation.py:26: note:     def num_args(x: str) -> str
+overloads_evaluation.py:35: error: No overload variant of "num_args" matches argument types "int", "int"  [call-overload]
+overloads_evaluation.py:35: note: Possible overload variants:
+overloads_evaluation.py:35: note:     def num_args(x: int, y: str) -> int
+overloads_evaluation.py:35: note:     def num_args(x: str) -> str
+overloads_evaluation.py:36: error: Expression is of type "Any", not "int"  [assert-type]
+overloads_evaluation.py:41: error: No overload variant of "num_args" matches argument type "int"  [call-overload]
+overloads_evaluation.py:41: note: Possible overload variants:
+overloads_evaluation.py:41: note:     def num_args(x: int, y: str) -> int
+overloads_evaluation.py:41: note:     def num_args(x: str) -> str
+overloads_evaluation.py:42: error: Expression is of type "Any", not "str"  [assert-type]
+"""

conformance/results/pyre/overloads_evaluation.toml

@@ -0,0 +1,9 @@
+conformant = "Pass"
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+overloads_evaluation.py:26:0 Missing argument [20]: Call `num_args` expects argument `x`.
+overloads_evaluation.py:35:19 Incompatible parameter type [6]: In call `num_args`, for 2nd positional argument, expected `str` but got `int`.
+overloads_evaluation.py:41:16 Incompatible parameter type [6]: In call `num_args`, for 1st positional argument, expected `str` but got `int`.
+"""

conformance/results/pyre/version.toml

@@ -1,2 +1,2 @@
 version = "pyre 0.9.23"
-test_duration = 6.6
+test_duration = 6.7

conformance/results/pyright/overloads_evaluation.toml

@@ -0,0 +1,12 @@
+conformant = "Pass"
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+overloads_evaluation.py:26:1 - error: No overloads for "num_args" match the provided arguments
+  Argument types: () (reportCallIssue)
+overloads_evaluation.py:35:20 - error: Argument of type "Literal[1]" cannot be assigned to parameter "y" of type "str" in function "num_args"
+  "Literal[1]" is not assignable to "str" (reportArgumentType)
+overloads_evaluation.py:41:17 - error: Argument of type "Literal[1]" cannot be assigned to parameter "x" of type "str" in function "num_args"
+  "Literal[1]" is not assignable to "str" (reportArgumentType)
+"""

conformance/results/pytype/overloads_evaluation.toml

@@ -0,0 +1,44 @@
+conformant = "Partial"
+notes = """
+Does not pick a winning overload based on arity, prior to considering argument types.
+"""
+conformance_automated = "Fail"
+errors_diff = """
+Line 24: Unexpected errors ['overloads_evaluation.py:24:12: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in num_args: bad return type [bad-return-type]']
+Line 36: Unexpected errors ['overloads_evaluation.py:36:1: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in <module>: Any [assert-type]']
+Line 42: Unexpected errors ['overloads_evaluation.py:42:1: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in <module>: Any [assert-type]']
+"""
+output = """
+overloads_evaluation.py:24:12: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in num_args: bad return type [bad-return-type]
+
+    return 1
+           \u001b[1m\u001b[31m~\u001b[39m\u001b[0m
+
+Called from (traceback):
+  line 38, in current file
+overloads_evaluation.py:26:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Missing parameter 'x' in call to function num_args [missing-parameter]
+
+num_args()  # E: no matching overload
+\u001b[1m\u001b[31m~~~~~~~~~~\u001b[39m\u001b[0m
+
+overloads_evaluation.py:35:8: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Function num_args was called with the wrong arguments [wrong-arg-types]
+
+ret2 = num_args(1, 1)  # E: Literal[1] not assignable to str
+       \u001b[1m\u001b[31m~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+
+overloads_evaluation.py:36:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Any [assert-type]
+
+assert_type(ret2, int)
+\u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+
+overloads_evaluation.py:41:8: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Function num_args was called with the wrong arguments [wrong-arg-types]
+
+ret4 = num_args(1)  # E: Literal[1] not assignable to str
+       \u001b[1m\u001b[31m~~~~~~~~~~~\u001b[39m\u001b[0m
+
+overloads_evaluation.py:42:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Any [assert-type]
+
+assert_type(ret4, str)
+\u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+
+"""

conformance/results/pytype/version.toml

@@ -1,2 +1,2 @@
 version = "pytype 2024.10.11"
-test_duration = 30.1
+test_duration = 30.7

conformance/results/results.html

@@ -165,10 +165,10 @@ <h3>Python Type System Conformance Test Results</h3>
 <div class='tc-time'>1.5sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyre 0.9.23</div>
-<div class='tc-time'>6.6sec</div>
+<div class='tc-time'>6.7sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pytype 2024.10.11</div>
-<div class='tc-time'>30.1sec</div>
+<div class='tc-time'>30.7sec</div>
 </th>
 </tr>
 <tr><th class="column" colspan="5">
@@ -690,6 +690,12 @@ <h3>Python Type System Conformance Test Results</h3>
 <th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Expects @final and @override to be present on all overloads, not just first.</p></span></div></th>
 <th class="column col2 not-conformant"><div class="hover-text">Fail<span class="tooltip-text" id="bottom"><p>Does not enforce any of the specified rules regarding overload definitions.</p></span></div></th>
 </tr>
+<tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;overloads_evaluation</th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not pick a winning overload based on arity, prior to considering argument types.</p></span></div></th>
+<th class="column col2 conformant">Pass</th>
+<th class="column col2 conformant">Pass</th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not pick a winning overload based on arity, prior to considering argument types.</p></span></div></th>
+</tr>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;overloads_overlap</th>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 conformant">Pass</th>

conformance/tests/overloads_evaluation.py

@@ -0,0 +1,44 @@
+"""
+Tests for evaluation of calls to overloaded functions.
+"""
+
+from typing import assert_type, overload
+
+
+# > Step 1: Examine the argument list to determine the number of
+# > positional and keyword arguments. Use this information to eliminate any
+# > overload candidates that are not plausible based on their
+# > input signatures.
+
+# > - If no candidate overloads remain, generate an error and stop.
+
+@overload
+def num_args(x: int, y: str) -> int:
+    ...
+
+@overload
+def num_args(x: str) -> str:
+    ...
+
+def num_args(x: int | str, y: str = "") -> int | str:
+    return 1
+
+num_args()  # E: no matching overload
+
+
+# > - If only one candidate overload remains, it is the winning match. Evaluate
+# >   it as if it were a non-overloaded function call and stop.
+
+ret1 = num_args(1, "")
+assert_type(ret1, int)
+
+ret2 = num_args(1, 1)  # E: Literal[1] not assignable to str
+assert_type(ret2, int)
+
+ret3 = num_args("")
+assert_type(ret3, str)
+
+ret4 = num_args(1)  # E: Literal[1] not assignable to str
+assert_type(ret4, str)
+
+