Update to ruff_python_parser 0.11 (RustPython#5615)

youknowone · web-flow · commit eaf4cdf5e13a · 2025-03-23T08:28:29.000+09:00
* Update to ruff_python_parser 0.11

* Try fix windows long paths error
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -132,6 +132,7 @@ jobs:
       - name: Set up the Windows environment
         shell: bash
         run: |
+          git config --system core.longpaths true
           cargo install --target-dir=target -v cargo-vcpkg
           cargo vcpkg -v build
         if: runner.os == 'Windows'
@@ -255,6 +256,7 @@ jobs:
       - name: Set up the Windows environment
         shell: bash
         run: |
+          git config --system core.longpaths true
           cargo install cargo-vcpkg
           cargo vcpkg build
         if: runner.os == 'Windows'
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -122,11 +122,11 @@ rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.
 rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" }
 rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
 
-ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
-ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
-ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
-ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
-ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
+ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
 # rustpython-literal = { version = "0.4.0" }
 # rustpython-parser-core = { version = "0.4.0" }
 # rustpython-parser = { version = "0.4.0" }
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
@@ -1619,8 +1619,6 @@ def test_nested_front():
         self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
                              ('Macdonalds', 'Cheeseburger')])
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_genexps(self):
         # generator expression tests
         g = ([x for x in range(10)] for x in range(1))
diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs
@@ -2029,8 +2029,7 @@ impl Compiler<'_> {
         if self.future_annotations {
             // FIXME: codegen?
             let ident = Default::default();
-            let codegen =
-                ruff_python_codegen::Generator::new(&ident, Default::default(), Default::default());
+            let codegen = ruff_python_codegen::Generator::new(&ident, Default::default());
             self.emit_load_const(ConstantData::Str {
                 value: codegen.expr(annotation),
             });
@@ -3595,18 +3594,19 @@ impl ToU32 for usize {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use ruff_python_ast::name::Name;
     use ruff_python_ast::*;
 
     /// Test if the compiler can correctly identify fstrings containing an `await` expression.
     #[test]
     fn test_fstring_contains_await() {
         let range = TextRange::default();
-        let flags = FStringFlags::default();
+        let flags = FStringFlags::empty();
 
         // f'{x}'
         let expr_x = Expr::Name(ExprName {
             range,
-            id: "x".to_owned(),
+            id: Name::new("x"),
             ctx: ExprContext::Load,
         });
         let not_present = &Expr::FString(ExprFString {
@@ -3631,7 +3631,7 @@ mod tests {
             range,
             value: Box::new(Expr::Name(ExprName {
                 range,
-                id: "x".to_owned(),
+                id: Name::new("x"),
                 ctx: ExprContext::Load,
             })),
         });
@@ -3655,14 +3655,14 @@ mod tests {
         // f'{x:{await y}}'
         let expr_x = Expr::Name(ExprName {
             range,
-            id: "x".to_owned(),
+            id: Name::new("x"),
             ctx: ExprContext::Load,
         });
         let expr_await_y = Expr::Await(ExprAwait {
             range,
             value: Box::new(Expr::Name(ExprName {
                 range,
-                id: "y".to_owned(),
+                id: Name::new("y"),
                 ctx: ExprContext::Load,
             })),
         });
diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs
@@ -119,7 +119,7 @@ fn _compile(
     mode: Mode,
     opts: CompileOpts,
 ) -> Result<CodeObject, CompileError> {
-    let parsed = parser::parse(source_code.text, mode.into())
+    let parsed = parser::parse(source_code.text, parser::Mode::from(mode).into())
         .map_err(|err| CompileError::from_ruff_parse_error(err, &source_code))?;
     let ast = parsed.into_syntax();
     compile::compile_top(ast, source_code, mode, opts).map_err(|e| e.into())
@@ -145,9 +145,8 @@ pub fn _compile_symtable(
             symboltable::SymbolTable::scan_program(&ast.into_syntax(), source_code.clone())
         }
         Mode::Eval => {
-            let ast =
-                ruff_python_parser::parse(source_code.text, ruff_python_parser::Mode::Expression)
-                    .map_err(|e| CompileError::from_ruff_parse_error(e, &source_code))?;
+            let ast = ruff_python_parser::parse(source_code.text, parser::Mode::Expression.into())
+                .map_err(|e| CompileError::from_ruff_parse_error(e, &source_code))?;
             symboltable::SymbolTable::scan_expr(
                 &ast.into_syntax().expect_expression(),
                 source_code.clone(),
diff --git a/src/shell.rs b/src/shell.rs
@@ -1,7 +1,7 @@
 mod helper;
 
 use rustpython_compiler::{
-    CompileError, ParseError, parser::ParseErrorType, parser::lexer::LexicalErrorType,
+    CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType,
 };
 use rustpython_vm::{
     AsObject, PyResult, VirtualMachine,
diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs
@@ -242,7 +242,7 @@ pub(crate) fn parse(
     mode: parser::Mode,
 ) -> Result<PyObjectRef, CompileError> {
     let source_code = SourceCodeOwned::new("".to_owned(), source.to_owned());
-    let top = parser::parse(source, mode)
+    let top = parser::parse(source, mode.into())
         .map_err(|parse_error| ParseError {
             error: parse_error.error,
             location: text_range_to_source_range(&source_code, parse_error.location)
diff --git a/vm/src/stdlib/ast/constant.rs b/vm/src/stdlib/ast/constant.rs
@@ -255,7 +255,7 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr {
                 value: ruff::StringLiteralValue::single(ruff::StringLiteral {
                     range,
                     value,
-                    flags: ruff::StringLiteralFlags::default().with_prefix(prefix),
+                    flags: ruff::StringLiteralFlags::empty().with_prefix(prefix),
                 }),
             })
         }
@@ -265,7 +265,7 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr {
                 value: ruff::BytesLiteralValue::single(ruff::BytesLiteral {
                     range,
                     value,
-                    flags: Default::default(), // TODO
+                    flags: ruff::BytesLiteralFlags::empty(), // TODO
                 }),
             })
         }
@@ -293,7 +293,7 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr {
             // idk lol
             func: Box::new(ruff::Expr::Name(ruff::ExprName {
                 range: TextRange::default(),
-                id: "frozenset".to_owned(),
+                id: ruff::name::Name::new_static("frozenset"),
                 ctx: ruff::ExprContext::Load,
             })),
             arguments: ruff::Arguments {
diff --git a/vm/src/stdlib/ast/expression.rs b/vm/src/stdlib/ast/expression.rs
@@ -977,7 +977,7 @@ impl Node for ruff::ExprName {
         object: PyObjectRef,
     ) -> PyResult<Self> {
         Ok(Self {
-            id: get_node_field(vm, &object, "id", "Name")?.try_into_value(vm)?,
+            id: Node::ast_from_object(vm, source_code, get_node_field(vm, &object, "id", "Name")?)?,
             ctx: Node::ast_from_object(
                 vm,
                 source_code,
diff --git a/vm/src/stdlib/ast/other.rs b/vm/src/stdlib/ast/other.rs
@@ -19,18 +19,22 @@ impl Node for ruff::ConversionFlag {
 }
 
 // /// This is just a string, not strictly an AST node. But it makes AST conversions easier.
-// impl Node for ruff::name::Name {
-//     fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
-//         vm.ctx.new_str(self.as_str()).to_pyobject(vm)
-//     }
+impl Node for ruff::name::Name {
+    fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
+        vm.ctx.new_str(self.as_str()).to_pyobject(vm)
+    }
 
-//     fn ast_from_object(vm: &VirtualMachine, source_code: &SourceCodeOwned, object: PyObjectRef) -> PyResult<Self> {
-//         match object.downcast::<PyStr>() {
-//             Ok(name) => Ok(Self::new(name)),
-//             Err(_) => Err(vm.new_value_error("expected str for name".to_owned())),
-//         }
-//     }
-// }
+    fn ast_from_object(
+        vm: &VirtualMachine,
+        _source_code: &SourceCodeOwned,
+        object: PyObjectRef,
+    ) -> PyResult<Self> {
+        match object.downcast::<PyStr>() {
+            Ok(name) => Ok(Self::new(name)),
+            Err(_) => Err(vm.new_value_error("expected str for name".to_owned())),
+        }
+    }
+}
 
 impl Node for ruff::Decorator {
     fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
diff --git a/vm/src/stdlib/ast/string.rs b/vm/src/stdlib/ast/string.rs
diff --git a/vm/src/vm/vm_new.rs b/vm/src/vm/vm_new.rs
diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs
