Backport more pure C parsers

.clangd

@@ -0,0 +1,2 @@
+CompileFlags:
+    CompilationDatabase: ext/rbs_extension

.gitignore

@@ -21,3 +21,7 @@ lib/**/*.dll
 doc/
 
 **/*.gem
+
+# For clangd's editor integration
+ext/rbs_extension/compile_commands.json
+ext/rbs_extension/.cache

Gemfile

@@ -20,6 +20,7 @@ gem "rdoc", "~> 6.11.0"
 gem "fileutils"
 gem "raap"
 gem "activesupport", "~> 7.0"
+gem "extconf_compile_commands_json"
 
 group :libs do
   # Libraries required for stdlib test

Gemfile.lock

@@ -39,6 +39,7 @@ GEM
     diff-lcs (1.6.1)
     digest (3.2.0)
     drb (2.2.1)
+    extconf_compile_commands_json (0.0.7)
     ffi (1.17.2)
     fileutils (1.7.3)
     goodcheck (3.1.0)
@@ -183,6 +184,7 @@ DEPENDENCIES
   csv
   dbm
   digest
+  extconf_compile_commands_json
   fileutils
   goodcheck
   json (~> 2.10.2)

Rakefile

@@ -11,6 +11,14 @@ bin = File.join(__dir__, "bin")
 
 Rake::ExtensionTask.new("rbs_extension")
 
+compile_task = Rake::Task[:compile]
+
+task :setup_extconf_compile_commands_json do
+  ENV["COMPILE_COMMANDS_JSON"] = "1"
+end
+
+compile_task.prerequisites.unshift(:setup_extconf_compile_commands_json)
+
 test_config = lambda do |t|
   t.libs << "test"
   t.libs << "lib"

config.yml

@@ -345,7 +345,7 @@ nodes:
   - name: RBS::Types::Bases::Top
   - name: RBS::Types::Bases::Void
   - name: RBS::Types::Block
-    expose_location: false
+    expose_location: true
     fields:
       - name: type
         c_type: rbs_node

ext/rbs_extension/ast_translation.c

@@ -785,6 +785,7 @@ VALUE rbs_struct_to_ruby_value(rbs_translation_context_t ctx, rbs_node_t *instan
         rbs_types_block_t *node = (rbs_types_block_t *) instance;
 
         VALUE h = rb_hash_new();
+        rb_hash_aset(h, ID2SYM(rb_intern("location")), rbs_loc_to_ruby_location(ctx, node->base.location));
         rb_hash_aset(h, ID2SYM(rb_intern("type")), rbs_struct_to_ruby_value(ctx, (rbs_node_t *) node->type)); // rbs_node
         rb_hash_aset(h, ID2SYM(rb_intern("required")), node->required ? Qtrue : Qfalse);
         rb_hash_aset(h, ID2SYM(rb_intern("self_type")), rbs_struct_to_ruby_value(ctx, (rbs_node_t *) node->self_type)); // rbs_node

ext/rbs_extension/extconf.rb

@@ -25,3 +25,11 @@
 end
 
 create_makefile 'rbs_extension'
+
+# Only generate compile_commands.json when compiling through Rake tasks
+# This is to avoid adding extconf_compile_commands_json as a runtime dependency
+if ENV["COMPILE_COMMANDS_JSON"]
+  require 'extconf_compile_commands_json'
+  ExtconfCompileCommandsJson.generate!
+  ExtconfCompileCommandsJson.symlink!
+end

ext/rbs_extension/main.c

@@ -267,6 +267,55 @@ static VALUE rbsparser_parse_signature(VALUE self, VALUE buffer, VALUE start_pos
     return result;
 }
 
+struct parse_type_params_arg {
+    VALUE buffer;
+    rb_encoding *encoding;
+    rbs_parser_t *parser;
+    VALUE module_type_params;
+};
+
+static VALUE parse_type_params_try(VALUE a) {
+    struct parse_type_params_arg *arg = (struct parse_type_params_arg *) a;
+    rbs_parser_t *parser = arg->parser;
+
+    if (parser->next_token.type == pEOF) {
+        return Qnil;
+    }
+
+    rbs_node_list_t *params = NULL;
+    rbs_parse_type_params(parser, arg->module_type_params, &params);
+
+    raise_error_if_any(parser, arg->buffer);
+
+    rbs_translation_context_t ctx = rbs_translation_context_create(
+        &parser->constant_pool,
+        arg->buffer,
+        arg->encoding
+    );
+
+    return rbs_node_list_to_ruby_array(ctx, params);
+}
+
+static VALUE rbsparser_parse_type_params(VALUE self, VALUE buffer, VALUE start_pos, VALUE end_pos, VALUE module_type_params) {
+    VALUE string = rb_funcall(buffer, rb_intern("content"), 0);
+    StringValue(string);
+    rb_encoding *encoding = rb_enc_get(string);
+
+    rbs_parser_t *parser = alloc_parser_from_buffer(buffer, FIX2INT(start_pos), FIX2INT(end_pos));
+    struct parse_type_params_arg arg = {
+        .buffer = buffer,
+        .encoding = encoding,
+        .parser = parser,
+        .module_type_params = module_type_params
+    };
+
+    VALUE result = rb_ensure(parse_type_params_try, (VALUE) &arg, ensure_free_parser, (VALUE) parser);
+
+    RB_GC_GUARD(string);
+
+    return result;
+}
+
 static VALUE rbsparser_lex(VALUE self, VALUE buffer, VALUE end_pos) {
     VALUE string = rb_funcall(buffer, rb_intern("content"), 0);
     StringValue(string);
@@ -304,6 +353,7 @@ void rbs__init_parser(void) {
     rb_define_singleton_method(RBS_Parser, "_parse_type", rbsparser_parse_type, 5);
     rb_define_singleton_method(RBS_Parser, "_parse_method_type", rbsparser_parse_method_type, 5);
     rb_define_singleton_method(RBS_Parser, "_parse_signature", rbsparser_parse_signature, 3);
+    rb_define_singleton_method(RBS_Parser, "_parse_type_params", rbsparser_parse_type_params, 4);
     rb_define_singleton_method(RBS_Parser, "_lex", rbsparser_lex, 2);
 }
 

include/rbs/parser.h

@@ -130,4 +130,6 @@ bool rbs_parse_type(rbs_parser_t *parser, rbs_node_t **type);
 bool rbs_parse_method_type(rbs_parser_t *parser, rbs_method_type_t **method_type);
 bool rbs_parse_signature(rbs_parser_t *parser, rbs_signature_t **signature);
 
+bool rbs_parse_type_params(rbs_parser_t *parser, bool module_type_params, rbs_node_list_t **params);
+
 #endif

lib/rbs/parser_aux.rb

@@ -35,6 +35,11 @@ def self.parse_signature(source)
       [buf, dirs, decls]
     end
 
+    def self.parse_type_params(source, module_type_params: true)
+      buf = buffer(source)
+      _parse_type_params(buf, 0, buf.last_position, module_type_params)
+    end
+
     def self.magic_comment(buf)
       start_pos = 0
 

lib/rbs/types.rb

@@ -1339,8 +1339,10 @@ class Block
       attr_reader :type
       attr_reader :required
       attr_reader :self_type
+      attr_reader :location
 
-      def initialize(type:, required:, self_type: nil)
+      def initialize(location: nil, type:, required:, self_type: nil)
+        @location = location
         @type = type
         @required = required ? true : false
         @self_type = self_type

sig/parser.rbs

@@ -68,6 +68,24 @@ module RBS
     #
     def self.parse_signature: (Buffer | String) -> [Buffer, Array[AST::Directives::t], Array[AST::Declarations::t]]
 
+    # Parse a list of type parameters and return it
+    #
+    # ```ruby
+    # RBS::Parser.parse_type_params("")                          # => nil
+    # RBS::Parser.parse_type_params("[U, V]")                    # => `[:U, :V]`
+    # RBS::Parser.parse_type_params("[in U, V < Integer]")       # => `[:U, :V]`
+    # ```
+    #
+    # When `module_type_params` is `false`, an error is raised if `unchecked`, `in` or `out` are used.
+    #
+    # ```ruby
+    # RBS::Parser.parse_type_params("[unchecked U]", module_type_params: false)   # => Raises an error
+    # RBS::Parser.parse_type_params("[out U]", module_type_params: false)         # => Raises an error
+    # RBS::Parser.parse_type_params("[in U]", module_type_params: false)          # => Raises an error
+    # ```
+    #
+    def self.parse_type_params: (Buffer | String, ?module_type_params: bool) -> Array[AST::TypeParam]
+
     # Returns the magic comment from the buffer
     #
     def self.magic_comment: (Buffer) -> AST::Directives::ResolveTypeNames?
@@ -92,6 +110,8 @@ module RBS
 
     def self._parse_signature: (Buffer, Integer start_pos, Integer end_pos) -> [Array[AST::Directives::t], Array[AST::Declarations::t]]
 
+    def self._parse_type_params: (Buffer, Integer start_pos, Integer end_pos, bool module_type_params) -> Array[AST::TypeParam]
+
     def self._lex: (Buffer, Integer end_pos) -> Array[[Symbol, Location[untyped, untyped]]]
 
     class LocatedValue

sig/types.rbs

@@ -518,7 +518,10 @@ module RBS
 
       attr_reader self_type: t?
 
-      def initialize: (type: function, ?self_type: t?, required: boolish) -> void
+      type loc = Location[bot, bot]
+      attr_reader location: loc?
+
+      def initialize: (?location: loc?, type: function, ?self_type: t?, required: boolish) -> void
 
       def ==: (untyped other) -> bool
 

src/parser.c

@@ -713,11 +713,17 @@ static bool parse_function(rbs_parser_t *parser, bool accept_type_binding, parse
     }
 
     bool required = true;
+    rbs_range_t block_range;
+
     if (parser->next_token.type == pQUESTION && parser->next_token2.type == pLBRACE) {
         // Optional block
+        block_range.start = parser->next_token.range.start;
         required = false;
         rbs_parser_advance(parser);
+    } else if (parser->next_token.type == pLBRACE) {
+        block_range.start = parser->next_token.range.start;
     }
+
     if (parser->next_token.type == pLBRACE) {
         rbs_parser_advance(parser);
 
@@ -737,9 +743,12 @@ static bool parse_function(rbs_parser_t *parser, bool accept_type_binding, parse
         rbs_node_t *block_return_type = NULL;
         CHECK_PARSE(parse_optional(parser, &block_return_type));
 
+        ADVANCE_ASSERT(parser, pRBRACE);
+
+        block_range.end = parser->current_token.range.end;
+        rbs_location_t *loc = rbs_location_new(ALLOCATOR(), block_range);
+
         rbs_node_t *block_function = NULL;
-        function_range.end = parser->current_token.range.end;
-        rbs_location_t *loc = rbs_location_new(ALLOCATOR(), function_range);
         if (rbs_is_untyped_params(&block_params)) {
             block_function = (rbs_node_t *) rbs_types_untyped_function_new(ALLOCATOR(), loc, block_return_type);
         } else {
@@ -758,8 +767,6 @@ static bool parse_function(rbs_parser_t *parser, bool accept_type_binding, parse
         }
 
         block = rbs_types_block_new(ALLOCATOR(), loc, block_function, required, self_type);
-
-        ADVANCE_ASSERT(parser, pRBRACE);
     }
 
     ADVANCE_ASSERT(parser, pARROW);
@@ -3228,6 +3235,26 @@ bool rbs_parse_signature(rbs_parser_t *parser, rbs_signature_t **signature) {
     return true;
 }
 
+bool rbs_parse_type_params(rbs_parser_t *parser, bool module_type_params, rbs_node_list_t **params) {
+    if (parser->next_token.type != pLBRACKET) {
+        rbs_parser_set_error(parser, parser->next_token, true, "expected a token `pLBRACKET`");
+        return false;
+    }
+
+    rbs_range_t rg = NULL_RANGE;
+    rbs_parser_push_typevar_table(parser, true);
+    bool res = parse_type_params(parser, &rg, module_type_params, params);
+    rbs_parser_push_typevar_table(parser, false);
+
+    rbs_parser_advance(parser);
+    if (parser->current_token.type != pEOF) {
+        rbs_parser_set_error(parser, parser->current_token, true, "expected a token `%s`", rbs_token_type_str(pEOF));
+        return false;
+    }
+
+    return res;
+}
+
 id_table *alloc_empty_table(rbs_allocator_t *allocator) {
     id_table *table = rbs_allocator_alloc(allocator, id_table);
 

test/rbs/parser_test.rb

@@ -733,6 +733,20 @@ def test_parse_method_type2
     end
   end
 
+  def test_parse_method_type_block
+    RBS::Parser.parse_method_type(buffer("{ -> void } -> void")).tap do |method_type|
+      assert_equal "{ -> void }", method_type.block.location.source
+    end
+
+    RBS::Parser.parse_method_type(buffer("(Integer) { (Integer) -> void } -> void")).tap do |method_type|
+      assert_equal "{ (Integer) -> void }", method_type.block.location.source
+    end
+
+    RBS::Parser.parse_method_type(buffer("() ?{ () -> void } -> void")).tap do |method_type|
+      assert_equal "?{ () -> void }", method_type.block.location.source
+    end
+  end
+
   def test_newline_inconsistency
     code = "module Test\r\nend"
 
@@ -820,6 +834,60 @@ def test_proc__untyped_function
     end
   end
 
+  def test_parse_type_params
+    RBS::Parser.parse_type_params(buffer("[T]")).tap do |params|
+      assert_equal 1, params.size
+      assert_equal :T, params[0].name
+      assert_nil params[0].upper_bound
+    end
+
+    RBS::Parser.parse_type_params(buffer("[T < Integer, U = String]")).tap do |params|
+      assert_equal 2, params.size
+      assert_equal :T, params[0].name
+      assert_equal "Integer", params[0].upper_bound.to_s
+      assert_equal :U, params[1].name
+      assert_equal "String", params[1].default_type.to_s
+    end
+
+    RBS::Parser.parse_type_params(buffer("[T, in U, out V]")).tap do |params|
+      assert_equal 3, params.size
+      assert_equal :T, params[0].name
+      assert_equal "invariant", params[0].variance.to_s
+      assert_equal :U, params[1].name
+      assert_equal "contravariant", params[1].variance.to_s
+      assert_equal :V, params[2].name
+      assert_equal "covariant", params[2].variance.to_s
+    end
+
+    RBS::Parser.parse_type_params(buffer("[T, unchecked U, unchecked out V = Integer]")).tap do |params|
+      assert_equal 3, params.size
+      assert_equal :T, params[0].name
+      refute params[0].unchecked?
+      assert_equal :U, params[1].name
+      assert params[1].unchecked?
+      assert_equal :V, params[2].name
+      assert params[2].unchecked?
+      assert_equal "covariant", params[2].variance.to_s
+      assert_equal "Integer", params[2].default_type.to_s
+    end
+
+    assert_raises RBS::ParsingError do
+      RBS::Parser.parse_type_params(buffer("[]"))
+    end
+
+    assert_raises RBS::ParsingError do
+      RBS::Parser.parse_type_params(buffer("[T]A"))
+    end
+
+    assert_raises RBS::ParsingError do
+      RBS::Parser.parse_type_params(buffer("[in T]"), module_type_params: false)
+    end
+
+    assert_raises RBS::ParsingError do
+      RBS::Parser.parse_type_params(buffer("[unchecked T]"), module_type_params: false)
+    end
+  end
+
   def test__lex
     content = <<~RBS
       # LineComment