diff --git a/lib/rdoc/generator/markup.rb b/lib/rdoc/generator/markup.rb index 1c39687040..8edc1dd882 100644 --- a/lib/rdoc/generator/markup.rb +++ b/lib/rdoc/generator/markup.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + ## # Handle common RDoc::Markup tasks for various CodeObjects # @@ -113,7 +114,8 @@ def add_line_numbers(src) def markup_code return '' unless @token_stream - src = RDoc::TokenStream.to_html @token_stream + # Convert tokens to HTML with language-appropriate highlighting + src = to_html # dedent the source indent = src.length diff --git a/lib/rdoc/generator/template/aliki/class.rhtml b/lib/rdoc/generator/template/aliki/class.rhtml index 593b96db22..16147ae264 100644 --- a/lib/rdoc/generator/template/aliki/class.rhtml +++ b/lib/rdoc/generator/template/aliki/class.rhtml @@ -164,7 +164,7 @@
<%- if method.token_stream then %>
- 
<%= method.markup_code %>
+ 
<%= method.markup_code %>
<%- end %> <%- if method.mixin_from then %> diff --git a/lib/rdoc/generator/template/aliki/css/rdoc.css b/lib/rdoc/generator/template/aliki/css/rdoc.css index 1efe3d7089..7381c9b2bb 100644 --- a/lib/rdoc/generator/template/aliki/css/rdoc.css +++ b/lib/rdoc/generator/template/aliki/css/rdoc.css @@ -829,6 +829,32 @@ main h5, main h6 { [data-theme="dark"] .ruby-value { color: var(--code-orange); } [data-theme="dark"] .ruby-string { color: var(--code-green); } +/* C Syntax Highlighting - Light Theme */ +.c-keyword { color: var(--code-red); } +.c-identifier { color: var(--code-blue); } +.c-operator { color: var(--code-green); } +.c-preprocessor { color: var(--code-purple); } +.c-value { color: var(--code-orange); } +.c-string { color: var(--code-green); } + +.c-comment { + color: var(--color-neutral-500); + font-style: italic; +} + +/* C Syntax Highlighting - Dark Theme */ +[data-theme="dark"] .c-keyword { color: var(--code-red); } +[data-theme="dark"] .c-identifier { color: var(--code-blue); } +[data-theme="dark"] .c-operator { color: var(--code-green); } +[data-theme="dark"] .c-preprocessor { color: var(--code-purple); } +[data-theme="dark"] .c-value { color: var(--code-orange); } +[data-theme="dark"] .c-string { color: var(--code-green); } + +[data-theme="dark"] .c-comment { + color: var(--color-neutral-400); + font-style: italic; +} + /* Emphasis */ em { text-decoration-color: var(--color-emphasis-decoration); diff --git a/lib/rdoc/markup/to_html.rb b/lib/rdoc/markup/to_html.rb index a949c53e41..0cb14be67d 100644 --- a/lib/rdoc/markup/to_html.rb +++ b/lib/rdoc/markup/to_html.rb @@ -224,7 +224,18 @@ def accept_verbatim(verbatim) klass = nil - content = if verbatim.ruby? or parseable? text then + content = if verbatim.c? + begin + tokens = RDoc::Parser::CStateLex.parse text + klass = ' class="c"' + + result = RDoc::TokenStream.to_html_c tokens + result = result + "\n" unless "\n" == result[-1] + result + rescue + CGI.escapeHTML text + end + elsif verbatim.ruby? || parseable?(text) begin tokens = RDoc::Parser::RipperStateLex.parse text klass = ' class="ruby"' diff --git a/lib/rdoc/markup/verbatim.rb b/lib/rdoc/markup/verbatim.rb index 606276dcf0..53bdaeb8a1 100644 --- a/lib/rdoc/markup/verbatim.rb +++ b/lib/rdoc/markup/verbatim.rb @@ -73,6 +73,13 @@ def ruby? @format == :ruby end + ## + # Is this verbatim section C code? + + def c? + @format == :c + end + ## # The text of the section diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb index 7e89507779..eccf975883 100644 --- a/lib/rdoc/parser/c.rb +++ b/lib/rdoc/parser/c.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true require 'tsort' +require_relative 'c_state_lex' ## # RDoc::Parser::C attempts to parse C extension files. It looks for @@ -622,9 +623,9 @@ def find_body(class_name, meth_name, meth_obj, file_content, quiet = false) find_modifiers comment, meth_obj if comment #meth_obj.params = params - meth_obj.start_collecting_tokens - tk = { :line_no => 1, :char_no => 1, :text => body } - meth_obj.add_token tk + meth_obj.start_collecting_tokens(:c) + c_tokens = RDoc::Parser::CStateLex.parse(body) + meth_obj.add_tokens(c_tokens) meth_obj.comment = comment meth_obj.line = file_content[0, offset].count("\n") + 1 @@ -638,9 +639,9 @@ def find_body(class_name, meth_name, meth_obj, file_content, quiet = false) find_modifiers comment, meth_obj - meth_obj.start_collecting_tokens - tk = { :line_no => 1, :char_no => 1, :text => body } - meth_obj.add_token tk + meth_obj.start_collecting_tokens(:c) + c_tokens = RDoc::Parser::CStateLex.parse(body) + meth_obj.add_tokens(c_tokens) meth_obj.comment = comment meth_obj.line = file_content[0, offset].count("\n") + 1 diff --git a/lib/rdoc/parser/c_state_lex.rb b/lib/rdoc/parser/c_state_lex.rb new file mode 100644 index 0000000000..c397dad24a --- /dev/null +++ b/lib/rdoc/parser/c_state_lex.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +## +# A simple C lexer for syntax highlighting in RDoc documentation. +# This lexer tokenizes C code into tokens compatible with RDoc::TokenStream. + +class RDoc::Parser::CStateLex # :nodoc: + Token = Struct.new(:line_no, :char_no, :kind, :text, :state) + + # C keywords + KEYWORDS = %w[ + auto break case char const continue default do double else enum extern + float for goto if inline int long register restrict return short signed + sizeof static struct switch typedef union unsigned void volatile while + _Alignas _Alignof _Atomic _Bool _Complex _Generic _Imaginary _Noreturn + _Static_assert _Thread_local + ].freeze + + # C preprocessor directives + PREPROCESSOR = /^[ \t]*#[ \t]*\w+/ + + ## + # Parse C code and return an array of tokens + + def self.parse(code) + new.parse(code) + end + + def initialize + @tokens = [] + @line_no = 1 + @char_no = 0 + end + + def parse(code) + @code = code + @pos = 0 + @line_no = 1 + @char_no = 0 + + while @pos < @code.length + case + when scan(/\/\/[^\n]*/) + add_token(:on_comment, matched) + when scan(/\/\*.*?\*\//m) + add_token(:on_comment, matched) + when scan(PREPROCESSOR) + add_token(:on_preprocessor, matched) + when scan(/"(?:[^"\\]|\\.)*"/) + add_token(:on_tstring, matched) + when scan(/'(?:[^'\\]|\\.)*'/) + add_token(:on_char, matched) + when scan(/\d+\.\d+([eE][+-]?\d+)?[fFlL]?/) + add_token(:on_float, matched) + when scan(/0[xX][0-9a-fA-F]+[uUlL]*/) + add_token(:on_int, matched) + when scan(/0[0-7]+[uUlL]*/) + add_token(:on_int, matched) + when scan(/\d+[uUlL]*/) + add_token(:on_int, matched) + when scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + word = matched + if KEYWORDS.include?(word) + add_token(:on_kw, word) + else + add_token(:on_ident, word) + end + when scan(/&&|\|\||<<|>>|\+\+|--|[+\-*\/%&|^~!<>=]=?/) + add_token(:on_op, matched) + when scan(/\n/) + add_token(:on_nl, matched) + @line_no += 1 + @char_no = 0 + next + when scan(/[ \t]+/) + add_token(:on_sp, matched) + when scan(/[{}()\[\];,.]/) + add_token(:on_punct, matched) + else + # Unknown character, consume it + advance + end + end + + @tokens + end + + private + + def scan(pattern) + if @code[@pos..-1] =~ /\A#{pattern}/ + @match = $& + advance(@match.length) + true + else + false + end + end + + def matched + @match + end + + def advance(count = 1) + @pos += count + @char_no += count + end + + def add_token(kind, text) + @tokens << Token.new(@line_no, @char_no - text.length, kind, text, nil) + end +end diff --git a/lib/rdoc/parser/prism_ruby.rb b/lib/rdoc/parser/prism_ruby.rb index 872eab9ec0..56da6ac227 100644 --- a/lib/rdoc/parser/prism_ruby.rb +++ b/lib/rdoc/parser/prism_ruby.rb @@ -203,7 +203,7 @@ def parse_comment_tomdoc(container, comment, line_no, start_line) meth.call_seq = signature return unless meth.name - meth.start_collecting_tokens + meth.start_collecting_tokens(:ruby) node = @line_nodes[line_no] tokens = node ? visible_tokens_from_location(node.location) : [file_line_comment_token(start_line)] tokens.each { |token| meth.token_stream << token } @@ -554,7 +554,7 @@ def add_method(method_name, receiver_name:, receiver_fallback_type:, visibility: meth.calls_super = calls_super meth.block_params ||= block_params if block_params record_location(meth) - meth.start_collecting_tokens + meth.start_collecting_tokens(:ruby) tokens.each do |token| meth.token_stream << token end diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index c97945392b..a801b5fb0d 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1137,7 +1137,7 @@ def parse_comment_ghost(container, text, name, column, line_no, # :nodoc: meth = RDoc::GhostMethod.new get_tkread, name record_location meth - meth.start_collecting_tokens + meth.start_collecting_tokens(:ruby) indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column) position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment) position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" @@ -1180,7 +1180,7 @@ def parse_comment_tomdoc(container, tk, comment) record_location meth meth.line = line_no - meth.start_collecting_tokens + meth.start_collecting_tokens(:ruby) indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column) position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment) position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" @@ -1363,7 +1363,7 @@ def parse_meta_method(container, single, tk, comment) remove_token_listener self - meth.start_collecting_tokens + meth.start_collecting_tokens(:ruby) indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column) position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment) position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" @@ -1471,7 +1471,7 @@ def parse_method(container, single, tk, comment) record_location meth meth.line = line_no - meth.start_collecting_tokens + meth.start_collecting_tokens(:ruby) indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column) token = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment) token[:text] = "# File #{@top_level.relative_name}, line #{line_no}" diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index e4583651b1..705972a6ba 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + ## # A TokenStream is a list of tokens, gathered during the parse of some entity # (say a method). Entities populate these streams by being registered with the @@ -71,6 +72,44 @@ def self.to_html(token_stream) end.join end + ## + # Converts C +token_stream+ to HTML wrapping various tokens with + # elements. Maps C token types to CSS class names. + + def self.to_html_c(token_stream) + token_stream.map do |t| + next unless t + + style = case t[:kind] + when :on_kw then 'c-keyword' + when :on_ident then 'c-identifier' + when :on_comment then 'c-comment' + when :on_tstring then 'c-string' + when :on_char then 'c-value' + when :on_int then 'c-value' + when :on_float then 'c-value' + when :on_op then 'c-operator' + when :on_preprocessor then 'c-preprocessor' + end + + comment_with_nl = false + if :on_comment == t[:kind] + comment_with_nl = true if "\n" == t[:text][-1] + text = t[:text].rstrip + else + text = t[:text] + end + + text = CGI.escapeHTML text + + if style then + "#{text}#{"\n" if comment_with_nl}" + else + text + end + end.join + end + ## # Adds +tokens+ to the collected tokens @@ -87,9 +126,12 @@ def add_token(token) ## # Starts collecting tokens + # + # @param language [Symbol] the language of the token stream (:ruby or :c) - def collect_tokens + def collect_tokens(language = :ruby) @token_stream = [] + @token_stream_language = language end alias start_collecting_tokens collect_tokens @@ -115,4 +157,29 @@ def tokens_to_s (token_stream or return '').compact.map { |token| token[:text] }.join '' end + ## + # Returns the source language of the token stream as a string + # + # Returns 'c' or 'ruby' + + def source_language + @token_stream_language == :c ? 'c' : 'ruby' + end + + ## + # Converts the token stream to HTML with appropriate syntax highlighting + # based on the source language + # + # Returns HTML string with syntax highlighting + + def to_html + return '' unless @token_stream + + if @token_stream_language == :c + RDoc::TokenStream.to_html_c @token_stream + else + RDoc::TokenStream.to_html @token_stream + end + end + end diff --git a/test/rdoc/code_object/any_method_test.rb b/test/rdoc/code_object/any_method_test.rb index 0c72fc479e..03bd44c1e7 100644 --- a/test/rdoc/code_object/any_method_test.rb +++ b/test/rdoc/code_object/any_method_test.rb @@ -192,6 +192,63 @@ def test_markup_code_empty assert_equal '', @c2_a.markup_code end + def test_markup_code_c_function + # C source code is tokenized immediately by the C parser + c_code = <<~C + static VALUE + rb_ary_all_p(int argc, VALUE *argv, VALUE ary) + { + return Qtrue; + } + C + + # Simulate what the C parser does: tokenize immediately and set language + c_tokens = RDoc::Parser::CStateLex.parse(c_code) + + @c2_a.collect_tokens(:c) + @c2_a.add_tokens(c_tokens) + @c2_a.c_function = 'rb_ary_all_p' + + result = @c2_a.markup_code + + # Verify C syntax highlighting is applied + assert_includes result, 'static' + assert_includes result, 'VALUE' # VALUE is a typedef, not a keyword + assert_includes result, 'rb_ary_all_p' + assert_includes result, 'int' + assert_includes result, 'argc' + assert_includes result, 'return' + assert_includes result, 'Qtrue' + end + + def test_markup_code_c_function_with_preprocessor + # Test C code with preprocessor directives + c_code = <<~C + #define rb_obj_singleton_method_added rb_obj_dummy1 + + static VALUE + bsock_do_not_rev_lookup(VALUE _) + { + return rsock_do_not_reverse_lookup?Qtrue:Qfalse; + } + C + + # Simulate what the C parser does: tokenize immediately and set language + c_tokens = RDoc::Parser::CStateLex.parse(c_code) + + @c2_a.collect_tokens(:c) + @c2_a.add_tokens(c_tokens) + @c2_a.c_function = 'bsock_do_not_rev_lookup' + + result = @c2_a.markup_code + + # Verify preprocessor directive is highlighted + assert_includes result, '#define' + assert_includes result, 'static' + assert_includes result, 'return' + assert_includes result, 'rsock_do_not_reverse_lookup' + end + def test_markup_code_with_variable_expansion m = RDoc::AnyMethod.new nil, 'method' m.parent = @c1 diff --git a/test/rdoc/markup/to_html_test.rb b/test/rdoc/markup/to_html_test.rb index caa11c3e7f..40f754d0e0 100644 --- a/test/rdoc/markup/to_html_test.rb +++ b/test/rdoc/markup/to_html_test.rb @@ -633,6 +633,22 @@ def test_accept_verbatim_ruby assert_equal expected, @to.res.join end + def test_accept_verbatim_c + verb = @RM::Verbatim.new("int x = 42;\n") + verb.format = :c + + @to.start_accepting + @to.accept_verbatim verb + + expected = <<-EXPECTED + +
int x = 42;
+
+ EXPECTED + + assert_equal expected, @to.res.join + end + def test_accept_verbatim_redefinable_operators functions = %w[| ^ & <=> == === =~ > >= < <= << >> + - * / % ** ~ +@ -@ [] []= ` ! != !~].flat_map { |redefinable_op| ["def #{redefinable_op}\n", "end\n"] diff --git a/test/rdoc/markup/verbatim_test.rb b/test/rdoc/markup/verbatim_test.rb index 59b40abaea..4f6c21f3fa 100644 --- a/test/rdoc/markup/verbatim_test.rb +++ b/test/rdoc/markup/verbatim_test.rb @@ -26,4 +26,14 @@ def test_ruby_eh assert verbatim.ruby? end + def test_c_eh + verbatim = RDoc::Markup::Verbatim.new + + refute verbatim.c? + + verbatim.format = :c + + assert verbatim.c? + end + end diff --git a/test/rdoc/parser/c_state_lex_test.rb b/test/rdoc/parser/c_state_lex_test.rb new file mode 100644 index 0000000000..05fb22c055 --- /dev/null +++ b/test/rdoc/parser/c_state_lex_test.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true +require_relative '../helper' + +class RDocParserCStateLexTest < RDoc::TestCase + + def test_parse_keywords + code = "int void return" + tokens = RDoc::Parser::CStateLex.parse(code) + + kw_tokens = tokens.select { |t| t[:kind] == :on_kw } + assert_equal 3, kw_tokens.length + assert_equal 'int', kw_tokens[0][:text] + assert_equal 'void', kw_tokens[1][:text] + assert_equal 'return', kw_tokens[2][:text] + end + + def test_parse_identifiers + code = "my_variable foo_bar" + tokens = RDoc::Parser::CStateLex.parse(code) + + assert_equal 3, tokens.length + assert_equal :on_ident, tokens[0][:kind] + assert_equal 'my_variable', tokens[0][:text] + assert_equal :on_ident, tokens[2][:kind] + assert_equal 'foo_bar', tokens[2][:text] + end + + def test_parse_numbers + code = "42 3.14 0xFF 0777" + tokens = RDoc::Parser::CStateLex.parse(code) + + int_tokens = tokens.select { |t| t[:kind] == :on_int } + float_tokens = tokens.select { |t| t[:kind] == :on_float } + + assert_equal 3, int_tokens.length + assert_equal 1, float_tokens.length + assert_equal '42', int_tokens[0][:text] + assert_equal '3.14', float_tokens[0][:text] + end + + def test_parse_strings + code = '"hello world" \'c\'' + tokens = RDoc::Parser::CStateLex.parse(code) + + string_tokens = tokens.select { |t| t[:kind] == :on_tstring } + char_tokens = tokens.select { |t| t[:kind] == :on_char } + + assert_equal 1, string_tokens.length + assert_equal 1, char_tokens.length + assert_equal '"hello world"', string_tokens[0][:text] + assert_equal "'c'", char_tokens[0][:text] + end + + def test_parse_comments + code = "// single line comment\n/* multi line\ncomment */" + tokens = RDoc::Parser::CStateLex.parse(code) + + comment_tokens = tokens.select { |t| t[:kind] == :on_comment } + + assert_equal 2, comment_tokens.length + assert_equal '// single line comment', comment_tokens[0][:text] + assert comment_tokens[1][:text].include?('multi line') + end + + def test_parse_preprocessor + code = "#include \n#define MAX 100" + tokens = RDoc::Parser::CStateLex.parse(code) + + pp_tokens = tokens.select { |t| t[:kind] == :on_preprocessor } + + assert_equal 2, pp_tokens.length + assert pp_tokens[0][:text].start_with?('#include') + assert pp_tokens[1][:text].start_with?('#define') + end + + def test_parse_operators + code = "+ - * / == != && ||" + tokens = RDoc::Parser::CStateLex.parse(code) + + op_tokens = tokens.select { |t| t[:kind] == :on_op } + + assert_equal 8, op_tokens.length + assert_equal '+', op_tokens[0][:text] + assert_equal '==', op_tokens[4][:text] + end + + def test_parse_complex_code + code = <<~C + #include + + int main() { + int x = 42; + printf("Hello, %d\\n", x); + return 0; + } + C + + tokens = RDoc::Parser::CStateLex.parse(code) + + refute_empty tokens + + preprocessor_tokens = tokens.select { |t| t[:kind] == :on_preprocessor } + assert_equal 1, preprocessor_tokens.length + assert preprocessor_tokens[0][:text].start_with?('#include') + + keyword_tokens = tokens.select { |t| t[:kind] == :on_kw } + assert_equal 3, keyword_tokens.length + assert_equal 'int', keyword_tokens[0][:text] + assert_equal 'int', keyword_tokens[1][:text] + assert_equal 'return', keyword_tokens[2][:text] + + identifier_tokens = tokens.select { |t| t[:kind] == :on_ident } + assert identifier_tokens.length >= 4 + assert identifier_tokens.any? { |t| t[:text] == 'main' } + assert identifier_tokens.any? { |t| t[:text] == 'printf' } + assert_equal 2, identifier_tokens.count { |t| t[:text] == 'x' } + + int_tokens = tokens.select { |t| t[:kind] == :on_int } + assert_equal 2, int_tokens.length + assert_equal '42', int_tokens[0][:text] + assert_equal '0', int_tokens[1][:text] + + string_tokens = tokens.select { |t| t[:kind] == :on_tstring } + assert_equal 1, string_tokens.length + assert string_tokens[0][:text].include?('Hello') + end + +end diff --git a/test/rdoc/parser/c_test.rb b/test/rdoc/parser/c_test.rb index 6794677008..413104e319 100644 --- a/test/rdoc/parser/c_test.rb +++ b/test/rdoc/parser/c_test.rb @@ -1174,7 +1174,8 @@ def test_find_body other_function.comment.text assert_equal '()', other_function.params - code = other_function.token_stream.first[:text] + # Token stream is now tokenized, not raw text + code = other_function.token_stream.map { |t| t[:text] }.join assert_equal "VALUE\nother_function() {\n}", code end @@ -1244,7 +1245,8 @@ def test_find_body_cast other_function.comment.text assert_equal '()', other_function.params - code = other_function.token_stream.first[:text] + # Token stream is now tokenized, not raw text + code = other_function.token_stream.map { |t| t[:text] }.join assert_equal "VALUE\nother_function() {\n}", code end @@ -1278,7 +1280,8 @@ def test_find_body_define assert_equal '()', other_function.params assert_equal 8, other_function.line - code = other_function.token_stream.first[:text] + # Token stream is now tokenized, not raw text + code = other_function.token_stream.map { |t| t[:text] }.join assert_equal "VALUE\nrb_other_function() {\n}", code end @@ -1311,7 +1314,8 @@ def test_find_body_define_comment assert_equal '()', other_function.params assert_equal 4, other_function.line - code = other_function.token_stream.first[:text] + # Token stream is now tokenized, not raw text + code = other_function.token_stream.map { |t| t[:text] }.join assert_equal "#define other_function rb_other_function", code end @@ -1451,7 +1455,8 @@ def test_find_body_macro other_function.comment.text assert_equal '()', other_function.params - code = other_function.token_stream.first[:text] + # Token stream is now tokenized, not raw text + code = other_function.token_stream.map { |t| t[:text] }.join assert_equal "DLL_LOCAL VALUE\nother_function() {\n}", code end @@ -1481,7 +1486,8 @@ def test_find_body_static_inline other_function.comment.text assert_equal '()', other_function.params - code = other_function.token_stream.first[:text] + # Token stream is now tokenized, not raw text + code = other_function.token_stream.map { |t| t[:text] }.join assert_equal "static inline VALUE\nother_function() {\n}", code end diff --git a/test/rdoc/rdoc_token_stream_test.rb b/test/rdoc/rdoc_token_stream_test.rb index a5f1930daa..b9355a18cd 100644 --- a/test/rdoc/rdoc_token_stream_test.rb +++ b/test/rdoc/rdoc_token_stream_test.rb @@ -39,6 +39,86 @@ def test_class_to_html_empty assert_equal '', RDoc::TokenStream.to_html([]) end + def test_class_to_html_c + tokens = [ + { line_no: 0, char_no: 0, kind: :on_kw, text: 'int' }, + { line_no: 0, char_no: 0, kind: :on_ident, text: 'main' }, + { line_no: 0, char_no: 0, kind: :on_op, text: '+' }, + { line_no: 0, char_no: 0, kind: :on_int, text: '42' }, + { line_no: 0, char_no: 0, kind: :on_tstring, text: '"hello"' }, + { line_no: 0, char_no: 0, kind: :on_char, text: "'c'" }, + { line_no: 0, char_no: 0, kind: :on_comment, text: '// comment' }, + { line_no: 0, char_no: 0, kind: :on_preprocessor, text: '#include' }, + { line_no: 0, char_no: 0, kind: :on_unknown, text: '\\' } + ] + + expected = [ + 'int', + 'main', + '+', + '42', + '"hello"', + ''c'', + '// comment', + '#include', + '\\' + ].join + + assert_equal expected, RDoc::TokenStream.to_html_c(tokens) + end + + def test_class_to_html_c_empty + assert_equal '', RDoc::TokenStream.to_html_c([]) + end + + def test_source_language_ruby + foo = Class.new do + include RDoc::TokenStream + end.new + + # Default is :ruby + foo.collect_tokens + assert_equal 'ruby', foo.source_language + + # Explicit :ruby + foo.collect_tokens(:ruby) + assert_equal 'ruby', foo.source_language + end + + def test_source_language_c + foo = Class.new do + include RDoc::TokenStream + end.new + + foo.collect_tokens(:c) + assert_equal 'c', foo.source_language + end + + def test_to_html_dispatches_based_on_language + foo = Class.new do + include RDoc::TokenStream + end.new + + # Ruby tokens + foo.collect_tokens(:ruby) + ruby_tokens = [ + { line_no: 1, char_no: 0, kind: :on_kw, text: 'def', state: nil }, + { line_no: 1, char_no: 4, kind: :on_ident, text: 'foo', state: nil } + ] + foo.add_tokens(ruby_tokens) + html = foo.to_html + assert_includes html, 'ruby-keyword' + + # C tokens + foo.collect_tokens(:c) + c_tokens = [ + { line_no: 1, char_no: 0, kind: :on_kw, text: 'int', state: nil } + ] + foo.add_tokens(c_tokens) + html = foo.to_html + assert_includes html, 'c-keyword' + end + def test_add_tokens foo = Class.new do include RDoc::TokenStream