Skip to content

[clang][deps] Properly capture the global module and '\n' for all module directives #148685

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 19, 2025
Merged

[clang][deps] Properly capture the global module and '\n' for all module directives #148685

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7a75927
[clang][deps] Recognize 'module;' in dependency directive scanner
naveen-seth Jul 14, 2025
25553f7
Add newline after 'module;' to be valid C++
naveen-seth Jul 14, 2025
586b438
[clang][deps] Properly capture the global module and '\n' for all mod…
naveen-seth Jul 16, 2025
e9a6b45
Address akyrtzi's review feedback
naveen-seth Jul 18, 2025
e755d18
For review only: additional `TryConsumeToken(tok::eod)`
naveen-seth Jul 18, 2025
0bc7245
Update Parser and Preprocessor to handle the added tok::eod
naveen-seth Jul 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 20 additions & 15 deletions clang/lib/Lex/DependencyDirectivesScanner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -560,15 +560,13 @@ bool Scanner::lexModuleDirectiveBody(DirectiveKind Kind, const char *&First,
if (Tok.is(tok::semi))
break;
}

const auto &Tok = lexToken(First, End);
pushDirective(Kind);
skipWhitespace(First, End);
if (First == End)
if (Tok.is(tok::eof) || Tok.is(tok::eod))
return false;
if (!isVerticalWhitespace(*First))
return reportError(
DirectiveLoc, diag::err_dep_source_scanner_unexpected_tokens_at_import);
skipNewline(First, End);
return false;
return reportError(DirectiveLoc,
diag::err_dep_source_scanner_unexpected_tokens_at_import);
}

dependency_directives_scan::Token &Scanner::lexToken(const char *&First,
Expand Down Expand Up @@ -728,6 +726,13 @@ bool Scanner::lexModule(const char *&First, const char *const End) {
return false;
break;
}
case ';': {
// Handle the global module fragment `module;`.
if (Id == "module" && !Export)
break;
skipLine(First, End);
return false;
}
case '<':
case '"':
break;
Expand Down Expand Up @@ -898,14 +903,6 @@ bool Scanner::lexPPLine(const char *&First, const char *const End) {
CurDirToks.clear();
});

// Handle "@import".
if (*First == '@')
return lexAt(First, End);

// Handle module directives for C++20 modules.
if (*First == 'i' || *First == 'e' || *First == 'm')
return lexModule(First, End);

if (*First == '_') {
if (isNextIdentifierOrSkipLine("_Pragma", First, End))
return lex_Pragma(First, End);
Expand All @@ -918,6 +915,14 @@ bool Scanner::lexPPLine(const char *&First, const char *const End) {
auto ScEx2 = make_scope_exit(
[&]() { TheLexer.setParsingPreprocessorDirective(false); });

// Handle "@import".
if (*First == '@')
return lexAt(First, End);

// Handle module directives for C++20 modules.
if (*First == 'i' || *First == 'e' || *First == 'm')
return lexModule(First, End);

// Lex '#'.
const dependency_directives_scan::Token &HashTok = lexToken(First, End);
if (HashTok.is(tok::hashhash)) {
Expand Down
2 changes: 2 additions & 0 deletions clang/lib/Lex/Preprocessor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -950,6 +950,8 @@ void Preprocessor::Lex(Token &Result) {
case tok::period:
ModuleDeclState.handlePeriod();
break;
case tok::eod:
break;
case tok::identifier:
// Check "import" and "module" when there is no open bracket. The two
// identifiers are not meaningful with open brackets.
Expand Down
1 change: 1 addition & 0 deletions clang/lib/Parse/Parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2519,6 +2519,7 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
break;
}
ExpectAndConsumeSemi(diag::err_module_expected_semi);
TryConsumeToken(tok::eod);

if (SeenError)
return nullptr;
Expand Down
29 changes: 18 additions & 11 deletions clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -640,14 +640,14 @@ TEST(MinimizeSourceToDependencyDirectivesTest, AtImport) {
EXPECT_STREQ("@import A;\n", Out.data());

ASSERT_FALSE(minimizeSourceToDependencyDirectives("@import A\n;", Out));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this check removed?

Copy link
Contributor Author

@naveen-seth naveen-seth Jul 16, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From P1857R3:

The entire import or module directive (including the closing ;) must be on a single logical line and for module must not come from an #include.

While this paper addresses C++, I've consulted with @Bigcheese on whether Objective-C modules should follow the same rules.
In the same spirit, a line-splice was added to this test:

llvm-project/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp

Lines 645 to 647 in 586b438

ASSERT_FALSE(minimizeSourceToDependencyDirectives(
"@import /*x*/ A /*x*/ . /*x*/ B /*x*/ \\n /*x*/ ; /*x*/", Out));
EXPECT_STREQ("@import A.B\\n;\n", Out.data());

This patch currently fails CI due to some clang-scan-deps test failures caused by these changes. I'll fix those issues now. Apologies for missing this earlier.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you keep the check but update what the result of the scanner for this input should be?

EXPECT_STREQ("@import A;\n", Out.data());
EXPECT_STREQ("@import A\n;\n", Out.data());

ASSERT_FALSE(minimizeSourceToDependencyDirectives("@import A.B;\n", Out));
EXPECT_STREQ("@import A.B;\n", Out.data());

ASSERT_FALSE(minimizeSourceToDependencyDirectives(
"@import /*x*/ A /*x*/ . /*x*/ B /*x*/ \n /*x*/ ; /*x*/", Out));
EXPECT_STREQ("@import A.B;\n", Out.data());
"@import /*x*/ A /*x*/ . /*x*/ B /*x*/ \\n /*x*/ ; /*x*/", Out));
EXPECT_STREQ("@import A.B\\n;\n", Out.data());
}

TEST(MinimizeSourceToDependencyDirectivesTest, EmptyIncludesAndImports) {
Expand Down Expand Up @@ -1122,16 +1122,23 @@ ort \
)";
ASSERT_FALSE(
minimizeSourceToDependencyDirectives(Source, Out, Tokens, Directives));
EXPECT_STREQ("#include \"textual-header.h\"\nexport module m;"
"exp\\\nort import:l[[rename]];"
"import<<=3;import a b d e d e f e;"
"import foo[[no_unique_address]];import foo();"
"import f(:sefse);import f(->a=3);"

EXPECT_STREQ("module;\n"
"#include \"textual-header.h\"\n"
"export module m;\n"
"exp\\\nort import:l[[rename]];\n"
"import<<=3;\n"
"import a b d e d e f e;\n"
"import foo[[no_unique_address]];\n"
"import foo();\n"
"import f(:sefse);\n"
"import f(->a=3);\n"
"<TokBeforeEOF>\n",
Out.data());
ASSERT_EQ(Directives.size(), 11u);
EXPECT_EQ(Directives[0].Kind, pp_include);
EXPECT_EQ(Directives[1].Kind, cxx_export_module_decl);
ASSERT_EQ(Directives.size(), 12u);
EXPECT_EQ(Directives[0].Kind, cxx_module_decl);
EXPECT_EQ(Directives[1].Kind, pp_include);
EXPECT_EQ(Directives[2].Kind, cxx_export_module_decl);
}

TEST(MinimizeSourceToDependencyDirectivesTest, ObjCMethodArgs) {
Expand Down
Loading