Skip to content

Commit f4441c3

Browse files
xinzhengzhangxinzhengzhang
committed
file based filter: Handle header lib
1 parent 0f25090 commit f4441c3

File tree

1 file changed

+106
-72
lines changed

1 file changed

+106
-72
lines changed

‎refresh.template.py

+106-72
Original file line numberDiff line numberDiff line change
@@ -855,88 +855,122 @@ def _get_commands(target: str, flags: str):
855855
Try adding them as flags in your refresh_compile_commands rather than targets.
856856
In a moment, Bazel will likely fail to parse.""")
857857

858-
# First, query Bazel's C-family compile actions for that configured target
859-
target_statment = f'deps({target})'
860-
if {exclude_external_sources}:
861-
# For efficiency, have bazel filter out external targets (and therefore actions) before they even get turned into actions or serialized and sent to us. Note: this is a different mechanism than is used for excluding just external headers.
862-
target_statment = f"filter('^(//|@//)',{target_statment})"
863-
if file_flags:
864-
file_path = file_flags[0]
865-
if file_path.endswith(_get_files.source_extensions):
866-
target_statment = f"inputs('{re.escape(file_path)}', {target_statment})"
867-
else:
868-
# For header files we try to find from hdrs and srcs to get the targets
869-
# Since attr function can't query with full path, get the file name to query
870-
fname = os.path.basename(file_path)
871-
target_statment = f"let v = {target_statment} in attr(hdrs, '{fname}', $v) + attr(srcs, '{fname}', $v)"
872-
aquery_args = [
873-
'bazel',
874-
'aquery',
875-
# Aquery docs if you need em: https://docs.bazel.build/versions/master/aquery.html
876-
# Aquery output proto reference: https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/analysis_v2.proto
877-
# One bummer, not described in the docs, is that aquery filters over *all* actions for a given target, rather than just those that would be run by a build to produce a given output. This mostly isn't a problem, but can sometimes surface extra, unnecessary, misconfigured actions. Chris has emailed the authors to discuss and filed an issue so anyone reading this could track it: https://github.com/bazelbuild/bazel/issues/14156.
878-
f"mnemonic('(Objc|Cpp)Compile', {target_statment})",
879-
# We switched to jsonproto instead of proto because of https://github.com/bazelbuild/bazel/issues/13404. We could change back when fixed--reverting most of the commit that added this line and tweaking the build file to depend on the target in that issue. That said, it's kinda nice to be free of the dependency, unless (OPTIMNOTE) jsonproto becomes a performance bottleneck compated to binary protos.
880-
'--output=jsonproto',
881-
# We'll disable artifact output for efficiency, since it's large and we don't use them. Small win timewise, but dramatically less json output from aquery.
882-
'--include_artifacts=false',
883-
# Shush logging. Just for readability.
884-
'--ui_event_filters=-info',
885-
'--noshow_progress',
886-
# Disable param files, which would obscure compile actions
887-
# Mostly, people enable param files on Windows to avoid the relatively short command length limit.
888-
# For more, see compiler_param_file in https://bazel.build/docs/windows
889-
# They are, however, technically supported on other platforms/compilers.
890-
# That's all well and good, but param files would prevent us from seeing compile actions before the param files had been generated by compilation.
891-
# Since clangd has no such length limit, we'll disable param files for our aquery run.
892-
'--features=-compiler_param_file',
893-
# Disable layering_check during, because it causes large-scale dependence on generated module map files that prevent header extraction before their generation
894-
# For more context, see https://github.com/hedronvision/bazel-compile-commands-extractor/issues/83
895-
# If https://github.com/clangd/clangd/issues/123 is resolved and we're not doing header extraction, we could try removing this, checking that there aren't erroneous red squigglies squigglies before the module maps are generated.
896-
# If Bazel starts supporting modules (https://github.com/bazelbuild/bazel/issues/4005), we'll probably need to make changes that subsume this.
897-
'--features=-layering_check',
898-
] + additional_flags
899-
900-
aquery_process = subprocess.run(
901-
aquery_args,
902-
capture_output=True,
903-
encoding=locale.getpreferredencoding(),
904-
check=False, # We explicitly ignore errors from `bazel aquery` and carry on.
905-
)
858+
@enum.unique
859+
class Stage(enum.Enum):
860+
INIT = 0
861+
FINDED = 1
862+
# For header files we try to find from hdrs and srcs to get the targets
863+
# Since attr function can't query with full path, get the file name to query
864+
FIND_HEADER_FROM_ATTRS = 2
865+
# If the header can not found from attrs then we processing query from all inputs which includes
866+
FIND_HEADER_FROM_INPUTS = 3
867+
# if still not found the query the whole tree
868+
FIND_HEADER_FROM_WHO_DEPS = 4
869+
870+
def get_next_stage(self):
871+
if self == Stage.FIND_HEADER_FROM_ATTRS:
872+
return Stage.FIND_HEADER_FROM_INPUTS
873+
elif self == Stage.FIND_HEADER_FROM_INPUTS:
874+
return Stage.FIND_HEADER_FROM_WHO_DEPS
875+
else:
876+
return Stage.FINDED
877+
compile_commands = []
878+
stage = Stage.INIT
879+
while stage != Stage.FINDED:
880+
# First, query Bazel's C-family compile actions for that configured target
881+
target_statment = f'deps({target})'
882+
if {exclude_external_sources}:
883+
# For efficiency, have bazel filter out external targets (and therefore actions) before they even get turned into actions or serialized and sent to us. Note: this is a different mechanism than is used for excluding just external headers.
884+
target_statment = f"filter('^(//|@//)',{target_statment})"
885+
if file_flags:
886+
file_path = file_flags[0]
887+
if file_path.endswith(_get_files.source_extensions):
888+
target_statment = f"inputs('{re.escape(file_path)}', {target_statment})"
889+
else:
890+
if stage == Stage.INIT:
891+
stage = Stage.FIND_HEADER_FROM_ATTRS
892+
if stage == Stage.FIND_HEADER_FROM_ATTRS:
893+
fname = os.path.basename(file_path)
894+
target_statment = f"let v = {target_statment} in attr(hdrs, '{fname}', $v) + attr(srcs, '{fname}', $v)"
895+
elif stage == Stage.FIND_HEADER_FROM_INPUTS:
896+
target_statment = f"inputs('{re.escape(file_path)}', {target_statment})"
897+
elif stage == Stage.FIND_HEADER_FROM_WHO_DEPS:
898+
target_statment = f'deps({target})'
899+
900+
aquery_args = [
901+
'bazel',
902+
'aquery',
903+
# Aquery docs if you need em: https://docs.bazel.build/versions/master/aquery.html
904+
# Aquery output proto reference: https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/analysis_v2.proto
905+
# One bummer, not described in the docs, is that aquery filters over *all* actions for a given target, rather than just those that would be run by a build to produce a given output. This mostly isn't a problem, but can sometimes surface extra, unnecessary, misconfigured actions. Chris has emailed the authors to discuss and filed an issue so anyone reading this could track it: https://github.com/bazelbuild/bazel/issues/14156.
906+
f"mnemonic('(Objc|Cpp)Compile', {target_statment})",
907+
# We switched to jsonproto instead of proto because of https://github.com/bazelbuild/bazel/issues/13404. We could change back when fixed--reverting most of the commit that added this line and tweaking the build file to depend on the target in that issue. That said, it's kinda nice to be free of the dependency, unless (OPTIMNOTE) jsonproto becomes a performance bottleneck compated to binary protos.
908+
'--output=jsonproto',
909+
# We'll disable artifact output for efficiency, since it's large and we don't use them. Small win timewise, but dramatically less json output from aquery.
910+
'--include_artifacts=false',
911+
# Shush logging. Just for readability.
912+
'--ui_event_filters=-info',
913+
'--noshow_progress',
914+
# Disable param files, which would obscure compile actions
915+
# Mostly, people enable param files on Windows to avoid the relatively short command length limit.
916+
# For more, see compiler_param_file in https://bazel.build/docs/windows
917+
# They are, however, technically supported on other platforms/compilers.
918+
# That's all well and good, but param files would prevent us from seeing compile actions before the param files had been generated by compilation.
919+
# Since clangd has no such length limit, we'll disable param files for our aquery run.
920+
'--features=-compiler_param_file',
921+
# Disable layering_check during, because it causes large-scale dependence on generated module map files that prevent header extraction before their generation
922+
# For more context, see https://github.com/hedronvision/bazel-compile-commands-extractor/issues/83
923+
# If https://github.com/clangd/clangd/issues/123 is resolved and we're not doing header extraction, we could try removing this, checking that there aren't erroneous red squigglies squigglies before the module maps are generated.
924+
# If Bazel starts supporting modules (https://github.com/bazelbuild/bazel/issues/4005), we'll probably need to make changes that subsume this.
925+
'--features=-layering_check',
926+
] + additional_flags
927+
928+
aquery_process = subprocess.run(
929+
aquery_args,
930+
capture_output=True,
931+
encoding=locale.getpreferredencoding(),
932+
check=False, # We explicitly ignore errors from `bazel aquery` and carry on.
933+
)
906934

907935

908-
# Filter aquery error messages to just those the user should care about.
909-
missing_targets_warning: typing.Pattern[str] = re.compile(r'(\(\d+:\d+:\d+\) )?(\033\[[\d;]+m)?WARNING: (\033\[[\d;]+m)?Targets were missing from graph:') # Regex handles --show_timestamps and --color=yes. Could use "in" if we ever need more flexibility.
910-
for line in aquery_process.stderr.splitlines():
911-
# Shush known warnings about missing graph targets.
912-
# The missing graph targets are not things we want to introspect anyway.
913-
# Tracking issue https://github.com/bazelbuild/bazel/issues/13007.
914-
if missing_targets_warning.match(line):
915-
continue
936+
# Filter aquery error messages to just those the user should care about.
937+
missing_targets_warning: typing.Pattern[str] = re.compile(r'(\(\d+:\d+:\d+\) )?(\033\[[\d;]+m)?WARNING: (\033\[[\d;]+m)?Targets were missing from graph:') # Regex handles --show_timestamps and --color=yes. Could use "in" if we ever need more flexibility.
938+
for line in aquery_process.stderr.splitlines():
939+
# Shush known warnings about missing graph targets.
940+
# The missing graph targets are not things we want to introspect anyway.
941+
# Tracking issue https://github.com/bazelbuild/bazel/issues/13007.
942+
if missing_targets_warning.match(line):
943+
continue
916944

917-
print(line, file=sys.stderr)
945+
print(line, file=sys.stderr)
918946

919947

920-
# Parse proto output from aquery
921-
try:
922-
# object_hook -> SimpleNamespace allows object.member syntax, like a proto, while avoiding the protobuf dependency
923-
parsed_aquery_output = json.loads(aquery_process.stdout, object_hook=lambda d: types.SimpleNamespace(**d))
924-
except json.JSONDecodeError:
925-
print("Bazel aquery failed. Command:", aquery_args, file=sys.stderr)
926-
log_warning(f">>> Failed extracting commands for {target}\n Continuing gracefully...")
927-
return
928-
929-
if not getattr(parsed_aquery_output, 'actions', None): # Unifies cases: No actions (or actions list is empty)
930-
log_warning(f""">>> Bazel lists no applicable compile commands for {target}
931-
If this is a header-only library, please instead specify a test or binary target that compiles it (search "header-only" in README.md).
932-
Continuing gracefully...""")
933-
return
948+
# Parse proto output from aquery
949+
try:
950+
# object_hook -> SimpleNamespace allows object.member syntax, like a proto, while avoiding the protobuf dependency
951+
parsed_aquery_output = json.loads(aquery_process.stdout, object_hook=lambda d: types.SimpleNamespace(**d))
952+
except json.JSONDecodeError:
953+
print("Bazel aquery failed. Command:", aquery_args, file=sys.stderr)
954+
log_warning(f">>> Failed extracting commands for {target}\n Continuing gracefully...")
955+
return
956+
957+
if not getattr(parsed_aquery_output, 'actions', None): # Unifies cases: No actions (or actions list is empty)
958+
log_warning(f""">>> Bazel lists no applicable compile commands for {target}
959+
If this is a header-only library, please instead specify a test or binary target that compiles it (search "header-only" in README.md).
960+
Continuing gracefully...""")
961+
return
934962

935-
yield from _convert_compile_commands(parsed_aquery_output)
963+
# Don't waste if we resolve other commands
964+
compile_commands.extend(_convert_compile_commands(parsed_aquery_output))
936965

966+
if file_flags:
967+
if any(command['file'].endswith(file_flags[0]) for command in compile_commands):
968+
stage = Stage.FINDED
969+
stage = stage.get_next_stage()
937970

938971
# Log clear completion messages
939972
log_success(f">>> Finished extracting commands for {target}")
973+
return compile_commands
940974

941975

942976
def _ensure_external_workspaces_link_exists():

0 commit comments

Comments
 (0)