From ce0fb9e8177c377c7bb050088a949b3ae2558930 Mon Sep 17 00:00:00 2001 From: Borja Lorente Date: Tue, 19 Mar 2024 11:20:51 +0000 Subject: [PATCH 01/45] fix(sync): Disable query-sync by default (#6210) * fix(sync): Disable query-sync by default Recently, upstream has started rolling out a new way of syncing a project (https://github.com/bazelbuild/intellij/pull/5646). This is still in beta, and we've seen some issues reported internally. We disable it for now, until they stabilize the feature. Users that want to try it can still do so, from their settings menu. Author: Borja Lorente * Address comments --- .../com/google/idea/common/experiments/experiment.properties | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/experiments/src/com/google/idea/common/experiments/experiment.properties b/common/experiments/src/com/google/idea/common/experiments/experiment.properties index c92c64e3237..08ad1e99898 100644 --- a/common/experiments/src/com/google/idea/common/experiments/experiment.properties +++ b/common/experiments/src/com/google/idea/common/experiments/experiment.properties @@ -11,4 +11,7 @@ formatter.api.buildifier=100 aswb.use.studio.deployer.2=100 # Show What's New pop-up at startup -blaze.assistant.whatsnew.popup=1 \ No newline at end of file +blaze.assistant.whatsnew.popup=1 + +# Disable until the feature is stabilized +query.sync=disabled From c2b5d114c58609c5a785931609c5563d86bb8302 Mon Sep 17 00:00:00 2001 From: Borja Lorente Date: Tue, 19 Mar 2024 17:17:17 +0000 Subject: [PATCH 02/45] (rebase) Support embedded go_proto_library (#6030) * Support embedded go_proto_library * [aspect] Add tests for go_proto_library * Delete duplicated rules in WORKSPACE * Clean up unnecessary files and deps * Fix CI * Address comments --------- Co-authored-by: jacqueline.lee Co-authored-by: Jacqueline Lee --- WORKSPACE.bzlmod | 48 ++++++++++++------ aspect/intellij_info_impl.bzl | 2 +- .../blaze/aspect/go/go_proto_library/BUILD | 50 +++++++++++++++++++ .../aspect/go/go_proto_library/GoTest.java | 45 +++++++++++++++++ .../go/go_proto_library/fooserver.proto | 27 ++++++++++ .../aspect/go/go_proto_library/translators.go | 11 ++++ examples/go/with_proto/proto/BUILD.bazel | 1 + examples/go/with_proto/proto/fooserver.proto | 4 ++ examples/go/with_proto/proto/translators.go | 12 +++++ 9 files changed, 185 insertions(+), 15 deletions(-) create mode 100644 aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/BUILD create mode 100644 aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/GoTest.java create mode 100644 aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/fooserver.proto create mode 100644 aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/translators.go create mode 100644 examples/go/with_proto/proto/translators.go diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod index e83a73bd814..30daa46fcae 100644 --- a/WORKSPACE.bzlmod +++ b/WORKSPACE.bzlmod @@ -391,20 +391,6 @@ http_archive( url = PYTHON_PLUGIN_233_URL, ) -http_archive( - name = "python_2023_1", - build_file_content = _PYTHON_CE_BUILD_FILE, - sha256 = "825c30d2cbcce405fd18fddf356eb1f425607e9c780f8eff95d21ac23f8d90fd", - url = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/PythonCore/231.8770.65/PythonCore-231.8770.65.zip", -) - -http_archive( - name = "python_2023_2", - build_file_content = _PYTHON_CE_BUILD_FILE, - sha256 = "e744349f353568c18a9e11ec5e3a205f62bbdc1b65c9abc96783c479fe2aa51b", - url = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/PythonCore/232.9921.47/PythonCore-232.9921.47.zip", -) - PYTHON_PLUGIN_241_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/PythonCore/241.14494.127/PythonCore-241.14494.127.zip" PYTHON_PLUGIN_241_SHA = "41d941d9198078df72583297ca643c2c465a58fa478beda595e18f79fccb7dca" @@ -774,3 +760,37 @@ grpc_java_repositories() # Register custom java 17 toolchain register_toolchains("//:custom_java_17_toolchain_definition") + +# Dependency needed for Go test library +http_archive( + name = "bazel_gazelle", + sha256 = "29218f8e0cebe583643cbf93cae6f971be8a2484cdcfa1e45057658df8d54002", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.32.0/bazel-gazelle-v0.32.0.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.32.0/bazel-gazelle-v0.32.0.tar.gz", + ], +) + +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") + +# gRPC is used to generate Go from proto in the aspect tests, in +# //aspect/testing/tests/src/com/google/idea/blaze/aspect/go/... +go_repository( + name = "org_golang_google_grpc", + build_file_proto_mode = "disable", + importpath = "google.golang.org/grpc", + sum = "h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=", + version = "v1.27.1", +) + +load("@io_bazel_rules_go//tests:grpc_repos.bzl", "grpc_dependencies") + +grpc_dependencies() + +load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") + +go_rules_dependencies() + +go_register_toolchains(version = "1.19.3") + +gazelle_dependencies(go_repository_default_config = "//:WORKSPACE.bzlmod") diff --git a/aspect/intellij_info_impl.bzl b/aspect/intellij_info_impl.bzl index e1f9c42154b..a861937d1dc 100644 --- a/aspect/intellij_info_impl.bzl +++ b/aspect/intellij_info_impl.bzl @@ -411,7 +411,7 @@ def collect_go_info(target, ctx, semantics, ide_info, ide_info_file, output_grou library_labels = [stringify_label(ctx.rule.attr.library.label)] elif getattr(ctx.rule.attr, "embed", None) != None: for library in ctx.rule.attr.embed: - if library.intellij_info.kind == "go_source": + if library.intellij_info.kind == "go_source" or library.intellij_info.kind == "go_proto_library": l = library.intellij_info.output_groups["intellij-sources-go-outputs"].to_list() sources += l generated += [f for f in l if not f.is_source] diff --git a/aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/BUILD b/aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/BUILD new file mode 100644 index 00000000000..2f17a56ba5d --- /dev/null +++ b/aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/BUILD @@ -0,0 +1,50 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load( + "//aspect/testing/rules:intellij_aspect_test_fixture.bzl", + "intellij_aspect_test_fixture", +) + +proto_library( + name = "fooproto_proto", + srcs = ["fooserver.proto"], + visibility = ["//visibility:public"], + deps = ["@com_google_protobuf//:empty_proto"], +) + +go_proto_library( + name = "fooproto_go_proto", + compilers = ["@io_bazel_rules_go//proto:go_grpc"], + importpath = "github.com/bazelbuild/intellij/examples/go/with_proto/proto", + proto = ":fooproto_proto", + visibility = ["//visibility:public"], +) + +go_library( + name = "proto", + srcs = ["translators.go"], + embed = [":fooproto_go_proto"], + importpath = "github.com/bazelbuild/intellij/examples/go/with_proto/proto", + visibility = ["//visibility:public"], +) + +intellij_aspect_test_fixture( + name = "simple_fixture", + deps = [":proto"], +) + +java_test( + name = "GoTest", + srcs = ["GoTest.java"], + data = [":simple_fixture"], + deps = [ + "//aspect/testing:BazelIntellijAspectTest", + "//aspect/testing:guava", + "//aspect/testing/rules:IntellijAspectTest", + "//aspect/testing/rules:intellij_aspect_test_fixture_java_proto", + "//intellij_platform_sdk:test_libs", + "//proto:intellij_ide_info_java_proto", + "@junit//jar", + ], +) diff --git a/aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/GoTest.java b/aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/GoTest.java new file mode 100644 index 00000000000..58358f81226 --- /dev/null +++ b/aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/GoTest.java @@ -0,0 +1,45 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.aspect.go.go_proto_library; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.devtools.intellij.IntellijAspectTestFixtureOuterClass.IntellijAspectTestFixture; +import com.google.devtools.intellij.ideinfo.IntellijIdeInfo.TargetIdeInfo; +import com.google.idea.blaze.BazelIntellijAspectTest; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + + +/** Tests go_proto_library */ +@RunWith(JUnit4.class) +public class GoTest extends BazelIntellijAspectTest { + @Test + public void testGoTest() throws Exception { + IntellijAspectTestFixture testFixture = loadTestFixture(":simple_fixture"); + TargetIdeInfo target = findTarget(testFixture, ":proto"); + assertThat(target.getKindString()).isEqualTo("go_library"); + assertThat(relativePathsForArtifacts(target.getGoIdeInfo().getSourcesList())) + .contains(testRelative("translators.go")); + assertThat(relativePathsForArtifacts(target.getGoIdeInfo().getSourcesList())) + .contains(testRelative("fooproto_go_proto_/github.com/bazelbuild/intellij/examples/go/with_proto/proto/fooserver.pb.go")); + } +} diff --git a/aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/fooserver.proto b/aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/fooserver.proto new file mode 100644 index 00000000000..ab63c861d34 --- /dev/null +++ b/aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/fooserver.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package fooproto; + +import "google/protobuf/empty.proto"; + +service FooService { + rpc SayHello (HelloRequest) returns (HelloReply) {} + // This function exists to validate that the well known types (such as google.protobuf.Empty) + // are picked up by the plugin. + // To validate that they are working, navigate to "External Libraries" > "Go Libraries" > + // "github.com/bazelbuild/intellij/examples/go/with_proto/proto/fooserver.pb.go", + // and validate that "google.golang.org/protobuf/types/known/emptypb" is resolved correctly. + rpc EmptyFunction (google.protobuf.Empty) returns (google.protobuf.Empty); +} + +message HelloRequest { + optional string name = 1; +} + +message HelloReply { + optional string message = 1; +} + +message Time { + string value = 1; +} diff --git a/aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/translators.go b/aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/translators.go new file mode 100644 index 00000000000..f79720cb504 --- /dev/null +++ b/aspect/testing/tests/src/com/google/idea/blaze/aspect/go/go_proto_library/translators.go @@ -0,0 +1,11 @@ +package proto + +import ( + "time" +) + +// Time translation functions. + +func ToProtoTime(t time.Time) Time { + return Time{Value: t.Format(time.RFC3339Nano)} +} diff --git a/examples/go/with_proto/proto/BUILD.bazel b/examples/go/with_proto/proto/BUILD.bazel index da57e4a9d30..5d6ba7a0a09 100644 --- a/examples/go/with_proto/proto/BUILD.bazel +++ b/examples/go/with_proto/proto/BUILD.bazel @@ -19,6 +19,7 @@ go_proto_library( go_library( name = "proto", + srcs = ["translators.go"], embed = [":fooproto_go_proto"], importpath = "github.com/bazelbuild/intellij/examples/go/with_proto/proto", visibility = ["//visibility:public"], diff --git a/examples/go/with_proto/proto/fooserver.proto b/examples/go/with_proto/proto/fooserver.proto index ccac31f0d8c..ab63c861d34 100644 --- a/examples/go/with_proto/proto/fooserver.proto +++ b/examples/go/with_proto/proto/fooserver.proto @@ -21,3 +21,7 @@ message HelloRequest { message HelloReply { optional string message = 1; } + +message Time { + string value = 1; +} diff --git a/examples/go/with_proto/proto/translators.go b/examples/go/with_proto/proto/translators.go new file mode 100644 index 00000000000..b90b57db861 --- /dev/null +++ b/examples/go/with_proto/proto/translators.go @@ -0,0 +1,12 @@ +package proto + +import ( + "time" +) + +// Time translation functions. +// If all symbols resolve correctly (i.e. there are no red squigglies after a sync) +// then this target is working. +func ToProtoTime(t time.Time) Time { + return Time{Value: t.Format(time.RFC3339Nano)} +} From 0a1dcd43300efbee1d120be7527be32096936531 Mon Sep 17 00:00:00 2001 From: Tomasz Pasternak Date: Wed, 20 Mar 2024 18:47:46 +0100 Subject: [PATCH 03/45] Bump JetBrains IDEs (#6301) --- WORKSPACE.bzlmod | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod index 30daa46fcae..d344f507fc9 100644 --- a/WORKSPACE.bzlmod +++ b/WORKSPACE.bzlmod @@ -84,9 +84,9 @@ http_archive( ) # The plugin api for intellij_ce_2024_1. This is required to build IJwB and run integration tests. -IC_241_SHA = "0a18e02b611562b4e81f2f5af570975c153874154ad17fdb026eae3b6818e422" +IC_241_SHA = "7dc317709f981bc772083f39bb4ea73a2a5e441bf7b9400b730004f786dcaed4" -IC_241_URL = "https://www.jetbrains.com/intellij-repository/snapshots/com/jetbrains/intellij/idea/ideaIC/241.14494.17-EAP-SNAPSHOT/ideaIC-241.14494.17-EAP-SNAPSHOT.zip" +IC_241_URL = "https://www.jetbrains.com/intellij-repository/snapshots/com/jetbrains/intellij/idea/ideaIC/241.14494.127-EAP-SNAPSHOT/ideaIC-241.14494.127-EAP-SNAPSHOT.zip" http_archive( name = "intellij_ce_2024_1", @@ -171,9 +171,9 @@ http_archive( url = IU_233_URL, ) -IU_241_SHA = "6b5df17f4a28844fd02fa500404308660f46f3dc72496f12f6c2eee8b95f4392" +IU_241_SHA = "9e884acb059d1e2cc2523b2f78c93251ee69c7762c9ca5532ca716a70a698c71" -IU_241_URL = "https://www.jetbrains.com/intellij-repository/snapshots/com/jetbrains/intellij/idea/ideaIU/241.14494.17-EAP-SNAPSHOT/ideaIU-241.14494.17-EAP-SNAPSHOT.zip" +IU_241_URL = "https://www.jetbrains.com/intellij-repository/snapshots/com/jetbrains/intellij/idea/ideaIU/241.14494.127-EAP-SNAPSHOT/ideaIU-241.14494.127-EAP-SNAPSHOT.zip" http_archive( name = "intellij_ue_2024_1", @@ -257,9 +257,9 @@ http_archive( url = CLION_233_URL, ) -CLION_241_SHA = "55c9130a6b6fda96a9e45766392ee7b2322d66c1d66e2ef89be38227fbbbcb96" +CLION_241_SHA = "28d38edd0f82674b19ee42f806690e2e5aadb9534a8ab30457cc6585284c80f5" -CLION_241_URL = "https://www.jetbrains.com/intellij-repository/snapshots/com/jetbrains/intellij/clion/clion/241.14494.27-EAP-SNAPSHOT/clion-241.14494.27-EAP-SNAPSHOT.zip" +CLION_241_URL = "https://www.jetbrains.com/intellij-repository/snapshots/com/jetbrains/intellij/clion/clion/241.14494.102-EAP-SNAPSHOT/clion-241.14494.102-EAP-SNAPSHOT.zip" http_archive( name = "clion_2024_1", @@ -276,9 +276,9 @@ java_import( ) """ -DEVKIT_233_SHA = "225165191a515ac97c30dad96889d80c88129ce6bf069b9401da6a4c04a123b4" +DEVKIT_233_SHA = "2523362a9735c72205f53b4be91c5e71e0e7b6b2dbc6dc28894ecaeb6121725d" -DEVKIT_233_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/DevKit/233.14808.24/DevKit-233.14808.24.zip" +DEVKIT_233_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/DevKit/233.14475.74/DevKit-233.14475.74.zip" http_archive( name = "devkit_2023_3", From efe6b2a540707ee79a6be7914ea9be4c232ef0df Mon Sep 17 00:00:00 2001 From: Myroslav Date: Fri, 22 Mar 2024 11:53:23 +0100 Subject: [PATCH 04/45] Reduced the logging level for the stack traces produced when resolving external artifacts. (#6310) In case of an IOException, only the message and the type of error is logged as a warning. The stack trace is logged only at the lowest level, so as to limit the amount of logs when there are many problematic artifacts. --- .../base/sync/workspace/ArtifactLocationDecoderImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/src/com/google/idea/blaze/base/sync/workspace/ArtifactLocationDecoderImpl.java b/base/src/com/google/idea/blaze/base/sync/workspace/ArtifactLocationDecoderImpl.java index 8489ceb0ac9..397219d5ee5 100644 --- a/base/src/com/google/idea/blaze/base/sync/workspace/ArtifactLocationDecoderImpl.java +++ b/base/src/com/google/idea/blaze/base/sync/workspace/ArtifactLocationDecoderImpl.java @@ -98,8 +98,9 @@ public File decode(ArtifactLocation artifactLocation) { return realFile; } } catch (IOException ioException) { - LOG.warn("Failed to resolve real path for " + artifactLocation.getExecutionRootRelativePath(), - ioException); + LOG.warn("Failed to resolve real path for " + artifactLocation.getExecutionRootRelativePath() + + "\n" + ioException.getClass().getSimpleName() + ": " + ioException.getMessage()); + LOG.trace(ioException); } } return null; From 58b4e407aefaed909f630752dae06f4461e8379b Mon Sep 17 00:00:00 2001 From: Tomasz Pasternak Date: Fri, 22 Mar 2024 18:34:48 +0100 Subject: [PATCH 05/45] chore: Bump JetBrains IDEs (#6312) Co-authored-by: Mai Hussien <70515749+mai93@users.noreply.github.com> --- WORKSPACE.bzlmod | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod index d344f507fc9..59e7cd11f0b 100644 --- a/WORKSPACE.bzlmod +++ b/WORKSPACE.bzlmod @@ -72,9 +72,9 @@ http_archive( ) # The plugin api for intellij_ce_2023_2. This is required to build IJwB and run integration tests. -IC_233_SHA = "ffa6c0f98bbf67286eefb8bfad0026029edcd334463fffc17f369d9a90156992" +IC_233_SHA = "c7c4064ef95f58503ecb0d1bcebc6d9e04587adaf407da085497e8b37b788125" -IC_233_URL = "https://www.jetbrains.com/intellij-repository/releases/com/jetbrains/intellij/idea/ideaIC/2023.3.5/ideaIC-2023.3.5.zip" +IC_233_URL = "https://www.jetbrains.com/intellij-repository/releases/com/jetbrains/intellij/idea/ideaIC/2023.3.6/ideaIC-2023.3.6.zip" http_archive( name = "intellij_ce_2023_3", @@ -84,9 +84,9 @@ http_archive( ) # The plugin api for intellij_ce_2024_1. This is required to build IJwB and run integration tests. -IC_241_SHA = "7dc317709f981bc772083f39bb4ea73a2a5e441bf7b9400b730004f786dcaed4" +IC_241_SHA = "1b2e9ea16b9f8bcfbfab0c67b082b2e42906b5b7ec83d5520cc3e719e5e1cff0" -IC_241_URL = "https://www.jetbrains.com/intellij-repository/snapshots/com/jetbrains/intellij/idea/ideaIC/241.14494.127-EAP-SNAPSHOT/ideaIC-241.14494.127-EAP-SNAPSHOT.zip" +IC_241_URL = "https://www.jetbrains.com/intellij-repository/snapshots/com/jetbrains/intellij/idea/ideaIC/241.14494.158-EAP-SNAPSHOT/ideaIC-241.14494.158-EAP-SNAPSHOT.zip" http_archive( name = "intellij_ce_2024_1", @@ -160,9 +160,9 @@ http_archive( url = IU_232_URL, ) -IU_233_SHA = "8ba41bcdb40d64c9257467fe4f7f999b666494cc10cdce8db894d91c0b16905d" +IU_233_SHA = "ba40dbd0347a86567f2e916271dac2532cb7e8f512a31cbb63552f563ba5de24" -IU_233_URL = "https://www.jetbrains.com/intellij-repository/releases/com/jetbrains/intellij/idea/ideaIU/2023.3.5/ideaIU-2023.3.5.zip" +IU_233_URL = "https://www.jetbrains.com/intellij-repository/releases/com/jetbrains/intellij/idea/ideaIU/2023.3.6/ideaIU-2023.3.6.zip" http_archive( name = "intellij_ue_2023_3", @@ -171,9 +171,9 @@ http_archive( url = IU_233_URL, ) -IU_241_SHA = "9e884acb059d1e2cc2523b2f78c93251ee69c7762c9ca5532ca716a70a698c71" +IU_241_SHA = "b60cb31bdd4f65f6885aa7da8dc2c65485e041cec2406b140e875b5040f2a051" -IU_241_URL = "https://www.jetbrains.com/intellij-repository/snapshots/com/jetbrains/intellij/idea/ideaIU/241.14494.127-EAP-SNAPSHOT/ideaIU-241.14494.127-EAP-SNAPSHOT.zip" +IU_241_URL = "https://www.jetbrains.com/intellij-repository/snapshots/com/jetbrains/intellij/idea/ideaIU/241.14494.158-EAP-SNAPSHOT/ideaIU-241.14494.158-EAP-SNAPSHOT.zip" http_archive( name = "intellij_ue_2024_1", @@ -257,9 +257,9 @@ http_archive( url = CLION_233_URL, ) -CLION_241_SHA = "28d38edd0f82674b19ee42f806690e2e5aadb9534a8ab30457cc6585284c80f5" +CLION_241_SHA = "5f64009afc5bbbfbe2b46aba5c9cd7571891827b5e4ece94c3614c5cfe7964b8" -CLION_241_URL = "https://www.jetbrains.com/intellij-repository/snapshots/com/jetbrains/intellij/clion/clion/241.14494.102-EAP-SNAPSHOT/clion-241.14494.102-EAP-SNAPSHOT.zip" +CLION_241_URL = "https://www.jetbrains.com/intellij-repository/snapshots/com/jetbrains/intellij/clion/clion/241.14494.154-EAP-SNAPSHOT/clion-241.14494.154-EAP-SNAPSHOT.zip" http_archive( name = "clion_2024_1", @@ -276,9 +276,9 @@ java_import( ) """ -DEVKIT_233_SHA = "2523362a9735c72205f53b4be91c5e71e0e7b6b2dbc6dc28894ecaeb6121725d" +DEVKIT_233_SHA = "4ed75336c016ed6f66856c95259140018aef62c376ac1bdb1a7e0b96c670ca43" -DEVKIT_233_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/DevKit/233.14475.74/DevKit-233.14475.74.zip" +DEVKIT_233_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/DevKit/233.15026.17/DevKit-233.15026.17.zip" http_archive( name = "devkit_2023_3", @@ -287,9 +287,9 @@ http_archive( url = DEVKIT_233_URL, ) -DEVKIT_241_SHA = "8a886ef1f1b2c68ad5352bf503551cea1d2c1c61d76ffc2c875c229674a69cde" +DEVKIT_241_SHA = "5fb34141dd8c75ff44dffcda845f22aa31f6c4cf0641eae18e08190eb7cdacd5" -DEVKIT_241_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/DevKit/241.14494.120/DevKit-241.14494.120.zip" +DEVKIT_241_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/DevKit/241.14494.158/DevKit-241.14494.158.zip" http_archive( name = "devkit_2024_1", @@ -391,9 +391,9 @@ http_archive( url = PYTHON_PLUGIN_233_URL, ) -PYTHON_PLUGIN_241_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/PythonCore/241.14494.127/PythonCore-241.14494.127.zip" +PYTHON_PLUGIN_241_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/PythonCore/241.14494.158/PythonCore-241.14494.158.zip" -PYTHON_PLUGIN_241_SHA = "41d941d9198078df72583297ca643c2c465a58fa478beda595e18f79fccb7dca" +PYTHON_PLUGIN_241_SHA = "3953f517290b1d8ceb504940c2980959c7040ab87752b8e0a36a75ef92aae3ad" http_archive( name = "python_2024_1", @@ -478,9 +478,9 @@ http_archive( url = GO_PLUGIN_232_URL, ) -GO_PLUGIN_233_SHA = "369c8acf0cc8a4dc28d2b859d389cd5a0375a14faba4a690ad91c57b2af7362e" +GO_PLUGIN_233_SHA = "6ec43880bfbedaead564def4b9f40baa13de4167d7d7c7c0522f7e7c5dd77a77" -GO_PLUGIN_233_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/org.jetbrains.plugins.go/233.14808.21/org.jetbrains.plugins.go-233.14808.21.zip" +GO_PLUGIN_233_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/org.jetbrains.plugins.go/233.15026.9/org.jetbrains.plugins.go-233.15026.9.zip" http_archive( name = "go_2023_3", @@ -489,9 +489,9 @@ http_archive( url = GO_PLUGIN_233_URL, ) -GO_PLUGIN_241_SHA = "b43ac565ce70fd68f502446ecec9546ddf748a77ac0df8d76a1c67213b354661" +GO_PLUGIN_241_SHA = "9150eb33b223c115b21a181a2a0aeb9dd89829f0aed95e976c430a6ca9d03eea" -GO_PLUGIN_241_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/org.jetbrains.plugins.go/241.14494.127/org.jetbrains.plugins.go-241.14494.127.zip" +GO_PLUGIN_241_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/org.jetbrains.plugins.go/241.14494.158/org.jetbrains.plugins.go-241.14494.158.zip" http_archive( name = "go_2024_1", @@ -579,9 +579,9 @@ http_archive( url = SCALA_PLUGIN_233_URL, ) -SCALA_PLUGIN_241_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/org.intellij.scala/2024.1.11/org.intellij.scala-2024.1.11.zip" +SCALA_PLUGIN_241_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/org.intellij.scala/2024.1.12/org.intellij.scala-2024.1.12.zip" -SCALA_PLUGIN_241_SHA = "e74febc21300db05f3697d55d79b017d80f7278a584d3bceebf07f1c8caf1d13" +SCALA_PLUGIN_241_SHA = "b28d565ee84cc1c39cf4b62eec685d3523934404422cb1426f4dffb496b98a50" http_archive( name = "scala_2024_1", @@ -598,9 +598,9 @@ java_import( ) """ -TOML_PLUGIN_241_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/org.toml.lang/241.14494.33/org.toml.lang-241.14494.33.zip" +TOML_PLUGIN_241_URL = "https://plugins.jetbrains.com/maven/com/jetbrains/plugins/org.toml.lang/241.14494.150/org.toml.lang-241.14494.150.zip" -TOML_PLUGIN_241_SHA = "356f480cb508dee613b98c71de1f0e1b871ebc5a33294199be483353523ff317" +TOML_PLUGIN_241_SHA = "f2ad603910c59b6f2b3623e52af9256f7b560ecf17f9f80b4fbc3060ee59aeee" # TOML required by Python since 241.13688.18 http_archive( From a144d339020d98d1727fc6bbd7432153c3f851c4 Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 11 Mar 2024 14:15:01 -0700 Subject: [PATCH 06/45] Automated rollback of commit ea8edb228a3a89203e4bbf76ca168638b2556af1. *** Reason for rollback *** Breaks Mac Build because of not finding IntellijExts class in the ext package. *** Original change description *** Fix IntellijExts package name (cherry picked from commit 6eba8b2f176b0b3c0fb5665b51fe62dc1fbbf868) --- ext/src/com/google/idea/blaze/ext/mac/IntelliJExts.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ext/src/com/google/idea/blaze/ext/mac/IntelliJExts.java b/ext/src/com/google/idea/blaze/ext/mac/IntelliJExts.java index 24feac74136..40d98c024a4 100644 --- a/ext/src/com/google/idea/blaze/ext/mac/IntelliJExts.java +++ b/ext/src/com/google/idea/blaze/ext/mac/IntelliJExts.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.idea.blaze.ext.mac; +package com.google.idea.blaze.ext; import io.netty.channel.Channel; import io.netty.channel.EventLoopGroup; @@ -24,9 +24,8 @@ import io.netty.util.concurrent.DefaultThreadFactory; /** - * Static utility methods relating to {@link com.google.idea.blaze.ext.IntelliJExtClient} and {@link - * com.google.idea.blaze.ext.IntelliJExtTestServer}. Provides platform-specific implementations to - * get ManagedChannel using the netty library for Mac. + * Static utility methods relating to {@link IntelliJExtClient} and {@link IntelliJExtTestServer}. + * Provides platform-specific implementations to get ManagedChannel using the netty library for Mac. */ public final class IntelliJExts { public static EventLoopGroup createGroup(DefaultThreadFactory threadFactory) { From bc9509cb581477ae8fcaa53fae6970d1b1bba818 Mon Sep 17 00:00:00 2001 From: Mai Hussien Date: Wed, 13 Mar 2024 11:19:54 -0700 Subject: [PATCH 07/45] Report the start of action execution in BEP (cherry picked from commit 9cd70a2314d03485d757820667b27fe91d6f1b2e) --- .../build/lib/buildeventstream/proto/build_event_stream.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/third_party/bazel/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto b/third_party/bazel/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto index 987f534b66b..d0b239f16af 100644 --- a/third_party/bazel/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto +++ b/third_party/bazel/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto @@ -1032,6 +1032,10 @@ message BuildMetrics { // elapsed time from the first action execution (excluding workspace status // actions) to the last. int64 execution_phase_time_in_ms = 4; + + // The elapsed wall time in milliseconds until the first action execution + // started (excluding workspace status actions). + int64 actions_execution_start_in_ms = 5; } TimingMetrics timing_metrics = 5; From 491751034584cb9f0008566abdecdc9d78e03c19 Mon Sep 17 00:00:00 2001 From: Googler Date: Thu, 14 Mar 2024 02:40:57 -0700 Subject: [PATCH 08/45] Add compiled artifact class jars to the project proto. This implements support for basic java deps with the new artifact tracker, and also adds appropriate support for more types of artifact in the future: - `ProjectProtoUpdate` is a helper class for making updates to the prooject proto, providing simpler access to the `.workspace` module and the like. - `ArtifactDirectoryBuilder` helps to construct an `ArtifactDirectoryContents` proto message and provides paths to the final artifact desinations for adding to other parts of the project proto. - `DependenciesProjectProtoUpdater` to hook this into the existing `ProjectProtoTransform`. (cherry picked from commit de40e787a2246c8a5d01318ca71f40944fb1cd41) --- .../idea/blaze/base/qsync/ProjectLoader.java | 2 + .../blaze/qsync/GraphToProjectConverter.java | 5 +- .../blaze/qsync/deps/AddCompiledJavaDeps.java | 54 ++++++++ .../qsync/deps/ArtifactDirectoryBuilder.java | 123 ++++++++++++++++++ .../com/google/idea/blaze/qsync/deps/BUILD | 2 + .../deps/DependenciesProjectProtoUpdater.java | 49 +++++++ .../blaze/qsync/deps/ProjectProtoUpdate.java | 90 +++++++++++++ .../deps/ProjectProtoUpdateOperation.java | 31 +++++ .../blaze/qsync/java/JavaArtifactInfo.java | 2 + .../blaze/qsync/GraphToProjectConverters.java | 3 +- .../idea/blaze/qsync/TestDataSyncRunner.java | 4 +- .../qsync/deps/AddCompiledJavaDepsTest.java | 103 +++++++++++++++ .../com/google/idea/blaze/qsync/deps/BUILD | 18 +++ .../blaze/qsync/testdata/ProjectProtos.java | 14 +- 14 files changed, 493 insertions(+), 7 deletions(-) create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/AddCompiledJavaDeps.java create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/ArtifactDirectoryBuilder.java create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdate.java create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdateOperation.java create mode 100644 querysync/javatests/com/google/idea/blaze/qsync/deps/AddCompiledJavaDepsTest.java diff --git a/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java b/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java index 20ddc85426f..688deeccf3b 100644 --- a/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java +++ b/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java @@ -53,6 +53,7 @@ import com.google.idea.blaze.qsync.ProjectRefresher; import com.google.idea.blaze.qsync.VcsStateDiffer; import com.google.idea.blaze.qsync.deps.ArtifactTracker; +import com.google.idea.blaze.qsync.deps.DependenciesProjectProtoUpdater; import com.google.idea.blaze.qsync.deps.NewArtifactTracker; import com.google.idea.blaze.qsync.java.PackageStatementParser; import com.google.idea.blaze.qsync.java.ParallelPackageReader; @@ -154,6 +155,7 @@ public QuerySyncProject loadProject(BlazeContext context) throws BuildException NewArtifactTracker tracker = new NewArtifactTracker<>( BlazeDataStorage.getProjectDataDir(importSettings).toPath(), artifactCache); + projectTransformRegistry.add(new DependenciesProjectProtoUpdater(tracker)); artifactTracker = tracker; renderJarArtifactTracker = new RenderJarArtifactTrackerImpl(); diff --git a/querysync/java/com/google/idea/blaze/qsync/GraphToProjectConverter.java b/querysync/java/com/google/idea/blaze/qsync/GraphToProjectConverter.java index ba41d5cd790..6126445f82e 100644 --- a/querysync/java/com/google/idea/blaze/qsync/GraphToProjectConverter.java +++ b/querysync/java/com/google/idea/blaze/qsync/GraphToProjectConverter.java @@ -114,7 +114,8 @@ public GraphToProjectConverter( Predicate fileExistenceCheck, Context context, ProjectDefinition projectDefinition, - ListeningExecutorService executor) { + ListeningExecutorService executor, + boolean useNewBuildArtifactLogic) { this.packageReader = packageReader; this.fileExistenceCheck = fileExistenceCheck; this.context = context; @@ -122,7 +123,7 @@ public GraphToProjectConverter( this.executor = executor; this.useNewResDirLogic = Suppliers.ofInstance(true); this.guessAndroidResPackages = Suppliers.ofInstance(false); - this.useNewBuildArtifactLogic = false; + this.useNewBuildArtifactLogic = useNewBuildArtifactLogic; } /** diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/AddCompiledJavaDeps.java b/querysync/java/com/google/idea/blaze/qsync/deps/AddCompiledJavaDeps.java new file mode 100644 index 00000000000..ea100aa9d5c --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/AddCompiledJavaDeps.java @@ -0,0 +1,54 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableCollection; +import com.google.idea.blaze.qsync.artifacts.BuildArtifact; +import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.project.ProjectProto.JarDirectory; +import java.nio.file.Path; + +/** Adds compiled jars from dependencies to the project. */ +public class AddCompiledJavaDeps implements ProjectProtoUpdateOperation { + private final Supplier> builtTargetsSupplier; + + public AddCompiledJavaDeps(Supplier> builtTargetsSupplier) { + this.builtTargetsSupplier = builtTargetsSupplier; + } + + @Override + public void update(ProjectProtoUpdate update) { + ArtifactDirectoryBuilder javaDepsDir = update.artifactDirectory(Path.of("javadeps")); + for (TargetBuildInfo target : builtTargetsSupplier.get()) { + if (target.javaInfo().isPresent()) { + JavaArtifactInfo javaInfo = target.javaInfo().get(); + for (BuildArtifact jar : javaInfo.jars()) { + javaDepsDir.addIfNewer(jar.path(), jar, target.buildContext()); + } + } + } + if (!javaDepsDir.isEmpty()) { + update + .library(JAVA_DEPS_LIB_NAME) + .addClassesJar( + JarDirectory.newBuilder().setPath(javaDepsDir.path().toString()).setRecursive(true)); + if (!update.workspaceModule().getLibraryNameList().contains(JAVA_DEPS_LIB_NAME)) { + update.workspaceModule().addLibraryName(JAVA_DEPS_LIB_NAME); + } + } + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactDirectoryBuilder.java b/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactDirectoryBuilder.java new file mode 100644 index 00000000000..7adbd428556 --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactDirectoryBuilder.java @@ -0,0 +1,123 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.google.idea.blaze.qsync.artifacts.BuildArtifact; +import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectPath.Root; +import com.google.idea.blaze.qsync.project.ProjectProto; +import com.google.idea.blaze.qsync.project.ProjectProto.ProjectArtifact.ArtifactTransform; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; + +/** Populates a {@link ProjectProto.ArtifactDirectoryContents} proto from build artifacts. */ +public class ArtifactDirectoryBuilder { + + @AutoValue + abstract static class Entry { + + abstract String destination(); + + abstract BuildArtifact source(); + + abstract DependencyBuildContext fromBuild(); + + abstract ArtifactTransform transform(); + + static Entry create( + Path destination, + BuildArtifact source, + DependencyBuildContext fromBuild, + ArtifactTransform transform) { + return new AutoValue_ArtifactDirectoryBuilder_Entry( + destination.toString(), source, fromBuild, transform); + } + + ProjectProto.ProjectArtifact toProto() { + return ProjectProto.ProjectArtifact.newBuilder() + .setBuildArtifact( + ProjectProto.BuildArtifact.newBuilder().setDigest(source().digest()).build()) + .setTransform(transform()) + .build(); + } + } + + private final Path path; + private final Map contents = Maps.newHashMap(); + + /** + * @param relativePath Project directory relative path of the destination directory. + */ + public ArtifactDirectoryBuilder(Path relativePath) { + Preconditions.checkState( + !relativePath.isAbsolute(), "Expected a relative path: %s", relativePath); + this.path = relativePath; + } + + /** Returns the project directory relative path of the destination directory. */ + public Path path() { + return path; + } + + /** Returns this directories root as a project path. */ + public ProjectPath root() { + return ProjectPath.projectRelative(path()); + } + + /** + * Adds a new artifact to the directory if it is not already present, or was produced by a more + * recent build that an existing artifact at the same location. + * + * @param relativePath Path to place the artifact at, relative to {@link #root()}. + * @param source The artifact to put there. + * @param fromBuild The build that produced the artifact. + * @return The path to the final artifact, if it was added. + */ + @CanIgnoreReturnValue + public Optional addIfNewer( + Path relativePath, BuildArtifact source, DependencyBuildContext fromBuild) { + Entry existing = contents.get(relativePath); + if (existing != null && existing.fromBuild().startTime().isAfter(fromBuild.startTime())) { + // we already have the same artifact from a more recent build. + return Optional.empty(); + } + Entry e = Entry.create(relativePath, source, fromBuild, ArtifactTransform.COPY); + contents.put(relativePath, e); + return Optional.of(ProjectPath.create(Root.PROJECT, path.resolve(relativePath))); + } + + public boolean isEmpty() { + return contents.isEmpty(); + } + + public ProjectProto.ArtifactDirectoryContents toProto() { + return ProjectProto.ArtifactDirectoryContents.newBuilder() + .putAllContents( + contents.values().stream().collect(toImmutableMap(Entry::destination, Entry::toProto))) + .build(); + } + + public void addTo(ProjectProto.ArtifactDirectories.Builder directories) { + directories.putDirectories(path.toString(), toProto()); + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/BUILD b/querysync/java/com/google/idea/blaze/qsync/deps/BUILD index 1dd06c12f59..fc7cbf621c5 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/BUILD +++ b/querysync/java/com/google/idea/blaze/qsync/deps/BUILD @@ -7,11 +7,13 @@ java_library( ], deps = [ ":artifact_tracker_state_java_proto", + "//querysync/java/com/google/idea/blaze/qsync", "//querysync/java/com/google/idea/blaze/qsync/artifacts", "//querysync/java/com/google/idea/blaze/qsync/cc:cc_compilation_info_java_proto", "//querysync/java/com/google/idea/blaze/qsync/java", "//querysync/java/com/google/idea/blaze/qsync/java:java_target_info_java_proto", "//querysync/java/com/google/idea/blaze/qsync/project", + "//querysync/java/com/google/idea/blaze/qsync/project:project_java_proto", "//shared", "//shared:artifact", "//shared:proto", diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java new file mode 100644 index 00000000000..9fc499003ac --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java @@ -0,0 +1,49 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import com.google.common.collect.ImmutableList; +import com.google.idea.blaze.common.Context; +import com.google.idea.blaze.exception.BuildException; +import com.google.idea.blaze.qsync.ProjectProtoTransform; +import com.google.idea.blaze.qsync.project.BuildGraphData; +import com.google.idea.blaze.qsync.project.ProjectProto.Project; + +/** + * A {@link ProjectProtoTransform} that adds built artifact information to the project proto, based + * on all artifacts that have been built. + */ +public class DependenciesProjectProtoUpdater implements ProjectProtoTransform { + private final ImmutableList updateOperations; + + public DependenciesProjectProtoUpdater(NewArtifactTracker dependencyTracker) { + ImmutableList.Builder updateOperations = + ImmutableList.builder() + .add(new AddCompiledJavaDeps(dependencyTracker::getBuiltDeps)); + this.updateOperations = updateOperations.build(); + } + + @Override + public Project apply(Project proto, BuildGraphData graph, Context context) + throws BuildException { + + ProjectProtoUpdate protoUpdate = new ProjectProtoUpdate(proto); + for (ProjectProtoUpdateOperation op : updateOperations) { + op.update(protoUpdate); + } + return protoUpdate.build(); + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdate.java b/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdate.java new file mode 100644 index 00000000000..98a6986ff2a --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdate.java @@ -0,0 +1,90 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import com.google.common.collect.Maps; +import com.google.idea.blaze.qsync.project.BlazeProjectDataStorage; +import com.google.idea.blaze.qsync.project.ProjectProto; +import com.google.idea.blaze.qsync.project.ProjectProto.Library; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; + +/** + * Helper class for making a number of updates to the project proto. + * + *

This class provides a convenient way of accessing and updating various interesting parts of + * the project proto, such as the {@code .workspace} module and libraries by name. + */ +public class ProjectProtoUpdate { + + private final ProjectProto.Project.Builder project; + private final ProjectProto.Module.Builder workspaceModule; + private final Map libraries = Maps.newHashMap(); + private final Map artifactDirs = Maps.newHashMap(); + + public ProjectProtoUpdate(ProjectProto.Project existingProject) { + this.project = existingProject.toBuilder(); + this.workspaceModule = getWorkspaceModuleBuilder(project); + } + + private static ProjectProto.Module.Builder getWorkspaceModuleBuilder( + ProjectProto.Project.Builder project) { + for (int i = 0; i < project.getModulesCount(); i++) { + if (project.getModules(i).getName().equals(BlazeProjectDataStorage.WORKSPACE_MODULE_NAME)) { + return project.getModulesBuilder(i); + } + } + throw new IllegalArgumentException( + "Module with name " + + BlazeProjectDataStorage.WORKSPACE_MODULE_NAME + + " not found in project proto."); + } + + public ProjectProto.Project.Builder project() { + return project; + } + + public ProjectProto.Module.Builder workspaceModule() { + return workspaceModule; + } + + /** Gets a builder for a library, creating it if it doesn't already exist. */ + public ProjectProto.Library.Builder library(String name) { + if (!libraries.containsKey(name)) { + Optional existingProto = + project.getLibraryBuilderList().stream() + .filter(l -> l.getName().equals(name)) + .findFirst(); + + if (existingProto.isPresent()) { + libraries.put(name, existingProto.get()); + } else { + libraries.put(name, project.addLibraryBuilder().setName(name)); + } + } + return libraries.get(name); + } + + public ArtifactDirectoryBuilder artifactDirectory(Path relativePath) { + return artifactDirs.computeIfAbsent(relativePath, ArtifactDirectoryBuilder::new); + } + + public ProjectProto.Project build() { + artifactDirs.values().forEach(d -> d.addTo(project.getArtifactDirectoriesBuilder())); + return project.build(); + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdateOperation.java b/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdateOperation.java new file mode 100644 index 00000000000..38919a7c9d8 --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdateOperation.java @@ -0,0 +1,31 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import com.google.common.collect.ImmutableSet; +import com.google.idea.blaze.exception.BuildException; + +/** + * An update to the project proto that operates on a {@link ProjectProtoUpdate}. Also defines some + * constants that are useful to implementations. + */ +public interface ProjectProtoUpdateOperation { + + String JAVA_DEPS_LIB_NAME = ".dependencies"; + ImmutableSet JAVA_ARCHIVE_EXTENSIONS = ImmutableSet.of("jar", "srcjar"); + + void update(ProjectProtoUpdate update) throws BuildException; +} diff --git a/querysync/java/com/google/idea/blaze/qsync/java/JavaArtifactInfo.java b/querysync/java/com/google/idea/blaze/qsync/java/JavaArtifactInfo.java index 321fee571fb..25856f5d008 100644 --- a/querysync/java/com/google/idea/blaze/qsync/java/JavaArtifactInfo.java +++ b/querysync/java/com/google/idea/blaze/qsync/java/JavaArtifactInfo.java @@ -64,6 +64,8 @@ public abstract class JavaArtifactInfo { public abstract String androidResourcesPackage(); + public abstract Builder toBuilder(); + public static Builder builder() { return new AutoValue_JavaArtifactInfo.Builder(); } diff --git a/querysync/javatests/com/google/idea/blaze/qsync/GraphToProjectConverters.java b/querysync/javatests/com/google/idea/blaze/qsync/GraphToProjectConverters.java index d1ecc6f6637..acd635ca63f 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/GraphToProjectConverters.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/GraphToProjectConverters.java @@ -80,7 +80,8 @@ public GraphToProjectConverter build() { info.projectExcludes(), info.languageClasses(), info.testSources()), - newDirectExecutorService()); + newDirectExecutorService(), + false); } } } diff --git a/querysync/javatests/com/google/idea/blaze/qsync/TestDataSyncRunner.java b/querysync/javatests/com/google/idea/blaze/qsync/TestDataSyncRunner.java index f4185f9e997..273ce4b5cd9 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/TestDataSyncRunner.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/TestDataSyncRunner.java @@ -41,7 +41,6 @@ public class TestDataSyncRunner { private final Context context; private final PackageReader packageReader; - public TestDataSyncRunner(Context context, PackageReader packageReader) { this.context = context; this.packageReader = packageReader; @@ -69,7 +68,8 @@ public BlazeProjectSnapshot sync(TestData testProject) throws IOException, Build Predicates.alwaysTrue(), context, projectDefinition, - newDirectExecutorService()); + newDirectExecutorService(), + false); Project project = converter.createProject(buildGraphData); return BlazeProjectSnapshot.builder() .queryData(pqsd) diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddCompiledJavaDepsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddCompiledJavaDepsTest.java new file mode 100644 index 00000000000..d4934aaa2dd --- /dev/null +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddCompiledJavaDepsTest.java @@ -0,0 +1,103 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.idea.blaze.common.Label; +import com.google.idea.blaze.qsync.artifacts.BuildArtifact; +import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.project.ProjectProto; +import com.google.idea.blaze.qsync.project.ProjectProto.ProjectArtifact.ArtifactTransform; +import com.google.idea.blaze.qsync.testdata.ProjectProtos; +import com.google.idea.blaze.qsync.testdata.TestData; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Optional; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class AddCompiledJavaDepsTest { + + @Test + public void no_deps_built() throws Exception { + ProjectProto.Project original = + ProjectProtos.forTestProjectWithNewArtifactLogic(TestData.JAVA_LIBRARY_EXTERNAL_DEP_QUERY); + + AddCompiledJavaDeps javaDeps = new AddCompiledJavaDeps(ImmutableList::of); + + ProjectProtoUpdate update = new ProjectProtoUpdate(original); + javaDeps.update(update); + ProjectProto.Project newProject = update.build(); + assertThat(newProject.getLibraryList()).isEqualTo(original.getLibraryList()); + assertThat(newProject.getModulesList()).isEqualTo(original.getModulesList()); + assertThat(newProject.getArtifactDirectories().getDirectoriesMap().keySet()) + .containsExactly("javadeps"); + assertThat( + newProject + .getArtifactDirectories() + .getDirectoriesMap() + .get("javadeps") + .getContentsMap()) + .isEmpty(); + } + + @Test + public void dep_built() throws Exception { + ProjectProto.Project original = + ProjectProtos.forTestProjectWithNewArtifactLogic(TestData.JAVA_LIBRARY_EXTERNAL_DEP_QUERY); + + TargetBuildInfo builtDep = + TargetBuildInfo.forJavaTarget( + JavaArtifactInfo.empty(Label.of("@com_google_guava_guava//jar:jar")).toBuilder() + .setJars( + ImmutableList.of( + BuildArtifact.create( + "jardigest", + Path.of("build-out/java/com/google/common/collect/libcollect.jar"), + Label.of("@com_google_guava_guava//jar:jar")))) + .build(), + DependencyBuildContext.create("abc-def", Instant.now(), Optional.empty())); + + AddCompiledJavaDeps javaDeps = new AddCompiledJavaDeps(() -> ImmutableList.of(builtDep)); + + ProjectProtoUpdate update = new ProjectProtoUpdate(original); + javaDeps.update(update); + ProjectProto.Project newProject = update.build(); + assertThat(newProject.getLibraryList()).hasSize(1); + assertThat(newProject.getLibrary(0).getName()).isEqualTo(".dependencies"); + assertThat(newProject.getLibrary(0).getClassesJarList()) + .containsExactly( + ProjectProto.JarDirectory.newBuilder().setPath("javadeps").setRecursive(true).build()); + assertThat(newProject.getArtifactDirectories().getDirectoriesMap().keySet()) + .containsExactly("javadeps"); + assertThat( + newProject + .getArtifactDirectories() + .getDirectoriesMap() + .get("javadeps") + .getContentsMap()) + .containsExactly( + "build-out/java/com/google/common/collect/libcollect.jar", + ProjectProto.ProjectArtifact.newBuilder() + .setBuildArtifact(ProjectProto.BuildArtifact.newBuilder().setDigest("jardigest")) + .setTransform(ArtifactTransform.COPY) + .build()); + } +} diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD index 47189cc0d7a..bcf23554ada 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD @@ -39,3 +39,21 @@ java_test( "@truth//jar", ], ) + +java_test( + name = "AddCompiledJavaDepsTest", + size = "small", + srcs = ["AddCompiledJavaDepsTest.java"], + deps = [ + "//querysync/java/com/google/idea/blaze/qsync/artifacts", + "//querysync/java/com/google/idea/blaze/qsync/deps", + "//querysync/java/com/google/idea/blaze/qsync/java", + "//querysync/java/com/google/idea/blaze/qsync/project:project_java_proto", + "//querysync/javatests/com/google/idea/blaze/qsync/testdata", + "//querysync/javatests/com/google/idea/blaze/qsync/testdata:project_protos", + "//shared", + "@com_google_guava_guava//jar", + "@junit//jar", + "@truth//jar", + ], +) diff --git a/querysync/javatests/com/google/idea/blaze/qsync/testdata/ProjectProtos.java b/querysync/javatests/com/google/idea/blaze/qsync/testdata/ProjectProtos.java index 2e3a00b0565..1997bfc503b 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/testdata/ProjectProtos.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/testdata/ProjectProtos.java @@ -40,6 +40,16 @@ public class ProjectProtos { private ProjectProtos() {} public static Project forTestProject(TestData project) throws IOException, BuildException { + return create(project, false); + } + + public static Project forTestProjectWithNewArtifactLogic(TestData project) + throws IOException, BuildException { + return create(project, true); + } + + private static Project create(TestData project, boolean useNewArtifactLogic) + throws IOException, BuildException { Path workspaceImportDirectory = project.getQueryOutputPath(); GraphToProjectConverter converter = new GraphToProjectConverter( @@ -51,8 +61,8 @@ public static Project forTestProject(TestData project) throws IOException, Build ImmutableSet.of(), ImmutableSet.of(QuerySyncLanguage.JAVA), ImmutableSet.of()), - newDirectExecutorService()); - + newDirectExecutorService(), + useNewArtifactLogic); return converter.createProject(BuildGraphs.forTestProject(project)); } } From 4843af001505e69076b73f04796f553908cb4487 Mon Sep 17 00:00:00 2001 From: Googler Date: Thu, 14 Mar 2024 03:00:16 -0700 Subject: [PATCH 09/45] Add dependency generated source jars to the project. This change implements this logic for the new artifact tracker, it already exists in the old codepath. (cherry picked from commit fe40bb9714a8356849c94ea606b1d5ae67c2443d) --- .../idea/blaze/base/qsync/ProjectLoader.java | 4 +- .../base/qsync/DependencyTrackerImplTest.java | 2 +- .../qsync/deps/AddDependencyGenSrcsJars.java | 85 ++++++++ .../deps/DependenciesProjectProtoUpdater.java | 27 ++- .../qsync/java/SrcJarInnerPathFinder.java | 105 +++++----- .../idea/blaze/qsync/TestDataSyncRunner.java | 8 +- .../deps/AddDependencyGenSrcsJarsTest.java | 181 ++++++++++++++++++ .../com/google/idea/blaze/qsync/deps/BUILD | 22 +++ 8 files changed, 385 insertions(+), 49 deletions(-) create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJars.java create mode 100644 querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java diff --git a/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java b/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java index 688deeccf3b..6aeb1c49fb8 100644 --- a/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java +++ b/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java @@ -155,7 +155,9 @@ public QuerySyncProject loadProject(BlazeContext context) throws BuildException NewArtifactTracker tracker = new NewArtifactTracker<>( BlazeDataStorage.getProjectDataDir(importSettings).toPath(), artifactCache); - projectTransformRegistry.add(new DependenciesProjectProtoUpdater(tracker)); + projectTransformRegistry.add( + new DependenciesProjectProtoUpdater( + tracker, latestProjectDef, artifactCache, QuerySync.ATTACH_DEP_SRCJARS::getValue)); artifactTracker = tracker; renderJarArtifactTracker = new RenderJarArtifactTrackerImpl(); diff --git a/base/tests/unittests/com/google/idea/blaze/base/qsync/DependencyTrackerImplTest.java b/base/tests/unittests/com/google/idea/blaze/base/qsync/DependencyTrackerImplTest.java index 9a2bd335b72..199dd465dcb 100644 --- a/base/tests/unittests/com/google/idea/blaze/base/qsync/DependencyTrackerImplTest.java +++ b/base/tests/unittests/com/google/idea/blaze/base/qsync/DependencyTrackerImplTest.java @@ -46,7 +46,7 @@ public class DependencyTrackerImplTest { public final BlazeContext context = BlazeContext.create(); public final TestDataSyncRunner syncRunner = - new TestDataSyncRunner(context, QuerySyncTestUtils.PATH_INFERRING_PACKAGE_READER); + new TestDataSyncRunner(context, QuerySyncTestUtils.PATH_INFERRING_PACKAGE_READER, false); public final BlazeProject blazeProject = new BlazeProject(); @Mock DependencyBuilder dependencyBuilder; diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJars.java b/querysync/java/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJars.java new file mode 100644 index 00000000000..a70597156b7 --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJars.java @@ -0,0 +1,85 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableCollection; +import com.google.idea.blaze.common.artifact.BuildArtifactCache; +import com.google.idea.blaze.exception.BuildException; +import com.google.idea.blaze.qsync.artifacts.BuildArtifact; +import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; +import com.google.idea.blaze.qsync.project.ProjectDefinition; +import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectProto.LibrarySource; +import java.nio.file.Path; + +/** + * Adds generated {@code .srcjar} files from external dependencies to the {@code .dependencies} + * library. This means that when navigating to these dependencies, we see the generated sources + * rather than decompiled code. + */ +public class AddDependencyGenSrcsJars implements ProjectProtoUpdateOperation { + + private final Supplier> builtTargetsSupplier; + private final BuildArtifactCache buildCache; + private final ProjectDefinition projectDefinition; + private final SrcJarInnerPathFinder srcJarInnerPathFinder; + + public AddDependencyGenSrcsJars( + Supplier> builtTargetsSupplier, + ProjectDefinition projectDefinition, + BuildArtifactCache buildCache, + SrcJarInnerPathFinder srcJarInnerPathFinder) { + this.builtTargetsSupplier = builtTargetsSupplier; + this.projectDefinition = projectDefinition; + this.buildCache = buildCache; + this.srcJarInnerPathFinder = srcJarInnerPathFinder; + } + + @Override + public void update(ProjectProtoUpdate update) throws BuildException { + for (TargetBuildInfo target : builtTargetsSupplier.get()) { + if (target.javaInfo().isEmpty()) { + continue; + } + JavaArtifactInfo javaInfo = target.javaInfo().get(); + if (projectDefinition.isIncluded(javaInfo.label())) { + continue; + } + for (BuildArtifact genSrc : javaInfo.genSrcs()) { + if (!JAVA_ARCHIVE_EXTENSIONS.contains(genSrc.getExtension())) { + continue; + } + + ProjectPath projectArtifact = + update + .artifactDirectory(Path.of("buildout")) + .addIfNewer(genSrc.path(), genSrc, target.buildContext()) + .orElse(null); + + if (projectArtifact != null) { + srcJarInnerPathFinder.findInnerJarPaths(genSrc.blockingGetFrom(buildCache)).stream() + .map(p -> p.path) + .map(projectArtifact::withInnerJarPath) + .map(ProjectPath::toProto) + .map(LibrarySource.newBuilder()::setSrcjar) + .forEach(update.library(JAVA_DEPS_LIB_NAME)::addSources); + } + } + } + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java index 9fc499003ac..4ae304fc7f4 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java @@ -15,11 +15,18 @@ */ package com.google.idea.blaze.qsync.deps; +import static com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder.AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY; + +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.idea.blaze.common.Context; +import com.google.idea.blaze.common.artifact.BuildArtifactCache; import com.google.idea.blaze.exception.BuildException; import com.google.idea.blaze.qsync.ProjectProtoTransform; +import com.google.idea.blaze.qsync.java.PackageStatementParser; +import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; import com.google.idea.blaze.qsync.project.BuildGraphData; +import com.google.idea.blaze.qsync.project.ProjectDefinition; import com.google.idea.blaze.qsync.project.ProjectProto.Project; /** @@ -29,10 +36,28 @@ public class DependenciesProjectProtoUpdater implements ProjectProtoTransform { private final ImmutableList updateOperations; - public DependenciesProjectProtoUpdater(NewArtifactTracker dependencyTracker) { + public DependenciesProjectProtoUpdater( + NewArtifactTracker dependencyTracker, + ProjectDefinition projectDefinition, + BuildArtifactCache artifactCache, + Supplier attachDepsSrcjarsExperiment) { + // Require empty package prefixes for srcjar inner paths, since the ultimate consumer of these + // paths does not support setting a package prefix (see `Library.ModifiableModel.addRoot`). + PackageStatementParser packageReader = new PackageStatementParser(); + SrcJarInnerPathFinder srcJarInnerPathFinder = + new SrcJarInnerPathFinder(packageReader, EMPTY_PACKAGE_PREFIXES_ONLY); + ImmutableList.Builder updateOperations = ImmutableList.builder() .add(new AddCompiledJavaDeps(dependencyTracker::getBuiltDeps)); + if (attachDepsSrcjarsExperiment.get()) { + updateOperations.add( + new AddDependencyGenSrcsJars( + dependencyTracker::getBuiltDeps, + projectDefinition, + artifactCache, + srcJarInnerPathFinder)); + } this.updateOperations = updateOperations.build(); } diff --git a/querysync/java/com/google/idea/blaze/qsync/java/SrcJarInnerPathFinder.java b/querysync/java/com/google/idea/blaze/qsync/java/SrcJarInnerPathFinder.java index 8d479161ee3..0c6b18a12ad 100644 --- a/querysync/java/com/google/idea/blaze/qsync/java/SrcJarInnerPathFinder.java +++ b/querysync/java/com/google/idea/blaze/qsync/java/SrcJarInnerPathFinder.java @@ -17,16 +17,18 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import com.google.common.io.ByteSource; +import java.io.BufferedInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; -import java.util.Enumeration; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; +import java.util.zip.ZipInputStream; /** Utility for finding inner paths of a source jar corresponding to package roots */ public class SrcJarInnerPathFinder { @@ -72,51 +74,66 @@ public SrcJarInnerPathFinder( } public ImmutableSet findInnerJarPaths(File jarFile) { + try (InputStream in = new FileInputStream(jarFile)) { + return findInnerJarPaths(in); + } catch (IOException ioe) { + logger.log(Level.WARNING, "Failed to examine " + jarFile, ioe); + // return the jar file root to ensure we don't ignore it. + return ImmutableSet.of(JarPath.create("", "")); + } + } + + public ImmutableSet findInnerJarPaths(ByteSource artifact) { + try (InputStream in = artifact.openBufferedStream()) { + return findInnerJarPaths(in); + } catch (IOException ioe) { + logger.log(Level.WARNING, "Failed to examine " + artifact, ioe); + // return the jar file root to ensure we don't ignore it. + return ImmutableSet.of(JarPath.create("", "")); + } + } + + private ImmutableSet findInnerJarPaths(InputStream jarFile) throws IOException { Set paths = Sets.newHashSet(); - try { - ZipFile zip = new ZipFile(jarFile); - Enumeration entries = zip.entries(); - Set topLevelPaths = Sets.newHashSet(); - while (entries.hasMoreElements()) { - ZipEntry e = entries.nextElement(); - if (e.isDirectory()) { - continue; - } - Path zipfilePath = Path.of(e.getName()); - if (!(zipfilePath.getFileName().toString().endsWith(".java") - || zipfilePath.getFileName().toString().endsWith(".kt"))) { - continue; - } - if (!topLevelPaths.add(zipfilePath.getName(0))) { - continue; - } - try (InputStream in = zip.getInputStream(e)) { - String pname = packageStatementParser.readPackage(in); - Path packageAsPath = Path.of(pname.replace('.', '/')); - Path zipPath = zipfilePath.getParent(); - if (zipPath == null) { - zipPath = Path.of(""); - } - if (zipPath.equals(packageAsPath)) { - // package root is the jar file root. - paths.add(JarPath.create("", "")); - } else if (zipPath.endsWith(packageAsPath)) { - paths.add( - JarPath.create( - zipPath.subpath(0, zipPath.getNameCount() - packageAsPath.getNameCount()), "")); - } else { - if (allowPackagePrefixes == AllowPackagePrefixes.ALLOW_NON_EMPTY_PACKAGE_PREFIXES) { - paths.add(JarPath.create(zipPath, pname)); - } else { - logger.log( - Level.WARNING, - "Java package name " + pname + " does not match srcjar path " + zipfilePath); - } - } + ZipInputStream zis = new ZipInputStream(new BufferedInputStream(jarFile)); + + ZipEntry e; + Set topLevelPaths = Sets.newHashSet(); + while ((e = zis.getNextEntry()) != null) { + if (e.isDirectory()) { + continue; + } + Path zipfilePath = Path.of(e.getName()); + if (!(zipfilePath.getFileName().toString().endsWith(".java") + || zipfilePath.getFileName().toString().endsWith(".kt"))) { + continue; + } + if (!topLevelPaths.add(zipfilePath.getName(0))) { + continue; + } + String pname = packageStatementParser.readPackage(zis); + Path packageAsPath = Path.of(pname.replace('.', '/')); + Path zipPath = zipfilePath.getParent(); + if (zipPath == null) { + zipPath = Path.of(""); + } + if (zipPath.equals(packageAsPath)) { + // package root is the jar file root. + paths.add(JarPath.create("", "")); + } else if (zipPath.endsWith(packageAsPath)) { + paths.add( + JarPath.create( + zipPath.subpath(0, zipPath.getNameCount() - packageAsPath.getNameCount()), "")); + } else { + if (allowPackagePrefixes == AllowPackagePrefixes.ALLOW_NON_EMPTY_PACKAGE_PREFIXES) { + paths.add(JarPath.create(zipPath, pname)); + } else { + logger.log( + Level.WARNING, + "Java package name " + pname + " does not match srcjar path " + zipfilePath); } } - } catch (IOException ioe) { - logger.log(Level.WARNING, "Failed to examine " + jarFile, ioe); + zis.closeEntry(); } if (paths.isEmpty()) { // we didn't find any java/kt sources. Add the jar file root to ensure we don't ignore it. diff --git a/querysync/javatests/com/google/idea/blaze/qsync/TestDataSyncRunner.java b/querysync/javatests/com/google/idea/blaze/qsync/TestDataSyncRunner.java index 273ce4b5cd9..b8fae119980 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/TestDataSyncRunner.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/TestDataSyncRunner.java @@ -41,9 +41,13 @@ public class TestDataSyncRunner { private final Context context; private final PackageReader packageReader; - public TestDataSyncRunner(Context context, PackageReader packageReader) { + private final boolean useNewArtifactLogic; + + public TestDataSyncRunner( + Context context, PackageReader packageReader, boolean useNewArtifactLogic) { this.context = context; this.packageReader = packageReader; + this.useNewArtifactLogic = useNewArtifactLogic; } public BlazeProjectSnapshot sync(TestData testProject) throws IOException, BuildException { @@ -69,7 +73,7 @@ public BlazeProjectSnapshot sync(TestData testProject) throws IOException, Build context, projectDefinition, newDirectExecutorService(), - false); + useNewArtifactLogic); Project project = converter.createProject(buildGraphData); return BlazeProjectSnapshot.builder() .queryData(pqsd) diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java new file mode 100644 index 00000000000..986534e1a66 --- /dev/null +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java @@ -0,0 +1,181 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.io.ByteSource; +import com.google.common.util.concurrent.Futures; +import com.google.idea.blaze.common.Label; +import com.google.idea.blaze.common.NoopContext; +import com.google.idea.blaze.common.artifact.BuildArtifactCache; +import com.google.idea.blaze.qsync.QuerySyncTestUtils; +import com.google.idea.blaze.qsync.TestDataSyncRunner; +import com.google.idea.blaze.qsync.artifacts.BuildArtifact; +import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.java.PackageStatementParser; +import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; +import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder.AllowPackagePrefixes; +import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.project.ProjectProto; +import com.google.idea.blaze.qsync.project.ProjectProto.Library; +import com.google.idea.blaze.qsync.project.ProjectProto.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectProto.ProjectPath.Base; +import com.google.idea.blaze.qsync.testdata.TestData; +import java.io.ByteArrayOutputStream; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Optional; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +@RunWith(JUnit4.class) +public class AddDependencyGenSrcsJarsTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock public BuildArtifactCache cache; + + private final TestDataSyncRunner syncer = + new TestDataSyncRunner( + new NoopContext(), QuerySyncTestUtils.PATH_INFERRING_PACKAGE_READER, true); + + @Test + public void no_deps_built() throws Exception { + + BlazeProjectSnapshot original = syncer.sync(TestData.JAVA_LIBRARY_EXTERNAL_DEP_QUERY); + + AddDependencyGenSrcsJars addGenSrcJars = + new AddDependencyGenSrcsJars( + ImmutableList::of, + original.queryData().projectDefinition(), + cache, + new SrcJarInnerPathFinder( + new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); + + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + + addGenSrcJars.update(update); + + ProjectProto.Project newProject = update.build(); + + assertThat(newProject.getLibraryList()).isEqualTo(original.project().getLibraryList()); + assertThat(newProject.getModulesList()).isEqualTo(original.project().getModulesList()); + assertThat(newProject.getArtifactDirectories().getDirectoriesMap().keySet()).isEmpty(); + } + + @Test + public void project_gensrcs_ignored() throws Exception { + + TestData testProject = TestData.JAVA_LIBRARY_EXTERNAL_DEP_QUERY; + + BlazeProjectSnapshot original = syncer.sync(testProject); + + TargetBuildInfo builtDep = + TargetBuildInfo.forJavaTarget( + JavaArtifactInfo.empty(testProject.getAssumedOnlyLabel()).toBuilder() + .setGenSrcs( + ImmutableList.of( + BuildArtifact.create( + "srcjardigest", + Path.of("output/path/to/in_project.srcjar"), + testProject.getAssumedOnlyLabel()))) + .build(), + DependencyBuildContext.create("abc-def", Instant.now(), Optional.empty())); + + AddDependencyGenSrcsJars addGenSrcJars = + new AddDependencyGenSrcsJars( + () -> ImmutableList.of(builtDep), + original.queryData().projectDefinition(), + cache, + new SrcJarInnerPathFinder( + new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); + + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + addGenSrcJars.update(update); + ProjectProto.Project newProject = update.build(); + + verify(cache, never()).get(ArgumentMatchers.any()); + + assertThat(newProject.getLibraryList()).isEqualTo(original.project().getLibraryList()); + assertThat(newProject.getModulesList()).isEqualTo(original.project().getModulesList()); + assertThat(newProject.getArtifactDirectories().getDirectoriesMap().keySet()).isEmpty(); + } + + @Test + public void external_gensrcs_added() throws Exception { + BlazeProjectSnapshot original = syncer.sync(TestData.JAVA_LIBRARY_EXTERNAL_DEP_QUERY); + + ByteArrayOutputStream zipFile = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(zipFile)) { + zos.putNextEntry(new ZipEntry("root/com/org/Class.java")); + zos.write("package com.org;\npublic class Class{}".getBytes(UTF_8)); + } + + when(cache.get("srcjardigest")) + .thenReturn(Optional.of(Futures.immediateFuture(ByteSource.wrap(zipFile.toByteArray())))); + + TargetBuildInfo builtDep = + TargetBuildInfo.forJavaTarget( + JavaArtifactInfo.empty(Label.of("@com_google_guava_guava//jar:jar")).toBuilder() + .setGenSrcs( + ImmutableList.of( + BuildArtifact.create( + "srcjardigest", + Path.of("output/path/to/external.srcjar"), + Label.of("@com_google_guava_guava//jar:jar")))) + .build(), + DependencyBuildContext.create("abc-def", Instant.now(), Optional.empty())); + + AddDependencyGenSrcsJars addGenSrcJars = + new AddDependencyGenSrcsJars( + () -> ImmutableList.of(builtDep), + original.queryData().projectDefinition(), + cache, + new SrcJarInnerPathFinder( + new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); + + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + addGenSrcJars.update(update); + ProjectProto.Project newProject = update.build(); + + assertThat(newProject.getLibraryList()).hasSize(1); + Library depsLib = newProject.getLibrary(0); + assertThat(depsLib.getName()).isEqualTo(".dependencies"); + assertThat(depsLib.getSourcesList()) + .containsExactly( + ProjectProto.LibrarySource.newBuilder() + .setSrcjar( + ProjectPath.newBuilder() + .setBase(Base.PROJECT) + .setPath("buildout/output/path/to/external.srcjar") + .setInnerPath("root")) + .build()); + } +} diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD index bcf23554ada..cac202d6a14 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD @@ -57,3 +57,25 @@ java_test( "@truth//jar", ], ) + +java_test( + name = "AddDependencyGenSrcsJarsTest", + size = "small", + srcs = ["AddDependencyGenSrcsJarsTest.java"], + deps = [ + "//querysync/java/com/google/idea/blaze/qsync/artifacts", + "//querysync/java/com/google/idea/blaze/qsync/deps", + "//querysync/java/com/google/idea/blaze/qsync/java", + "//querysync/java/com/google/idea/blaze/qsync/project", + "//querysync/java/com/google/idea/blaze/qsync/project:project_java_proto", + "//querysync/javatests/com/google/idea/blaze/qsync:TestUtils", + "//querysync/javatests/com/google/idea/blaze/qsync/testdata", + "//shared", + "//shared:artifact", + "//shared/javatests/com/google/idea/blaze/common:test_utils", + "@com_google_guava_guava//jar", + "@junit//jar", + "@mockito//jar", + "@truth//jar", + ], +) From 1901a0e482e3cb9f8e5e7e021ac70d5d8729551c Mon Sep 17 00:00:00 2001 From: Googler Date: Thu, 14 Mar 2024 03:06:07 -0700 Subject: [PATCH 10/45] Add checked-in source jars from dependencies to the project. (cherry picked from commit 45360ae49c1946ad0993c164943de9abb427c0a8) --- .../idea/blaze/base/qsync/ProjectLoader.java | 6 +- .../qsync/deps/AddDependencySrcJars.java | 72 +++++++++ .../deps/DependenciesProjectProtoUpdater.java | 8 + .../qsync/deps/AddDependencySrcJarsTest.java | 141 ++++++++++++++++++ .../com/google/idea/blaze/qsync/deps/BUILD | 19 +++ 5 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/AddDependencySrcJars.java create mode 100644 querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java diff --git a/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java b/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java index 6aeb1c49fb8..ec3cff9aefb 100644 --- a/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java +++ b/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java @@ -157,7 +157,11 @@ public QuerySyncProject loadProject(BlazeContext context) throws BuildException BlazeDataStorage.getProjectDataDir(importSettings).toPath(), artifactCache); projectTransformRegistry.add( new DependenciesProjectProtoUpdater( - tracker, latestProjectDef, artifactCache, QuerySync.ATTACH_DEP_SRCJARS::getValue)); + tracker, + latestProjectDef, + artifactCache, + projectPathResolver, + QuerySync.ATTACH_DEP_SRCJARS::getValue)); artifactTracker = tracker; renderJarArtifactTracker = new RenderJarArtifactTrackerImpl(); diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/AddDependencySrcJars.java b/querysync/java/com/google/idea/blaze/qsync/deps/AddDependencySrcJars.java new file mode 100644 index 00000000000..45464465b5a --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/AddDependencySrcJars.java @@ -0,0 +1,72 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableCollection; +import com.google.idea.blaze.exception.BuildException; +import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; +import com.google.idea.blaze.qsync.project.ProjectDefinition; +import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectProto.LibrarySource; +import java.nio.file.Path; + +/** + * Adds checked-in {@code .srcjar} files from external dependencies to the project proto. This + * allows those sources to be shown in the IDE instead of decompiled class files. + */ +public class AddDependencySrcJars implements ProjectProtoUpdateOperation { + + private final Supplier> builtTargetsSupplier; + private final ProjectDefinition projectDefinition; + private final ProjectPath.Resolver pathResolver; + private final SrcJarInnerPathFinder srcJarInnerPathFinder; + + public AddDependencySrcJars( + Supplier> builtTargetsSupplier, + ProjectDefinition projectDefinition, + ProjectPath.Resolver pathResolver, + SrcJarInnerPathFinder srcJarInnerPathFinder) { + this.builtTargetsSupplier = builtTargetsSupplier; + this.projectDefinition = projectDefinition; + this.pathResolver = pathResolver; + this.srcJarInnerPathFinder = srcJarInnerPathFinder; + } + + @Override + public void update(ProjectProtoUpdate update) throws BuildException { + for (TargetBuildInfo target : builtTargetsSupplier.get()) { + if (target.javaInfo().isEmpty()) { + continue; + } + JavaArtifactInfo javaInfo = target.javaInfo().get(); + if (projectDefinition.isIncluded(javaInfo.label())) { + continue; + } + for (Path srcJar : javaInfo.srcJars()) { + // these are workspace relative srcjar paths. + ProjectPath jarPath = ProjectPath.workspaceRelative(srcJar); + srcJarInnerPathFinder.findInnerJarPaths(pathResolver.resolve(jarPath).toFile()).stream() + .map(p -> p.path) + .map(jarPath::withInnerJarPath) + .map(ProjectPath::toProto) + .map(LibrarySource.newBuilder()::setSrcjar) + .forEach(update.library(JAVA_DEPS_LIB_NAME)::addSources); + } + } + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java index 4ae304fc7f4..dba40d7210c 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java @@ -27,6 +27,7 @@ import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; import com.google.idea.blaze.qsync.project.BuildGraphData; import com.google.idea.blaze.qsync.project.ProjectDefinition; +import com.google.idea.blaze.qsync.project.ProjectPath; import com.google.idea.blaze.qsync.project.ProjectProto.Project; /** @@ -40,6 +41,7 @@ public DependenciesProjectProtoUpdater( NewArtifactTracker dependencyTracker, ProjectDefinition projectDefinition, BuildArtifactCache artifactCache, + ProjectPath.Resolver pathResolver, Supplier attachDepsSrcjarsExperiment) { // Require empty package prefixes for srcjar inner paths, since the ultimate consumer of these // paths does not support setting a package prefix (see `Library.ModifiableModel.addRoot`). @@ -51,6 +53,12 @@ public DependenciesProjectProtoUpdater( ImmutableList.builder() .add(new AddCompiledJavaDeps(dependencyTracker::getBuiltDeps)); if (attachDepsSrcjarsExperiment.get()) { + updateOperations.add( + new AddDependencySrcJars( + dependencyTracker::getBuiltDeps, + projectDefinition, + pathResolver, + srcJarInnerPathFinder)); updateOperations.add( new AddDependencyGenSrcsJars( dependencyTracker::getBuiltDeps, diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java new file mode 100644 index 00000000000..69c755d48bc --- /dev/null +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java @@ -0,0 +1,141 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.idea.blaze.common.Label; +import com.google.idea.blaze.common.NoopContext; +import com.google.idea.blaze.qsync.QuerySyncTestUtils; +import com.google.idea.blaze.qsync.TestDataSyncRunner; +import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.java.PackageStatementParser; +import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; +import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder.AllowPackagePrefixes; +import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectProto; +import com.google.idea.blaze.qsync.project.ProjectProto.Library; +import com.google.idea.blaze.qsync.project.ProjectProto.ProjectPath.Base; +import com.google.idea.blaze.qsync.testdata.TestData; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Optional; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class AddDependencySrcJarsTest { + + @Rule public final TemporaryFolder tempDir = new TemporaryFolder(); + private Path workspaceRoot; + private ProjectPath.Resolver pathResolver; + + private final TestDataSyncRunner syncer = + new TestDataSyncRunner( + new NoopContext(), QuerySyncTestUtils.PATH_INFERRING_PACKAGE_READER, true); + + @Before + public void createDirs() throws IOException { + workspaceRoot = tempDir.newFolder("workspace").toPath(); + pathResolver = + ProjectPath.Resolver.create(workspaceRoot, tempDir.newFolder("project").toPath()); + } + + @Test + public void no_deps_built() throws Exception { + BlazeProjectSnapshot original = syncer.sync(TestData.JAVA_LIBRARY_EXTERNAL_DEP_QUERY); + + AddDependencySrcJars addSrcJars = + new AddDependencySrcJars( + ImmutableList::of, + original.queryData().projectDefinition(), + pathResolver, + new SrcJarInnerPathFinder( + new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); + + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + + addSrcJars.update(update); + + ProjectProto.Project newProject = update.build(); + + assertThat(newProject.getLibraryList()).isEqualTo(original.project().getLibraryList()); + assertThat(newProject.getModulesList()).isEqualTo(original.project().getModulesList()); + assertThat(newProject.getArtifactDirectories().getDirectoriesMap().keySet()).isEmpty(); + } + + @Test + public void external_srcjar_added() throws Exception { + BlazeProjectSnapshot original = syncer.sync(TestData.JAVA_LIBRARY_EXTERNAL_DEP_QUERY); + + try (ZipOutputStream zos = + new ZipOutputStream( + new FileOutputStream( + Files.createDirectories(workspaceRoot.resolve("source/path")) + .resolve("external.srcjar") + .toFile()))) { + zos.putNextEntry(new ZipEntry("root/com/pkg/Class.java")); + zos.write("package com.pkg;\nclass Class {}".getBytes(UTF_8)); + } + + TargetBuildInfo builtDep = + TargetBuildInfo.forJavaTarget( + JavaArtifactInfo.empty(Label.of("@com_google_guava_guava//jar:jar")).toBuilder() + .setSrcJars(ImmutableSet.of(Path.of("source/path/external.srcjar"))) + .build(), + DependencyBuildContext.create("abc-def", Instant.now(), Optional.empty())); + + AddDependencySrcJars addSrcJars = + new AddDependencySrcJars( + () -> ImmutableList.of(builtDep), + original.queryData().projectDefinition(), + pathResolver, + new SrcJarInnerPathFinder( + new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); + + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + + addSrcJars.update(update); + + ProjectProto.Project newProject = update.build(); + + assertThat(newProject.getLibraryList()).hasSize(1); + Library depsLib = newProject.getLibrary(0); + assertThat(depsLib.getName()).isEqualTo(".dependencies"); + assertThat(depsLib.getSourcesList()) + .containsExactly( + ProjectProto.LibrarySource.newBuilder() + .setSrcjar( + ProjectProto.ProjectPath.newBuilder() + .setBase(Base.WORKSPACE) + .setPath("source/path/external.srcjar") + .setInnerPath("root")) + .build()); + } +} diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD index cac202d6a14..be4e9a255ac 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD @@ -79,3 +79,22 @@ java_test( "@truth//jar", ], ) + +java_test( + name = "AddDependencySrcJarsTest", + size = "small", + srcs = ["AddDependencySrcJarsTest.java"], + deps = [ + "//querysync/java/com/google/idea/blaze/qsync/deps", + "//querysync/java/com/google/idea/blaze/qsync/java", + "//querysync/java/com/google/idea/blaze/qsync/project", + "//querysync/java/com/google/idea/blaze/qsync/project:project_java_proto", + "//querysync/javatests/com/google/idea/blaze/qsync:TestUtils", + "//querysync/javatests/com/google/idea/blaze/qsync/testdata", + "//shared", + "//shared/javatests/com/google/idea/blaze/common:test_utils", + "@com_google_guava_guava//jar", + "@junit//jar", + "@truth//jar", + ], +) From 93abc31d12d8170ea4f2072cae9091d278524f54 Mon Sep 17 00:00:00 2001 From: Googler Date: Thu, 14 Mar 2024 03:14:07 -0700 Subject: [PATCH 11/45] Support for in-project generated srcjars. Add any generated srcjars to the project so that the contained sources are available in the IDE. (cherry picked from commit 5e24640f6e6ce52095431e79eed1602a0f8f705f) --- .../qsync/deps/AddProjectGenSrcJars.java | 94 ++++++++++ .../deps/DependenciesProjectProtoUpdater.java | 8 +- .../qsync/deps/AddProjectGenSrcJarsTest.java | 161 ++++++++++++++++++ .../com/google/idea/blaze/qsync/deps/BUILD | 23 +++ 4 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJars.java create mode 100644 querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJars.java b/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJars.java new file mode 100644 index 00000000000..96a41e45862 --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJars.java @@ -0,0 +1,94 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableCollection; +import com.google.idea.blaze.common.artifact.BuildArtifactCache; +import com.google.idea.blaze.exception.BuildException; +import com.google.idea.blaze.qsync.TestSourceGlobMatcher; +import com.google.idea.blaze.qsync.artifacts.BuildArtifact; +import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; +import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder.JarPath; +import com.google.idea.blaze.qsync.project.ProjectDefinition; +import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectProto; +import java.nio.file.Path; + +/** + * Adds in-project generated {@code .srcjar} files to the project proto. This allows these sources + * to be resolved and viewed. + */ +public class AddProjectGenSrcJars implements ProjectProtoUpdateOperation { + + private final Supplier> builtTargetsSupplier; + private final BuildArtifactCache buildCache; + private final ProjectDefinition projectDefinition; + private final SrcJarInnerPathFinder srcJarInnerPathFinder; + private final TestSourceGlobMatcher testSourceMatcher; + + public AddProjectGenSrcJars( + Supplier> builtTargetsSupplier, + ProjectDefinition projectDefinition, + BuildArtifactCache buildCache, + SrcJarInnerPathFinder srcJarInnerPathFinder) { + this.builtTargetsSupplier = builtTargetsSupplier; + this.projectDefinition = projectDefinition; + this.buildCache = buildCache; + this.srcJarInnerPathFinder = srcJarInnerPathFinder; + testSourceMatcher = TestSourceGlobMatcher.create(projectDefinition); + } + + @Override + public void update(ProjectProtoUpdate update) throws BuildException { + for (TargetBuildInfo target : builtTargetsSupplier.get()) { + if (target.javaInfo().isEmpty()) { + continue; + } + JavaArtifactInfo javaInfo = target.javaInfo().get(); + if (!projectDefinition.isIncluded(javaInfo.label())) { + continue; + } + for (BuildArtifact genSrc : javaInfo.genSrcs()) { + if (JAVA_ARCHIVE_EXTENSIONS.contains(genSrc.getExtension())) { + // a zip of generated sources + ProjectPath added = + update + .artifactDirectory(Path.of("buildout")) + .addIfNewer(genSrc.path(), genSrc, target.buildContext()) + .orElse(null); + if (added != null) { + ProjectProto.ContentEntry.Builder genSrcJarContentEntry = + ProjectProto.ContentEntry.newBuilder().setRoot(added.toProto()); + for (JarPath innerPath : + srcJarInnerPathFinder.findInnerJarPaths(genSrc.blockingGetFrom(buildCache))) { + + genSrcJarContentEntry.addSources( + ProjectProto.SourceFolder.newBuilder() + .setProjectPath(added.withInnerJarPath(innerPath.path).toProto()) + .setIsGenerated(true) + .setIsTest(testSourceMatcher.matches(genSrc.target().getPackage())) + .setPackagePrefix(innerPath.packagePrefix) + .build()); + } + update.workspaceModule().addContentEntries(genSrcJarContentEntry.build()); + } + } + } + } + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java index dba40d7210c..cddab88569d 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java @@ -51,7 +51,13 @@ public DependenciesProjectProtoUpdater( ImmutableList.Builder updateOperations = ImmutableList.builder() - .add(new AddCompiledJavaDeps(dependencyTracker::getBuiltDeps)); + .add(new AddCompiledJavaDeps(dependencyTracker::getBuiltDeps)) + .add( + new AddProjectGenSrcJars( + dependencyTracker::getBuiltDeps, + projectDefinition, + artifactCache, + srcJarInnerPathFinder)); if (attachDepsSrcjarsExperiment.get()) { updateOperations.add( new AddDependencySrcJars( diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java new file mode 100644 index 00000000000..ef1aac0b974 --- /dev/null +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java @@ -0,0 +1,161 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.io.ByteSource; +import com.google.idea.blaze.common.Label; +import com.google.idea.blaze.common.NoopContext; +import com.google.idea.blaze.common.artifact.BuildArtifactCache; +import com.google.idea.blaze.qsync.QuerySyncTestUtils; +import com.google.idea.blaze.qsync.TestDataSyncRunner; +import com.google.idea.blaze.qsync.artifacts.BuildArtifact; +import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.java.PackageStatementParser; +import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; +import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder.AllowPackagePrefixes; +import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.project.ProjectProto; +import com.google.idea.blaze.qsync.project.ProjectProto.ContentEntry; +import com.google.idea.blaze.qsync.project.ProjectProto.Module; +import com.google.idea.blaze.qsync.testdata.TestData; +import com.google.protobuf.TextFormat; +import java.io.ByteArrayOutputStream; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Optional; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +@RunWith(JUnit4.class) +public class AddProjectGenSrcJarsTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock public BuildArtifactCache cache; + + private final TestDataSyncRunner syncer = + new TestDataSyncRunner( + new NoopContext(), QuerySyncTestUtils.PATH_INFERRING_PACKAGE_READER, true); + + @Test + public void external_srcjar_ignored() throws Exception { + BlazeProjectSnapshot original = syncer.sync(TestData.JAVA_LIBRARY_EXTERNAL_DEP_QUERY); + + TargetBuildInfo builtDep = + TargetBuildInfo.forJavaTarget( + JavaArtifactInfo.empty(Label.of("@com_google_guava_guava//jar:jar")).toBuilder() + .setGenSrcs( + ImmutableList.of( + BuildArtifact.create( + "srcjardigest", + Path.of("output/path/to/external.srcjar"), + Label.of("@com_google_guava_guava//jar:jar")))) + .build(), + DependencyBuildContext.create("abc-def", Instant.now(), Optional.empty())); + + AddProjectGenSrcJars javaDeps = + new AddProjectGenSrcJars( + () -> ImmutableList.of(builtDep), + original.queryData().projectDefinition(), + cache, + new SrcJarInnerPathFinder( + new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); + + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + javaDeps.update(update); + ProjectProto.Project newProject = update.build(); + assertThat(newProject.getLibraryList()).isEqualTo(original.project().getLibraryList()); + assertThat(newProject.getModulesList()).isEqualTo(original.project().getModulesList()); + assertThat(newProject.getArtifactDirectories().getDirectoriesMap().keySet()).isEmpty(); + verifyNoInteractions(cache); + } + + @Test + public void project_srcjar_ignored() throws Exception { + TestData testData = TestData.JAVA_LIBRARY_EXTERNAL_DEP_QUERY; + BlazeProjectSnapshot original = syncer.sync(testData); + + ByteArrayOutputStream zipFile = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(zipFile)) { + zos.putNextEntry(new ZipEntry("root/com/org/Class.java")); + zos.write("package com.org;\npublic class Class{}".getBytes(UTF_8)); + } + when(cache.get("srcjardigest")) + .thenReturn(Optional.of(immediateFuture(ByteSource.wrap(zipFile.toByteArray())))); + + TargetBuildInfo builtDep = + TargetBuildInfo.forJavaTarget( + JavaArtifactInfo.empty(testData.getAssumedOnlyLabel()).toBuilder() + .setGenSrcs( + ImmutableList.of( + BuildArtifact.create( + "srcjardigest", + Path.of("output/path/to/project.srcjar"), + testData.getAssumedOnlyLabel()))) + .build(), + DependencyBuildContext.create("abc-def", Instant.now(), Optional.empty())); + + AddProjectGenSrcJars javaDeps = + new AddProjectGenSrcJars( + () -> ImmutableList.of(builtDep), + original.queryData().projectDefinition(), + cache, + new SrcJarInnerPathFinder( + new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); + + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + javaDeps.update(update); + ProjectProto.Project newProject = update.build(); + assertThat(newProject.getLibraryList()).isEqualTo(original.project().getLibraryList()); + Module workspace = newProject.getModules(0); + // check our assumptions: + assertThat(workspace.getName()).isEqualTo(".workspace"); + + assertThat(workspace.getContentEntriesList()) + .contains( + TextFormat.parse( + Joiner.on("\n") + .join( + "root {", + " path: \"buildout/output/path/to/project.srcjar\"", + " base: PROJECT", + " }", + " sources {", + " is_generated: true", + " project_path {", + " path: \"buildout/output/path/to/project.srcjar\"", + " base: PROJECT", + " inner_path: \"root\"", + " }", + " }"), + ContentEntry.class)); + } +} diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD index be4e9a255ac..683084c2bd3 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD @@ -98,3 +98,26 @@ java_test( "@truth//jar", ], ) + +java_test( + name = "AddProjectGenSrcJarsTest", + size = "small", + srcs = ["AddProjectGenSrcJarsTest.java"], + deps = [ + "//querysync/java/com/google/idea/blaze/qsync/artifacts", + "//querysync/java/com/google/idea/blaze/qsync/deps", + "//querysync/java/com/google/idea/blaze/qsync/java", + "//querysync/java/com/google/idea/blaze/qsync/project", + "//querysync/java/com/google/idea/blaze/qsync/project:project_java_proto", + "//querysync/javatests/com/google/idea/blaze/qsync:TestUtils", + "//querysync/javatests/com/google/idea/blaze/qsync/testdata", + "//shared", + "//shared:artifact", + "//shared/javatests/com/google/idea/blaze/common:test_utils", + "@com_google_guava_guava//jar", + "@com_google_protobuf//:protobuf_java", + "@junit//jar", + "@mockito//jar", + "@truth//jar", + ], +) From 84aea272940ba8e6a03543c8b36ad34cb8a55424 Mon Sep 17 00:00:00 2001 From: Googler Date: Thu, 14 Mar 2024 03:17:07 -0700 Subject: [PATCH 12/45] Support for in-project generated sources. This implements this logic for the new artifact tracker. It also makes some improvements on the old logic, made possible by the new structure: - we now detect conflicts between sequential builds, not just within one build - we provide more context about the conflictng source files. Example output from conflicts resolution: ``` WARNING: your project contains conflicting generated java sources for: com/example/BuildConfig.java From: k8-opt/genfiles/third_party/intellij/bazel/plugin/aswb/testdata/projects/gensrc_conflict/java/com/example/deb/com/example/BuildConfig.java (//third_party/intellij/bazel/plugin/aswb/testdata/projects/gensrc_conflict/java/com/example:test2 built 1 seconds ago) k8-opt/genfiles/third_party/intellij/bazel/plugin/aswb/testdata/projects/gensrc_conflict/java/com/example/rel/com/example/BuildConfig.java (//third_party/intellij/bazel/plugin/aswb/testdata/projects/gensrc_conflict/java/com/example:test1 built 1 Hours ago) ``` (cherry picked from commit e9504a78bc6dd07b44e9a25b96c9defe37d0250a) --- .../blaze/qsync/deps/AddProjectGenSrcs.java | 196 +++++++++++++++ .../deps/DependenciesProjectProtoUpdater.java | 10 +- .../blaze/qsync/deps/ProjectProtoUpdate.java | 9 +- .../qsync/deps/AddCompiledJavaDepsTest.java | 5 +- .../deps/AddDependencyGenSrcsJarsTest.java | 6 +- .../qsync/deps/AddDependencySrcJarsTest.java | 4 +- .../qsync/deps/AddProjectGenSrcJarsTest.java | 4 +- .../qsync/deps/AddProjectGenSrcsTest.java | 233 ++++++++++++++++++ .../com/google/idea/blaze/qsync/deps/BUILD | 24 ++ 9 files changed, 479 insertions(+), 12 deletions(-) create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcs.java create mode 100644 querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcsTest.java diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcs.java b/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcs.java new file mode 100644 index 00000000000..ad1c6470538 --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcs.java @@ -0,0 +1,196 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import static java.util.stream.Collectors.joining; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Supplier; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.idea.blaze.common.PrintOutput; +import com.google.idea.blaze.common.artifact.BuildArtifactCache; +import com.google.idea.blaze.exception.BuildException; +import com.google.idea.blaze.qsync.TestSourceGlobMatcher; +import com.google.idea.blaze.qsync.artifacts.BuildArtifact; +import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.java.PackageStatementParser; +import com.google.idea.blaze.qsync.project.ProjectDefinition; +import com.google.idea.blaze.qsync.project.ProjectProto; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collection; + +/** + * Adds generated java and kotlin source files to the project proto. + * + *

This class also resolves conflicts between multiple generated source files that resolve to the + * same output path, i.e. that have the same java class name. + */ +public class AddProjectGenSrcs implements ProjectProtoUpdateOperation { + + private static final ImmutableSet JAVA_SRC_EXTENSIONS = ImmutableSet.of("java", "kt"); + + private final Supplier> builtTargetsSupplier; + private final BuildArtifactCache buildCache; + private final ProjectDefinition projectDefinition; + private final PackageStatementParser packageReader; + private final TestSourceGlobMatcher testSourceMatcher; + + public AddProjectGenSrcs( + Supplier> builtTargetsSupplier, + ProjectDefinition projectDefinition, + BuildArtifactCache buildCache, + PackageStatementParser packageReader) { + this.builtTargetsSupplier = builtTargetsSupplier; + this.projectDefinition = projectDefinition; + this.buildCache = buildCache; + this.packageReader = packageReader; + testSourceMatcher = TestSourceGlobMatcher.create(projectDefinition); + } + + /** + * A simple holder class for a build artifact and info about the build that produced it. This is + * used to resolve conflicts between source files. + */ + @AutoValue + abstract static class ArtifactWithOrigin { + abstract BuildArtifact artifact(); + + abstract DependencyBuildContext origin(); + + static ArtifactWithOrigin create(BuildArtifact artifact, DependencyBuildContext origin) { + return new AutoValue_AddProjectGenSrcs_ArtifactWithOrigin(artifact, origin); + } + + /** + * When we find conflicting generated sources (same java source path), we resolve the conflict + * by selecting the per this method. + * + *

If the files were produced by different build invocations, select the most recent. + * Otherwise, disambiguate using the target string. + */ + int compareTo(ArtifactWithOrigin other) { + // Note: we do a reverse comparison for start time to ensure the newest build "wins". + int compare = other.origin().startTime().compareTo(origin().startTime()); + if (compare == 0) { + compare = artifact().target().toString().compareTo(other.artifact().target().toString()); + } + return compare; + } + } + + @Override + public void update(ProjectProtoUpdate update) throws BuildException { + ArtifactDirectoryBuilder javaSrc = update.artifactDirectory(Path.of("gensrc/java")); + ArtifactDirectoryBuilder javatestsSrc = update.artifactDirectory(Path.of("gensrc/javatests")); + ArrayListMultimap srcsByJavaPath = ArrayListMultimap.create(); + for (TargetBuildInfo target : builtTargetsSupplier.get()) { + if (target.javaInfo().isEmpty()) { + continue; + } + JavaArtifactInfo javaInfo = target.javaInfo().get(); + if (!projectDefinition.isIncluded(javaInfo.label())) { + continue; + } + for (BuildArtifact genSrc : javaInfo.genSrcs()) { + if (JAVA_SRC_EXTENSIONS.contains(genSrc.getExtension())) { + String javaPackage = readJavaPackage(genSrc); + Path finalDest = + Path.of(javaPackage.replace('.', '/')).resolve(genSrc.path().getFileName()); + srcsByJavaPath.put(finalDest, ArtifactWithOrigin.create(genSrc, target.buildContext())); + } + } + } + for (var entry : srcsByJavaPath.asMap().entrySet()) { + Path finalDest = entry.getKey(); + Collection candidates = entry.getValue(); + if (candidates.size() > 1) { + update + .context() + .output( + PrintOutput.error( + "WARNING: your project contains conflicting generated java sources for:\n" + + " %s\n" + + "From:\n" + + " %s", + finalDest, + candidates.stream() + .map( + a -> + String.format( + "%s (%s built %s ago)", + a.artifact().path(), + a.artifact().target(), + formatDuration( + Duration.between(a.origin().startTime(), Instant.now())))) + .collect(joining("\n ")))); + update.context().setHasWarnings(); + } + + ArtifactWithOrigin chosen = candidates.stream().min((a, b) -> a.compareTo(b)).orElseThrow(); + if (testSourceMatcher.matches(chosen.artifact().target().getPackage())) { + javatestsSrc.addIfNewer(finalDest, chosen.artifact(), chosen.origin()); + } else { + javaSrc.addIfNewer(finalDest, chosen.artifact(), chosen.origin()); + } + } + for (ArtifactDirectoryBuilder gensrcDir : ImmutableList.of(javaSrc, javatestsSrc)) { + if (!gensrcDir.isEmpty()) { + ProjectProto.ProjectPath pathProto = gensrcDir.root().toProto(); + ProjectProto.ContentEntry.Builder genSourcesContentEntry = + ProjectProto.ContentEntry.newBuilder().setRoot(pathProto); + genSourcesContentEntry.addSources( + ProjectProto.SourceFolder.newBuilder() + .setProjectPath(pathProto) + .setIsGenerated(true) + .setIsTest(gensrcDir == javatestsSrc) + .setPackagePrefix("")); + update.workspaceModule().addContentEntries(genSourcesContentEntry.build()); + } + } + } + + /** + * A simple inexact duration format, returning a duration in whichever unit of (days, hours, + * minutes, seconds) is the first to get a non-zero figure. + */ + private String formatDuration(Duration p) { + for (ChronoUnit unit : + ImmutableList.of(ChronoUnit.DAYS, ChronoUnit.HOURS, ChronoUnit.MINUTES)) { + long durationInUnits = p.getSeconds() / unit.getDuration().getSeconds(); + if (durationInUnits > 0) { + return String.format("%d %s", durationInUnits, unit); + } + } + return String.format("%d seconds", p.getSeconds()); + } + + /** Parses the java package statement from a build artifact. */ + private String readJavaPackage(BuildArtifact genSrc) throws BuildException { + try (InputStream javaSrcStream = genSrc.blockingGetFrom(buildCache).openStream()) { + return packageReader.readPackage(javaSrcStream); + } catch (IOException e) { + throw new BuildException("Failed to read package name for " + genSrc, e); + } + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java index cddab88569d..7027abda176 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java @@ -57,7 +57,13 @@ public DependenciesProjectProtoUpdater( dependencyTracker::getBuiltDeps, projectDefinition, artifactCache, - srcJarInnerPathFinder)); + srcJarInnerPathFinder)) + .add( + new AddProjectGenSrcs( + dependencyTracker::getBuiltDeps, + projectDefinition, + artifactCache, + packageReader)); if (attachDepsSrcjarsExperiment.get()) { updateOperations.add( new AddDependencySrcJars( @@ -79,7 +85,7 @@ public DependenciesProjectProtoUpdater( public Project apply(Project proto, BuildGraphData graph, Context context) throws BuildException { - ProjectProtoUpdate protoUpdate = new ProjectProtoUpdate(proto); + ProjectProtoUpdate protoUpdate = new ProjectProtoUpdate(proto, context); for (ProjectProtoUpdateOperation op : updateOperations) { op.update(protoUpdate); } diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdate.java b/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdate.java index 98a6986ff2a..d5e6cd66171 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdate.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdate.java @@ -16,6 +16,7 @@ package com.google.idea.blaze.qsync.deps; import com.google.common.collect.Maps; +import com.google.idea.blaze.common.Context; import com.google.idea.blaze.qsync.project.BlazeProjectDataStorage; import com.google.idea.blaze.qsync.project.ProjectProto; import com.google.idea.blaze.qsync.project.ProjectProto.Library; @@ -32,12 +33,14 @@ public class ProjectProtoUpdate { private final ProjectProto.Project.Builder project; + private final Context context; private final ProjectProto.Module.Builder workspaceModule; private final Map libraries = Maps.newHashMap(); private final Map artifactDirs = Maps.newHashMap(); - public ProjectProtoUpdate(ProjectProto.Project existingProject) { + public ProjectProtoUpdate(ProjectProto.Project existingProject, Context context) { this.project = existingProject.toBuilder(); + this.context = context; this.workspaceModule = getWorkspaceModuleBuilder(project); } @@ -54,6 +57,10 @@ private static ProjectProto.Module.Builder getWorkspaceModuleBuilder( + " not found in project proto."); } + public Context context() { + return context; + } + public ProjectProto.Project.Builder project() { return project; } diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddCompiledJavaDepsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddCompiledJavaDepsTest.java index d4934aaa2dd..a7e6122d86f 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddCompiledJavaDepsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddCompiledJavaDepsTest.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableList; import com.google.idea.blaze.common.Label; +import com.google.idea.blaze.common.NoopContext; import com.google.idea.blaze.qsync.artifacts.BuildArtifact; import com.google.idea.blaze.qsync.java.JavaArtifactInfo; import com.google.idea.blaze.qsync.project.ProjectProto; @@ -42,7 +43,7 @@ public void no_deps_built() throws Exception { AddCompiledJavaDeps javaDeps = new AddCompiledJavaDeps(ImmutableList::of); - ProjectProtoUpdate update = new ProjectProtoUpdate(original); + ProjectProtoUpdate update = new ProjectProtoUpdate(original, new NoopContext()); javaDeps.update(update); ProjectProto.Project newProject = update.build(); assertThat(newProject.getLibraryList()).isEqualTo(original.getLibraryList()); @@ -77,7 +78,7 @@ public void dep_built() throws Exception { AddCompiledJavaDeps javaDeps = new AddCompiledJavaDeps(() -> ImmutableList.of(builtDep)); - ProjectProtoUpdate update = new ProjectProtoUpdate(original); + ProjectProtoUpdate update = new ProjectProtoUpdate(original, new NoopContext()); javaDeps.update(update); ProjectProto.Project newProject = update.build(); assertThat(newProject.getLibraryList()).hasSize(1); diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java index 986534e1a66..0c53fce227f 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java @@ -79,7 +79,7 @@ public void no_deps_built() throws Exception { new SrcJarInnerPathFinder( new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); addGenSrcJars.update(update); @@ -117,7 +117,7 @@ public void project_gensrcs_ignored() throws Exception { new SrcJarInnerPathFinder( new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); addGenSrcJars.update(update); ProjectProto.Project newProject = update.build(); @@ -161,7 +161,7 @@ public void external_gensrcs_added() throws Exception { new SrcJarInnerPathFinder( new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); addGenSrcJars.update(update); ProjectProto.Project newProject = update.build(); diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java index 69c755d48bc..1b174e582f4 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java @@ -79,7 +79,7 @@ public void no_deps_built() throws Exception { new SrcJarInnerPathFinder( new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); addSrcJars.update(update); @@ -119,7 +119,7 @@ public void external_srcjar_added() throws Exception { new SrcJarInnerPathFinder( new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); addSrcJars.update(update); diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java index ef1aac0b974..6bd457ebb5a 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java @@ -89,7 +89,7 @@ public void external_srcjar_ignored() throws Exception { new SrcJarInnerPathFinder( new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); javaDeps.update(update); ProjectProto.Project newProject = update.build(); assertThat(newProject.getLibraryList()).isEqualTo(original.project().getLibraryList()); @@ -131,7 +131,7 @@ public void project_srcjar_ignored() throws Exception { new SrcJarInnerPathFinder( new PackageStatementParser(), AllowPackagePrefixes.EMPTY_PACKAGE_PREFIXES_ONLY)); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project()); + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); javaDeps.update(update); ProjectProto.Project newProject = update.build(); assertThat(newProject.getLibraryList()).isEqualTo(original.project().getLibraryList()); diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcsTest.java new file mode 100644 index 00000000000..4f7f2a31913 --- /dev/null +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcsTest.java @@ -0,0 +1,233 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.io.ByteSource; +import com.google.idea.blaze.common.Context; +import com.google.idea.blaze.common.Label; +import com.google.idea.blaze.common.NoopContext; +import com.google.idea.blaze.common.artifact.BuildArtifactCache; +import com.google.idea.blaze.qsync.QuerySyncTestUtils; +import com.google.idea.blaze.qsync.TestDataSyncRunner; +import com.google.idea.blaze.qsync.artifacts.BuildArtifact; +import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.java.PackageStatementParser; +import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.project.ProjectProto; +import com.google.idea.blaze.qsync.project.ProjectProto.ArtifactDirectoryContents; +import com.google.idea.blaze.qsync.project.ProjectProto.ContentEntry; +import com.google.idea.blaze.qsync.project.ProjectProto.Module; +import com.google.idea.blaze.qsync.testdata.TestData; +import com.google.protobuf.TextFormat; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Optional; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +@RunWith(JUnit4.class) +public class AddProjectGenSrcsTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock public BuildArtifactCache cache; + + @Mock Context context; + + private final TestDataSyncRunner syncer = + new TestDataSyncRunner( + new NoopContext(), QuerySyncTestUtils.PATH_INFERRING_PACKAGE_READER, true); + + @Test + public void generated_source_added() throws Exception { + TestData testData = TestData.JAVA_LIBRARY_EXTERNAL_DEP_QUERY; + BlazeProjectSnapshot original = syncer.sync(testData); + + when(cache.get("gensrcdigest")) + .thenReturn( + Optional.of( + immediateFuture( + ByteSource.wrap("package com.org;\nclass Class {}".getBytes(UTF_8))))); + + TargetBuildInfo builtDep = + TargetBuildInfo.forJavaTarget( + JavaArtifactInfo.empty(testData.getAssumedOnlyLabel()).toBuilder() + .setGenSrcs( + ImmutableList.of( + BuildArtifact.create( + "gensrcdigest", + Path.of("output/path/com/org/Class.java"), + testData.getAssumedOnlyLabel()))) + .build(), + DependencyBuildContext.create("abc-def", Instant.now(), Optional.empty())); + + AddProjectGenSrcs addGensrcs = + new AddProjectGenSrcs( + () -> ImmutableList.of(builtDep), + original.queryData().projectDefinition(), + cache, + new PackageStatementParser()); + + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), context); + addGensrcs.update(update); + ProjectProto.Project newProject = update.build(); + + Module workspace = newProject.getModules(0); + // check our above assumption: + assertThat(workspace.getName()).isEqualTo(".workspace"); + assertThat(workspace.getContentEntriesList()) + .contains( + TextFormat.parse( + Joiner.on("\n") + .join( + "root {", + " path: \"gensrc/java\"", + " base: PROJECT", + " }", + " sources {", + " is_generated: true", + " project_path {", + " path: \"gensrc/java\"", + " base: PROJECT", + " }", + " }"), + ContentEntry.class)); + assertThat(newProject.getArtifactDirectories().getDirectoriesMap()) + .containsEntry( + "gensrc/java", + TextFormat.parse( + Joiner.on("\n") + .join( + "contents {", + " key: \"com/org/Class.java\"", + " value {", + " transform: COPY", + " build_artifact {", + " digest: \"gensrcdigest\"", + " }", + " }", + " }"), + ArtifactDirectoryContents.class)); + verify(context, never()).setHasWarnings(); + } + + @Test + public void conflict_last_build_taken() throws Exception { + TestData testData = TestData.JAVA_LIBRARY_EXTERNAL_DEP_QUERY; + BlazeProjectSnapshot original = syncer.sync(testData); + + when(cache.get("gensrc1")) + .thenReturn( + Optional.of( + immediateFuture( + ByteSource.wrap("package com.org;\nclass Class {}".getBytes(UTF_8))))); + when(cache.get("gensrc2")) + .thenReturn( + Optional.of( + immediateFuture( + ByteSource.wrap("package com.org;\nclass Class {}".getBytes(UTF_8))))); + + Label testLabel = testData.getAssumedOnlyLabel(); + + TargetBuildInfo genSrc1 = + TargetBuildInfo.forJavaTarget( + JavaArtifactInfo.empty(testLabel.siblingWithName("genSrc1")).toBuilder() + .setGenSrcs( + ImmutableList.of( + BuildArtifact.create( + "gensrc1", + Path.of("output/path/com/org/Class.java"), + testLabel.siblingWithName("genSrc1")))) + .build(), + DependencyBuildContext.create( + "abc-def", Instant.now().minusSeconds(60), Optional.empty())); + + TargetBuildInfo genSrc2 = + TargetBuildInfo.forJavaTarget( + JavaArtifactInfo.empty(testLabel.siblingWithName("genSrc2")).toBuilder() + .setGenSrcs( + ImmutableList.of( + BuildArtifact.create( + "gensrc2", + Path.of("output/otherpath/com/org/Class.java"), + testLabel.siblingWithName("genSrc2")))) + .build(), + DependencyBuildContext.create("abc-def", Instant.now(), Optional.empty())); + + AddProjectGenSrcs addGenSrcs = + new AddProjectGenSrcs( + () -> ImmutableList.of(genSrc1, genSrc2), + original.queryData().projectDefinition(), + cache, + new PackageStatementParser()); + + ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), context); + addGenSrcs.update(update); + ProjectProto.Project newProject = update.build(); + + Module workspace = newProject.getModules(0); + // check our above assumption: + assertThat(workspace.getName()).isEqualTo(".workspace"); + + assertThat(workspace.getContentEntriesList()) + .contains( + TextFormat.parse( + Joiner.on("\n") + .join( + "root {", + " path: \"gensrc/java\"", + " base: PROJECT", + " }", + " sources {", + " is_generated: true", + " project_path {", + " path: \"gensrc/java\"", + " base: PROJECT", + " }", + " }"), + ContentEntry.class)); + assertThat(newProject.getArtifactDirectories().getDirectoriesMap()) + .containsEntry( + "gensrc/java", + TextFormat.parse( + Joiner.on("\n") + .join( + "contents {", + " key: \"com/org/Class.java\"", + " value {", + " transform: COPY", + " build_artifact {", + " digest: \"gensrc2\"", + " }", + " }", + " }"), + ArtifactDirectoryContents.class)); + verify(context).setHasWarnings(); + } +} diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD index 683084c2bd3..9d2a2be2806 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD @@ -52,6 +52,7 @@ java_test( "//querysync/javatests/com/google/idea/blaze/qsync/testdata", "//querysync/javatests/com/google/idea/blaze/qsync/testdata:project_protos", "//shared", + "//shared/javatests/com/google/idea/blaze/common:test_utils", "@com_google_guava_guava//jar", "@junit//jar", "@truth//jar", @@ -121,3 +122,26 @@ java_test( "@truth//jar", ], ) + +java_test( + name = "AddProjectGenSrcsTest", + size = "small", + srcs = ["AddProjectGenSrcsTest.java"], + deps = [ + "//querysync/java/com/google/idea/blaze/qsync/artifacts", + "//querysync/java/com/google/idea/blaze/qsync/deps", + "//querysync/java/com/google/idea/blaze/qsync/java", + "//querysync/java/com/google/idea/blaze/qsync/project", + "//querysync/java/com/google/idea/blaze/qsync/project:project_java_proto", + "//querysync/javatests/com/google/idea/blaze/qsync:TestUtils", + "//querysync/javatests/com/google/idea/blaze/qsync/testdata", + "//shared", + "//shared:artifact", + "//shared/javatests/com/google/idea/blaze/common:test_utils", + "@com_google_guava_guava//jar", + "@com_google_protobuf//:protobuf_java", + "@junit//jar", + "@mockito//jar", + "@truth//jar", + ], +) From 46d39919c3fff330f7bc39fdb93bfb327d7c7b81 Mon Sep 17 00:00:00 2001 From: Googler Date: Thu, 14 Mar 2024 10:31:04 -0700 Subject: [PATCH 13/45] Change default value for android resource package (cherry picked from commit 59944f066d5349d535c4395da3e16fa7240ed8e6) --- aspect/build_dependencies.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aspect/build_dependencies.bzl b/aspect/build_dependencies.bzl index 58c0dbe7752..fab28340fac 100644 --- a/aspect/build_dependencies.bzl +++ b/aspect/build_dependencies.bzl @@ -237,7 +237,7 @@ def _collect_own_java_artifacts( own_gensrc_files = [] own_src_files = [] own_srcjar_files = [] - resource_package = None + resource_package = "" if must_build_main_artifacts: # For rules that we do not follow dependencies of (either because they don't From 514e4648e9626cfee9e794f98df325fb21feb747 Mon Sep 17 00:00:00 2001 From: Googler Date: Thu, 14 Mar 2024 10:55:24 -0700 Subject: [PATCH 14/45] Fix bazel build error bazel will throw `Every .bzl file must have a corresponding package, but '//:.aswb.bzl' does not have one. Please create a BUILD file in the same or any parent directory. Note that this BUILD file does not need to do anything except exist.` if there's no BUILD file in the directory that aswb.bzl exists. It can be fixed manually but it's better to avoid other interaction when user using query sync (cherry picked from commit 5ce8cd138865842230a2318c9aa588258ad4eff1) --- .../blaze/base/qsync/BazelDependencyBuilder.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/base/src/com/google/idea/blaze/base/qsync/BazelDependencyBuilder.java b/base/src/com/google/idea/blaze/base/qsync/BazelDependencyBuilder.java index 4553f508f59..379d2f83740 100644 --- a/base/src/com/google/idea/blaze/base/qsync/BazelDependencyBuilder.java +++ b/base/src/com/google/idea/blaze/base/qsync/BazelDependencyBuilder.java @@ -237,11 +237,15 @@ protected Path getBundledAspectPath() { */ protected String prepareAspect(BlazeContext context) throws IOException, BuildException { Path aspect = getBundledAspectPath(); + Path projectPath = workspaceRoot.directory().toPath().resolve(".aswb"); Files.copy( - aspect, - workspaceRoot.directory().toPath().resolve(".aswb.bzl"), - StandardCopyOption.REPLACE_EXISTING); - return "//:.aswb.bzl"; + aspect, projectPath.resolve("build_dependencies.bzl"), StandardCopyOption.REPLACE_EXISTING); + // bazel asks BUILD file exists with the .bzl file. It's ok that BUILD file contains nothing. + Path buildPath = projectPath.resolve("BUILD"); + if (!Files.exists(buildPath)) { + Files.createFile(buildPath); + } + return "//.aswb:build_dependencies.bzl"; } private OutputInfo createOutputInfo( From a4e793cbadfb18226123decf944d0ced88811588 Mon Sep 17 00:00:00 2001 From: Googler Date: Fri, 15 Mar 2024 12:20:47 -0700 Subject: [PATCH 15/45] Remove the previous proto for piper and re-enable fixed test Remove the preivous proto for piper service as its replaced by protos in perforce/*. Re-enable fixed piper_test. (cherry picked from commit 14cfbf37417f03ba973fed5362c9ff28e931604b) --- ext/proto/piper.proto | 285 ------------------------------------------ 1 file changed, 285 deletions(-) delete mode 100644 ext/proto/piper.proto diff --git a/ext/proto/piper.proto b/ext/proto/piper.proto deleted file mode 100644 index b2f573c1679..00000000000 --- a/ext/proto/piper.proto +++ /dev/null @@ -1,285 +0,0 @@ -/* Copyright 2023 The Bazel Authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; - -package com.google.idea.blaze.ext; - -option java_multiple_files = true; -option java_outer_classname = 'PiperServiceProto'; -option java_package = "com.google.idea.blaze.ext"; - - -service PiperService { - // Get workspaces - rpc GetWorkspaces(GetWorkspacesRequest) - returns (stream.com.google.idea.blaze.ext.GetWorkspacesResponse) {} - - // Creates or updates a Piper workspace. - rpc UpdateWorkspace(UpdateWorkspaceRequest) - returns (UpdateWorkspaceResponse) {} - - // Retrieves a list of changes - rpc GetChanges(GetChangesRequest) - returns (stream.com.google.idea.blaze.ext.GetChangesResponse) {} - - // Resolves conflicts in a Piper workspace after sync or integrate. - rpc ResolveConflicts(ResolveConflictsRequest) - returns (stream.com.google.idea.blaze.ext.ResolveConflictsResponse) {} - - // Read the content of a depot or workspace file. - rpc ReadFiles(ReadFilesRequest) - returns (stream.com.google.idea.blaze.ext.ReadFilesResponse) {} - - // Update the state and type of files that are open in a Piper workspace. - rpc UpdateFileStates(UpdateFileStatesRequest) - returns (stream.com.google.idea.blaze.ext.UpdateFileStatesResponse) {} -} - -message GetWorkspacesRequest { - optional RequestBase request_base = 1; - repeated WorkspaceId workspace_ids = 2; - optional bool return_default_workspace_if_missing = 3; -} - -message GetWorkspacesResponse { - PiperStatus piper_status = 1; - Workspace workspace = 2; -} - -message PiperStatus { - int32 error_code = 1; - string error_message = 2; - int32 canonical_code = 3; -} - -message RequestBase { - optional WorkspaceId workspace_id = 1; - optional string current_directory = 2; -} - -message WorkspaceId { - optional string workspace_name = 1; - optional int32 revision_number = 2; -} - -message UpdateWorkspaceRequest { - optional RequestBase request_base = 1; - Workspace workspace = 2; - optional string new_citc_alias = 3; -} - -message UpdateWorkspaceResponse { - PiperStatus piper_status = 1; - WorkspaceId workspace_id = 2; -} - -message Workspace { - optional bool all_files_writable = 1; - optional WorkspaceId workspace_id = 2; - optional OverlayBranchId overlay_branch_id = 3; - repeated ViewMapping view_mappings = 4; - repeated string root_directories = 5; - optional ChangeId sync_change_id = 6; - optional bool multichange = 7; - optional string citc_alias = 8; -} - -message OverlayBranchId { - optional string branch_prefix = 1; - optional string branch_suffix = 2; -} - -message ViewMapping { - optional bool exclude = 1; - string depot_path = 2; - string workspace_path = 3; -} - -message ChangeId { - optional int64 change_number = 1; - optional WorkspaceId workspace_id = 2; - optional int32 revision_number = 3; -} - -message GetChangesRequest { - optional RequestBase request_base = 1; - repeated ChangeId change_ids = 2; - optional bool include_resolved_info = 3; - optional bool follow_submitted_changes = 4; -} - -message GetChangesResponse { - PiperStatus piper_status = 1; - Change change = 2; -} - -message Change { - enum ChangeStatus { - // This is the client's default change (no change number). - STATUS_DEFAULT = 0; - // A numbered pending change. - STATUS_PENDING = 1; - // A submitted change. - STATUS_SUBMITTED = 2; - } - ChangeId change_id = 1; - ChangeDescription description = 2; - ChangeStatus change_status = 3; - string user_name = 4; - int64 create_time_micros = 5; - repeated SourceFile source_files = 6; -} - -message ChangeDescription { - string text = 1; - message Tag { - optional string key = 1; - optional string value = 2; - } - repeated Tag tags = 2; -} - -message SourceFile { - enum FileOperation { - OPERATION_NONE = 0; - OPERATION_INTEGRATE_BRANCH = 1; - OPERATION_ADD = 2; - OPERATION_INTEGRATE_BRANCH_EDIT = 3; - OPERATION_INTEGRATE_ADD = 4; - OPERATION_DELETE = 5; - OPERATION_INTEGRATE_DELETE = 6; - OPERATION_EDIT = 7; - OPERATION_INTEGRATE_EDIT = 8; - OPERATION_INTEGRATE = 9; - OPERATION_UNRESOLVED_INTEGRATE_DELETE = 10; - } - optional int64 size = 1; - optional SourceFileId source_file_id = 2; - optional FileOperation operation = 3; - optional ChangeId change_id = 4; - optional int32 baseline_revision_number = 5; -} - -message SourceFileId { - optional string depot_path = 1; - optional string workspace_path = 2; - optional string filesystem_path = 3; - int32 revision_number = 4; -} - -message ResolveConflictsRequest { - enum ResolveMode { - DO_NOT_RESOLVE = 0; - ACCEPT_YOURS = 1; - ACCEPT_THEIRS = 2; - MERGE_DEFAULT = 3; - MERGE_SAFE = 4; - MERGE_FORCED = 5; - ACCEPT_WORKSPACE_FILE = 6; - ACCEPT_EDITED_WORKSPACE_FILE = 7; - } - RequestBase request_base = 1; - ResolveMode resolve_mode = 2; - repeated ConflictPath conflict_paths = 3; -} - -message ConflictPath { - string path = 1; - repeated string conflict_ids = 2; -} - -message ResolveConflictsResponse { - PiperStatus piper_status = 1; - optional Conflict resolve_skipped_conflict = 2; -} - -message Conflict { - SourceFile yours_file = 1; - optional SourceFile theirs_file = 2; - optional SourceFile base_file = 3; -} - -message ReadFilesRequest { - enum ReadMode { - NONE = 0; - DEPOT = 1; - WORKSPACE = 2; - } - optional RequestBase request_base = 1; - optional ReadMode read_mode = 2; - repeated SourceFileId source_file_ids = 3; - repeated PathExpression path_expressions = 4; -} - -message ReadFilesResponse { - PiperStatus piper_status = 1; - bytes content = 2; -} - -message PathExpression { - string base_path = 1; - optional bool escaped = 2; - repeated RevisionPoint revision_points = 3; -} - -message RevisionPoint { - enum Operation { - UNKNOWN = 0; - LESS_OR_EQUAL = 1; - LESS = 2; - EQUAL = 3; - GREATER_OR_EQUAL = 4; - GREATER = 5; - } - Operation operation = 1; - ChangeId change_id = 2; -} - -message UpdateFileStatesRequest { - optional RequestBase request_base = 1; - repeated FileState file_states = 2; - optional string ignore_path_regexp = 3; - optional ChangeId change_id = 4; - optional bool dry_run = 5; - optional bool reconcile_unmodified_files = 6; -} - -message FileState { - enum Mode { - NONE = 0; - SET_STATE = 1; - IMPLICIT_STATE_FROM_WORKSPACE = 2; - } - optional PathExpression path_expression = 1; - optional Mode mode = 2; - optional SourceFile.FileOperation file_operation = 3; -} - -message UpdateFileStatesResponse { - PiperStatus piper_status = 1; - SourceFile source_file = 3; - optional SourceFile.FileOperation operation_prior_to_revert = 4; - optional RevertOperation updated_revert_operation = 5; -} - -// The revert operation indicates whether the file was reverted and what -// happened to the file in the file system. -enum RevertOperation { - REVERT_NONE = 0; - REVERT_STANDARD = 1; - REVERT_ABANDON = 2; - REVERT_CLEAR = 3; - REVERT_DELETE = 4; -} From 6eb7037d7256f926fd1682a5a8e7a6a552dda1d2 Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 18 Mar 2024 04:03:35 -0700 Subject: [PATCH 16/45] Add an extension point to support providing `ProjectProtoTransform` implementations. This allows IDE/language specific transforms to be added, needed for some Android functionality. Also add appropriate classes & getters to `QuerySyncProject` to allow such transforms to be implemented. (cherry picked from commit e38345806b368696c8a281f2868682f884f0bf92) --- base/src/META-INF/blaze-base.xml | 3 ++ .../idea/blaze/base/qsync/ProjectLoader.java | 7 ++++ .../qsync/ProjectProtoTransformProvider.java | 40 +++++++++++++++++++ .../blaze/base/qsync/QuerySyncProject.java | 16 ++++++++ .../blaze/qsync/ProjectProtoTransform.java | 5 +++ .../qsync/deps/BuiltDependenciesSupplier.java | 30 ++++++++++++++ .../blaze/qsync/deps/NewArtifactTracker.java | 4 ++ 7 files changed, 105 insertions(+) create mode 100644 base/src/com/google/idea/blaze/base/qsync/ProjectProtoTransformProvider.java create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/BuiltDependenciesSupplier.java diff --git a/base/src/META-INF/blaze-base.xml b/base/src/META-INF/blaze-base.xml index 174a566dc9c..e51c3fb4fc2 100644 --- a/base/src/META-INF/blaze-base.xml +++ b/base/src/META-INF/blaze-base.xml @@ -494,6 +494,9 @@ interface="com.google.idea.blaze.base.qsync.HandledRulesProvider" /> + + diff --git a/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java b/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java index ec3cff9aefb..ebedd58a538 100644 --- a/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java +++ b/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java @@ -53,6 +53,7 @@ import com.google.idea.blaze.qsync.ProjectRefresher; import com.google.idea.blaze.qsync.VcsStateDiffer; import com.google.idea.blaze.qsync.deps.ArtifactTracker; +import com.google.idea.blaze.qsync.deps.BuiltDependenciesSupplier; import com.google.idea.blaze.qsync.deps.DependenciesProjectProtoUpdater; import com.google.idea.blaze.qsync.deps.NewArtifactTracker; import com.google.idea.blaze.qsync.java.PackageStatementParser; @@ -149,6 +150,7 @@ public QuerySyncProject loadProject(BlazeContext context) throws BuildException ideProjectBasePath.resolve("buildcache"), artifactFetcher, executor); ArtifactTracker artifactTracker; + BuiltDependenciesSupplier builtDepsSupplier; RenderJarArtifactTracker renderJarArtifactTracker; AppInspectorArtifactTracker appInspectorArtifactTracker; if (QuerySync.USE_NEW_BUILD_ARTIFACT_MANAGEMENT) { @@ -166,6 +168,7 @@ public QuerySyncProject loadProject(BlazeContext context) throws BuildException artifactTracker = tracker; renderJarArtifactTracker = new RenderJarArtifactTrackerImpl(); appInspectorArtifactTracker = new AppInspectorArtifactTrackerImpl(); + builtDepsSupplier = tracker.getBuiltDepsSupplier(); } else { ArtifactTrackerImpl impl = new ArtifactTrackerImpl( @@ -180,6 +183,7 @@ public QuerySyncProject loadProject(BlazeContext context) throws BuildException artifactTracker = impl; renderJarArtifactTracker = impl; appInspectorArtifactTracker = impl; + builtDepsSupplier = BuiltDependenciesSupplier.NONE; } RenderJarTracker renderJarTracker = new RenderJarTrackerImpl(graph, renderJarBuilder, renderJarArtifactTracker); @@ -217,6 +221,8 @@ public QuerySyncProject loadProject(BlazeContext context) throws BuildException importSettings, workspaceRoot, artifactTracker, + builtDepsSupplier, + artifactCache, artifactStore, renderJarArtifactTracker, appInspectorArtifactTracker, @@ -235,6 +241,7 @@ public QuerySyncProject loadProject(BlazeContext context) throws BuildException buildSystem, projectTransformRegistry); BlazeProjectListenerProvider.registerListenersFor(querySyncProject); + projectTransformRegistry.addAll(ProjectProtoTransformProvider.getAll(querySyncProject)); return querySyncProject; } diff --git a/base/src/com/google/idea/blaze/base/qsync/ProjectProtoTransformProvider.java b/base/src/com/google/idea/blaze/base/qsync/ProjectProtoTransformProvider.java new file mode 100644 index 00000000000..2914b3ec9e9 --- /dev/null +++ b/base/src/com/google/idea/blaze/base/qsync/ProjectProtoTransformProvider.java @@ -0,0 +1,40 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.base.qsync; + +import com.google.common.collect.ImmutableList; +import com.google.idea.blaze.qsync.ProjectProtoTransform; +import com.intellij.openapi.extensions.ExtensionPointName; +import java.util.List; + +/** + * Provides a {@link ProjectProtoTransform} that requires AS specific dependencies so cannot be + * created directly by the base code. + */ +public interface ProjectProtoTransformProvider { + + ExtensionPointName EP_NAME = + new ExtensionPointName<>("com.google.idea.blaze.base.qsync.ProjectProtoTransformProvider"); + + static ImmutableList getAll(QuerySyncProject project) { + return EP_NAME.getExtensionList().stream() + .map(ep -> ep.createTransforms(project)) + .flatMap(List::stream) + .collect(ImmutableList.toImmutableList()); + } + + List createTransforms(QuerySyncProject project); +} diff --git a/base/src/com/google/idea/blaze/base/qsync/QuerySyncProject.java b/base/src/com/google/idea/blaze/base/qsync/QuerySyncProject.java index 7b420b0f796..03f78616fa7 100644 --- a/base/src/com/google/idea/blaze/base/qsync/QuerySyncProject.java +++ b/base/src/com/google/idea/blaze/base/qsync/QuerySyncProject.java @@ -45,12 +45,14 @@ import com.google.idea.blaze.base.util.SaveUtil; import com.google.idea.blaze.common.Label; import com.google.idea.blaze.common.PrintOutput; +import com.google.idea.blaze.common.artifact.BuildArtifactCache; import com.google.idea.blaze.common.vcs.VcsState; import com.google.idea.blaze.exception.BuildException; import com.google.idea.blaze.qsync.BlazeProject; import com.google.idea.blaze.qsync.BlazeProjectSnapshotBuilder; import com.google.idea.blaze.qsync.ProjectProtoTransform; import com.google.idea.blaze.qsync.deps.ArtifactTracker; +import com.google.idea.blaze.qsync.deps.BuiltDependenciesSupplier; import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.PostQuerySyncData; import com.google.idea.blaze.qsync.project.ProjectDefinition; @@ -89,6 +91,8 @@ public class QuerySyncProject { private final BlazeImportSettings importSettings; private final WorkspaceRoot workspaceRoot; private final ArtifactTracker artifactTracker; + private final BuiltDependenciesSupplier builtArtifactsSupplier; + private final BuildArtifactCache buildArtifactCache; private final ProjectArtifactStore artifactStore; private final RenderJarArtifactTracker renderJarArtifactTracker; private final AppInspectorArtifactTracker appInspectorArtifactTracker; @@ -118,6 +122,8 @@ public QuerySyncProject( BlazeImportSettings importSettings, WorkspaceRoot workspaceRoot, ArtifactTracker artifactTracker, + BuiltDependenciesSupplier builtArtifactsSupplier, + BuildArtifactCache buildArtifactCache, ProjectArtifactStore artifactStore, RenderJarArtifactTracker renderJarArtifactTracker, AppInspectorArtifactTracker appInspectorArtifactTracker, @@ -141,6 +147,8 @@ public QuerySyncProject( this.importSettings = importSettings; this.workspaceRoot = workspaceRoot; this.artifactTracker = artifactTracker; + this.builtArtifactsSupplier = builtArtifactsSupplier; + this.buildArtifactCache = buildArtifactCache; this.artifactStore = artifactStore; this.renderJarArtifactTracker = renderJarArtifactTracker; this.appInspectorArtifactTracker = appInspectorArtifactTracker; @@ -193,6 +201,14 @@ public ProjectPath.Resolver getProjectPathResolver() { return projectPathResolver; } + public BuiltDependenciesSupplier getBuiltArtifactsSupplier() { + return builtArtifactsSupplier; + } + + public BuildArtifactCache getBuildArtifactCache() { + return buildArtifactCache; + } + public ProjectArtifactStore getArtifactStore() { return artifactStore; } diff --git a/querysync/java/com/google/idea/blaze/qsync/ProjectProtoTransform.java b/querysync/java/com/google/idea/blaze/qsync/ProjectProtoTransform.java index 145431c7947..e7556910f3e 100644 --- a/querysync/java/com/google/idea/blaze/qsync/ProjectProtoTransform.java +++ b/querysync/java/com/google/idea/blaze/qsync/ProjectProtoTransform.java @@ -15,6 +15,7 @@ */ package com.google.idea.blaze.qsync; +import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.idea.blaze.common.Context; @@ -59,6 +60,10 @@ public void add(ProjectProtoTransform transform) { transforms.add(transform); } + public void addAll(ImmutableCollection transforms) { + this.transforms.addAll(transforms); + } + public ProjectProtoTransform getComposedTransform() { return compose(ImmutableList.copyOf(transforms)); } diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/BuiltDependenciesSupplier.java b/querysync/java/com/google/idea/blaze/qsync/deps/BuiltDependenciesSupplier.java new file mode 100644 index 00000000000..02acd3e2f49 --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/BuiltDependenciesSupplier.java @@ -0,0 +1,30 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; + +/** + * Interface for accessing the list of {@link TargetBuildInfo} instances relating to all dependency + * builds that have been completed. + */ +public interface BuiltDependenciesSupplier { + + BuiltDependenciesSupplier NONE = ImmutableList::of; + + ImmutableCollection get(); +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/NewArtifactTracker.java b/querysync/java/com/google/idea/blaze/qsync/deps/NewArtifactTracker.java index 6cbee5fe3a3..e361ad26e03 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/NewArtifactTracker.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/NewArtifactTracker.java @@ -79,6 +79,10 @@ public NewArtifactTracker(Path projectDirectory, BuildArtifactCache artifactCach loadState(); } + public BuiltDependenciesSupplier getBuiltDepsSupplier() { + return this::getBuiltDeps; + } + public ImmutableCollection getBuiltDeps() { synchronized (stateLock) { return ImmutableList.copyOf(builtDeps.values()); From adf2eecc7735fd1c63162e17f763c1624cc48054 Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 18 Mar 2024 04:18:38 -0700 Subject: [PATCH 17/45] Support external android resources with the new artifact logic. Add external android libraries to the project proto. Populate this in a new `AddDependencyAars` updater which also places the aars in the project artifact cache (with an unzip transform). We also parse the embedded manifest here, which may not be ideal but works for now. Update `BlazeModuleSystemBase` to build studio `ExternalAndroidLibrary` class instances based on the proto contents and returns them to the IDE. (cherry picked from commit 4cfab1bd8187e382042e694b05453e5c3004b462) --- aswb/src/META-INF/aswb.xml | 4 + .../projectsystem/BlazeModuleSystemBase.java | 57 +++++++++- .../qsync/AndroidProjectProtoTransform.java | 63 +++++++++++ .../projectsystem/BlazeModuleSystemTest.java | 1 + .../projectsystem/BlazeModuleSystemTest.java | 1 + .../projectsystem/BlazeModuleSystemTest.java | 1 + .../projectsystem/BlazeModuleSystemTest.java | 1 + .../blaze/qsync/deps/AddDependencyAars.java | 106 ++++++++++++++++++ .../qsync/deps/AndroidManifestParser.java | 27 +++++ .../qsync/deps/ArtifactDirectoryBuilder.java | 11 +- .../idea/blaze/qsync/project/ProjectPath.java | 6 + .../idea/blaze/qsync/project/project.proto | 18 +++ 12 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/AddDependencyAars.java create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/AndroidManifestParser.java diff --git a/aswb/src/META-INF/aswb.xml b/aswb/src/META-INF/aswb.xml index 1b102d5493a..76f4d09cd11 100644 --- a/aswb/src/META-INF/aswb.xml +++ b/aswb/src/META-INF/aswb.xml @@ -119,4 +119,8 @@ + + + + diff --git a/aswb/src/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemBase.java b/aswb/src/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemBase.java index 08195904ecf..aff635053f7 100644 --- a/aswb/src/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemBase.java +++ b/aswb/src/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemBase.java @@ -44,6 +44,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.idea.blaze.android.compose.ComposeStatusProvider; import com.google.idea.blaze.android.libraries.UnpackedAars; import com.google.idea.blaze.android.npw.project.BlazeAndroidModuleTemplate; @@ -65,10 +66,13 @@ import com.google.idea.blaze.base.model.primitives.WorkspaceRoot; import com.google.idea.blaze.base.projectview.ProjectViewManager; import com.google.idea.blaze.base.qsync.DependencyTracker; +import com.google.idea.blaze.base.qsync.QuerySync; import com.google.idea.blaze.base.qsync.QuerySyncManager; +import com.google.idea.blaze.base.qsync.QuerySyncProject; import com.google.idea.blaze.base.scope.BlazeContext; import com.google.idea.blaze.base.settings.Blaze; import com.google.idea.blaze.base.settings.BlazeImportSettings.ProjectType; +import com.google.idea.blaze.base.settings.BlazeImportSettingsManager; import com.google.idea.blaze.base.sync.SyncCache; import com.google.idea.blaze.base.sync.data.BlazeDataStorage; import com.google.idea.blaze.base.sync.data.BlazeProjectDataManager; @@ -77,7 +81,11 @@ import com.google.idea.blaze.base.targetmaps.ReverseDependencyMap; import com.google.idea.blaze.base.targetmaps.TransitiveDependencyMap; import com.google.idea.blaze.common.Label; +import com.google.idea.blaze.qsync.BlazeProject; import com.google.idea.blaze.qsync.deps.ArtifactTracker; +import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectProto; import com.google.idea.common.experiments.BoolExperiment; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.diagnostic.Logger; @@ -122,6 +130,7 @@ abstract class BlazeModuleSystemBase implements AndroidModuleSystem { protected static final Logger logger = Logger.getInstance(BlazeModuleSystem.class); protected Module module; protected final Project project; + private final ProjectPath.Resolver pathResolver; SampleDataDirectoryProvider sampleDataDirectoryProvider; RenderJarClassFileFinder classFileFinder; final boolean isWorkspaceModule; @@ -130,10 +139,18 @@ abstract class BlazeModuleSystemBase implements AndroidModuleSystem { BlazeModuleSystemBase(Module module) { this.module = module; this.project = module.getProject(); + this.pathResolver = + ProjectPath.Resolver.create( + WorkspaceRoot.fromProject(project).path(), + Path.of( + BlazeImportSettingsManager.getInstance(project) + .getImportSettings() + .getProjectDataDirectory())); classFileFinder = new RenderJarClassFileFinder(module); sampleDataDirectoryProvider = new BlazeSampleDataDirectoryProvider(module); isWorkspaceModule = module.getName().equals(BlazeDataStorage.WORKSPACE_MODULE_NAME); - if (Blaze.getProjectType(project) == ProjectType.QUERY_SYNC) { + if (Blaze.getProjectType(project) == ProjectType.QUERY_SYNC + && !QuerySync.USE_NEW_BUILD_ARTIFACT_MANAGEMENT) { androidExternalLibraryManager = new AndroidExternalLibraryManager( () -> { @@ -565,7 +582,30 @@ public boolean getUsesCompose() { public Collection getDependentLibraries() { if (Blaze.getProjectType(project) == ProjectType.QUERY_SYNC) { - return androidExternalLibraryManager.getExternalLibraries(); + if (QuerySync.USE_NEW_BUILD_ARTIFACT_MANAGEMENT) { + ProjectProto.Project projectProto = + QuerySyncManager.getInstance(project) + .getLoadedProject() + .map(QuerySyncProject::getSnapshotHolder) + .flatMap(BlazeProject::getCurrent) + .map(BlazeProjectSnapshot::project) + .orElse(null); + if (projectProto == null) { + return ImmutableList.of(); + } + ImmutableList matchingModules = + projectProto.getModulesList().stream() + .filter(m -> m.getName().equals(module.getName())) + .collect(ImmutableList.toImmutableList()); + if (matchingModules.isEmpty()) { + return ImmutableList.of(); + } + return Iterables.getOnlyElement(matchingModules).getAndroidExternalLibrariesList().stream() + .map(this::fromProto) + .collect(toImmutableList()); + } else { + return androidExternalLibraryManager.getExternalLibraries(); + } } BlazeProjectData blazeProjectData = BlazeProjectDataManager.getInstance(project).getBlazeProjectData(); @@ -609,6 +649,19 @@ public Collection getDependentLibraries() { return libraries.build(); } + private ExternalAndroidLibrary fromProto(ProjectProto.ExternalAndroidLibrary proto) { + return new ExternalLibraryImpl(proto.getName()) + .withLocation(toPathString(proto.getLocation())) + .withManifestFile(toPathString(proto.getManifestFile())) + .withResFolder(new SelectiveResourceFolder(toPathString(proto.getResFolder()), null)) + .withSymbolFile(toPathString(proto.getSymbolFile())) + .withPackageName(proto.getPackageName()); + } + + private PathString toPathString(ProjectProto.ProjectPath projectPath) { + return new PathString(pathResolver.resolve(ProjectPath.create(projectPath))); + } + private static ImmutableList getLibrariesForWorkspaceModule( Project project, BlazeProjectData blazeProjectData) { ArtifactLocationDecoder decoder = blazeProjectData.getArtifactLocationDecoder(); diff --git a/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java b/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java new file mode 100644 index 00000000000..c30fa3a95d6 --- /dev/null +++ b/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java @@ -0,0 +1,63 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.android.qsync; + +import com.google.common.collect.ImmutableList; +import com.google.idea.blaze.android.manifest.ManifestParser; +import com.google.idea.blaze.base.qsync.ProjectProtoTransformProvider; +import com.google.idea.blaze.base.qsync.QuerySyncProject; +import com.google.idea.blaze.common.Context; +import com.google.idea.blaze.exception.BuildException; +import com.google.idea.blaze.qsync.ProjectProtoTransform; +import com.google.idea.blaze.qsync.deps.AddDependencyAars; +import com.google.idea.blaze.qsync.deps.ProjectProtoUpdate; +import com.google.idea.blaze.qsync.project.BuildGraphData; +import com.google.idea.blaze.qsync.project.ProjectProto.Project; +import java.util.List; + +/** A {@link ProjectProtoTransform} that adds android specific information to the project proto. */ +public class AndroidProjectProtoTransform implements ProjectProtoTransform { + + /** + * Provides a {@link ProjectProtoTransform} that adds android specific information to the project + * proto. + */ + public static class Provider implements ProjectProtoTransformProvider { + @Override + public List createTransforms(QuerySyncProject project) { + return ImmutableList.of(new AndroidProjectProtoTransform(project)); + } + } + + private final AddDependencyAars addAars; + + private AndroidProjectProtoTransform(QuerySyncProject project) { + addAars = + new AddDependencyAars( + project.getBuiltArtifactsSupplier(), + project.getBuildArtifactCache(), + project.getProjectDefinition(), + in -> ManifestParser.parseManifestFromInputStream(in).packageName); + } + + @Override + public Project apply(Project proto, BuildGraphData graph, Context context) + throws BuildException { + ProjectProtoUpdate update = new ProjectProtoUpdate(proto, context); + addAars.update(update); + return update.build(); + } +} diff --git a/aswb/tests/unittests/sdkcompat/as223/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java b/aswb/tests/unittests/sdkcompat/as223/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java index 3ff113a9526..1b7194a7e31 100644 --- a/aswb/tests/unittests/sdkcompat/as223/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java +++ b/aswb/tests/unittests/sdkcompat/as223/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java @@ -95,6 +95,7 @@ protected void initTest(Container applicationServices, Container projectServices mockBlazeImportSettings(projectServices); // For Blaze.isBlazeProject. createMocksForAddDependency(applicationServices, projectServices); + project.setBaseDir(new MockVirtualFile("/")); service = new BlazeProjectSystem(project); } diff --git a/aswb/tests/unittests/sdkcompat/as231/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java b/aswb/tests/unittests/sdkcompat/as231/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java index 3ff113a9526..1b7194a7e31 100644 --- a/aswb/tests/unittests/sdkcompat/as231/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java +++ b/aswb/tests/unittests/sdkcompat/as231/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java @@ -95,6 +95,7 @@ protected void initTest(Container applicationServices, Container projectServices mockBlazeImportSettings(projectServices); // For Blaze.isBlazeProject. createMocksForAddDependency(applicationServices, projectServices); + project.setBaseDir(new MockVirtualFile("/")); service = new BlazeProjectSystem(project); } diff --git a/aswb/tests/unittests/sdkcompat/as232/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java b/aswb/tests/unittests/sdkcompat/as232/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java index faaa9dcfd00..ed87ed760d9 100644 --- a/aswb/tests/unittests/sdkcompat/as232/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java +++ b/aswb/tests/unittests/sdkcompat/as232/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java @@ -99,6 +99,7 @@ protected void initTest(Container applicationServices, Container projectServices mockBlazeImportSettings(projectServices); // For Blaze.isBlazeProject. createMocksForAddDependency(applicationServices, projectServices); + project.setBaseDir(new MockVirtualFile("/")); service = new BlazeProjectSystem(project); } diff --git a/aswb/tests/unittests/sdkcompat/asdev/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java b/aswb/tests/unittests/sdkcompat/asdev/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java index 3ff113a9526..1b7194a7e31 100644 --- a/aswb/tests/unittests/sdkcompat/asdev/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java +++ b/aswb/tests/unittests/sdkcompat/asdev/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemTest.java @@ -95,6 +95,7 @@ protected void initTest(Container applicationServices, Container projectServices mockBlazeImportSettings(projectServices); // For Blaze.isBlazeProject. createMocksForAddDependency(applicationServices, projectServices); + project.setBaseDir(new MockVirtualFile("/")); service = new BlazeProjectSystem(project); } diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/AddDependencyAars.java b/querysync/java/com/google/idea/blaze/qsync/deps/AddDependencyAars.java new file mode 100644 index 00000000000..e84a2f432f1 --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/AddDependencyAars.java @@ -0,0 +1,106 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import com.google.idea.blaze.common.artifact.BuildArtifactCache; +import com.google.idea.blaze.exception.BuildException; +import com.google.idea.blaze.qsync.artifacts.BuildArtifact; +import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.project.ProjectDefinition; +import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectProto.ExternalAndroidLibrary; +import com.google.idea.blaze.qsync.project.ProjectProto.ProjectArtifact.ArtifactTransform; +import java.io.IOException; +import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +/** + * Adds external {@code .aar} files to the project proto as {@link ExternalAndroidLibrary}s. This + * allows resources references to external libraries to be resolved in Android Studio. + */ +public class AddDependencyAars implements ProjectProtoUpdateOperation { + + private final BuiltDependenciesSupplier builtTargetsSupplier; + private final BuildArtifactCache buildCache; + private final ProjectDefinition projectDefinition; + private final AndroidManifestParser manifestParser; + + public AddDependencyAars( + BuiltDependenciesSupplier builtTargetsSupplier, + BuildArtifactCache buildCache, + ProjectDefinition projectDefinition, + AndroidManifestParser manifestParser) { + this.builtTargetsSupplier = builtTargetsSupplier; + this.buildCache = buildCache; + this.projectDefinition = projectDefinition; + this.manifestParser = manifestParser; + } + + @Override + public void update(ProjectProtoUpdate update) throws BuildException { + ArtifactDirectoryBuilder aarDir = null; + for (TargetBuildInfo target : builtTargetsSupplier.get()) { + if (target.javaInfo().isEmpty()) { + continue; + } + JavaArtifactInfo javaInfo = target.javaInfo().get(); + if (projectDefinition.isIncluded(javaInfo.label())) { + continue; + } + for (BuildArtifact aar : javaInfo.ideAars()) { + if (aarDir == null) { + aarDir = update.artifactDirectory(Path.of("buildout")); + } + String packageName = readPackageFromAarManifest(aar); + ProjectPath dest = + aarDir + .addIfNewer(aar.path(), aar, target.buildContext(), ArtifactTransform.UNZIP) + .orElse(null); + if (dest != null) { + update + .workspaceModule() + .addAndroidExternalLibraries( + ExternalAndroidLibrary.newBuilder() + .setName(aar.path().toString().replace('/', '_')) + .setLocation(dest.toProto()) + .setManifestFile(dest.resolveChild(Path.of("AndroidManifest.xml")).toProto()) + .setResFolder(dest.resolveChild(Path.of("res")).toProto()) + .setSymbolFile(dest.resolveChild(Path.of("R.txt")).toProto()) + .setPackageName(packageName)); + } + } + } + } + + public String readPackageFromAarManifest(BuildArtifact aar) throws BuildException { + try (ZipInputStream zis = + new ZipInputStream(aar.blockingGetFrom(buildCache).openBufferedStream())) { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (entry.getName().equals("AndroidManifest.xml")) { + return manifestParser.readPackageNameFrom(zis); + } + } + } catch (IOException e) { + throw new BuildException( + String.format("Failed to read aar file %s (built by %s)", aar.path(), aar.target()), e); + } + throw new BuildException( + String.format( + "Failed to find AndroidManifest.xml in %s (built by %s)", aar.path(), aar.target())); + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/AndroidManifestParser.java b/querysync/java/com/google/idea/blaze/qsync/deps/AndroidManifestParser.java new file mode 100644 index 00000000000..8627dc67d1c --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/AndroidManifestParser.java @@ -0,0 +1,27 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Interface to an android manifest parser. This is used to read android manifest package names from + * {@code .aar} files. + */ +public interface AndroidManifestParser { + String readPackageNameFrom(InputStream in) throws IOException; +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactDirectoryBuilder.java b/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactDirectoryBuilder.java index 7adbd428556..8521bffba2f 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactDirectoryBuilder.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactDirectoryBuilder.java @@ -96,12 +96,21 @@ public ProjectPath root() { @CanIgnoreReturnValue public Optional addIfNewer( Path relativePath, BuildArtifact source, DependencyBuildContext fromBuild) { + return addIfNewer(relativePath, source, fromBuild, ArtifactTransform.COPY); + } + + @CanIgnoreReturnValue + public Optional addIfNewer( + Path relativePath, + BuildArtifact source, + DependencyBuildContext fromBuild, + ArtifactTransform transform) { Entry existing = contents.get(relativePath); if (existing != null && existing.fromBuild().startTime().isAfter(fromBuild.startTime())) { // we already have the same artifact from a more recent build. return Optional.empty(); } - Entry e = Entry.create(relativePath, source, fromBuild, ArtifactTransform.COPY); + Entry e = Entry.create(relativePath, source, fromBuild, transform); contents.put(relativePath, e); return Optional.of(ProjectPath.create(Root.PROJECT, path.resolve(relativePath))); } diff --git a/querysync/java/com/google/idea/blaze/qsync/project/ProjectPath.java b/querysync/java/com/google/idea/blaze/qsync/project/ProjectPath.java index faade1265b4..899f4eb0fc1 100644 --- a/querysync/java/com/google/idea/blaze/qsync/project/ProjectPath.java +++ b/querysync/java/com/google/idea/blaze/qsync/project/ProjectPath.java @@ -17,6 +17,7 @@ import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.idea.blaze.qsync.project.ProjectProto.ProjectPath.Base; import java.nio.file.Path; @@ -43,6 +44,11 @@ public ProjectPath withInnerJarPath(Path inner) { return create(rootType(), relativePath(), inner); } + public ProjectPath resolveChild(Path child) { + Preconditions.checkState(!child.isAbsolute(), child); + return create(rootType(), relativePath().resolve(child)); + } + public ProjectProto.ProjectPath toProto() { ProjectProto.ProjectPath.Builder proto = ProjectProto.ProjectPath.newBuilder(); switch (rootType()) { diff --git a/querysync/java/com/google/idea/blaze/qsync/project/project.proto b/querysync/java/com/google/idea/blaze/qsync/project/project.proto index a47f3a82e67..1f85d55984e 100644 --- a/querysync/java/com/google/idea/blaze/qsync/project/project.proto +++ b/querysync/java/com/google/idea/blaze/qsync/project/project.proto @@ -45,6 +45,10 @@ message Module { // Separate custom package with source packages as it's only used by R class. repeated string android_custom_packages = 7; + + // Android libs are kept separate to regular libraries reflecting the separatation between the + // APIs we use to configure them. + repeated ExternalAndroidLibrary android_external_libraries = 8; } /** @@ -132,6 +136,20 @@ message LibrarySource { ProjectPath srcjar = 2; } +/** + * An external android library. + * + * Corresponds to the AS class com.android.projectmodel.ExternalAndroidLibrary. + */ +message ExternalAndroidLibrary { + string name = 1; + ProjectPath location = 2; + ProjectPath manifest_file = 3; + ProjectPath res_folder = 4; + ProjectPath symbol_file = 5; + string package_name = 6; +} + /** * A single compiler flag that supports resolving to an absolute path when * configuring the project model. From be246fd669a16ba28f67fe8e1f2d0b5d517ff7b1 Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 18 Mar 2024 04:21:05 -0700 Subject: [PATCH 18/45] Reinstate android res packages functionality with new artifact tracker. Use the new `ProjectProtoUpdateOperation` for this to configure the packages in the project proto based on the output from a dependencies build. (cherry picked from commit 54f81c0b96362166e7766a5fb63f574db3ae48cc) --- .../qsync/AndroidProjectProtoTransform.java | 22 +++++--- .../qsync/deps/AddAndroidResPackages.java | 51 +++++++++++++++++++ 2 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/AddAndroidResPackages.java diff --git a/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java b/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java index c30fa3a95d6..4ee09d4b2c7 100644 --- a/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java +++ b/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java @@ -22,8 +22,10 @@ import com.google.idea.blaze.common.Context; import com.google.idea.blaze.exception.BuildException; import com.google.idea.blaze.qsync.ProjectProtoTransform; +import com.google.idea.blaze.qsync.deps.AddAndroidResPackages; import com.google.idea.blaze.qsync.deps.AddDependencyAars; import com.google.idea.blaze.qsync.deps.ProjectProtoUpdate; +import com.google.idea.blaze.qsync.deps.ProjectProtoUpdateOperation; import com.google.idea.blaze.qsync.project.BuildGraphData; import com.google.idea.blaze.qsync.project.ProjectProto.Project; import java.util.List; @@ -42,22 +44,26 @@ public List createTransforms(QuerySyncProject project) { } } - private final AddDependencyAars addAars; + private final ImmutableList updateOperations; private AndroidProjectProtoTransform(QuerySyncProject project) { - addAars = - new AddDependencyAars( - project.getBuiltArtifactsSupplier(), - project.getBuildArtifactCache(), - project.getProjectDefinition(), - in -> ManifestParser.parseManifestFromInputStream(in).packageName); + updateOperations = + ImmutableList.of( + new AddDependencyAars( + project.getBuiltArtifactsSupplier(), + project.getBuildArtifactCache(), + project.getProjectDefinition(), + in -> ManifestParser.parseManifestFromInputStream(in).packageName), + new AddAndroidResPackages(project.getBuiltArtifactsSupplier())); } @Override public Project apply(Project proto, BuildGraphData graph, Context context) throws BuildException { ProjectProtoUpdate update = new ProjectProtoUpdate(proto, context); - addAars.update(update); + for (var op : updateOperations) { + op.update(update); + } return update.build(); } } diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/AddAndroidResPackages.java b/querysync/java/com/google/idea/blaze/qsync/deps/AddAndroidResPackages.java new file mode 100644 index 00000000000..170189c38a8 --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/AddAndroidResPackages.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import static java.util.function.Predicate.not; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.idea.blaze.exception.BuildException; +import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import java.util.Optional; + +/** + * Updates the project proto with the android resources packages extracted by the aspect in a + * dependencies build. + */ +public class AddAndroidResPackages implements ProjectProtoUpdateOperation { + + private final BuiltDependenciesSupplier builtDepsSupplier; + + public AddAndroidResPackages(BuiltDependenciesSupplier builtDepsSupplier) { + this.builtDepsSupplier = builtDepsSupplier; + } + + @Override + public void update(ProjectProtoUpdate update) throws BuildException { + update + .workspaceModule() + .addAllAndroidSourcePackages( + builtDepsSupplier.get().stream() + .map(TargetBuildInfo::javaInfo) + .flatMap(Optional::stream) + .map(JavaArtifactInfo::androidResourcesPackage) + .filter(not(Strings::isNullOrEmpty)) + .distinct() + .collect(ImmutableList.toImmutableList())); + } +} From e3a12ace4869df3d72167eb8cd50049ef9dc6eac Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 18 Mar 2024 04:38:46 -0700 Subject: [PATCH 19/45] Move some classes between the `qsync` and `qsync.deps` packages. `BlazeProjectSnapshot` moves from the `project` package into the top level `qsync`. This allows it to include state from the `deps` package in future without creating a circular dependency. To enable this, move `ProjectProtoTransform` from the parent `qsync` into the `project` package. This makes sense, since it operates on the project proto defined in the same package. Also move `TestSourceGlobMatcher` which it depends on. (cherry picked from commit 57403b00c330855ed11f358a44d3119968234b65) --- .../android/cppimpl/BlazeNdkSupportEnabler.java | 2 +- .../android/projectsystem/BlazeModuleSystemBase.java | 2 +- .../android/qsync/AndroidProjectProtoTransform.java | 2 +- .../idea/blaze/base/qsync/DependencyTrackerImpl.java | 2 +- .../google/idea/blaze/base/qsync/ProjectLoader.java | 4 ++-- .../base/qsync/ProjectProtoTransformProvider.java | 2 +- .../idea/blaze/base/qsync/ProjectStatsLogger.java | 2 +- .../google/idea/blaze/base/qsync/ProjectUpdater.java | 2 +- .../idea/blaze/base/qsync/QuerySyncManager.java | 2 +- .../idea/blaze/base/qsync/QuerySyncProject.java | 4 ++-- .../idea/blaze/base/qsync/QuerySyncProjectData.java | 2 +- .../blaze/base/qsync/QuerySyncSourceToTargetMap.java | 2 +- .../idea/blaze/base/qsync/RenderJarTrackerImpl.java | 2 +- .../action/BuildDependenciesForProjectAction.java | 2 +- .../base/qsync/artifacts/ProjectArtifactStore.java | 2 +- .../blaze/base/qsync/cache/ArtifactTrackerImpl.java | 4 ++-- .../blaze/base/qsync/cc/CcProjectProtoTransform.java | 2 +- .../blaze/base/qsync/DependencyTrackerImplTest.java | 2 +- .../base/qsync/cache/ArtifactTrackerImplTest.java | 2 +- .../idea/blaze/cpp/qsync/CcProjectModelUpdater.java | 2 +- .../run/producers/JavaBinaryContextProvider.java | 2 +- .../com/google/idea/blaze/qsync/BlazeProject.java | 1 - .../idea/blaze/qsync/BlazeProjectListener.java | 1 - .../qsync/{project => }/BlazeProjectSnapshot.java | 6 +++++- .../blaze/qsync/BlazeProjectSnapshotBuilder.java | 2 +- .../idea/blaze/qsync/GraphToProjectConverter.java | 1 + .../google/idea/blaze/qsync/NoopProjectRefresh.java | 1 - .../google/idea/blaze/qsync/ProjectRefresher.java | 1 - .../google/idea/blaze/qsync/RefreshOperation.java | 1 - .../idea/blaze/qsync/deps/AddProjectGenSrcJars.java | 2 +- .../idea/blaze/qsync/deps/AddProjectGenSrcs.java | 2 +- .../java/com/google/idea/blaze/qsync/deps/BUILD | 1 - .../qsync/deps/DependenciesProjectProtoUpdater.java | 2 +- .../qsync/{ => project}/ProjectProtoTransform.java | 4 +--- .../qsync/{ => project}/TestSourceGlobMatcher.java | 3 +-- .../javatests/com/google/idea/blaze/qsync/BUILD | 12 ------------ .../idea/blaze/qsync/ProjectRefresherTest.java | 1 - .../google/idea/blaze/qsync/TestDataSyncRunner.java | 1 - .../qsync/deps/AddDependencyGenSrcsJarsTest.java | 2 +- .../blaze/qsync/deps/AddDependencySrcJarsTest.java | 2 +- .../blaze/qsync/deps/AddProjectGenSrcJarsTest.java | 2 +- .../idea/blaze/qsync/deps/AddProjectGenSrcsTest.java | 2 +- .../javatests/com/google/idea/blaze/qsync/deps/BUILD | 7 ++++--- .../com/google/idea/blaze/qsync/project/BUILD | 12 ++++++++++++ .../{ => project}/TestSourceGlobMatcherTest.java | 2 +- 45 files changed, 57 insertions(+), 62 deletions(-) rename querysync/java/com/google/idea/blaze/qsync/{project => }/BlazeProjectSnapshot.java (93%) rename querysync/java/com/google/idea/blaze/qsync/{ => project}/ProjectProtoTransform.java (94%) rename querysync/java/com/google/idea/blaze/qsync/{ => project}/TestSourceGlobMatcher.java (95%) rename querysync/javatests/com/google/idea/blaze/qsync/{ => project}/TestSourceGlobMatcherTest.java (98%) diff --git a/aswb/src/com/google/idea/blaze/android/cppimpl/BlazeNdkSupportEnabler.java b/aswb/src/com/google/idea/blaze/android/cppimpl/BlazeNdkSupportEnabler.java index 9657c0ebc17..29cd9527b38 100644 --- a/aswb/src/com/google/idea/blaze/android/cppimpl/BlazeNdkSupportEnabler.java +++ b/aswb/src/com/google/idea/blaze/android/cppimpl/BlazeNdkSupportEnabler.java @@ -32,7 +32,7 @@ import com.google.idea.blaze.base.sync.SyncResult; import com.google.idea.blaze.common.Context; import com.google.idea.blaze.qsync.BlazeProjectListener; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.QuerySyncLanguage; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.application.TransactionGuard; diff --git a/aswb/src/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemBase.java b/aswb/src/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemBase.java index aff635053f7..36f5b94a83e 100644 --- a/aswb/src/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemBase.java +++ b/aswb/src/com/google/idea/blaze/android/projectsystem/BlazeModuleSystemBase.java @@ -82,8 +82,8 @@ import com.google.idea.blaze.base.targetmaps.TransitiveDependencyMap; import com.google.idea.blaze.common.Label; import com.google.idea.blaze.qsync.BlazeProject; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.deps.ArtifactTracker; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.ProjectPath; import com.google.idea.blaze.qsync.project.ProjectProto; import com.google.idea.common.experiments.BoolExperiment; diff --git a/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java b/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java index 4ee09d4b2c7..776de76230b 100644 --- a/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java +++ b/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java @@ -21,13 +21,13 @@ import com.google.idea.blaze.base.qsync.QuerySyncProject; import com.google.idea.blaze.common.Context; import com.google.idea.blaze.exception.BuildException; -import com.google.idea.blaze.qsync.ProjectProtoTransform; import com.google.idea.blaze.qsync.deps.AddAndroidResPackages; import com.google.idea.blaze.qsync.deps.AddDependencyAars; import com.google.idea.blaze.qsync.deps.ProjectProtoUpdate; import com.google.idea.blaze.qsync.deps.ProjectProtoUpdateOperation; import com.google.idea.blaze.qsync.project.BuildGraphData; import com.google.idea.blaze.qsync.project.ProjectProto.Project; +import com.google.idea.blaze.qsync.project.ProjectProtoTransform; import java.util.List; /** A {@link ProjectProtoTransform} that adds android specific information to the project proto. */ diff --git a/base/src/com/google/idea/blaze/base/qsync/DependencyTrackerImpl.java b/base/src/com/google/idea/blaze/base/qsync/DependencyTrackerImpl.java index bfac552ec67..79f0b43786c 100644 --- a/base/src/com/google/idea/blaze/base/qsync/DependencyTrackerImpl.java +++ b/base/src/com/google/idea/blaze/base/qsync/DependencyTrackerImpl.java @@ -31,9 +31,9 @@ import com.google.idea.blaze.common.PrintOutput; import com.google.idea.blaze.exception.BuildException; import com.google.idea.blaze.qsync.BlazeProject; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.deps.ArtifactTracker; import com.google.idea.blaze.qsync.deps.OutputInfo; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.DependencyTrackingBehavior; import com.google.idea.blaze.qsync.project.ProjectDefinition; import com.google.idea.blaze.qsync.project.RequestedTargets; diff --git a/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java b/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java index ebedd58a538..3e3f7b54159 100644 --- a/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java +++ b/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java @@ -48,8 +48,6 @@ import com.google.idea.blaze.exception.BuildException; import com.google.idea.blaze.qsync.BlazeProject; import com.google.idea.blaze.qsync.BlazeProjectSnapshotBuilder; -import com.google.idea.blaze.qsync.ProjectProtoTransform; -import com.google.idea.blaze.qsync.ProjectProtoTransform.Registry; import com.google.idea.blaze.qsync.ProjectRefresher; import com.google.idea.blaze.qsync.VcsStateDiffer; import com.google.idea.blaze.qsync.deps.ArtifactTracker; @@ -60,6 +58,8 @@ import com.google.idea.blaze.qsync.java.ParallelPackageReader; import com.google.idea.blaze.qsync.project.ProjectDefinition; import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectProtoTransform; +import com.google.idea.blaze.qsync.project.ProjectProtoTransform.Registry; import com.intellij.openapi.project.Project; import com.intellij.openapi.util.ModificationTracker; import com.intellij.openapi.util.SimpleModificationTracker; diff --git a/base/src/com/google/idea/blaze/base/qsync/ProjectProtoTransformProvider.java b/base/src/com/google/idea/blaze/base/qsync/ProjectProtoTransformProvider.java index 2914b3ec9e9..8e1985d49e6 100644 --- a/base/src/com/google/idea/blaze/base/qsync/ProjectProtoTransformProvider.java +++ b/base/src/com/google/idea/blaze/base/qsync/ProjectProtoTransformProvider.java @@ -16,7 +16,7 @@ package com.google.idea.blaze.base.qsync; import com.google.common.collect.ImmutableList; -import com.google.idea.blaze.qsync.ProjectProtoTransform; +import com.google.idea.blaze.qsync.project.ProjectProtoTransform; import com.intellij.openapi.extensions.ExtensionPointName; import java.util.List; diff --git a/base/src/com/google/idea/blaze/base/qsync/ProjectStatsLogger.java b/base/src/com/google/idea/blaze/base/qsync/ProjectStatsLogger.java index 8dc5df94d12..2ee57c02323 100644 --- a/base/src/com/google/idea/blaze/base/qsync/ProjectStatsLogger.java +++ b/base/src/com/google/idea/blaze/base/qsync/ProjectStatsLogger.java @@ -24,8 +24,8 @@ import com.google.idea.blaze.base.projectview.section.sections.TryImportSection; import com.google.idea.blaze.common.Context; import com.google.idea.blaze.qsync.BlazeProjectListener; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.deps.ArtifactTracker; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import java.util.Optional; /** Updates project info according to the newly generated build graph. */ diff --git a/base/src/com/google/idea/blaze/base/qsync/ProjectUpdater.java b/base/src/com/google/idea/blaze/base/qsync/ProjectUpdater.java index bd86e423785..416d20450b1 100644 --- a/base/src/com/google/idea/blaze/base/qsync/ProjectUpdater.java +++ b/base/src/com/google/idea/blaze/base/qsync/ProjectUpdater.java @@ -35,7 +35,7 @@ import com.google.idea.blaze.common.PrintOutput; import com.google.idea.blaze.exception.BuildException; import com.google.idea.blaze.qsync.BlazeProjectListener; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.ProjectPath; import com.google.idea.blaze.qsync.project.ProjectProto; import com.google.idea.blaze.qsync.project.ProjectProto.LibrarySource; diff --git a/base/src/com/google/idea/blaze/base/qsync/QuerySyncManager.java b/base/src/com/google/idea/blaze/base/qsync/QuerySyncManager.java index d307ce12910..3352467bea1 100644 --- a/base/src/com/google/idea/blaze/base/qsync/QuerySyncManager.java +++ b/base/src/com/google/idea/blaze/base/qsync/QuerySyncManager.java @@ -44,8 +44,8 @@ import com.google.idea.blaze.common.Label; import com.google.idea.blaze.common.PrintOutput; import com.google.idea.blaze.exception.BuildException; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.deps.ArtifactTracker; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.PostQuerySyncData; import com.google.idea.blaze.qsync.project.ProjectDefinition; import com.google.idea.blaze.qsync.project.TargetsToBuild; diff --git a/base/src/com/google/idea/blaze/base/qsync/QuerySyncProject.java b/base/src/com/google/idea/blaze/base/qsync/QuerySyncProject.java index 03f78616fa7..75a5422a5c4 100644 --- a/base/src/com/google/idea/blaze/base/qsync/QuerySyncProject.java +++ b/base/src/com/google/idea/blaze/base/qsync/QuerySyncProject.java @@ -49,14 +49,14 @@ import com.google.idea.blaze.common.vcs.VcsState; import com.google.idea.blaze.exception.BuildException; import com.google.idea.blaze.qsync.BlazeProject; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.BlazeProjectSnapshotBuilder; -import com.google.idea.blaze.qsync.ProjectProtoTransform; import com.google.idea.blaze.qsync.deps.ArtifactTracker; import com.google.idea.blaze.qsync.deps.BuiltDependenciesSupplier; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.PostQuerySyncData; import com.google.idea.blaze.qsync.project.ProjectDefinition; import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectProtoTransform; import com.google.idea.blaze.qsync.project.SnapshotDeserializer; import com.google.idea.blaze.qsync.project.SnapshotSerializer; import com.google.idea.blaze.qsync.project.TargetsToBuild; diff --git a/base/src/com/google/idea/blaze/base/qsync/QuerySyncProjectData.java b/base/src/com/google/idea/blaze/base/qsync/QuerySyncProjectData.java index 3d4f99e714e..51eaa969df7 100644 --- a/base/src/com/google/idea/blaze/base/qsync/QuerySyncProjectData.java +++ b/base/src/com/google/idea/blaze/base/qsync/QuerySyncProjectData.java @@ -29,7 +29,7 @@ import com.google.idea.blaze.base.sync.projectview.WorkspaceLanguageSettings; import com.google.idea.blaze.base.sync.workspace.ArtifactLocationDecoder; import com.google.idea.blaze.base.sync.workspace.WorkspacePathResolver; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.ProjectTarget; import com.google.idea.blaze.qsync.project.QuerySyncLanguage; import com.intellij.openapi.diagnostic.Logger; diff --git a/base/src/com/google/idea/blaze/base/qsync/QuerySyncSourceToTargetMap.java b/base/src/com/google/idea/blaze/base/qsync/QuerySyncSourceToTargetMap.java index c0791b1e993..1a7de63f93a 100644 --- a/base/src/com/google/idea/blaze/base/qsync/QuerySyncSourceToTargetMap.java +++ b/base/src/com/google/idea/blaze/base/qsync/QuerySyncSourceToTargetMap.java @@ -22,7 +22,7 @@ import com.google.idea.blaze.base.model.primitives.Label; import com.google.idea.blaze.base.targetmaps.SourceToTargetMap; import com.google.idea.blaze.qsync.BlazeProject; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.intellij.openapi.diagnostic.Logger; import java.io.File; import java.nio.file.Path; diff --git a/base/src/com/google/idea/blaze/base/qsync/RenderJarTrackerImpl.java b/base/src/com/google/idea/blaze/base/qsync/RenderJarTrackerImpl.java index a29f5ccefcc..a0142ee5f87 100644 --- a/base/src/com/google/idea/blaze/base/qsync/RenderJarTrackerImpl.java +++ b/base/src/com/google/idea/blaze/base/qsync/RenderJarTrackerImpl.java @@ -24,7 +24,7 @@ import com.google.idea.blaze.common.PrintOutput; import com.google.idea.blaze.exception.BuildException; import com.google.idea.blaze.qsync.BlazeProject; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import java.io.IOException; import java.nio.file.Path; import java.util.HashSet; diff --git a/base/src/com/google/idea/blaze/base/qsync/action/BuildDependenciesForProjectAction.java b/base/src/com/google/idea/blaze/base/qsync/action/BuildDependenciesForProjectAction.java index 56587319f93..82f0d66b505 100644 --- a/base/src/com/google/idea/blaze/base/qsync/action/BuildDependenciesForProjectAction.java +++ b/base/src/com/google/idea/blaze/base/qsync/action/BuildDependenciesForProjectAction.java @@ -19,7 +19,7 @@ import com.google.idea.blaze.base.logging.utils.querysync.QuerySyncActionStatsScope; import com.google.idea.blaze.base.qsync.QuerySyncManager; import com.google.idea.blaze.base.qsync.QuerySyncManager.TaskOrigin; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.intellij.openapi.actionSystem.ActionUpdateThread; import com.intellij.openapi.actionSystem.AnActionEvent; import com.intellij.openapi.diagnostic.Logger; diff --git a/base/src/com/google/idea/blaze/base/qsync/artifacts/ProjectArtifactStore.java b/base/src/com/google/idea/blaze/base/qsync/artifacts/ProjectArtifactStore.java index 2cd9788512b..18555ae98d1 100644 --- a/base/src/com/google/idea/blaze/base/qsync/artifacts/ProjectArtifactStore.java +++ b/base/src/com/google/idea/blaze/base/qsync/artifacts/ProjectArtifactStore.java @@ -21,8 +21,8 @@ import com.google.idea.blaze.common.Context; import com.google.idea.blaze.common.artifact.BuildArtifactCache; import com.google.idea.blaze.exception.BuildException; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.artifacts.ArtifactDirectoryUpdate; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.ProjectProto.ArtifactDirectories; import com.google.idea.blaze.qsync.project.ProjectProto.ArtifactDirectoryContents; import java.io.IOException; diff --git a/base/src/com/google/idea/blaze/base/qsync/cache/ArtifactTrackerImpl.java b/base/src/com/google/idea/blaze/base/qsync/cache/ArtifactTrackerImpl.java index f2e8dea58e9..cd20f8af70c 100644 --- a/base/src/com/google/idea/blaze/base/qsync/cache/ArtifactTrackerImpl.java +++ b/base/src/com/google/idea/blaze/base/qsync/cache/ArtifactTrackerImpl.java @@ -70,8 +70,6 @@ import com.google.idea.blaze.common.artifact.OutputArtifact; import com.google.idea.blaze.common.proto.ProtoStringInterner; import com.google.idea.blaze.exception.BuildException; -import com.google.idea.blaze.qsync.ProjectProtoTransform; -import com.google.idea.blaze.qsync.TestSourceGlobMatcher; import com.google.idea.blaze.qsync.artifacts.BuildArtifact; import com.google.idea.blaze.qsync.cc.CcDependenciesInfo; import com.google.idea.blaze.qsync.deps.ArtifactTracker; @@ -89,6 +87,8 @@ import com.google.idea.blaze.qsync.project.ProjectDefinition; import com.google.idea.blaze.qsync.project.ProjectPath; import com.google.idea.blaze.qsync.project.ProjectProto; +import com.google.idea.blaze.qsync.project.ProjectProtoTransform; +import com.google.idea.blaze.qsync.project.TestSourceGlobMatcher; import com.google.protobuf.ExtensionRegistry; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.io.FileUtilRt; diff --git a/base/src/com/google/idea/blaze/base/qsync/cc/CcProjectProtoTransform.java b/base/src/com/google/idea/blaze/base/qsync/cc/CcProjectProtoTransform.java index 9cf942d0361..62ccb9faee8 100644 --- a/base/src/com/google/idea/blaze/base/qsync/cc/CcProjectProtoTransform.java +++ b/base/src/com/google/idea/blaze/base/qsync/cc/CcProjectProtoTransform.java @@ -18,11 +18,11 @@ import com.google.common.base.Supplier; import com.google.idea.blaze.common.Context; import com.google.idea.blaze.exception.BuildException; -import com.google.idea.blaze.qsync.ProjectProtoTransform; import com.google.idea.blaze.qsync.cc.CcDependenciesInfo; import com.google.idea.blaze.qsync.cc.CcWorkspaceBuilder; import com.google.idea.blaze.qsync.project.BuildGraphData; import com.google.idea.blaze.qsync.project.ProjectProto.Project; +import com.google.idea.blaze.qsync.project.ProjectProtoTransform; /** * Adds cc workspace information to the project proto, based on the output from cc dependency diff --git a/base/tests/unittests/com/google/idea/blaze/base/qsync/DependencyTrackerImplTest.java b/base/tests/unittests/com/google/idea/blaze/base/qsync/DependencyTrackerImplTest.java index 199dd465dcb..c5387f0fcbc 100644 --- a/base/tests/unittests/com/google/idea/blaze/base/qsync/DependencyTrackerImplTest.java +++ b/base/tests/unittests/com/google/idea/blaze/base/qsync/DependencyTrackerImplTest.java @@ -24,10 +24,10 @@ import com.google.idea.blaze.base.scope.BlazeContext; import com.google.idea.blaze.common.Label; import com.google.idea.blaze.qsync.BlazeProject; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.QuerySyncTestUtils; import com.google.idea.blaze.qsync.TestDataSyncRunner; import com.google.idea.blaze.qsync.deps.ArtifactTracker; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.testdata.TestData; import org.junit.Rule; import org.junit.Test; diff --git a/base/tests/unittests/com/google/idea/blaze/base/qsync/cache/ArtifactTrackerImplTest.java b/base/tests/unittests/com/google/idea/blaze/base/qsync/cache/ArtifactTrackerImplTest.java index e1fd5e71e21..b6bf9773858 100644 --- a/base/tests/unittests/com/google/idea/blaze/base/qsync/cache/ArtifactTrackerImplTest.java +++ b/base/tests/unittests/com/google/idea/blaze/base/qsync/cache/ArtifactTrackerImplTest.java @@ -35,13 +35,13 @@ import com.google.idea.blaze.common.artifact.ArtifactFetcher; import com.google.idea.blaze.common.artifact.OutputArtifact; import com.google.idea.blaze.common.artifact.TestOutputArtifact; -import com.google.idea.blaze.qsync.ProjectProtoTransform; import com.google.idea.blaze.qsync.deps.OutputGroup; import com.google.idea.blaze.qsync.deps.OutputInfo; import com.google.idea.blaze.qsync.java.JavaTargetInfo.JavaArtifacts; import com.google.idea.blaze.qsync.java.JavaTargetInfo.JavaTargetArtifacts; import com.google.idea.blaze.qsync.project.ProjectDefinition; import com.google.idea.blaze.qsync.project.ProjectPath.Resolver; +import com.google.idea.blaze.qsync.project.ProjectProtoTransform; import java.nio.file.Path; import java.util.Map; import java.util.Optional; diff --git a/cpp/src/com/google/idea/blaze/cpp/qsync/CcProjectModelUpdater.java b/cpp/src/com/google/idea/blaze/cpp/qsync/CcProjectModelUpdater.java index 002c7a485e0..3081830e20d 100644 --- a/cpp/src/com/google/idea/blaze/cpp/qsync/CcProjectModelUpdater.java +++ b/cpp/src/com/google/idea/blaze/cpp/qsync/CcProjectModelUpdater.java @@ -19,7 +19,7 @@ import com.google.idea.blaze.base.qsync.QuerySyncProject; import com.google.idea.blaze.common.Context; import com.google.idea.blaze.qsync.BlazeProjectListener; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.ProjectPath; import com.google.idea.blaze.qsync.project.ProjectProto; import com.intellij.openapi.application.WriteAction; diff --git a/java/src/com/google/idea/blaze/java/run/producers/JavaBinaryContextProvider.java b/java/src/com/google/idea/blaze/java/run/producers/JavaBinaryContextProvider.java index a256822b955..9f5acbdfdf0 100644 --- a/java/src/com/google/idea/blaze/java/run/producers/JavaBinaryContextProvider.java +++ b/java/src/com/google/idea/blaze/java/run/producers/JavaBinaryContextProvider.java @@ -36,7 +36,7 @@ import com.google.idea.blaze.base.sync.SyncCache; import com.google.idea.blaze.common.Label; import com.google.idea.blaze.java.run.RunUtil; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.BuildGraphData; import com.google.idea.blaze.qsync.project.ProjectTarget; import com.intellij.execution.JavaExecutionUtil; diff --git a/querysync/java/com/google/idea/blaze/qsync/BlazeProject.java b/querysync/java/com/google/idea/blaze/qsync/BlazeProject.java index 4311484448b..28fd7fa74d9 100644 --- a/querysync/java/com/google/idea/blaze/qsync/BlazeProject.java +++ b/querysync/java/com/google/idea/blaze/qsync/BlazeProject.java @@ -19,7 +19,6 @@ import com.google.common.collect.Lists; import com.google.idea.blaze.common.Context; import com.google.idea.blaze.exception.BuildException; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import java.util.List; import java.util.Optional; import javax.annotation.Nullable; diff --git a/querysync/java/com/google/idea/blaze/qsync/BlazeProjectListener.java b/querysync/java/com/google/idea/blaze/qsync/BlazeProjectListener.java index 762fa5b6fcc..ea84e43fb9b 100644 --- a/querysync/java/com/google/idea/blaze/qsync/BlazeProjectListener.java +++ b/querysync/java/com/google/idea/blaze/qsync/BlazeProjectListener.java @@ -17,7 +17,6 @@ import com.google.idea.blaze.common.Context; import com.google.idea.blaze.exception.BuildException; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; /** A listener interface for new project snapshots. */ public interface BlazeProjectListener { diff --git a/querysync/java/com/google/idea/blaze/qsync/project/BlazeProjectSnapshot.java b/querysync/java/com/google/idea/blaze/qsync/BlazeProjectSnapshot.java similarity index 93% rename from querysync/java/com/google/idea/blaze/qsync/project/BlazeProjectSnapshot.java rename to querysync/java/com/google/idea/blaze/qsync/BlazeProjectSnapshot.java index e985d6d6f8d..fa302d84afb 100644 --- a/querysync/java/com/google/idea/blaze/qsync/project/BlazeProjectSnapshot.java +++ b/querysync/java/com/google/idea/blaze/qsync/BlazeProjectSnapshot.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.idea.blaze.qsync.project; +package com.google.idea.blaze.qsync; import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; @@ -21,7 +21,11 @@ import com.google.common.collect.ImmutableSet; import com.google.idea.blaze.common.BuildTarget; import com.google.idea.blaze.common.Label; +import com.google.idea.blaze.qsync.project.BuildGraphData; +import com.google.idea.blaze.qsync.project.PostQuerySyncData; +import com.google.idea.blaze.qsync.project.ProjectProto; import com.google.idea.blaze.qsync.project.ProjectProto.Project; +import com.google.idea.blaze.qsync.project.ProjectTarget; import com.google.idea.blaze.qsync.query.PackageSet; import java.nio.file.Path; import javax.annotation.Nullable; diff --git a/querysync/java/com/google/idea/blaze/qsync/BlazeProjectSnapshotBuilder.java b/querysync/java/com/google/idea/blaze/qsync/BlazeProjectSnapshotBuilder.java index acab55eefd0..effbd9c2776 100644 --- a/querysync/java/com/google/idea/blaze/qsync/BlazeProjectSnapshotBuilder.java +++ b/querysync/java/com/google/idea/blaze/qsync/BlazeProjectSnapshotBuilder.java @@ -21,10 +21,10 @@ import com.google.idea.blaze.exception.BuildException; import com.google.idea.blaze.qsync.java.PackageReader; import com.google.idea.blaze.qsync.java.WorkspaceResolvingPackageReader; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.BuildGraphData; import com.google.idea.blaze.qsync.project.PostQuerySyncData; import com.google.idea.blaze.qsync.project.ProjectProto.Project; +import com.google.idea.blaze.qsync.project.ProjectProtoTransform; import com.google.idea.blaze.qsync.query.QuerySummary; import java.nio.file.Path; import java.util.function.Supplier; diff --git a/querysync/java/com/google/idea/blaze/qsync/GraphToProjectConverter.java b/querysync/java/com/google/idea/blaze/qsync/GraphToProjectConverter.java index 6126445f82e..d666640f237 100644 --- a/querysync/java/com/google/idea/blaze/qsync/GraphToProjectConverter.java +++ b/querysync/java/com/google/idea/blaze/qsync/GraphToProjectConverter.java @@ -51,6 +51,7 @@ import com.google.idea.blaze.qsync.project.ProjectProto.ProjectPath.Base; import com.google.idea.blaze.qsync.project.ProjectTarget; import com.google.idea.blaze.qsync.project.ProjectTarget.SourceType; +import com.google.idea.blaze.qsync.project.TestSourceGlobMatcher; import com.google.idea.blaze.qsync.query.PackageSet; import java.io.IOException; import java.nio.file.Files; diff --git a/querysync/java/com/google/idea/blaze/qsync/NoopProjectRefresh.java b/querysync/java/com/google/idea/blaze/qsync/NoopProjectRefresh.java index c09ff5534de..1590bec9f06 100644 --- a/querysync/java/com/google/idea/blaze/qsync/NoopProjectRefresh.java +++ b/querysync/java/com/google/idea/blaze/qsync/NoopProjectRefresh.java @@ -16,7 +16,6 @@ package com.google.idea.blaze.qsync; import com.google.idea.blaze.common.vcs.VcsState; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.PostQuerySyncData; import com.google.idea.blaze.qsync.query.QuerySpec; import com.google.idea.blaze.qsync.query.QuerySummary; diff --git a/querysync/java/com/google/idea/blaze/qsync/ProjectRefresher.java b/querysync/java/com/google/idea/blaze/qsync/ProjectRefresher.java index 575f0fbd858..d3d9ec95afe 100644 --- a/querysync/java/com/google/idea/blaze/qsync/ProjectRefresher.java +++ b/querysync/java/com/google/idea/blaze/qsync/ProjectRefresher.java @@ -19,7 +19,6 @@ import com.google.idea.blaze.common.PrintOutput; import com.google.idea.blaze.common.vcs.VcsState; import com.google.idea.blaze.exception.BuildException; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.PostQuerySyncData; import com.google.idea.blaze.qsync.project.ProjectDefinition; import java.nio.file.Path; diff --git a/querysync/java/com/google/idea/blaze/qsync/RefreshOperation.java b/querysync/java/com/google/idea/blaze/qsync/RefreshOperation.java index a7e5c192baf..9ba7487eed4 100644 --- a/querysync/java/com/google/idea/blaze/qsync/RefreshOperation.java +++ b/querysync/java/com/google/idea/blaze/qsync/RefreshOperation.java @@ -15,7 +15,6 @@ */ package com.google.idea.blaze.qsync; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.PostQuerySyncData; import com.google.idea.blaze.qsync.query.QuerySpec; import com.google.idea.blaze.qsync.query.QuerySummary; diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJars.java b/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJars.java index 96a41e45862..7fefa8ad1ad 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJars.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJars.java @@ -19,7 +19,6 @@ import com.google.common.collect.ImmutableCollection; import com.google.idea.blaze.common.artifact.BuildArtifactCache; import com.google.idea.blaze.exception.BuildException; -import com.google.idea.blaze.qsync.TestSourceGlobMatcher; import com.google.idea.blaze.qsync.artifacts.BuildArtifact; import com.google.idea.blaze.qsync.java.JavaArtifactInfo; import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; @@ -27,6 +26,7 @@ import com.google.idea.blaze.qsync.project.ProjectDefinition; import com.google.idea.blaze.qsync.project.ProjectPath; import com.google.idea.blaze.qsync.project.ProjectProto; +import com.google.idea.blaze.qsync.project.TestSourceGlobMatcher; import java.nio.file.Path; /** diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcs.java b/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcs.java index ad1c6470538..7faae92b3fa 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcs.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/AddProjectGenSrcs.java @@ -26,12 +26,12 @@ import com.google.idea.blaze.common.PrintOutput; import com.google.idea.blaze.common.artifact.BuildArtifactCache; import com.google.idea.blaze.exception.BuildException; -import com.google.idea.blaze.qsync.TestSourceGlobMatcher; import com.google.idea.blaze.qsync.artifacts.BuildArtifact; import com.google.idea.blaze.qsync.java.JavaArtifactInfo; import com.google.idea.blaze.qsync.java.PackageStatementParser; import com.google.idea.blaze.qsync.project.ProjectDefinition; import com.google.idea.blaze.qsync.project.ProjectProto; +import com.google.idea.blaze.qsync.project.TestSourceGlobMatcher; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/BUILD b/querysync/java/com/google/idea/blaze/qsync/deps/BUILD index fc7cbf621c5..396ce9c6069 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/BUILD +++ b/querysync/java/com/google/idea/blaze/qsync/deps/BUILD @@ -7,7 +7,6 @@ java_library( ], deps = [ ":artifact_tracker_state_java_proto", - "//querysync/java/com/google/idea/blaze/qsync", "//querysync/java/com/google/idea/blaze/qsync/artifacts", "//querysync/java/com/google/idea/blaze/qsync/cc:cc_compilation_info_java_proto", "//querysync/java/com/google/idea/blaze/qsync/java", diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java index 7027abda176..c073eb68251 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java @@ -22,13 +22,13 @@ import com.google.idea.blaze.common.Context; import com.google.idea.blaze.common.artifact.BuildArtifactCache; import com.google.idea.blaze.exception.BuildException; -import com.google.idea.blaze.qsync.ProjectProtoTransform; import com.google.idea.blaze.qsync.java.PackageStatementParser; import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; import com.google.idea.blaze.qsync.project.BuildGraphData; import com.google.idea.blaze.qsync.project.ProjectDefinition; import com.google.idea.blaze.qsync.project.ProjectPath; import com.google.idea.blaze.qsync.project.ProjectProto.Project; +import com.google.idea.blaze.qsync.project.ProjectProtoTransform; /** * A {@link ProjectProtoTransform} that adds built artifact information to the project proto, based diff --git a/querysync/java/com/google/idea/blaze/qsync/ProjectProtoTransform.java b/querysync/java/com/google/idea/blaze/qsync/project/ProjectProtoTransform.java similarity index 94% rename from querysync/java/com/google/idea/blaze/qsync/ProjectProtoTransform.java rename to querysync/java/com/google/idea/blaze/qsync/project/ProjectProtoTransform.java index e7556910f3e..68c4613f8f6 100644 --- a/querysync/java/com/google/idea/blaze/qsync/ProjectProtoTransform.java +++ b/querysync/java/com/google/idea/blaze/qsync/project/ProjectProtoTransform.java @@ -13,15 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.idea.blaze.qsync; +package com.google.idea.blaze.qsync.project; import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.idea.blaze.common.Context; import com.google.idea.blaze.exception.BuildException; -import com.google.idea.blaze.qsync.project.BuildGraphData; -import com.google.idea.blaze.qsync.project.ProjectProto; import java.util.List; /** Applies a transform to a project proto instance, yielding a new instance. */ diff --git a/querysync/java/com/google/idea/blaze/qsync/TestSourceGlobMatcher.java b/querysync/java/com/google/idea/blaze/qsync/project/TestSourceGlobMatcher.java similarity index 95% rename from querysync/java/com/google/idea/blaze/qsync/TestSourceGlobMatcher.java rename to querysync/java/com/google/idea/blaze/qsync/project/TestSourceGlobMatcher.java index 84c3f2fba6a..612251a14e2 100644 --- a/querysync/java/com/google/idea/blaze/qsync/TestSourceGlobMatcher.java +++ b/querysync/java/com/google/idea/blaze/qsync/project/TestSourceGlobMatcher.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.idea.blaze.qsync; +package com.google.idea.blaze.qsync.project; import static com.google.common.collect.ImmutableSet.toImmutableSet; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; -import com.google.idea.blaze.qsync.project.ProjectDefinition; import java.io.File; import java.nio.file.Path; diff --git a/querysync/javatests/com/google/idea/blaze/qsync/BUILD b/querysync/javatests/com/google/idea/blaze/qsync/BUILD index 41f78057b91..7780796747b 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/BUILD +++ b/querysync/javatests/com/google/idea/blaze/qsync/BUILD @@ -109,15 +109,3 @@ java_test( "@truth//jar", ], ) - -java_test( - name = "TestSourceGlobMatcherTest", - size = "small", - srcs = ["TestSourceGlobMatcherTest.java"], - deps = [ - "//querysync/java/com/google/idea/blaze/qsync", - "@com_google_guava_guava//jar", - "@junit//jar", - "@truth//jar", - ], -) diff --git a/querysync/javatests/com/google/idea/blaze/qsync/ProjectRefresherTest.java b/querysync/javatests/com/google/idea/blaze/qsync/ProjectRefresherTest.java index b665378bba0..067343f7cae 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/ProjectRefresherTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/ProjectRefresherTest.java @@ -24,7 +24,6 @@ import com.google.idea.blaze.common.vcs.VcsState; import com.google.idea.blaze.common.vcs.WorkspaceFileChange; import com.google.idea.blaze.common.vcs.WorkspaceFileChange.Operation; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.PostQuerySyncData; import com.google.idea.blaze.qsync.project.ProjectDefinition; import com.google.idea.blaze.qsync.project.QuerySyncLanguage; diff --git a/querysync/javatests/com/google/idea/blaze/qsync/TestDataSyncRunner.java b/querysync/javatests/com/google/idea/blaze/qsync/TestDataSyncRunner.java index b8fae119980..042658b1217 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/TestDataSyncRunner.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/TestDataSyncRunner.java @@ -23,7 +23,6 @@ import com.google.idea.blaze.common.Context; import com.google.idea.blaze.exception.BuildException; import com.google.idea.blaze.qsync.java.PackageReader; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.BuildGraphData; import com.google.idea.blaze.qsync.project.PostQuerySyncData; import com.google.idea.blaze.qsync.project.ProjectDefinition; diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java index 0c53fce227f..2cda7b7d240 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java @@ -27,6 +27,7 @@ import com.google.idea.blaze.common.Label; import com.google.idea.blaze.common.NoopContext; import com.google.idea.blaze.common.artifact.BuildArtifactCache; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.QuerySyncTestUtils; import com.google.idea.blaze.qsync.TestDataSyncRunner; import com.google.idea.blaze.qsync.artifacts.BuildArtifact; @@ -34,7 +35,6 @@ import com.google.idea.blaze.qsync.java.PackageStatementParser; import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder.AllowPackagePrefixes; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.ProjectProto; import com.google.idea.blaze.qsync.project.ProjectProto.Library; import com.google.idea.blaze.qsync.project.ProjectProto.ProjectPath; diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java index 1b174e582f4..0e9ced69fa3 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java @@ -22,13 +22,13 @@ import com.google.common.collect.ImmutableSet; import com.google.idea.blaze.common.Label; import com.google.idea.blaze.common.NoopContext; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.QuerySyncTestUtils; import com.google.idea.blaze.qsync.TestDataSyncRunner; import com.google.idea.blaze.qsync.java.JavaArtifactInfo; import com.google.idea.blaze.qsync.java.PackageStatementParser; import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder.AllowPackagePrefixes; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.ProjectPath; import com.google.idea.blaze.qsync.project.ProjectProto; import com.google.idea.blaze.qsync.project.ProjectProto.Library; diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java index 6bd457ebb5a..15254a86264 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java @@ -27,6 +27,7 @@ import com.google.idea.blaze.common.Label; import com.google.idea.blaze.common.NoopContext; import com.google.idea.blaze.common.artifact.BuildArtifactCache; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.QuerySyncTestUtils; import com.google.idea.blaze.qsync.TestDataSyncRunner; import com.google.idea.blaze.qsync.artifacts.BuildArtifact; @@ -34,7 +35,6 @@ import com.google.idea.blaze.qsync.java.PackageStatementParser; import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder; import com.google.idea.blaze.qsync.java.SrcJarInnerPathFinder.AllowPackagePrefixes; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.ProjectProto; import com.google.idea.blaze.qsync.project.ProjectProto.ContentEntry; import com.google.idea.blaze.qsync.project.ProjectProto.Module; diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcsTest.java index 4f7f2a31913..941bddee2f1 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcsTest.java @@ -29,12 +29,12 @@ import com.google.idea.blaze.common.Label; import com.google.idea.blaze.common.NoopContext; import com.google.idea.blaze.common.artifact.BuildArtifactCache; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.QuerySyncTestUtils; import com.google.idea.blaze.qsync.TestDataSyncRunner; import com.google.idea.blaze.qsync.artifacts.BuildArtifact; import com.google.idea.blaze.qsync.java.JavaArtifactInfo; import com.google.idea.blaze.qsync.java.PackageStatementParser; -import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.project.ProjectProto; import com.google.idea.blaze.qsync.project.ProjectProto.ArtifactDirectoryContents; import com.google.idea.blaze.qsync.project.ProjectProto.ContentEntry; diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD index 9d2a2be2806..bb6f387336d 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD @@ -64,10 +64,10 @@ java_test( size = "small", srcs = ["AddDependencyGenSrcsJarsTest.java"], deps = [ + "//querysync/java/com/google/idea/blaze/qsync", "//querysync/java/com/google/idea/blaze/qsync/artifacts", "//querysync/java/com/google/idea/blaze/qsync/deps", "//querysync/java/com/google/idea/blaze/qsync/java", - "//querysync/java/com/google/idea/blaze/qsync/project", "//querysync/java/com/google/idea/blaze/qsync/project:project_java_proto", "//querysync/javatests/com/google/idea/blaze/qsync:TestUtils", "//querysync/javatests/com/google/idea/blaze/qsync/testdata", @@ -86,6 +86,7 @@ java_test( size = "small", srcs = ["AddDependencySrcJarsTest.java"], deps = [ + "//querysync/java/com/google/idea/blaze/qsync", "//querysync/java/com/google/idea/blaze/qsync/deps", "//querysync/java/com/google/idea/blaze/qsync/java", "//querysync/java/com/google/idea/blaze/qsync/project", @@ -105,10 +106,10 @@ java_test( size = "small", srcs = ["AddProjectGenSrcJarsTest.java"], deps = [ + "//querysync/java/com/google/idea/blaze/qsync", "//querysync/java/com/google/idea/blaze/qsync/artifacts", "//querysync/java/com/google/idea/blaze/qsync/deps", "//querysync/java/com/google/idea/blaze/qsync/java", - "//querysync/java/com/google/idea/blaze/qsync/project", "//querysync/java/com/google/idea/blaze/qsync/project:project_java_proto", "//querysync/javatests/com/google/idea/blaze/qsync:TestUtils", "//querysync/javatests/com/google/idea/blaze/qsync/testdata", @@ -128,10 +129,10 @@ java_test( size = "small", srcs = ["AddProjectGenSrcsTest.java"], deps = [ + "//querysync/java/com/google/idea/blaze/qsync", "//querysync/java/com/google/idea/blaze/qsync/artifacts", "//querysync/java/com/google/idea/blaze/qsync/deps", "//querysync/java/com/google/idea/blaze/qsync/java", - "//querysync/java/com/google/idea/blaze/qsync/project", "//querysync/java/com/google/idea/blaze/qsync/project:project_java_proto", "//querysync/javatests/com/google/idea/blaze/qsync:TestUtils", "//querysync/javatests/com/google/idea/blaze/qsync/testdata", diff --git a/querysync/javatests/com/google/idea/blaze/qsync/project/BUILD b/querysync/javatests/com/google/idea/blaze/qsync/project/BUILD index c9690e99004..a216826be1b 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/project/BUILD +++ b/querysync/javatests/com/google/idea/blaze/qsync/project/BUILD @@ -56,3 +56,15 @@ java_test( "@truth//jar", ], ) + +java_test( + name = "TestSourceGlobMatcherTest", + size = "small", + srcs = ["TestSourceGlobMatcherTest.java"], + deps = [ + "//querysync/java/com/google/idea/blaze/qsync/project", + "@com_google_guava_guava//jar", + "@junit//jar", + "@truth//jar", + ], +) diff --git a/querysync/javatests/com/google/idea/blaze/qsync/TestSourceGlobMatcherTest.java b/querysync/javatests/com/google/idea/blaze/qsync/project/TestSourceGlobMatcherTest.java similarity index 98% rename from querysync/javatests/com/google/idea/blaze/qsync/TestSourceGlobMatcherTest.java rename to querysync/javatests/com/google/idea/blaze/qsync/project/TestSourceGlobMatcherTest.java index 37f58e82201..e571d2fb1e7 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/TestSourceGlobMatcherTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/project/TestSourceGlobMatcherTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.idea.blaze.qsync; +package com.google.idea.blaze.qsync.project; import static com.google.common.truth.Truth.assertThat; From 459a595c892e67a3d838554c9ae342f87d80ec0f Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 18 Mar 2024 04:51:06 -0700 Subject: [PATCH 20/45] Create a new class to store immutable artifact tracker state. Keep an instance of this inside `BlazeProjectSnapshot` so that artifact state is accessible alongside the other project state. This allows artifact info to be looked up in future CLs. (cherry picked from commit 4620108d34d7007eb360532ccf2ec5c28bdced2d) --- .../qsync/AndroidProjectProtoTransform.java | 4 +-- .../idea/blaze/base/qsync/ProjectLoader.java | 5 ---- .../blaze/base/qsync/QuerySyncProject.java | 14 ++++----- .../base/qsync/cache/ArtifactTrackerImpl.java | 5 ++++ .../java/com/google/idea/blaze/qsync/BUILD | 1 + .../blaze/qsync/BlazeProjectSnapshot.java | 9 ++++-- .../qsync/BlazeProjectSnapshotBuilder.java | 3 ++ .../qsync/deps/AddAndroidResPackages.java | 10 ++++--- .../blaze/qsync/deps/AddDependencyAars.java | 9 +++--- .../blaze/qsync/deps/ArtifactTracker.java | 22 ++++++++++++++ .../qsync/deps/BuiltDependenciesSupplier.java | 30 ------------------- .../blaze/qsync/deps/NewArtifactTracker.java | 13 +++++--- .../com/google/idea/blaze/qsync/BUILD | 1 + .../idea/blaze/qsync/TestDataSyncRunner.java | 2 ++ 14 files changed, 68 insertions(+), 60 deletions(-) delete mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/BuiltDependenciesSupplier.java diff --git a/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java b/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java index 776de76230b..aced873e292 100644 --- a/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java +++ b/aswb/src/com/google/idea/blaze/android/qsync/AndroidProjectProtoTransform.java @@ -50,11 +50,11 @@ private AndroidProjectProtoTransform(QuerySyncProject project) { updateOperations = ImmutableList.of( new AddDependencyAars( - project.getBuiltArtifactsSupplier(), + project.getArtifactTracker()::getStateSnapshot, project.getBuildArtifactCache(), project.getProjectDefinition(), in -> ManifestParser.parseManifestFromInputStream(in).packageName), - new AddAndroidResPackages(project.getBuiltArtifactsSupplier())); + new AddAndroidResPackages(project.getArtifactTracker()::getStateSnapshot)); } @Override diff --git a/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java b/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java index 3e3f7b54159..bf2ba032caa 100644 --- a/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java +++ b/base/src/com/google/idea/blaze/base/qsync/ProjectLoader.java @@ -51,7 +51,6 @@ import com.google.idea.blaze.qsync.ProjectRefresher; import com.google.idea.blaze.qsync.VcsStateDiffer; import com.google.idea.blaze.qsync.deps.ArtifactTracker; -import com.google.idea.blaze.qsync.deps.BuiltDependenciesSupplier; import com.google.idea.blaze.qsync.deps.DependenciesProjectProtoUpdater; import com.google.idea.blaze.qsync.deps.NewArtifactTracker; import com.google.idea.blaze.qsync.java.PackageStatementParser; @@ -150,7 +149,6 @@ public QuerySyncProject loadProject(BlazeContext context) throws BuildException ideProjectBasePath.resolve("buildcache"), artifactFetcher, executor); ArtifactTracker artifactTracker; - BuiltDependenciesSupplier builtDepsSupplier; RenderJarArtifactTracker renderJarArtifactTracker; AppInspectorArtifactTracker appInspectorArtifactTracker; if (QuerySync.USE_NEW_BUILD_ARTIFACT_MANAGEMENT) { @@ -168,7 +166,6 @@ public QuerySyncProject loadProject(BlazeContext context) throws BuildException artifactTracker = tracker; renderJarArtifactTracker = new RenderJarArtifactTrackerImpl(); appInspectorArtifactTracker = new AppInspectorArtifactTrackerImpl(); - builtDepsSupplier = tracker.getBuiltDepsSupplier(); } else { ArtifactTrackerImpl impl = new ArtifactTrackerImpl( @@ -183,7 +180,6 @@ public QuerySyncProject loadProject(BlazeContext context) throws BuildException artifactTracker = impl; renderJarArtifactTracker = impl; appInspectorArtifactTracker = impl; - builtDepsSupplier = BuiltDependenciesSupplier.NONE; } RenderJarTracker renderJarTracker = new RenderJarTrackerImpl(graph, renderJarBuilder, renderJarArtifactTracker); @@ -221,7 +217,6 @@ public QuerySyncProject loadProject(BlazeContext context) throws BuildException importSettings, workspaceRoot, artifactTracker, - builtDepsSupplier, artifactCache, artifactStore, renderJarArtifactTracker, diff --git a/base/src/com/google/idea/blaze/base/qsync/QuerySyncProject.java b/base/src/com/google/idea/blaze/base/qsync/QuerySyncProject.java index 75a5422a5c4..1b3e35fd581 100644 --- a/base/src/com/google/idea/blaze/base/qsync/QuerySyncProject.java +++ b/base/src/com/google/idea/blaze/base/qsync/QuerySyncProject.java @@ -52,7 +52,6 @@ import com.google.idea.blaze.qsync.BlazeProjectSnapshot; import com.google.idea.blaze.qsync.BlazeProjectSnapshotBuilder; import com.google.idea.blaze.qsync.deps.ArtifactTracker; -import com.google.idea.blaze.qsync.deps.BuiltDependenciesSupplier; import com.google.idea.blaze.qsync.project.PostQuerySyncData; import com.google.idea.blaze.qsync.project.ProjectDefinition; import com.google.idea.blaze.qsync.project.ProjectPath; @@ -91,7 +90,6 @@ public class QuerySyncProject { private final BlazeImportSettings importSettings; private final WorkspaceRoot workspaceRoot; private final ArtifactTracker artifactTracker; - private final BuiltDependenciesSupplier builtArtifactsSupplier; private final BuildArtifactCache buildArtifactCache; private final ProjectArtifactStore artifactStore; private final RenderJarArtifactTracker renderJarArtifactTracker; @@ -122,7 +120,6 @@ public QuerySyncProject( BlazeImportSettings importSettings, WorkspaceRoot workspaceRoot, ArtifactTracker artifactTracker, - BuiltDependenciesSupplier builtArtifactsSupplier, BuildArtifactCache buildArtifactCache, ProjectArtifactStore artifactStore, RenderJarArtifactTracker renderJarArtifactTracker, @@ -147,7 +144,6 @@ public QuerySyncProject( this.importSettings = importSettings; this.workspaceRoot = workspaceRoot; this.artifactTracker = artifactTracker; - this.builtArtifactsSupplier = builtArtifactsSupplier; this.buildArtifactCache = buildArtifactCache; this.artifactStore = artifactStore; this.renderJarArtifactTracker = renderJarArtifactTracker; @@ -201,10 +197,6 @@ public ProjectPath.Resolver getProjectPathResolver() { return projectPathResolver; } - public BuiltDependenciesSupplier getBuiltArtifactsSupplier() { - return builtArtifactsSupplier; - } - public BuildArtifactCache getBuildArtifactCache() { return buildArtifactCache; } @@ -269,7 +261,10 @@ public void sync(BlazeContext parentContext, Optional lastQue : projectQuerier.update(projectDefinition, lastQuery.get(), context); BlazeProjectSnapshot newSnapshot = blazeProjectSnapshotBuilder.createBlazeProjectSnapshot( - context, postQuerySyncData, projectProtoTransforms.getComposedTransform()); + context, + postQuerySyncData, + artifactTracker.getStateSnapshot(), + projectProtoTransforms.getComposedTransform()); onNewSnapshot(context, newSnapshot); // TODO: Revisit SyncListeners once we switch fully to qsync @@ -348,6 +343,7 @@ public void build(BlazeContext parentContext, DependencyTracker.DependencyBuildR blazeProjectSnapshotBuilder.createBlazeProjectSnapshot( context, snapshotHolder.getCurrent().orElseThrow().queryData(), + artifactTracker.getStateSnapshot(), projectProtoTransforms.getComposedTransform()); onNewSnapshot(context, newSnapshot); } diff --git a/base/src/com/google/idea/blaze/base/qsync/cache/ArtifactTrackerImpl.java b/base/src/com/google/idea/blaze/base/qsync/cache/ArtifactTrackerImpl.java index cd20f8af70c..69b39834523 100644 --- a/base/src/com/google/idea/blaze/base/qsync/cache/ArtifactTrackerImpl.java +++ b/base/src/com/google/idea/blaze/base/qsync/cache/ArtifactTrackerImpl.java @@ -739,6 +739,11 @@ private ProjectProto.Project updateProjectProtoForJavaDeps(ProjectProto.Project return projectProto; } + @Override + public ArtifactTracker.State getStateSnapshot() { + return State.EMPTY; + } + @Override public Set

For each of {@code srcJars}, sets the {@link ProjectPath#innerJarPath()} to the java source * root within that jar file, if necessary. */ - private ImmutableList resolveSrcJarInnerPaths(Collection srcJars) { + private ImmutableList resolveSrcJarInnerPaths( + Collection srcJars, AllowPackagePrefixes allowPackagePrefixes) { ImmutableList.Builder newSrcJars = ImmutableList.builder(); for (ProjectPath srcJar : srcJars) { Path jarFile = resolver.resolve(srcJar); - srcJarInnerPathFinder.findInnerJarPaths(jarFile.toFile()).stream() + srcJarInnerPathFinder.findInnerJarPaths(jarFile.toFile(), allowPackagePrefixes).stream() .map(p -> p.path) .map(srcJar::withInnerJarPath) .forEach(newSrcJars::add); From a13c9cd69de781fbed3a2b75ddf10201f6ea8402 Mon Sep 17 00:00:00 2001 From: Mai Hussien Date: Tue, 19 Mar 2024 10:25:25 -0700 Subject: [PATCH 25/45] Add missing `kt_android_library` Based on https://github.com/bazelbuild/intellij/pull/6291 (cherry picked from commit 2b48c2eba3c7a47497a31ae3865b35733d1949ea) --- aspect/intellij_info_impl.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aspect/intellij_info_impl.bzl b/aspect/intellij_info_impl.bzl index a861937d1dc..6e384a93a27 100644 --- a/aspect/intellij_info_impl.bzl +++ b/aspect/intellij_info_impl.bzl @@ -833,7 +833,7 @@ def _collect_android_ide_info(target, ctx, semantics, ide_info, ide_info_file, o consistent functionality with the previous condition of the presence of the .android legacy provider. """ - if ctx.rule.kind not in ["android_library", "android_binary"]: + if ctx.rule.kind not in ["android_library", "android_binary", "kt_android_library"]: return False android_semantics = semantics.android if hasattr(semantics, "android") else None From 116d76751c718871e7c84b456fe183e845ae5e23 Mon Sep 17 00:00:00 2001 From: Mai Hussien Date: Tue, 19 Mar 2024 10:30:58 -0700 Subject: [PATCH 26/45] Upgrade the version of AS 2023.2 (cherry picked from commit 94af216da4d6485c9c35cf64529b3fa1dc41fdf9) --- WORKSPACE.bzlmod | 4 ++-- .../com/google/idea/blaze/android/resources/BlazeRClass.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod index 59e7cd11f0b..6c1d831ee84 100644 --- a/WORKSPACE.bzlmod +++ b/WORKSPACE.bzlmod @@ -630,8 +630,8 @@ http_archive( http_archive( name = "android_studio_2023_2", build_file = "@//intellij_platform_sdk:BUILD.android_studio232", - sha256 = "f2ccc445fb5c87525627ae81725241ab90d9707d577f5732563d3c5a49cba12f", - url = "https://dl.google.com/dl/android/studio/ide-zips/2023.2.1.14/android-studio-2023.2.1.14-linux.tar.gz", + sha256 = "0026427572849c9cbb0c94d6f9718ea08bc345dccfe3b372b54100a95fff99b5", + url = "https://dl.google.com/dl/android/studio/ide-zips/2023.2.1.24/android-studio-2023.2.1.24-linux.tar.gz", ) # The plugin api for android_studio_2023_1 android_studio. This is required to build ASwB and run integration tests diff --git a/aswb/sdkcompat/as232/com/google/idea/blaze/android/resources/BlazeRClass.java b/aswb/sdkcompat/as232/com/google/idea/blaze/android/resources/BlazeRClass.java index 3c64683d517..37895c5a1cc 100644 --- a/aswb/sdkcompat/as232/com/google/idea/blaze/android/resources/BlazeRClass.java +++ b/aswb/sdkcompat/as232/com/google/idea/blaze/android/resources/BlazeRClass.java @@ -21,7 +21,7 @@ import com.android.ide.common.rendering.api.ResourceNamespace; import com.android.tools.idea.res.ResourceRepositoryRClass; import com.android.tools.idea.res.StudioResourceRepositoryManager; -import com.android.tools.res.LocalResourceRepository; +import com.android.tools.res.CacheableResourceRepository; import com.intellij.openapi.module.Module; import com.intellij.openapi.module.ModulePointerManager; import com.intellij.openapi.vfs.VirtualFile; @@ -54,7 +54,7 @@ public StudioResourceRepositoryManager getResourceRepositoryManager() { } @Override - public LocalResourceRepository getResourceRepository() { + public CacheableResourceRepository getResourceRepository() { return StudioResourceRepositoryManager.getAppResources(androidFacet); } From 76300c47352eeb8e87071e1d7df1d711d2ddc854 Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Tue, 19 Mar 2024 10:46:24 -0700 Subject: [PATCH 27/45] Implement StartBuild [ (cherry picked from commit e28f73ed9e8726cccce07f4895a0678b477a8a72) --- third_party/bazel/src/main/protobuf/BUILD | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/third_party/bazel/src/main/protobuf/BUILD b/third_party/bazel/src/main/protobuf/BUILD index f3201f5faf1..9fe4393476d 100644 --- a/third_party/bazel/src/main/protobuf/BUILD +++ b/third_party/bazel/src/main/protobuf/BUILD @@ -186,6 +186,7 @@ proto_library( srcs = ["bazel_output_service.proto"], deps = [ "@com_google_protobuf//:any_proto", + "@googleapis//:google_rpc_status_proto", ], ) @@ -245,6 +246,9 @@ filegroup( name = "dist_jars", srcs = [s + "_java_proto_srcs" for s in FILES] + [ ":analysis_v2_java_proto_srcs", + ":bazel_output_service_java_grpc_srcs", + ":bazel_output_service_java_proto_srcs", + ":bazel_output_service_rev2_java_proto_srcs", ":cache_salt_java_proto_srcs", ":command_line_java_proto_srcs", ":command_server_java_grpc_srcs", From 41c118c60861a9bd61dd3f1c4f7298c271ae5189 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 20 Mar 2024 06:30:01 -0700 Subject: [PATCH 28/45] Add CC information to the artifact tracker state. Extract this from the proto that is output from the aspect, and store it in new data classes in a higher level format. Add proto definitions to persist it with the rest of the artifact tracker state and subsequently restore it later on. The data is not used to update the project proto yet, so this CL has no user visible behaviour changes. That will follow in a later CL. (cherry picked from commit 279baf3d5f5aeb7f7b2d3091692569a3e4e9c995) --- .../blaze/qsync/cc/CcIncludeDirectories.java | 46 +++++++ .../blaze/qsync/deps/ArtifactTracker.java | 13 +- .../ArtifactTrackerStateDeserializer.java | 69 +++++++++-- .../deps/ArtifactTrackerStateSerializer.java | 50 ++++++++ .../com/google/idea/blaze/qsync/deps/BUILD | 2 + .../blaze/qsync/deps/CcCompilationInfo.java | 115 ++++++++++++++++++ .../idea/blaze/qsync/deps/CcToolchain.java | 111 +++++++++++++++++ .../blaze/qsync/deps/NewArtifactTracker.java | 28 ++++- .../blaze/qsync/deps/TargetBuildInfo.java | 24 +++- .../qsync/deps/artifact_tracker_state.proto | 23 ++++ ...ArtifactTrackerStateSerializationTest.java | 76 +++++++++++- .../com/google/idea/blaze/qsync/deps/BUILD | 1 + 12 files changed, 539 insertions(+), 19 deletions(-) create mode 100644 querysync/java/com/google/idea/blaze/qsync/cc/CcIncludeDirectories.java create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/CcCompilationInfo.java create mode 100644 querysync/java/com/google/idea/blaze/qsync/deps/CcToolchain.java diff --git a/querysync/java/com/google/idea/blaze/qsync/cc/CcIncludeDirectories.java b/querysync/java/com/google/idea/blaze/qsync/cc/CcIncludeDirectories.java new file mode 100644 index 00000000000..71b186b74dc --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/cc/CcIncludeDirectories.java @@ -0,0 +1,46 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.cc; + +import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectPath.Root; +import java.nio.file.Path; + +/** Static helpers for managing C/C++ include paths. */ +public class CcIncludeDirectories { + + public static final ProjectPath GEN_INCLUDE_BASE = + ProjectPath.create(Root.PROJECT, Path.of("buildout")); + + private CcIncludeDirectories() {} + + /** + * Constructs a project path for a given include dir flag value. This can then be used to ensure + * that the flag passed to the IDE points to the correct location. + */ + public static ProjectPath projectPathFor(String includeDir) { + Path includePath = Path.of(includeDir); + if (includePath.startsWith("blaze-out") || includePath.startsWith("bazel-out")) { + // The directories given by blaze include the "bazel-out" component, but that is not present + // in the paths of the generated headers themselves due to the legacy semantics of + // OutputArtifactInfo which strips it. Since that determines their location in the + // artifact store, remove here too it to ensure consistency: + return GEN_INCLUDE_BASE.resolveChild(includePath.getName(0).relativize(includePath)); + } else { + return ProjectPath.WORKSPACE_ROOT.resolveChild(includePath); + } + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTracker.java b/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTracker.java index 14f216c8aba..93d61bc44e8 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTracker.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTracker.java @@ -38,12 +38,16 @@ public interface ArtifactTracker> { @AutoValue abstract class State { - public static final State EMPTY = create(ImmutableMap.of()); + public static final State EMPTY = create(ImmutableMap.of(), ImmutableMap.of()); abstract ImmutableMap depsMap(); - static State create(ImmutableMap map) { - return new AutoValue_ArtifactTracker_State(map); + abstract ImmutableMap ccToolchainMap(); + + static State create( + ImmutableMap map, + ImmutableMap ccToolchainMap) { + return new AutoValue_ArtifactTracker_State(map, ccToolchainMap); } public Optional getJavaInfo(Label label) { @@ -57,7 +61,8 @@ public static State forJavaArtifacts(ImmutableCollection infos .collect( toImmutableMap( JavaArtifactInfo::label, - j -> TargetBuildInfo.forJavaTarget(j, DependencyBuildContext.NONE)))); + j -> TargetBuildInfo.forJavaTarget(j, DependencyBuildContext.NONE))), + ImmutableMap.of()); } } diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTrackerStateDeserializer.java b/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTrackerStateDeserializer.java index d99a928b6a0..35983b08b1b 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTrackerStateDeserializer.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTrackerStateDeserializer.java @@ -18,7 +18,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -27,6 +26,7 @@ import com.google.idea.blaze.qsync.artifacts.BuildArtifact; import com.google.idea.blaze.qsync.java.ArtifactTrackerProto; import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.project.ProjectPath; import com.google.idea.blaze.qsync.project.SnapshotDeserializer; import java.nio.file.Path; import java.time.Instant; @@ -37,16 +37,22 @@ /** Deserializes {@link NewArtifactTracker} state from a proto. */ public class ArtifactTrackerStateDeserializer { - private final ImmutableMap.Builder map = ImmutableMap.builder(); + private final ImmutableMap.Builder depsMap = ImmutableMap.builder(); + private final ImmutableMap.Builder ccToolchainMap = ImmutableMap.builder(); private final Map buildContexts = Maps.newHashMap(); public void visit(ArtifactTrackerProto.ArtifactTrackerState proto) { proto.getBuildContextsList().forEach(this::visitBuildContext); proto.getBuiltDepsMap().entrySet().forEach(this::visitTargetBuildInfo); + proto.getCcToolchainsMap().forEach(this::visitCcToolchain); } public ImmutableMap getBuiltDepsMap() { - return map.buildOrThrow(); + return depsMap.buildOrThrow(); + } + + public ImmutableMap getCcToolchainMap() { + return ccToolchainMap.buildOrThrow(); } private void visitBuildContext(ArtifactTrackerProto.BuildContext buildContext) { @@ -64,15 +70,34 @@ private void visitBuildContext(ArtifactTrackerProto.BuildContext buildContext) { private void visitTargetBuildInfo(Map.Entry entry) { ArtifactTrackerProto.TargetBuildInfo proto = entry.getValue(); - DependencyBuildContext buildContext = - Preconditions.checkNotNull(buildContexts.get(proto.getBuildId())); + TargetBuildInfo.Builder builder = + TargetBuildInfo.builder().buildContext(buildContexts.get(proto.getBuildId())); Label owner = Label.of(entry.getKey()); if (proto.hasJavaArtifacts()) { - map.put( - owner, - TargetBuildInfo.forJavaTarget( - convertJavaArtifactInfo(owner, proto.getJavaArtifacts()), buildContext)); + builder.javaInfo(convertJavaArtifactInfo(owner, proto.getJavaArtifacts())); } + if (proto.hasCcInfo()) { + builder.ccInfo(convertCcCompilationInfo(owner, proto.getCcInfo())); + } + depsMap.put(owner, builder.build()); + } + + private void visitCcToolchain(String id, ArtifactTrackerProto.CcToolchain proto) { + ccToolchainMap.put( + id, + CcToolchain.builder() + .id(id) + .compiler(proto.getCompiler()) + .compilerExecutable(ProjectPath.create(proto.getCompilerExecutable())) + .cpu(proto.getCpu()) + .targetGnuSystemName(proto.getTargetGnuSystemName()) + .builtInIncludeDirectories( + proto.getBuiltInIncludeDirectoriesList().stream() + .map(ProjectPath::create) + .collect(toImmutableList())) + .cOptions(ImmutableList.copyOf(proto.getCOptionsList())) + .cppOptions(ImmutableList.copyOf(proto.getCppOptionsList())) + .build()); } private JavaArtifactInfo convertJavaArtifactInfo( @@ -88,6 +113,32 @@ private JavaArtifactInfo convertJavaArtifactInfo( .build(); } + private CcCompilationInfo convertCcCompilationInfo( + Label owner, ArtifactTrackerProto.CcCompilationInfo proto) { + return CcCompilationInfo.builder() + .target(owner) + .defines(proto.getDefinesList()) + .includeDirectories( + proto.getIncludeDirectoriesList().stream() + .map(ProjectPath::create) + .collect(toImmutableList())) + .quoteIncludeDirectories( + proto.getQuoteIncludeDirectoriesList().stream() + .map(ProjectPath::create) + .collect(toImmutableList())) + .systemIncludeDirectories( + proto.getSysytemIncludeDirectoriesList().stream() + .map(ProjectPath::create) + .collect(toImmutableList())) + .frameworkIncludeDirectories( + proto.getFrameworkIncludeDirectoriesList().stream() + .map(ProjectPath::create) + .collect(toImmutableList())) + .genHeaders(toArtifactList(proto.getGenHeadersList(), owner)) + .toolchainId(proto.getToolchainId()) + .build(); + } + private ImmutableList toArtifactList( List protos, Label owner) { return protos.stream() diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTrackerStateSerializer.java b/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTrackerStateSerializer.java index 4969900c024..31f87baf994 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTrackerStateSerializer.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTrackerStateSerializer.java @@ -27,6 +27,7 @@ import com.google.idea.blaze.qsync.java.ArtifactTrackerProto.ArtifactTrackerState; import com.google.idea.blaze.qsync.java.ArtifactTrackerProto.BuildContext; import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.project.ProjectPath; import com.google.idea.blaze.qsync.project.SnapshotSerializer; import java.nio.file.Path; import java.util.Map; @@ -47,6 +48,12 @@ public ArtifactTrackerStateSerializer visitDepsMap(Map b return this; } + @CanIgnoreReturnValue + public ArtifactTrackerStateSerializer visitToolchainMap(Map toolchainMap) { + toolchainMap.values().forEach(this::visitCcToolchain); + return this; + } + public ArtifactTrackerState toProto() { return proto.build(); } @@ -58,6 +65,7 @@ private void visitTargetBuildInfo(Label target, TargetBuildInfo targetBuildInfo) ArtifactTrackerProto.TargetBuildInfo.newBuilder(); builder.setBuildId(targetBuildInfo.buildContext().buildId()); targetBuildInfo.javaInfo().ifPresent(ji -> visitJavaInfo(ji, builder)); + targetBuildInfo.ccInfo().ifPresent(cc -> visitCcInfo(cc, builder)); proto.putBuiltDeps(target.toString(), builder.build()); } @@ -96,4 +104,46 @@ private ImmutableList toProtos(ImmutableList artifacts) .build()) .collect(toImmutableList()); } + + private void visitCcInfo( + CcCompilationInfo ccInfo, ArtifactTrackerProto.TargetBuildInfo.Builder builder) { + builder + .getCcInfoBuilder() + .addAllDefines(ccInfo.defines()) + .addAllIncludeDirectories( + ccInfo.includeDirectories().stream() + .map(ProjectPath::toProto) + .collect(toImmutableList())) + .addAllQuoteIncludeDirectories( + ccInfo.quoteIncludeDirectories().stream() + .map(ProjectPath::toProto) + .collect(toImmutableList())) + .addAllSysytemIncludeDirectories( + ccInfo.systemIncludeDirectories().stream() + .map(ProjectPath::toProto) + .collect(toImmutableList())) + .addAllFrameworkIncludeDirectories( + ccInfo.frameworkIncludeDirectories().stream() + .map(ProjectPath::toProto) + .collect(toImmutableList())) + .addAllGenHeaders(toProtos(ccInfo.genHeaders())) + .setToolchainId(ccInfo.toolchainId()); + } + + private void visitCcToolchain(CcToolchain toolchain) { + proto.putCcToolchains( + toolchain.id(), + ArtifactTrackerProto.CcToolchain.newBuilder() + .setCompiler(toolchain.compiler()) + .setCompilerExecutable(toolchain.compilerExecutable().toProto()) + .setCpu(toolchain.cpu()) + .setTargetGnuSystemName(toolchain.targetGnuSystemName()) + .addAllBuiltInIncludeDirectories( + toolchain.builtInIncludeDirectories().stream() + .map(ProjectPath::toProto) + .collect(toImmutableList())) + .addAllCOptions(toolchain.cOptions()) + .addAllCppOptions(toolchain.cppOptions()) + .build()); + } } diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/BUILD b/querysync/java/com/google/idea/blaze/qsync/deps/BUILD index 396ce9c6069..73940ca9b46 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/BUILD +++ b/querysync/java/com/google/idea/blaze/qsync/deps/BUILD @@ -8,6 +8,7 @@ java_library( deps = [ ":artifact_tracker_state_java_proto", "//querysync/java/com/google/idea/blaze/qsync/artifacts", + "//querysync/java/com/google/idea/blaze/qsync/cc", "//querysync/java/com/google/idea/blaze/qsync/cc:cc_compilation_info_java_proto", "//querysync/java/com/google/idea/blaze/qsync/java", "//querysync/java/com/google/idea/blaze/qsync/java:java_target_info_java_proto", @@ -28,6 +29,7 @@ proto_library( name = "artifact_tracker_state_proto", srcs = ["artifact_tracker_state.proto"], deps = [ + "//querysync/java/com/google/idea/blaze/qsync/project:project_proto", "//querysync/java/com/google/idea/blaze/qsync/project:snapshot_proto", ], ) diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/CcCompilationInfo.java b/querysync/java/com/google/idea/blaze/qsync/deps/CcCompilationInfo.java new file mode 100644 index 00000000000..059bf8c5f37 --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/CcCompilationInfo.java @@ -0,0 +1,115 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.idea.blaze.common.Interners; +import com.google.idea.blaze.common.Label; +import com.google.idea.blaze.qsync.artifacts.BuildArtifact; +import com.google.idea.blaze.qsync.cc.CcIncludeDirectories; +import com.google.idea.blaze.qsync.java.cc.CcCompilationInfoOuterClass.CcTargetInfo; +import com.google.idea.blaze.qsync.project.ProjectPath; +import java.nio.file.Path; +import java.util.List; +import java.util.function.Function; + +/** + * C/C++ compilation information. This stores information required to compile C or C++ targets. The + * information is extracted from the build at build deps time. + */ +@AutoValue +abstract class CcCompilationInfo { + abstract Label target(); + + abstract ImmutableList defines(); + + abstract ImmutableList includeDirectories(); + + abstract ImmutableList quoteIncludeDirectories(); + + abstract ImmutableList systemIncludeDirectories(); + + abstract ImmutableList frameworkIncludeDirectories(); + + abstract ImmutableList genHeaders(); + + abstract String toolchainId(); + + static CcCompilationInfo.Builder builder() { + return new AutoValue_CcCompilationInfo.Builder(); + } + + static CcCompilationInfo create(CcTargetInfo targetInfo, Function digestMap) { + Label target = Label.of(targetInfo.getLabel()); + return builder() + .target(target) + .defines(ImmutableList.copyOf(targetInfo.getDefinesList())) + .includeDirectories( + targetInfo.getIncludeDirectoriesList().stream() + .map(CcIncludeDirectories::projectPathFor) + .collect(toImmutableList())) + .quoteIncludeDirectories( + targetInfo.getQuoteIncludeDirectoriesList().stream() + .map(CcIncludeDirectories::projectPathFor) + .collect(toImmutableList())) + .systemIncludeDirectories( + targetInfo.getSystemIncludeDirectoriesList().stream() + .map(CcIncludeDirectories::projectPathFor) + .collect(toImmutableList())) + .frameworkIncludeDirectories( + targetInfo.getFrameworkIncludeDirectoriesList().stream() + .map(CcIncludeDirectories::projectPathFor) + .collect(toImmutableList())) + .genHeaders(toArtifacts(targetInfo.getGenHdrsList(), digestMap, target)) + .toolchainId(targetInfo.getToolchainId()) + .build(); + } + + // TODO share this with the java class from where it was copied. + private static ImmutableList toArtifacts( + List paths, Function digestMap, Label target) { + return paths.stream() + .map(Interners::pathOf) + .map(p -> BuildArtifact.create(p, target, digestMap)) + .collect(toImmutableList()); + } + + /** Builder for {@link CcCompilationInfo}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder target(Label value); + + public abstract Builder defines(List value); + + public abstract Builder includeDirectories(List value); + + public abstract Builder quoteIncludeDirectories(List value); + + public abstract Builder systemIncludeDirectories(List value); + + public abstract Builder frameworkIncludeDirectories(List value); + + public abstract Builder genHeaders(List value); + + public abstract Builder toolchainId(String value); + + public abstract CcCompilationInfo build(); + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/CcToolchain.java b/querysync/java/com/google/idea/blaze/qsync/deps/CcToolchain.java new file mode 100644 index 00000000000..54c616d34cb --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/CcToolchain.java @@ -0,0 +1,111 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.idea.blaze.qsync.cc.CcIncludeDirectories; +import com.google.idea.blaze.qsync.java.cc.CcCompilationInfoOuterClass.CcToolchainInfo; +import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectPath.Root; +import java.nio.file.Path; + +/** + * Information about a C/C++ toolchain. The information is extracted from the build at build deps + * time. + */ +@AutoValue +public abstract class CcToolchain { + + /** Unique ID for this toolchain. */ + abstract String id(); + + /** The compiler name, as reported by bazel API {@code CcToolchainInfo.compiler}. */ + abstract String compiler(); + + /** + * Path to the compiler executable, as reported by bazel API {@code + * CcToolchainInfo.compiler_executable}. + */ + abstract ProjectPath compilerExecutable(); + + /** Target CPU of the C++ toolchain, as reported by bazel API {@code CcToolchainInfo.cpu}. */ + abstract String cpu(); + + /** As reported by bazel API {@code CcToolchainInfo.target_gnu_system_name}. */ + abstract String targetGnuSystemName(); + + /** As reported by bazel API {@code CcToolchainInfo.built_in_include_directories}. */ + abstract ImmutableList builtInIncludeDirectories(); + + /** + * C compiler options, as reported by bazel API {@code + * cc_common.get_memory_inefficient_command_line} for {@code action_name=C_COMPILE_ACTION_NAME}. + */ + abstract ImmutableList cOptions(); + + /** + * C++ compiler options, as reported by bazel API {@code + * cc_common.get_memory_inefficient_command_line} for {@code action_name=CPP_COMPILE_ACTION_NAME}. + */ + abstract ImmutableList cppOptions(); + + public static Builder builder() { + return new AutoValue_CcToolchain.Builder(); + } + + public static CcToolchain create(CcToolchainInfo proto) { + return builder() + .id(proto.getId()) + .compiler(proto.getCompiler()) + .compilerExecutable( + ProjectPath.create(Root.WORKSPACE, Path.of(proto.getCompilerExecutable()))) + .cpu(proto.getCpu()) + .targetGnuSystemName(proto.getTargetName()) + .builtInIncludeDirectories( + proto.getBuiltInIncludeDirectoriesList().stream() + .map(CcIncludeDirectories::projectPathFor) + .collect(toImmutableList())) + .cOptions(ImmutableList.copyOf(proto.getCOptionsList())) + .cppOptions(ImmutableList.copyOf(proto.getCppOptionsList())) + .build(); + } + + /** Builder for {@link CcToolchain}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder id(String value); + + public abstract Builder compiler(String value); + + public abstract Builder compilerExecutable(ProjectPath value); + + public abstract Builder cpu(String value); + + public abstract Builder targetGnuSystemName(String value); + + public abstract Builder builtInIncludeDirectories(ImmutableList value); + + public abstract Builder cOptions(ImmutableList value); + + public abstract Builder cppOptions(ImmutableList value); + + public abstract CcToolchain build(); + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/NewArtifactTracker.java b/querysync/java/com/google/idea/blaze/qsync/deps/NewArtifactTracker.java index 23fe993c9cc..be3cf88d152 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/NewArtifactTracker.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/NewArtifactTracker.java @@ -35,6 +35,9 @@ import com.google.idea.blaze.qsync.java.JavaArtifactInfo; import com.google.idea.blaze.qsync.java.JavaTargetInfo.JavaArtifacts; import com.google.idea.blaze.qsync.java.JavaTargetInfo.JavaTargetArtifacts; +import com.google.idea.blaze.qsync.java.cc.CcCompilationInfoOuterClass; +import com.google.idea.blaze.qsync.java.cc.CcCompilationInfoOuterClass.CcTargetInfo; +import com.google.idea.blaze.qsync.java.cc.CcCompilationInfoOuterClass.CcToolchainInfo; import com.google.protobuf.ExtensionRegistry; import java.io.IOException; import java.io.InputStream; @@ -75,6 +78,9 @@ public class NewArtifactTracker> implements ArtifactTracker @GuardedBy("stateLock") private final Map builtDeps = Maps.newHashMap(); + @GuardedBy("stateLock") + private final Map ccToolchainMap = Maps.newHashMap(); + public NewArtifactTracker(Path projectDirectory, BuildArtifactCache artifactCache) { this.artifactCache = artifactCache; this.stateFile = projectDirectory.resolve("artifact_state"); @@ -90,7 +96,7 @@ public ImmutableCollection getBuiltDeps() { @Override public State getStateSnapshot() { synchronized (stateLock) { - return State.create(ImmutableMap.copyOf(builtDeps)); + return State.create(ImmutableMap.copyOf(builtDeps), ImmutableMap.copyOf(ccToolchainMap)); } } @@ -123,6 +129,20 @@ public void update(Set

    + *
  • The BES paths do not contain the {@code bazel-out} component + *
  • The cc compilation API does contain {@code bazel-out}, both in the include path flags and + * in the list of generated headers. + *
+ * + * To workaround this, we strip the {@code bazel-out} prefix when looking up generated headers in + * the digest map to ensure we can find them. + */ + static Function stripBazelOutPrefix(Function digestMap) { + return digestMap.compose( + p -> { + if (p.startsWith("bazel-out") || p.startsWith("blaze-out")) { + return p.getName(0).relativize(p); + } + return p; + }); + } + static CcCompilationInfo create(CcTargetInfo targetInfo, Function digestMap) { Label target = Label.of(targetInfo.getLabel()); return builder() @@ -76,7 +99,8 @@ static CcCompilationInfo create(CcTargetInfo targetInfo, Function targetInfo.getFrameworkIncludeDirectoriesList().stream() .map(CcIncludeDirectories::projectPathFor) .collect(toImmutableList())) - .genHeaders(toArtifacts(targetInfo.getGenHdrsList(), digestMap, target)) + .genHeaders( + toArtifacts(targetInfo.getGenHdrsList(), stripBazelOutPrefix(digestMap), target)) .toolchainId(targetInfo.getToolchainId()) .build(); } diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/ConfigureCcCompilation.java b/querysync/java/com/google/idea/blaze/qsync/deps/ConfigureCcCompilation.java new file mode 100644 index 00000000000..c4e9437e23e --- /dev/null +++ b/querysync/java/com/google/idea/blaze/qsync/deps/ConfigureCcCompilation.java @@ -0,0 +1,279 @@ +/* + * Copyright 2023 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; + +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.idea.blaze.common.PrintOutput; +import com.google.idea.blaze.exception.BuildException; +import com.google.idea.blaze.qsync.artifacts.BuildArtifact; +import com.google.idea.blaze.qsync.cc.CcIncludeDirectories; +import com.google.idea.blaze.qsync.deps.ArtifactTracker.State; +import com.google.idea.blaze.qsync.project.LanguageClassProto.LanguageClass; +import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectProto.CcCompilationContext; +import com.google.idea.blaze.qsync.project.ProjectProto.CcCompilerFlag; +import com.google.idea.blaze.qsync.project.ProjectProto.CcCompilerFlagSet; +import com.google.idea.blaze.qsync.project.ProjectProto.CcCompilerSettings; +import com.google.idea.blaze.qsync.project.ProjectProto.CcLanguage; +import com.google.idea.blaze.qsync.project.ProjectProto.CcSourceFile; +import com.google.idea.blaze.qsync.project.ProjectTarget; +import com.google.idea.blaze.qsync.project.ProjectTarget.SourceType; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +/** Adds C/C++ compilation information and headers to the project proto. */ +public class ConfigureCcCompilation { + + /** An update operation to configure CC compilation. */ + public static class UpdateOperation implements ProjectProtoUpdateOperation { + + private final Supplier artifactStateSupplier; + + public UpdateOperation(Supplier artifactStateSupplier) { + this.artifactStateSupplier = artifactStateSupplier; + } + + @Override + public void update(ProjectProtoUpdate update) throws BuildException { + new ConfigureCcCompilation(artifactStateSupplier.get(), update).update(); + } + } + + private static final AtomicInteger nextFlagSetId = new AtomicInteger(0); + private final ArtifactTracker.State artifactState; + private final ProjectProtoUpdate update; + + /* Map from toolchain ID -> language -> flags for that toolchain & language. */ + private final Map> toolchainLanguageFlags = + Maps.newHashMap(); + + /* Map of unique sets of compiler flags to an ID to identify them. + * We do this as the downstream code turns each set of flags into a CidrCompilerSwitches instance + * which can have a large memory footprint. */ + private final Map, String> uniqueFlagSetIds = Maps.newHashMap(); + + ConfigureCcCompilation(ArtifactTracker.State artifactState, ProjectProtoUpdate projectUpdate) { + this.artifactState = artifactState; + this.update = projectUpdate; + } + + public void update() throws BuildException { + + visitToolchainMap(artifactState.ccToolchainMap()); + + for (TargetBuildInfo target : artifactState.depsMap().values()) { + if (target.ccInfo().isEmpty()) { + continue; + } + CcCompilationInfo ccInfo = target.ccInfo().get(); + visitTarget(ccInfo, target.buildContext()); + } + if (update.project().getCcWorkspaceBuilder().getContextsCount() > 0) { + update.project().addActiveLanguages(LanguageClass.LANGUAGE_CLASS_CC); + } + } + + private void visitToolchainMap(Map toolchainInfoMap) { + toolchainInfoMap.values().forEach(this::visitToolchain); + } + + private void visitToolchain(CcToolchain toolchain) { + + ImmutableList commonFlags = + toolchain.builtInIncludeDirectories().stream() + .map(p -> makePathFlag("-I", p)) + .collect(toImmutableList()); + + toolchainLanguageFlags.put( + toolchain.id(), + ImmutableListMultimap.builder() + .putAll( + CcLanguage.C, + ImmutableList.builder() + .addAll(commonFlags) + .addAll( + toolchain.cOptions().stream().map(f -> makeStringFlag(f, "")).iterator()) + .build()) + .putAll( + CcLanguage.CPP, + ImmutableList.builder() + .addAll(commonFlags) + .addAll( + toolchain.cppOptions().stream().map(f -> makeStringFlag(f, "")).iterator()) + .build()) + .build()); + } + + private void visitTarget(CcCompilationInfo ccInfo, DependencyBuildContext buildContext) { + ProjectTarget projectTarget = update.buildGraph().targetMap().get(ccInfo.target()); + if (projectTarget == null) { + // This target is no longer present in the project. Ignore it. + // We should really clean up the dependency cache itself to remove any artifacts relating to + // no-longer-present targets, but that will be a lot more work. For now, just ensure we + // don't crash. + return; + } + CcToolchain toolchain = + Preconditions.checkNotNull(artifactState.ccToolchainMap().get(ccInfo.toolchainId())); + + ImmutableList targetFlags = + ImmutableList.builder() + .addAll(projectTarget.copts().stream().map(d -> makeStringFlag(d, "")).iterator()) + .addAll(ccInfo.defines().stream().map(d -> makeStringFlag("-D", d)).iterator()) + .addAll(ccInfo.includeDirectories().stream().map(p -> makePathFlag("-I", p)).iterator()) + .addAll( + ccInfo.quoteIncludeDirectories().stream() + .map(p -> makePathFlag("-iquote", p)) + .iterator()) + .addAll( + ccInfo.systemIncludeDirectories().stream() + .map(p -> makePathFlag("-isystem", p)) + .iterator()) + .addAll( + ccInfo.frameworkIncludeDirectories().stream() + .map(p -> makePathFlag("-F", p)) + .iterator()) + .build(); + + ImmutableList.Builder srcsBuilder = ImmutableList.builder(); + // TODO(mathewi): The handling of flag sets here is not optimal, since we recalculate an + // identical flag set for each source of the same language, then immediately de-dupe them in + // the addFlagSet call. For large flag sets this may be slow. + for (Path srcPath : update.buildGraph().getTargetSources(ccInfo.target(), SourceType.all())) { + Optional lang = getLanguage(srcPath); + if (lang.isPresent()) { + srcsBuilder.add( + CcSourceFile.newBuilder() + .setLanguage(lang.get()) + .setWorkspacePath(srcPath.toString()) + .setCompilerSettings( + CcCompilerSettings.newBuilder() + .setCompilerExecutablePath(toolchain.compilerExecutable().toProto()) + .setFlagSetId( + addFlagSet( + ImmutableList.builder() + .addAll(targetFlags) + .addAll( + toolchainLanguageFlags.get(toolchain.id()).get(lang.get())) + .build()))) + .build()); + } + } + ImmutableList srcs = srcsBuilder.build(); + + CcCompilationContext targetContext = + CcCompilationContext.newBuilder() + .setId(ccInfo.target() + "%" + toolchain.targetGnuSystemName()) + .setHumanReadableName(ccInfo.target() + " - " + toolchain.targetGnuSystemName()) + .addAllSources(srcs) + .putAllLanguageToCompilerSettings( + toolchainLanguageFlags.get(toolchain.id()).asMap().entrySet().stream() + .collect( + toImmutableMap( + e -> e.getKey().getValueDescriptor().getName(), + e -> + CcCompilerSettings.newBuilder() + .setCompilerExecutablePath( + toolchain.compilerExecutable().toProto()) + .setFlagSetId(addFlagSet(e.getValue())) + .build()))) + .build(); + update.project().getCcWorkspaceBuilder().addContexts(targetContext); + + ArtifactDirectoryBuilder headersDir = + update.artifactDirectory(CcIncludeDirectories.GEN_INCLUDE_BASE.relativePath()); + for (BuildArtifact artifact : ccInfo.genHeaders()) { + headersDir.addIfNewer(artifact.path(), artifact, buildContext); + } + } + + /** Ensure that the given flagset exists, adding it if necessary, and return its unique ID. */ + private String addFlagSet(Collection flags) { + // Create a set so that two flags sets are considered equivalent if their flag order differs. + ImmutableSet canonicalFlagSet = ImmutableSet.copyOf(flags); + String flagSetId = uniqueFlagSetIds.get(canonicalFlagSet); + + if (flagSetId == null) { + flagSetId = Integer.toString(nextFlagSetId.incrementAndGet()); + uniqueFlagSetIds.put(canonicalFlagSet, flagSetId); + update + .project() + .getCcWorkspaceBuilder() + .putFlagSets(flagSetId, CcCompilerFlagSet.newBuilder().addAllFlags(flags).build()); + } + return flagSetId; + } + + private CcCompilerFlag makeStringFlag(String flag, String value) { + return CcCompilerFlag.newBuilder().setFlag(flag).setPlainValue(value).build(); + } + + private CcCompilerFlag makePathFlag(String flag, ProjectPath path) { + return CcCompilerFlag.newBuilder().setFlag(flag).setPath(path.toProto()).build(); + } + + private static final ImmutableMap EXTENSION_TO_LANGUAGE_MAP = + ImmutableMap.of( + "c", CcLanguage.C, + "cc", CcLanguage.CPP, + "cpp", CcLanguage.CPP, + "cxx", CcLanguage.CPP, + "c++", CcLanguage.CPP, + "C", CcLanguage.C); + /* Files we ignore because they are not top level source files: */ + + private static final ImmutableSet IGNORE_SRC_FILE_EXTENSIONS = + ImmutableSet.of("h", "hh", "hpp", "hxx", "inc", "inl", "H", "S", "a", "lo", "so", "o"); + + private Optional getLanguage(Path srcPath) { + // logic in here based on https://bazel.build/reference/be/c-cpp#cc_library.srcs + int lastDot = srcPath.getFileName().toString().lastIndexOf('.'); + if (lastDot < 0) { + // default to cpp + update + .context() + .output(PrintOutput.log("No extension for c/c++ source file %s; assuming cpp", srcPath)); + return Optional.of(CcLanguage.CPP); + } + String ext = srcPath.getFileName().toString().substring(lastDot + 1); + if (IGNORE_SRC_FILE_EXTENSIONS.contains(ext)) { + return Optional.empty(); + } + if (EXTENSION_TO_LANGUAGE_MAP.containsKey(ext)) { + return Optional.of(EXTENSION_TO_LANGUAGE_MAP.get(ext)); + } + update + .context() + .output( + PrintOutput.log( + "Unrecognized extension %s for c/c++ source file %s; assuming cpp", ext, srcPath)); + return Optional.of(CcLanguage.CPP); + } +} diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java index 9ac3c2f345b..66ebfb88709 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/DependenciesProjectProtoUpdater.java @@ -61,7 +61,8 @@ public DependenciesProjectProtoUpdater( dependencyTracker::getBuiltDeps, projectDefinition, artifactCache, - packageReader)); + packageReader)) + .add(new ConfigureCcCompilation.UpdateOperation(dependencyTracker::getStateSnapshot)); if (attachDepsSrcjarsExperiment.get()) { updateOperations.add( new AddDependencySrcJars( @@ -83,7 +84,7 @@ public DependenciesProjectProtoUpdater( public Project apply(Project proto, BuildGraphData graph, Context context) throws BuildException { - ProjectProtoUpdate protoUpdate = new ProjectProtoUpdate(proto, context); + ProjectProtoUpdate protoUpdate = new ProjectProtoUpdate(proto, graph, context); for (ProjectProtoUpdateOperation op : updateOperations) { op.update(protoUpdate); } diff --git a/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdate.java b/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdate.java index d5e6cd66171..8dddbfd71d2 100644 --- a/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdate.java +++ b/querysync/java/com/google/idea/blaze/qsync/deps/ProjectProtoUpdate.java @@ -18,6 +18,7 @@ import com.google.common.collect.Maps; import com.google.idea.blaze.common.Context; import com.google.idea.blaze.qsync.project.BlazeProjectDataStorage; +import com.google.idea.blaze.qsync.project.BuildGraphData; import com.google.idea.blaze.qsync.project.ProjectProto; import com.google.idea.blaze.qsync.project.ProjectProto.Library; import java.nio.file.Path; @@ -33,13 +34,16 @@ public class ProjectProtoUpdate { private final ProjectProto.Project.Builder project; + private final BuildGraphData buildGraph; private final Context context; private final ProjectProto.Module.Builder workspaceModule; private final Map libraries = Maps.newHashMap(); private final Map artifactDirs = Maps.newHashMap(); - public ProjectProtoUpdate(ProjectProto.Project existingProject, Context context) { + public ProjectProtoUpdate( + ProjectProto.Project existingProject, BuildGraphData graph, Context context) { this.project = existingProject.toBuilder(); + this.buildGraph = graph; this.context = context; this.workspaceModule = getWorkspaceModuleBuilder(project); } @@ -65,6 +69,10 @@ public ProjectProto.Project.Builder project() { return project; } + public BuildGraphData buildGraph() { + return buildGraph; + } + public ProjectProto.Module.Builder workspaceModule() { return workspaceModule; } diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddCompiledJavaDepsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddCompiledJavaDepsTest.java index 85f19a2b970..47199fdb4c5 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddCompiledJavaDepsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddCompiledJavaDepsTest.java @@ -22,6 +22,7 @@ import com.google.idea.blaze.common.NoopContext; import com.google.idea.blaze.qsync.artifacts.BuildArtifact; import com.google.idea.blaze.qsync.java.JavaArtifactInfo; +import com.google.idea.blaze.qsync.project.BuildGraphData; import com.google.idea.blaze.qsync.project.ProjectProto; import com.google.idea.blaze.qsync.project.ProjectProto.ProjectArtifact.ArtifactTransform; import com.google.idea.blaze.qsync.testdata.ProjectProtos; @@ -43,7 +44,8 @@ public void no_deps_built() throws Exception { AddCompiledJavaDeps javaDeps = new AddCompiledJavaDeps(ImmutableList::of); - ProjectProtoUpdate update = new ProjectProtoUpdate(original, new NoopContext()); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original, BuildGraphData.EMPTY, new NoopContext()); javaDeps.update(update); ProjectProto.Project newProject = update.build(); assertThat(newProject.getLibraryList()).isEqualTo(original.getLibraryList()); @@ -78,7 +80,8 @@ public void dep_built() throws Exception { AddCompiledJavaDeps javaDeps = new AddCompiledJavaDeps(() -> ImmutableList.of(builtDep)); - ProjectProtoUpdate update = new ProjectProtoUpdate(original, new NoopContext()); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original, BuildGraphData.EMPTY, new NoopContext()); javaDeps.update(update); ProjectProto.Project newProject = update.build(); assertThat(newProject.getLibraryList()).hasSize(1); diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java index 2a6a906f303..c495d77d4ef 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencyGenSrcsJarsTest.java @@ -77,7 +77,8 @@ public void no_deps_built() throws Exception { cache, new SrcJarInnerPathFinder(new PackageStatementParser())); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original.project(), original.graph(), new NoopContext()); addGenSrcJars.update(update); @@ -114,7 +115,8 @@ public void project_gensrcs_ignored() throws Exception { cache, new SrcJarInnerPathFinder(new PackageStatementParser())); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original.project(), original.graph(), new NoopContext()); addGenSrcJars.update(update); ProjectProto.Project newProject = update.build(); @@ -157,7 +159,8 @@ public void external_gensrcs_added() throws Exception { cache, new SrcJarInnerPathFinder(new PackageStatementParser())); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original.project(), original.graph(), new NoopContext()); addGenSrcJars.update(update); ProjectProto.Project newProject = update.build(); diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java index 458c65b37f9..673c1808daf 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddDependencySrcJarsTest.java @@ -77,7 +77,8 @@ public void no_deps_built() throws Exception { pathResolver, new SrcJarInnerPathFinder(new PackageStatementParser())); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original.project(), original.graph(), new NoopContext()); addSrcJars.update(update); @@ -116,7 +117,8 @@ public void external_srcjar_added() throws Exception { pathResolver, new SrcJarInnerPathFinder(new PackageStatementParser())); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original.project(), original.graph(), new NoopContext()); addSrcJars.update(update); diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java index 27183aa947f..292d456a461 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcJarsTest.java @@ -87,7 +87,8 @@ public void external_srcjar_ignored() throws Exception { cache, new SrcJarInnerPathFinder(new PackageStatementParser())); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original.project(), original.graph(), new NoopContext()); javaDeps.update(update); ProjectProto.Project newProject = update.build(); assertThat(newProject.getLibraryList()).isEqualTo(original.project().getLibraryList()); @@ -128,7 +129,8 @@ public void project_srcjar_ignored() throws Exception { cache, new SrcJarInnerPathFinder(new PackageStatementParser())); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), new NoopContext()); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original.project(), original.graph(), new NoopContext()); javaDeps.update(update); ProjectProto.Project newProject = update.build(); assertThat(newProject.getLibraryList()).isEqualTo(original.project().getLibraryList()); diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcsTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcsTest.java index bcb427bb51b..45cd066cbda 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcsTest.java +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/AddProjectGenSrcsTest.java @@ -94,7 +94,8 @@ public void generated_source_added() throws Exception { cache, new PackageStatementParser()); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), context); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original.project(), original.graph(), context); addGensrcs.update(update); ProjectProto.Project newProject = update.build(); @@ -189,7 +190,8 @@ public void conflict_last_build_taken() throws Exception { cache, new PackageStatementParser()); - ProjectProtoUpdate update = new ProjectProtoUpdate(original.project(), context); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original.project(), original.graph(), context); addGenSrcs.update(update); ProjectProto.Project newProject = update.build(); diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD index 8dea076a616..4348e167c30 100644 --- a/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/BUILD @@ -49,6 +49,7 @@ java_test( "//querysync/java/com/google/idea/blaze/qsync/artifacts", "//querysync/java/com/google/idea/blaze/qsync/deps", "//querysync/java/com/google/idea/blaze/qsync/java", + "//querysync/java/com/google/idea/blaze/qsync/project", "//querysync/java/com/google/idea/blaze/qsync/project:project_java_proto", "//querysync/javatests/com/google/idea/blaze/qsync/testdata", "//querysync/javatests/com/google/idea/blaze/qsync/testdata:project_protos", @@ -147,3 +148,26 @@ java_test( "@truth//jar", ], ) + +java_test( + name = "ConfigureCcCompilationTest", + size = "small", + srcs = ["ConfigureCcCompilationTest.java"], + deps = [ + "//querysync/java/com/google/idea/blaze/qsync", + "//querysync/java/com/google/idea/blaze/qsync/cc", + "//querysync/java/com/google/idea/blaze/qsync/cc:cc_compilation_info_java_proto", + "//querysync/java/com/google/idea/blaze/qsync/deps", + "//querysync/java/com/google/idea/blaze/qsync/java", + "//querysync/java/com/google/idea/blaze/qsync/project", + "//querysync/java/com/google/idea/blaze/qsync/project:language_class_java_proto", + "//querysync/java/com/google/idea/blaze/qsync/project:project_java_proto", + "//querysync/javatests/com/google/idea/blaze/qsync:TestUtils", + "//querysync/javatests/com/google/idea/blaze/qsync/testdata", + "//shared", + "//shared/javatests/com/google/idea/blaze/common:test_utils", + "@com_google_guava_guava//jar", + "@junit//jar", + "@truth//jar", + ], +) diff --git a/querysync/javatests/com/google/idea/blaze/qsync/deps/ConfigureCcCompilationTest.java b/querysync/javatests/com/google/idea/blaze/qsync/deps/ConfigureCcCompilationTest.java new file mode 100644 index 00000000000..7563b6553e9 --- /dev/null +++ b/querysync/javatests/com/google/idea/blaze/qsync/deps/ConfigureCcCompilationTest.java @@ -0,0 +1,269 @@ +/* + * Copyright 2023 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.qsync.deps; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.Iterables.getOnlyElement; +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.base.Functions; +import com.google.common.collect.ImmutableList; +import com.google.common.truth.Truth8; +import com.google.idea.blaze.common.Context; +import com.google.idea.blaze.common.Label; +import com.google.idea.blaze.common.NoopContext; +import com.google.idea.blaze.qsync.BlazeProjectSnapshot; +import com.google.idea.blaze.qsync.TestDataSyncRunner; +import com.google.idea.blaze.qsync.cc.FlagResolver; +import com.google.idea.blaze.qsync.java.PackageStatementParser; +import com.google.idea.blaze.qsync.java.cc.CcCompilationInfoOuterClass.CcCompilationInfo; +import com.google.idea.blaze.qsync.java.cc.CcCompilationInfoOuterClass.CcTargetInfo; +import com.google.idea.blaze.qsync.java.cc.CcCompilationInfoOuterClass.CcToolchainInfo; +import com.google.idea.blaze.qsync.project.LanguageClassProto.LanguageClass; +import com.google.idea.blaze.qsync.project.ProjectPath; +import com.google.idea.blaze.qsync.project.ProjectProto; +import com.google.idea.blaze.qsync.project.ProjectProto.CcCompilationContext; +import com.google.idea.blaze.qsync.project.ProjectProto.CcCompilerSettings; +import com.google.idea.blaze.qsync.project.ProjectProto.CcLanguage; +import com.google.idea.blaze.qsync.project.ProjectProto.CcSourceFile; +import com.google.idea.blaze.qsync.project.ProjectProto.CcWorkspace; +import com.google.idea.blaze.qsync.project.ProjectProto.ProjectPath.Base; +import com.google.idea.blaze.qsync.testdata.TestData; +import java.nio.file.Path; +import java.util.List; +import java.util.function.Function; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ConfigureCcCompilationTest { + + private final Context context = new NoopContext(); + private final TestDataSyncRunner syncRunner = + new TestDataSyncRunner(context, new PackageStatementParser(), true); + + private static ArtifactTracker.State toArtifactState(CcCompilationInfo proto) { + Function digestMap = p -> Integer.toHexString(p.hashCode()); + return ArtifactTracker.State.create( + proto.getTargetsList().stream() + .map(t -> com.google.idea.blaze.qsync.deps.CcCompilationInfo.create(t, digestMap)) + .collect( + toImmutableMap( + cc -> cc.target(), + cc -> TargetBuildInfo.forCcTarget(cc, DependencyBuildContext.NONE))), + proto.getToolchainsList().stream() + .map(CcToolchain::create) + .collect(toImmutableMap(CcToolchain::id, Functions.identity()))); + } + + @Test + public void empty() throws Exception { + BlazeProjectSnapshot original = syncRunner.sync(TestData.CC_LIBRARY_QUERY); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original.project(), original.graph(), context); + ConfigureCcCompilation ccConfig = + new ConfigureCcCompilation(ArtifactTracker.State.EMPTY, update); + ccConfig.update(); + ProjectProto.Project project = update.build(); + assertThat(project.getCcWorkspace()).isEqualTo(CcWorkspace.getDefaultInstance()); + } + + @Test + public void basics() throws Exception { + BlazeProjectSnapshot original = syncRunner.sync(TestData.CC_LIBRARY_QUERY); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original.project(), original.graph(), context); + + Label ccTargetLabel = getOnlyElement(TestData.CC_LIBRARY_QUERY.getAssumedLabels()); + CcCompilationInfo compilationInfo = + CcCompilationInfo.newBuilder() + .addToolchains( + CcToolchainInfo.newBuilder() + .setId("//my/cc_toolchain") + .setCompiler("clang") + .setCompilerExecutable("workspace/path/to/clang") + .setCpu("k8") + .setTargetName("k8-debug") + .addBuiltInIncludeDirectories("bazel-out/builtin/include/directory") + .addBuiltInIncludeDirectories("src/builtin/include/directory") + .addAllCOptions(ImmutableList.of("--sharedopt", "--conlyopt")) + .addAllCppOptions(ImmutableList.of("--sharedopt", "--cppopt")) + .build()) + .addTargets( + CcTargetInfo.newBuilder() + .setLabel(ccTargetLabel.toString()) + .setToolchainId("//my/cc_toolchain") + .addDefines("DEBUG") + .addIncludeDirectories("bazel-out/include/directory") + .addIncludeDirectories("src/include/directory") + .addQuoteIncludeDirectories("bazel-out/quote/include/directory") + .addQuoteIncludeDirectories("src/quote/include/directory") + .addSystemIncludeDirectories("bazel-out/system/include/directory") + .addSystemIncludeDirectories("src/system/include/directory") + .addFrameworkIncludeDirectories("bazel-out/framework/include/directory") + .addFrameworkIncludeDirectories("src/framework/include/directory") + .addAllGenHdrs( + ImmutableList.of( + "bazel-out/include/directory/include_header.h", + "bazel-out/quote/include/directory/quote_include_header.h", + "bazel-out/system/include/directory/system_include_header.h", + "bazel-out/framework/include/directory/framework_include_header", + "bazel-out/builtin/include/directory/builtin_include.h"))) + .build(); + + ConfigureCcCompilation ccConfig = + new ConfigureCcCompilation(toArtifactState(compilationInfo), update); + ccConfig.update(); + + ProjectProto.Project project = update.build(); + + assertThat(project.getActiveLanguagesList()).contains(LanguageClass.LANGUAGE_CLASS_CC); + ProjectProto.CcWorkspace workspace = project.getCcWorkspace(); + CcCompilationContext context = getOnlyElement(workspace.getContextsList()); + assertThat(context.getHumanReadableName()).isNotEmpty(); + CcSourceFile sourceFile = getOnlyElement(context.getSourcesList()); + assertThat(sourceFile.getLanguage()).isEqualTo(CcLanguage.CPP); + assertThat(sourceFile.getWorkspacePath()) + .isEqualTo( + TestData.CC_LIBRARY_QUERY.getOnlySourcePath().resolve("TestClass.cc").toString()); + CcCompilerSettings compilerSettings = sourceFile.getCompilerSettings(); + FlagResolver resolver = + new FlagResolver( + ProjectPath.Resolver.create(Path.of("/workspace"), Path.of("/project")), false); + assertThat(resolver.resolveAll(workspace.getFlagSetsOrThrow(compilerSettings.getFlagSetId()))) + .containsExactly( + "-DDEBUG", + "-I/project/buildout/bazel-out/builtin/include/directory", + "-I/workspace/src/builtin/include/directory", + "-I/project/buildout/bazel-out/include/directory", + "-I/workspace/src/include/directory", + "-iquote/project/buildout/bazel-out/quote/include/directory", + "-iquote/workspace/src/quote/include/directory", + "-isystem/project/buildout/bazel-out/system/include/directory", + "-isystem/workspace/src/system/include/directory", + "-F/project/buildout/bazel-out/framework/include/directory", + "-F/workspace/src/framework/include/directory", + "-w", // This is defined in `copts` of the test project build rule. + "--sharedopt", + "--cppopt"); + + assertThat(compilerSettings.getCompilerExecutablePath().getBase()).isEqualTo(Base.WORKSPACE); + assertThat(compilerSettings.getCompilerExecutablePath().getPath()) + .isEqualTo("workspace/path/to/clang"); + + Truth8.assertThat( + context.getLanguageToCompilerSettingsMap().keySet().stream() + .map(l -> CcLanguage.valueOf(CcLanguage.getDescriptor().findValueByName(l)))) + .containsExactly(CcLanguage.CPP, CcLanguage.C); + + assertThat( + resolver.resolveAll( + workspace.getFlagSetsOrThrow( + context + .getLanguageToCompilerSettingsMap() + .get(CcLanguage.CPP.name()) + .getFlagSetId()))) + .containsExactly( + "-I/project/buildout/bazel-out/builtin/include/directory", + "-I/workspace/src/builtin/include/directory", + "--sharedopt", + "--cppopt"); + + assertThat( + resolver.resolveAll( + workspace.getFlagSetsOrThrow( + context + .getLanguageToCompilerSettingsMap() + .get(CcLanguage.C.name()) + .getFlagSetId()))) + .containsExactly( + "-I/project/buildout/bazel-out/builtin/include/directory", + "-I/workspace/src/builtin/include/directory", + "--sharedopt", + "--conlyopt"); + + assertThat( + project + .getArtifactDirectories() + .getDirectoriesMap() + .get("buildout") + .getContentsMap() + .keySet()) + .containsExactly( + "bazel-out/include/directory/include_header.h", + "bazel-out/quote/include/directory/quote_include_header.h", + "bazel-out/system/include/directory/system_include_header.h", + "bazel-out/framework/include/directory/framework_include_header", + "bazel-out/builtin/include/directory/builtin_include.h"); + } + + @Test + public void multi_srcs_share_flagset() throws Exception { + BlazeProjectSnapshot original = syncRunner.sync(TestData.CC_MULTISRC_QUERY); + ProjectProtoUpdate update = + new ProjectProtoUpdate(original.project(), original.graph(), context); + Path pkgPath = getOnlyElement(TestData.CC_MULTISRC_QUERY.getRelativeSourcePaths()); + ImmutableList