diff --git a/base/src/com/google/idea/blaze/base/sync/SyncProjectTargetsHelper.java b/base/src/com/google/idea/blaze/base/sync/SyncProjectTargetsHelper.java index 86327a130f2..ec66c56d27f 100644 --- a/base/src/com/google/idea/blaze/base/sync/SyncProjectTargetsHelper.java +++ b/base/src/com/google/idea/blaze/base/sync/SyncProjectTargetsHelper.java @@ -89,7 +89,7 @@ private static boolean shouldDeriveSyncTargetsFromDirectories(ProjectViewSet vie return viewSet.getScalarValue(AutomaticallyDeriveTargetsSection.KEY).orElse(false); } - private static boolean shouldSyncManualTargets(ProjectViewSet viewSet) { + public static boolean shouldSyncManualTargets(ProjectViewSet viewSet) { return viewSet.getScalarValue(SyncManualTargetsSection.KEY).orElse(false); } diff --git a/base/src/com/google/idea/blaze/base/sync/sharding/BlazeBuildTargetSharder.java b/base/src/com/google/idea/blaze/base/sync/sharding/BlazeBuildTargetSharder.java index aab3d93d0c5..b6fbfada0e6 100644 --- a/base/src/com/google/idea/blaze/base/sync/sharding/BlazeBuildTargetSharder.java +++ b/base/src/com/google/idea/blaze/base/sync/sharding/BlazeBuildTargetSharder.java @@ -31,6 +31,7 @@ import com.google.idea.blaze.base.projectview.ProjectViewManager; import com.google.idea.blaze.base.projectview.ProjectViewSet; import com.google.idea.blaze.base.projectview.section.sections.ShardBlazeBuildsSection; +import com.google.idea.blaze.base.projectview.section.sections.SyncManualTargetsSection; import com.google.idea.blaze.base.projectview.section.sections.TargetShardSizeSection; import com.google.idea.blaze.base.scope.BlazeContext; import com.google.idea.blaze.base.scope.Scope; @@ -87,7 +88,9 @@ static boolean shardingRequested(Project project) { } private static boolean shardingRequested(ProjectViewSet projectViewSet) { - return projectViewSet.getScalarValue(ShardBlazeBuildsSection.KEY).orElse(false); + // We need to perform expansion of query targets if we are to allow for manual targets to be synced. + return projectViewSet.getScalarValue(ShardBlazeBuildsSection.KEY).orElse(false) || + projectViewSet.getScalarValue(SyncManualTargetsSection.KEY).orElse(false); } /** Number of individual targets per blaze build shard. */ diff --git a/base/src/com/google/idea/blaze/base/sync/sharding/WildcardTargetExpander.java b/base/src/com/google/idea/blaze/base/sync/sharding/WildcardTargetExpander.java index 884d30c8e19..cac6ed9be41 100644 --- a/base/src/com/google/idea/blaze/base/sync/sharding/WildcardTargetExpander.java +++ b/base/src/com/google/idea/blaze/base/sync/sharding/WildcardTargetExpander.java @@ -40,6 +40,7 @@ import com.google.idea.blaze.base.scope.scopes.TimingScope; import com.google.idea.blaze.base.scope.scopes.TimingScope.EventType; import com.google.idea.blaze.base.settings.Blaze; +import com.google.idea.blaze.base.sync.SyncProjectTargetsHelper; import com.google.idea.blaze.base.sync.aspects.BuildResult; import com.google.idea.blaze.base.sync.aspects.BuildResult.Status; import com.google.idea.blaze.base.sync.projectview.LanguageSupport; @@ -64,6 +65,7 @@ /** Expands wildcard target patterns into individual blaze targets. */ public class WildcardTargetExpander { + public static final String MANUAL_EXCLUDE_TAG = "^((?!manual).)*$"; private static final BoolExperiment filterByRuleType = new BoolExperiment("blaze.build.filter.by.rule.type", true); @@ -195,7 +197,7 @@ private static boolean excludeManualTargets( BlazeCommandName.BUILD, context, BlazeInvocationContext.SYNC_CONTEXT) - .contains("--build_manual_tests"); + .contains("--build_manual_tests") && !SyncProjectTargetsHelper.shouldSyncManualTargets(projectView); } /** Runs a blaze query to expand the input target patterns to individual blaze targets. */ @@ -272,7 +274,7 @@ private static String queryString(List targets, boolean exclud return targetList; } return excludeManualTargets - ? String.format("attr('tags', '^((?!manual).)*$', %s)", targetList) + ? String.format("attr('tags', '%s', %s)", MANUAL_EXCLUDE_TAG, targetList) : targetList; } } diff --git a/base/tests/unittests/com/google/idea/blaze/base/sync/sharding/BlazeBuildTargetSharderTest.java b/base/tests/unittests/com/google/idea/blaze/base/sync/sharding/BlazeBuildTargetSharderTest.java index 3a1ab4bf45a..8f97ac757b0 100644 --- a/base/tests/unittests/com/google/idea/blaze/base/sync/sharding/BlazeBuildTargetSharderTest.java +++ b/base/tests/unittests/com/google/idea/blaze/base/sync/sharding/BlazeBuildTargetSharderTest.java @@ -18,6 +18,8 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.truth.Truth.assertThat; import static java.nio.charset.StandardCharsets.UTF_8; +import static junit.framework.TestCase.assertFalse; +import static junit.framework.TestCase.assertTrue; import static junit.framework.TestCase.fail; import com.google.common.base.Preconditions; @@ -55,6 +57,7 @@ import com.google.idea.blaze.base.projectview.ProjectViewSet; import com.google.idea.blaze.base.projectview.section.ScalarSection; import com.google.idea.blaze.base.projectview.section.sections.ShardBlazeBuildsSection; +import com.google.idea.blaze.base.projectview.section.sections.SyncManualTargetsSection; import com.google.idea.blaze.base.projectview.section.sections.TargetShardSizeSection; import com.google.idea.blaze.base.scope.BlazeContext; import com.google.idea.blaze.base.scope.BlazeScope; @@ -267,7 +270,7 @@ public void shardTargetsRetainingOrdering_testShardWithOnlyExcludedTargetsIsDrop public void expandAndShardTargets_shardingApproachPartitionWithoutExpanding() { List targets = ImmutableList.of(target("//java/com/google:foo")); ShardedTargetsResult result = - expandAndShardTargets(SyncStrategy.SERIAL, ProjectView.builder().build(), targets); + expandAndShardTargets(SyncStrategy.SERIAL, ProjectView.builder().build(), targets, fakeWildCardTargetExpanderBlazeCommandRunner); assertThat(result.buildResult.exitCode).isEqualTo(0); assertThat(result.shardedTargets.shardStats.shardingApproach()) @@ -281,7 +284,7 @@ public void expandAndShardTargets_remoteBuild_buildBatchingServiceIsUsed() { .setFailToBatchTarget(false); List targets = ImmutableList.of(target("//java/com/google:foo")); ShardedTargetsResult result = - expandAndShardTargets(SyncStrategy.PARALLEL, ProjectView.builder().build(), targets); + expandAndShardTargets(SyncStrategy.PARALLEL, ProjectView.builder().build(), targets, fakeWildCardTargetExpanderBlazeCommandRunner); assertThat(result.buildResult.exitCode).isEqualTo(0); assertThat(result.shardedTargets.shardStats.shardingApproach()) @@ -301,7 +304,7 @@ public void expandAndShardTargets_localBuild_buildBatchingServiceIsUsed() { .add(ScalarSection.builder(ShardBlazeBuildsSection.KEY).set(true)) .add(ScalarSection.builder(TargetShardSizeSection.KEY).set(500)) .build(), - targets); + targets, fakeWildCardTargetExpanderBlazeCommandRunner); assertThat(result.buildResult.exitCode).isEqualTo(0); ShardStats shardStats = result.shardedTargets.shardStats; @@ -323,7 +326,7 @@ public void expandAndShardTargets_failToExpand_shardingApproachError() { .add(ScalarSection.builder(ShardBlazeBuildsSection.KEY).set(true)) .add(ScalarSection.builder(TargetShardSizeSection.KEY).set(500)) .build(), - targets); + targets, fakeWildCardTargetExpanderBlazeCommandRunner); assertThat(result.buildResult.exitCode).isEqualTo(BuildResult.FATAL_ERROR.exitCode); assertThat(result.shardedTargets.shardStats.shardingApproach()) @@ -344,7 +347,7 @@ public void expandAndShardTargets_failToBatchingTargets_shardingApproachError() .add(ScalarSection.builder(ShardBlazeBuildsSection.KEY).set(true)) .add(ScalarSection.builder(TargetShardSizeSection.KEY).set(500)) .build(), - targets); + targets, fakeWildCardTargetExpanderBlazeCommandRunner); assertThat(result.buildResult.exitCode).isEqualTo(0); assertThat(result.shardedTargets.shardStats.shardingApproach()) @@ -372,7 +375,7 @@ public void expandAndShardTargets_expandWildcardTargets() { .add(ScalarSection.builder(ShardBlazeBuildsSection.KEY).set(true)) .add(ScalarSection.builder(TargetShardSizeSection.KEY).set(500)) .build(), - targets); + targets, fakeWildCardTargetExpanderBlazeCommandRunner); ShardStats shardStats = result.shardedTargets.shardStats; assertThat(shardStats.suggestedTargetSizePerShard()).isEqualTo(500); @@ -381,8 +384,90 @@ public void expandAndShardTargets_expandWildcardTargets() { .containsExactly(ImmutableList.of(target(expectedLabel1), target(expectedLabel2))); } + @Test + public void expandAndShardTargets_expandWildcardTargetsNoExcludeManualTag() { + String expectedLabel1 = "//java/com/google:one"; + FakeWildCardTargetExpanderBlazeCommandRunner commandRunner = new FakeWildCardTargetExpanderBlazeCommandRunner() { + + @Override + public InputStream runQuery(Project project, BlazeCommand.Builder blazeCommandBuilder, BuildResultHelper buildResultHelper, BlazeContext context) throws BuildException { + // We need to confirm within the query runner because there are no public methods currently to + // perform this check downstream. + for (String argument : blazeCommandBuilder.build().toArgumentList()) { + assertFalse(argument.contains(WildcardTargetExpander.MANUAL_EXCLUDE_TAG)); + } + return super.runQuery(project, blazeCommandBuilder, buildResultHelper, context); + } + }; + + fakeWildCardTargetExpanderExternalTaskProvider + .setReturnVal(0) + .setOutputMessage("sh_library rule " + expectedLabel1); + commandRunner.setOutputMessages( + ImmutableList.of("sh_library rule " + expectedLabel1)); + fakeBuildBatchingService + .setShardingApproach(ShardingApproach.LEXICOGRAPHIC_TARGET_SHARDER) + .setFailToBatchTarget(false); + + List targets = ImmutableList.of(target("//java/com/google/...")); + ProjectView projectView = ProjectView.builder() + .add(ScalarSection.builder(SyncManualTargetsSection.KEY).set(true)) + .add(ScalarSection.builder(ShardBlazeBuildsSection.KEY).set(true)) + .add(ScalarSection.builder(TargetShardSizeSection.KEY).set(500)) + .build(); + + ShardedTargetsResult result = + expandAndShardTargets( + SyncStrategy.PARALLEL, + projectView, + targets, commandRunner); + + assertThat(result.shardedTargets.shardedTargets) + .containsExactly(ImmutableList.of(target(expectedLabel1))); + } + + @Test + public void expandAndShardTargets_expandWildcardTargetsIncludesManualTag() { + String expectedLabel1 = "//java/com/google:one"; + FakeWildCardTargetExpanderBlazeCommandRunner commandRunner = new FakeWildCardTargetExpanderBlazeCommandRunner() { + + @Override + public InputStream runQuery(Project project, BlazeCommand.Builder blazeCommandBuilder, BuildResultHelper buildResultHelper, BlazeContext context) throws BuildException { + // We need to confirm within the query runner because there are no public methods currently to + // perform this check downstream. + assertTrue(blazeCommandBuilder.build().toArgumentList().stream().anyMatch(argument -> argument.contains(WildcardTargetExpander.MANUAL_EXCLUDE_TAG))); + return super.runQuery(project, blazeCommandBuilder, buildResultHelper, context); + } + }; + + fakeWildCardTargetExpanderExternalTaskProvider + .setReturnVal(0) + .setOutputMessage("sh_library rule " + expectedLabel1); + commandRunner.setOutputMessages( + ImmutableList.of("sh_library rule " + expectedLabel1)); + fakeBuildBatchingService + .setShardingApproach(ShardingApproach.LEXICOGRAPHIC_TARGET_SHARDER) + .setFailToBatchTarget(false); + + List targets = ImmutableList.of(target("//java/com/google/...")); + ProjectView projectView = ProjectView.builder() + .add(ScalarSection.builder(SyncManualTargetsSection.KEY).set(false)) + .add(ScalarSection.builder(ShardBlazeBuildsSection.KEY).set(true)) + .add(ScalarSection.builder(TargetShardSizeSection.KEY).set(500)) + .build(); + + ShardedTargetsResult result = + expandAndShardTargets( + SyncStrategy.PARALLEL, + projectView, + targets, commandRunner); + + assertThat(result.shardedTargets.shardedTargets) + .containsExactly(ImmutableList.of(target(expectedLabel1))); + } + private ShardedTargetsResult expandAndShardTargets( - SyncStrategy syncStrategy, ProjectView projectView, List targets) { + SyncStrategy syncStrategy, ProjectView projectView, List targets, FakeBlazeCommandRunner commandRunner) { WorkspaceRoot workspaceRoot = new WorkspaceRoot(new File("workspaceRoot")); return BlazeBuildTargetSharder.expandAndShardTargets( getProject(), @@ -392,7 +477,7 @@ private ShardedTargetsResult expandAndShardTargets( targets, FakeBuildInvoker.builder() .type(BuildBinaryType.BAZEL) - .commandRunner(fakeWildCardTargetExpanderBlazeCommandRunner) + .commandRunner(commandRunner) .build(), syncStrategy); }