Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sync process optimizations #5219

Merged
merged 2 commits into from
Sep 4, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions base/src/META-INF/blaze-base.xml
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@
<daemon.highlightInfoFilter implementation="com.google.idea.blaze.base.qsync.QuerySyncHighlightingFilter"/>
<trafficLightRendererContributor implementation="com.google.idea.blaze.base.qsync.QuerySyncTrafficLightRendererContributor"/>

<projectService serviceImplementation="com.google.idea.blaze.base.command.info.BlazeInfoProvider"/>
<projectService serviceImplementation="com.google.idea.blaze.base.sync.SyncPhaseCoordinator"/>
<projectService serviceInterface="com.google.idea.blaze.base.sync.status.BlazeSyncStatus"
serviceImplementation="com.google.idea.blaze.base.sync.status.BlazeSyncStatusImpl"/>
Expand Down Expand Up @@ -535,6 +536,7 @@
<SyncListener implementation="com.google.idea.blaze.base.sync.libraries.ExternalLibraryManager$StartSyncListener"/>
<SyncListener implementation="com.google.idea.blaze.base.sync.autosync.ProjectTargetManagerImpl$TargetSyncListener"/>
<SyncListener implementation="com.google.idea.blaze.base.lang.buildfile.language.semantics.BuildLanguageSpecProviderImpl$Listener"/>
<SyncListener implementation="com.google.idea.blaze.base.command.info.BlazeInfoProvider$Invalidator"/>
<SyncPlugin implementation="com.google.idea.blaze.base.lang.buildfile.sync.BuildLangSyncPlugin"/>
<SyncPlugin implementation="com.google.idea.blaze.base.sync.libraries.ExternalLibraryManager$SyncPlugin"/>
<BuildFlagsProvider implementation="com.google.idea.blaze.base.command.BuildFlagsProviderImpl"/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import com.google.idea.blaze.base.command.buildresult.BuildResultHelper;
import com.google.idea.blaze.base.command.buildresult.BuildResultHelperBep;
import com.google.idea.blaze.base.command.info.BlazeInfo;
import com.google.idea.blaze.base.command.info.BlazeInfoProvider;
import com.google.idea.blaze.base.command.info.BlazeInfoRunner;
import com.google.idea.blaze.base.projectview.ProjectViewManager;
import com.google.idea.blaze.base.projectview.ProjectViewSet;
Expand Down Expand Up @@ -144,7 +145,15 @@ private ListenableFuture<BlazeInfo> runBlazeInfo() {
BlazeCommandName.INFO,
blazeContext,
BlazeInvocationContext.SYNC_CONTEXT);
if (BlazeInfoProvider.isEnabled()) {
return BlazeInfoProvider.getInstance(project)
.getBlazeInfo(blazeContext, syncFlags);
}
return BlazeInfoRunner.getInstance()
.runBlazeInfo(project, this, blazeContext, buildSystem.getName(), syncFlags);
.runBlazeInfo(project,
this,
blazeContext,
buildSystem.getName(),
syncFlags);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/*
idanakav marked this conversation as resolved.
Show resolved Hide resolved
* Copyright 2023 The Bazel Authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.idea.blaze.base.command.info;

import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.idea.blaze.base.async.executor.BlazeExecutor;
import com.google.idea.blaze.base.bazel.BuildSystem;
import com.google.idea.blaze.base.scope.BlazeContext;
import com.google.idea.blaze.base.settings.Blaze;
import com.google.idea.blaze.base.settings.BlazeImportSettings;
import com.google.idea.blaze.base.settings.BlazeImportSettingsManager;
import com.google.idea.blaze.base.sync.SyncListener;
import com.google.idea.blaze.base.sync.SyncMode;
import com.google.idea.blaze.base.sync.SyncResult;
import com.google.idea.common.experiments.BoolExperiment;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.project.Project;
import org.jetbrains.annotations.Nullable;

import java.util.List;
import java.util.concurrent.ExecutionException;

/**
* <p> Providing a {@link com.google.idea.blaze.base.command.info.BlazeInfo}, the result will be cached and invalidated on any sync failure. </p>
*
* @see BlazeInfoRunner
*/
public class BlazeInfoProvider {
private static final Logger logger = Logger.getInstance(BlazeInfoProvider.class);
private static final BoolExperiment enabled =
new BoolExperiment("blaze.info.provider.enabled", false);

private final Project project;
private volatile BlazeInfo blazeInfo;

public BlazeInfoProvider(Project project) {
this.project = project;
}

public static BlazeInfoProvider getInstance(Project project) {
return project.getService(BlazeInfoProvider.class);
}

public static Boolean isEnabled() {
return enabled.getValue();
}

public ListenableFuture<BlazeInfo> getBlazeInfo(
BlazeContext context,
List<String> blazeFlags) {
return BlazeExecutor.getInstance().submit(() -> {
BlazeInfo info = getCachedBlazeInfo(context,
blazeFlags);
if (info == null) {
throw new BlazeInfoException("Unable to get blaze info");
}
return info;
});
}

private @Nullable BlazeInfo getCachedBlazeInfo(
BlazeContext context,
List<String> blazeFlags) {
if (blazeInfo != null) {
logger.debug("Returning cached BlazeInfo");
return blazeInfo;
}
try {
BlazeImportSettings importSettings = BlazeImportSettingsManager.getInstance(project).getImportSettings();
if (importSettings == null) {
return null;
}
BuildSystem.BuildInvoker buildInvoker = Blaze.getBuildSystemProvider(project)
.getBuildSystem()
.getDefaultInvoker(project, context);
logger.debug("Running bazel info");
blazeInfo = BlazeInfoRunner.getInstance().runBlazeInfo(
project,
buildInvoker,
BlazeContext.create(),
importSettings.getBuildSystem(),
blazeFlags
).get();
return blazeInfo;
} catch (InterruptedException | ExecutionException e) {
logger.warn("Unable to run blaze info", e);
return null;
}
}

public void invalidate() {
logger.debug("invalidating");
idanakav marked this conversation as resolved.
Show resolved Hide resolved
blazeInfo = null;
}

public static final class Invalidator implements SyncListener {
@Override
public void afterSync(
Project project,
BlazeContext context,
SyncMode syncMode,
SyncResult syncResult,
ImmutableSet<Integer> buildIds) {
if(!isEnabled()) {
return;
}
BlazeInfoProvider provider = BlazeInfoProvider.getInstance(project);
if(provider != null) {
provider.invalidate();
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ public void requestProjectSync(BlazeSyncParams syncParams) {
try {
projectState =
ProjectStateSyncTask.collectProjectState(
project, context);
project, context, syncParams);
} catch (SyncCanceledException | SyncFailedException e) {
ApplicationManager.getApplication()
.invokeLater(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.idea.blaze.base.async.FutureUtil;
Expand All @@ -25,6 +26,7 @@
import com.google.idea.blaze.base.command.BlazeFlags;
import com.google.idea.blaze.base.command.BlazeInvocationContext;
import com.google.idea.blaze.base.command.info.BlazeInfo;
import com.google.idea.blaze.base.command.info.BlazeInfoProvider;
import com.google.idea.blaze.base.command.info.BlazeInfoRunner;
import com.google.idea.blaze.base.io.FileOperationProvider;
import com.google.idea.blaze.base.model.BlazeVersionData;
Expand Down Expand Up @@ -61,10 +63,10 @@
/** Collects information about the project state (VCS, blaze info, .blazeproject contents, etc.). */
final class ProjectStateSyncTask {

static SyncProjectState collectProjectState(Project project, BlazeContext context)
static SyncProjectState collectProjectState(Project project, BlazeContext context, BlazeSyncParams syncParams)
throws SyncCanceledException, SyncFailedException {
ProjectStateSyncTask task = new ProjectStateSyncTask(project);
return task.getProjectState(context);
return task.getProjectState(context, syncParams);
}

private final Project project;
Expand All @@ -77,7 +79,7 @@ private ProjectStateSyncTask(Project project) {
this.workspaceRoot = WorkspaceRoot.fromImportSettings(importSettings);
}

private SyncProjectState getProjectState(BlazeContext context)
private SyncProjectState getProjectState(BlazeContext context, BlazeSyncParams params)
throws SyncFailedException, SyncCanceledException {
if (!FileOperationProvider.getInstance().exists(workspaceRoot.directory())) {
String message = String.format("Workspace '%s' doesn't exist.", workspaceRoot.directory());
Expand Down Expand Up @@ -114,17 +116,14 @@ private SyncProjectState getProjectState(BlazeContext context)
BlazeInvocationContext.SYNC_CONTEXT);

ListenableFuture<BlazeInfo> blazeInfoFuture =
BlazeInfoRunner.getInstance()
.runBlazeInfo(
project,
Blaze.getBuildSystemProvider(project)
.getBuildSystem()
.getDefaultInvoker(project, context),
context,
importSettings.getBuildSystem(),
syncFlags);
createBazelInfoFuture(context, syncFlags, params.syncMode());

ListenableFuture<WorkingSet> workingSetFuture = vcsHandler.getWorkingSet(context, executor);
ListenableFuture<WorkingSet> workingSetFuture;
if(params.addWorkingSet() || params.syncMode() == SyncMode.FULL) {
workingSetFuture = vcsHandler.getWorkingSet(context, executor);
} else {
workingSetFuture = Futures.immediateFuture(null);
}

BlazeInfo blazeInfo =
FutureUtil.waitForFuture(context, blazeInfoFuture)
Expand Down Expand Up @@ -183,6 +182,29 @@ private SyncProjectState getProjectState(BlazeContext context)
.build();
}

private ListenableFuture<BlazeInfo> createBazelInfoFuture(
BlazeContext context,
List<String> syncFlags,
SyncMode syncMode) {
boolean useBazelInfoRunner = !BlazeInfoProvider.isEnabled() || syncMode == SyncMode.FULL;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we could just pass the sync mode to the BlazeInfoProvider and have it invalidate itself (or pretend that it has invalidated) on a full sync? I quite like the idea of "everything goes through the provider".

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The idea was to keep BlazeInfoProvider useful to other use cases apart from the sync, but if we are not planning to extend it, we can make it more coupled to the sync and pass the sync mode as well.

if (useBazelInfoRunner) {
return BlazeInfoRunner.getInstance()
.runBlazeInfo(
project,
Blaze.getBuildSystemProvider(project)
.getBuildSystem()
.getDefaultInvoker(project,
context),
context,
importSettings.getBuildSystem(),
syncFlags);
}
return BlazeInfoProvider.getInstance(project)
.getBlazeInfo(
context,
syncFlags);
}

private static class WorkspacePathResolverAndProjectView {
final WorkspacePathResolver workspacePathResolver;
final ProjectViewSet projectViewSet;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -361,7 +361,7 @@ private void doFilterProjectTargets(
context,
childContext -> {
SyncProjectState projectState =
ProjectStateSyncTask.collectProjectState(project, context);
ProjectStateSyncTask.collectProjectState(project, context, params);
if (projectState == null) {
return;
}
Expand Down Expand Up @@ -436,7 +436,7 @@ void runSync(BlazeSyncParams params, boolean singleThreaded, BlazeContext contex
SyncStats.builder());
return;
}
SyncProjectState projectState = ProjectStateSyncTask.collectProjectState(project, context);
SyncProjectState projectState = ProjectStateSyncTask.collectProjectState(project, context, params);
BlazeSyncBuildResult buildResult =
BuildPhaseSyncTask.runBuildPhase(
project, params, projectState, buildId, context, buildSystem);
Expand Down
Loading