WIP: Add a lazy indexing mode to gazelle

cmd/gazelle/fix-update.go

@@ -136,7 +136,11 @@ func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) erro
 		uc.dirs[i] = dir
 	}
 
-	if ucr.recursive && c.IndexLibraries {
+	if ucr.recursive && c.LazyIndex {
+		uc.walkMode = walk.UpdateSubdirsMode
+	} else if c.LazyIndex {
+		uc.walkMode = walk.UpdateDirsMode
+	} else if ucr.recursive && c.IndexLibraries {
 		uc.walkMode = walk.VisitAllUpdateSubdirsMode
 	} else if c.IndexLibraries {
 		uc.walkMode = walk.VisitAllUpdateDirsMode
@@ -240,6 +244,8 @@ type visitRecord struct {
 	// empty is a list of empty Go rules that may be deleted.
 	empty []*rule.Rule
 
+	toIndex map[string]struct{}
+
 	// file is the build file being processed.
 	file *rule.File
 
@@ -311,8 +317,11 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
 		}
 	}()
 
+	visitedPkgs := make(map[string]struct{})
+
 	var errorsFromWalk []error
 	walk.Walk(c, cexts, uc.dirs, uc.walkMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) {
+		visitedPkgs[rel] = struct{}{}
 		// If this file is ignored or if Gazelle was not asked to update this
 		// directory, just index the build file and move on.
 		if !update {
@@ -337,6 +346,7 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
 		// Generate rules.
 		var empty, gen []*rule.Rule
 		var imports []interface{}
+		toIndex := map[string]struct{}{}
 		for _, l := range filterLanguages(c, languages) {
 			res := l.GenerateRules(language.GenerateArgs{
 				Config:       c,
@@ -355,6 +365,9 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
 			empty = append(empty, res.Empty...)
 			gen = append(gen, res.Gen...)
 			imports = append(imports, res.Imports...)
+			for _, pkg := range res.PackagesToIndex {
+				toIndex[pkg] = struct{}{}
+			}
 		}
 		if f == nil && len(gen) == 0 {
 			return
@@ -439,6 +452,7 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
 			c:              c,
 			rules:          gen,
 			imports:        imports,
+			toIndex:        toIndex,
 			empty:          empty,
 			file:           f,
 			mappedKinds:    mappedKinds,
@@ -472,6 +486,35 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
 		return fmt.Errorf("encountered multiple errors: %w, %v", errorsFromWalk[0], strings.Join(additionalErrors, ", "))
 	}
 
+	// Index the directories that we were asked to index
+	if c.LazyIndex {
+		allToIndex := map[string]interface{}{}
+		for _, v := range visits {
+			for pkg := range v.toIndex {
+				if _, ok := visitedPkgs[pkg]; !ok {
+					allToIndex[pkg] = struct{}{}
+				}
+			}
+		}
+
+		dirsToWalk := make([]string, 0, len(allToIndex))
+		for pkg := range allToIndex {
+			// TODO: Maybe want to do a similar EvalSymlinks logic that is done
+			// in CheckFlags to convert args to filepaths.
+			dirsToWalk = append(dirsToWalk, filepath.Join(c.WorkDir, pkg))
+		}
+		walk.Walk(c, cexts, dirsToWalk, walk.UpdateDirsMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) {
+			if f != nil {
+				for _, repl := range c.KindMap {
+					mrslv.MappedKind(rel, repl)
+				}
+				for _, r := range f.Rules {
+					ruleIndex.AddRule(c, r, f)
+				}
+			}
+		})
+	}
+
 	// Finish building the index for dependency resolution.
 	ruleIndex.Finish()
 

config/config.go

@@ -27,6 +27,7 @@ limitations under the License.
 package config
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"log"
@@ -85,6 +86,7 @@ type Config struct {
 	// IndexLibraries determines whether Gazelle should build an index of
 	// libraries in the workspace for dependency resolution
 	IndexLibraries bool
+	LazyIndex      bool
 
 	// KindMap maps from a kind name to its replacement. It provides a way for
 	// users to customize the kind of rules created by Gazelle, via
@@ -199,7 +201,7 @@ type Configurer interface {
 // i.e., those that apply to Config itself and not to Config.Exts.
 type CommonConfigurer struct {
 	repoRoot, buildFileNames, readBuildFilesDir, writeBuildFilesDir string
-	indexLibraries, strict                                          bool
+	indexLibraries, strict, lazyIndex                               bool
 	langCsv                                                         string
 	bzlmod                                                          bool
 }
@@ -208,6 +210,7 @@ func (cc *CommonConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *Confi
 	fs.StringVar(&cc.repoRoot, "repo_root", "", "path to a directory which corresponds to go_prefix, otherwise gazelle searches for it.")
 	fs.StringVar(&cc.buildFileNames, "build_file_name", strings.Join(DefaultValidBuildFileNames, ","), "comma-separated list of valid build file names.\nThe first element of the list is the name of output build files to generate.")
 	fs.BoolVar(&cc.indexLibraries, "index", true, "when true, gazelle will build an index of libraries in the workspace for dependency resolution")
+	fs.BoolVar(&cc.lazyIndex, "lazy_index", false, "when true, gazelle will lazily index")
 	fs.BoolVar(&cc.strict, "strict", false, "when true, gazelle will exit with none-zero value for build file syntax errors or unknown directives")
 	fs.StringVar(&cc.readBuildFilesDir, "experimental_read_build_files_dir", "", "path to a directory where build files should be read from (instead of -repo_root)")
 	fs.StringVar(&cc.writeBuildFilesDir, "experimental_write_build_files_dir", "", "path to a directory where build files should be written to (instead of -repo_root)")
@@ -250,7 +253,11 @@ func (cc *CommonConfigurer) CheckFlags(fs *flag.FlagSet, c *Config) error {
 			c.WriteBuildFilesDir = filepath.Join(c.WorkDir, cc.writeBuildFilesDir)
 		}
 	}
+	if cc.lazyIndex && !cc.indexLibraries {
+		return errors.New("Using -lazy_index requires -index=true")
+	}
 	c.IndexLibraries = cc.indexLibraries
+	c.LazyIndex = cc.lazyIndex
 	c.Strict = cc.strict
 	if len(cc.langCsv) > 0 {
 		c.Langs = strings.Split(cc.langCsv, ",")

language/lang.go

@@ -189,4 +189,7 @@ type GenerateResult struct {
 	// correspond. These values are passed to Resolve after merge. The type
 	// is opaque since different languages may use different representations.
 	Imports []interface{}
+
+    // Additional packages that should be indexed.
+    PackagesToIndex []string
 }