Make it possible to index Go stdlib

.github/workflows/test.yaml

@@ -67,6 +67,11 @@ jobs:
             ref: v0.50.0
             expect_file: "cmd/gazelle/main.go"
             gopackagesdriver: true
+          - name: golang
+            repo: golang/go
+            ref: go1.26.2
+            expect_file: "fmt/print.go"
+            module_root: src
 
     runs-on: ubuntu-latest
     name: ${{ matrix.name }}
@@ -99,7 +104,7 @@ jobs:
       - uses: actions/setup-go@v6
         with:
           go-version: stable
-          cache-dependency-path: target/go.sum
+          cache-dependency-path: target/${{ matrix.module_root || '.' }}/go.sum
 
       - name: Set up GOPACKAGESDRIVER
         if: matrix.gopackagesdriver
@@ -114,11 +119,16 @@ jobs:
           chmod +x "$driver"
           echo "GOPACKAGESDRIVER=$driver" >> "$GITHUB_ENV"
 
+      - name: Point GOROOT at target stdlib
+        if: matrix.name == 'golang'
+        run: echo "GOROOT=$GITHUB_WORKSPACE/target" >> "$GITHUB_ENV"
+
       - name: Run scip-go
         working-directory: target
         run: |
           /usr/bin/time -v -o "$GITHUB_WORKSPACE/metrics.txt" \
-            scip-go -o "$GITHUB_WORKSPACE/index.scip" -VV
+            scip-go --module-root="${{ matrix.module_root || '.' }}" \
+              -o "$GITHUB_WORKSPACE/index.scip" -VV
 
       - name: Validate index
         working-directory: target
@@ -127,7 +137,8 @@ jobs:
           echo "Index size: $size bytes"
           [ "$size" -ge 1024 ] || { echo "FAIL: index too small"; exit 1; }
 
-          scip stats --from "$GITHUB_WORKSPACE/index.scip" --project-root . \
+          scip stats --from "$GITHUB_WORKSPACE/index.scip" \
+              --project-root "${{ matrix.module_root || '.' }}" \
             | jq -e '.documents > 0 and .occurrences > 0 and .definitions > 0'
 
           scip print --json "$GITHUB_WORKSPACE/index.scip" \

go.mod

@@ -5,8 +5,8 @@ go 1.25.0
 require (
 	github.com/alecthomas/kong v1.14.0
 	github.com/scip-code/scip/bindings/go/scip v0.7.1
-	golang.org/x/mod v0.34.0
-	golang.org/x/tools v0.43.0
+	golang.org/x/mod v0.35.0
+	golang.org/x/tools v0.44.0
 	golang.org/x/tools/go/vcs v0.1.0-deprecated
 	google.golang.org/protobuf v1.36.11
 )
@@ -21,6 +21,6 @@ require (
 	github.com/sourcegraph/beaut v0.0.0-20240611013027-627e4c25335a // indirect
 	github.com/stretchr/testify v1.11.1 // indirect
 	golang.org/x/sync v0.20.0 // indirect
-	golang.org/x/sys v0.42.0 // indirect
+	golang.org/x/sys v0.43.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -34,13 +34,19 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
 golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
+golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
+golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
 golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
 golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
 golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
+golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
 golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
+golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
+golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
 golang.org/x/tools/go/vcs v0.1.0-deprecated h1:cOIJqWBl99H1dH5LWizPa+0ImeeJq3t3cJjaeOWUAL4=
 golang.org/x/tools/go/vcs v0.1.0-deprecated/go.mod h1:zUrvATBAvEI9535oC0yWYsLsHIV4Z7g63sNPVMtuBy8=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=

internal/index/scip.go

@@ -22,6 +22,7 @@ import (
 	"github.com/scip-code/scip-go/internal/symbols"
 	"github.com/scip-code/scip-go/internal/visitors"
 	"github.com/scip-code/scip/bindings/go/scip"
+	"golang.org/x/tools/go/packages"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -204,6 +205,13 @@ func indexVisitPackages(
 		for _, pkgID := range lookupIDs {
 			pkg := projectPackages[pkgID]
 			slog.Debug("Visiting package", "path", pkg.PkgPath)
+
+			if len(pkg.Syntax) == 0 {
+				slog.Debug("Skipping package with no syntax files", "path", pkg.PkgPath)
+				atomic.AddUint64(&count, 1)
+				continue
+			}
+
 			visitors.VisitPackageSyntax(opts.ModuleRoot, pkg, pathToDocuments, globalSymbols)
 
 			pkgSymbol, _ := globalSymbols.GetPkgSymbol(pkg)
@@ -218,12 +226,19 @@ func indexVisitPackages(
 					Text:     "package " + pkg.Name,
 				},
 			}
-			firstFile := pkg.Syntax[0]
-			firstDoc := pathToDocuments[pkg.Fset.File(firstFile.Package).Name()]
+			firstFile, firstDoc := firstSyntaxWithDocument(pkg, pathToDocuments)
+			if firstDoc == nil {
+				slog.Debug("Skipping package with no in-tree syntax files", "path", pkg.PkgPath)
+				atomic.AddUint64(&count, 1)
+				continue
+			}
 			firstDoc.SetSymbolInformation(firstFile.Name.NamePos, symInfo)
 
 			for _, f := range pkg.Syntax {
 				doc := pathToDocuments[pkg.Fset.File(f.Package).Name()]
+				if doc == nil {
+					continue
+				}
 				position := pkg.Fset.Position(f.Name.NamePos)
 
 				doc.PackageOccurrence = &scip.Occurrence{
@@ -241,3 +256,18 @@ func indexVisitPackages(
 
 	return pathToDocuments, globalSymbols
 }
+
+// firstSyntaxWithDocument returns the first parsed file in pkg.Syntax that has
+// an associated *document.Document in pathToDocuments. Files outside the module
+// root are skipped during visiting and won't appear in pathToDocuments.
+func firstSyntaxWithDocument(
+	pkg *packages.Package,
+	pathToDocuments map[string]*document.Document,
+) (*ast.File, *document.Document) {
+	for _, f := range pkg.Syntax {
+		if doc := pathToDocuments[pkg.Fset.File(f.Package).Name()]; doc != nil {
+			return f, doc
+		}
+	}
+	return nil, nil
+}

internal/loader/loader.go

@@ -122,11 +122,43 @@ func LoadPackages(
 			projectPackages[newtypes.GetID(pkg)] = allPackages[newtypes.GetID(pkg)]
 		}
 
+		// Promote any in-tree dependency to a project package.
+		//
+		// `go list ./...` deliberately exclude vendor/ directories, so vendored
+		// dependencies arrive only via the transitive import graph. Their source
+		// lives in the repository being indexed, however, so they should be indexed
+		// just like first-party code. The most prominent case is the standard
+		// library, which vendors golang.org/x/{crypto,net,...} under src/vendor/.
+		for id, pkg := range allPackages {
+			if _, ok := projectPackages[id]; ok {
+				continue
+			}
+			if isInTree(pkg, moduleRoot) {
+				projectPackages[id] = pkg
+			}
+		}
+
 		return nil
 	})
 	return projectPackages, allPackages, err
 }
 
+// isInTree reports whether pkg's source files live under root. All files
+// of a Go package share a directory, so checking the first parsed file
+// is sufficient. We deliberately use Syntax (rather than GoFiles) because
+// some loaders report empty GoFiles even when source is available on disk.
+func isInTree(pkg *packages.Package, root string) bool {
+	if pkg == nil || len(pkg.Syntax) == 0 || pkg.Fset == nil {
+		return false
+	}
+	f := pkg.Fset.File(pkg.Syntax[0].Package)
+	if f == nil {
+		return false
+	}
+	rel, err := filepath.Rel(root, f.Name())
+	return err == nil && filepath.IsLocal(rel)
+}
+
 func isStandardLib(pkg *packages.Package) bool {
 	return pkg.Module == nil || pkg.Module.Path == "std"
 }

internal/visitors/visitor_file.go

@@ -6,6 +6,7 @@ import (
 	"go/token"
 	"go/types"
 	"log/slog"
+	"strconv"
 
 	"github.com/scip-code/scip-go/internal/document"
 	"github.com/scip-code/scip-go/internal/lookup"
@@ -109,7 +110,15 @@ func (v *fileVisitor) Visit(n ast.Node) ast.Visitor {
 			return nil
 		}
 
-		importedPackage := v.pkg.Imports[pkgName.Imported().Path()]
+		// pkg.Imports is keyed by the import path string as it appears
+		// in source, not by the resolved path returned by the type checker
+		// (the two differ for vendored imports).
+		sourcePath, err := strconv.Unquote(node.Path.Value)
+		if err != nil {
+			slog.Warn("Could not find node", "node.Path", node.Path)
+			return nil
+		}
+		importedPackage := v.pkg.Imports[sourcePath]
 		if importedPackage == nil {
 			slog.Warn("Could not find node", "node.Path", node.Path)
 			return nil

internal/visitors/visitors.go

@@ -26,7 +26,10 @@ func VisitPackageSyntax(
 	for _, f := range pkg.Syntax {
 
 		abs := pkg.Fset.File(f.Package).Name()
-		relative, _ := filepath.Rel(moduleRoot, abs)
+		relative, err := filepath.Rel(moduleRoot, abs)
+		if err != nil || !filepath.IsLocal(relative) {
+			continue
+		}
 
 		doc := visitSyntax(pkg, pkgSymbols, f, relative)