diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 72d0102..2bb71f1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -67,6 +67,11 @@ jobs: ref: v0.50.0 expect_file: "cmd/gazelle/main.go" gopackagesdriver: true + - name: golang + repo: golang/go + ref: go1.26.2 + expect_file: "fmt/print.go" + module_root: src runs-on: ubuntu-latest name: ${{ matrix.name }} @@ -99,7 +104,7 @@ jobs: - uses: actions/setup-go@v6 with: go-version: stable - cache-dependency-path: target/go.sum + cache-dependency-path: target/${{ matrix.module_root || '.' }}/go.sum - name: Set up GOPACKAGESDRIVER if: matrix.gopackagesdriver @@ -114,11 +119,16 @@ jobs: chmod +x "$driver" echo "GOPACKAGESDRIVER=$driver" >> "$GITHUB_ENV" + - name: Point GOROOT at target stdlib + if: matrix.name == 'golang' + run: echo "GOROOT=$GITHUB_WORKSPACE/target" >> "$GITHUB_ENV" + - name: Run scip-go working-directory: target run: | /usr/bin/time -v -o "$GITHUB_WORKSPACE/metrics.txt" \ - scip-go -o "$GITHUB_WORKSPACE/index.scip" -VV + scip-go --module-root="${{ matrix.module_root || '.' }}" \ + -o "$GITHUB_WORKSPACE/index.scip" -VV - name: Validate index working-directory: target @@ -127,7 +137,8 @@ jobs: echo "Index size: $size bytes" [ "$size" -ge 1024 ] || { echo "FAIL: index too small"; exit 1; } - scip stats --from "$GITHUB_WORKSPACE/index.scip" --project-root . \ + scip stats --from "$GITHUB_WORKSPACE/index.scip" \ + --project-root "${{ matrix.module_root || '.' }}" \ | jq -e '.documents > 0 and .occurrences > 0 and .definitions > 0' scip print --json "$GITHUB_WORKSPACE/index.scip" \ diff --git a/go.mod b/go.mod index 1680c04..8bbf6d1 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.25.0 require ( github.com/alecthomas/kong v1.14.0 github.com/scip-code/scip/bindings/go/scip v0.7.1 - golang.org/x/mod v0.34.0 - golang.org/x/tools v0.43.0 + golang.org/x/mod v0.35.0 + golang.org/x/tools v0.44.0 golang.org/x/tools/go/vcs v0.1.0-deprecated google.golang.org/protobuf v1.36.11 ) @@ -21,6 +21,6 @@ require ( github.com/sourcegraph/beaut v0.0.0-20240611013027-627e4c25335a // indirect github.com/stretchr/testify v1.11.1 // indirect golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.42.0 // indirect + golang.org/x/sys v0.43.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 27969da..36195f0 100644 --- a/go.sum +++ b/go.sum @@ -34,13 +34,19 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/tools/go/vcs v0.1.0-deprecated h1:cOIJqWBl99H1dH5LWizPa+0ImeeJq3t3cJjaeOWUAL4= golang.org/x/tools/go/vcs v0.1.0-deprecated/go.mod h1:zUrvATBAvEI9535oC0yWYsLsHIV4Z7g63sNPVMtuBy8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/internal/index/scip.go b/internal/index/scip.go index 5c7b27c..fd33ffb 100644 --- a/internal/index/scip.go +++ b/internal/index/scip.go @@ -22,6 +22,7 @@ import ( "github.com/scip-code/scip-go/internal/symbols" "github.com/scip-code/scip-go/internal/visitors" "github.com/scip-code/scip/bindings/go/scip" + "golang.org/x/tools/go/packages" "google.golang.org/protobuf/proto" ) @@ -204,6 +205,13 @@ func indexVisitPackages( for _, pkgID := range lookupIDs { pkg := projectPackages[pkgID] slog.Debug("Visiting package", "path", pkg.PkgPath) + + if len(pkg.Syntax) == 0 { + slog.Debug("Skipping package with no syntax files", "path", pkg.PkgPath) + atomic.AddUint64(&count, 1) + continue + } + visitors.VisitPackageSyntax(opts.ModuleRoot, pkg, pathToDocuments, globalSymbols) pkgSymbol, _ := globalSymbols.GetPkgSymbol(pkg) @@ -218,12 +226,19 @@ func indexVisitPackages( Text: "package " + pkg.Name, }, } - firstFile := pkg.Syntax[0] - firstDoc := pathToDocuments[pkg.Fset.File(firstFile.Package).Name()] + firstFile, firstDoc := firstSyntaxWithDocument(pkg, pathToDocuments) + if firstDoc == nil { + slog.Debug("Skipping package with no in-tree syntax files", "path", pkg.PkgPath) + atomic.AddUint64(&count, 1) + continue + } firstDoc.SetSymbolInformation(firstFile.Name.NamePos, symInfo) for _, f := range pkg.Syntax { doc := pathToDocuments[pkg.Fset.File(f.Package).Name()] + if doc == nil { + continue + } position := pkg.Fset.Position(f.Name.NamePos) doc.PackageOccurrence = &scip.Occurrence{ @@ -241,3 +256,18 @@ func indexVisitPackages( return pathToDocuments, globalSymbols } + +// firstSyntaxWithDocument returns the first parsed file in pkg.Syntax that has +// an associated *document.Document in pathToDocuments. Files outside the module +// root are skipped during visiting and won't appear in pathToDocuments. +func firstSyntaxWithDocument( + pkg *packages.Package, + pathToDocuments map[string]*document.Document, +) (*ast.File, *document.Document) { + for _, f := range pkg.Syntax { + if doc := pathToDocuments[pkg.Fset.File(f.Package).Name()]; doc != nil { + return f, doc + } + } + return nil, nil +} diff --git a/internal/loader/loader.go b/internal/loader/loader.go index 087f2e1..912cf0e 100644 --- a/internal/loader/loader.go +++ b/internal/loader/loader.go @@ -122,11 +122,43 @@ func LoadPackages( projectPackages[newtypes.GetID(pkg)] = allPackages[newtypes.GetID(pkg)] } + // Promote any in-tree dependency to a project package. + // + // `go list ./...` deliberately exclude vendor/ directories, so vendored + // dependencies arrive only via the transitive import graph. Their source + // lives in the repository being indexed, however, so they should be indexed + // just like first-party code. The most prominent case is the standard + // library, which vendors golang.org/x/{crypto,net,...} under src/vendor/. + for id, pkg := range allPackages { + if _, ok := projectPackages[id]; ok { + continue + } + if isInTree(pkg, moduleRoot) { + projectPackages[id] = pkg + } + } + return nil }) return projectPackages, allPackages, err } +// isInTree reports whether pkg's source files live under root. All files +// of a Go package share a directory, so checking the first parsed file +// is sufficient. We deliberately use Syntax (rather than GoFiles) because +// some loaders report empty GoFiles even when source is available on disk. +func isInTree(pkg *packages.Package, root string) bool { + if pkg == nil || len(pkg.Syntax) == 0 || pkg.Fset == nil { + return false + } + f := pkg.Fset.File(pkg.Syntax[0].Package) + if f == nil { + return false + } + rel, err := filepath.Rel(root, f.Name()) + return err == nil && filepath.IsLocal(rel) +} + func isStandardLib(pkg *packages.Package) bool { return pkg.Module == nil || pkg.Module.Path == "std" } diff --git a/internal/visitors/visitor_file.go b/internal/visitors/visitor_file.go index 08c440a..1cc206e 100644 --- a/internal/visitors/visitor_file.go +++ b/internal/visitors/visitor_file.go @@ -6,6 +6,7 @@ import ( "go/token" "go/types" "log/slog" + "strconv" "github.com/scip-code/scip-go/internal/document" "github.com/scip-code/scip-go/internal/lookup" @@ -109,7 +110,15 @@ func (v *fileVisitor) Visit(n ast.Node) ast.Visitor { return nil } - importedPackage := v.pkg.Imports[pkgName.Imported().Path()] + // pkg.Imports is keyed by the import path string as it appears + // in source, not by the resolved path returned by the type checker + // (the two differ for vendored imports). + sourcePath, err := strconv.Unquote(node.Path.Value) + if err != nil { + slog.Warn("Could not find node", "node.Path", node.Path) + return nil + } + importedPackage := v.pkg.Imports[sourcePath] if importedPackage == nil { slog.Warn("Could not find node", "node.Path", node.Path) return nil diff --git a/internal/visitors/visitors.go b/internal/visitors/visitors.go index c30fe3f..8082ab0 100644 --- a/internal/visitors/visitors.go +++ b/internal/visitors/visitors.go @@ -26,7 +26,10 @@ func VisitPackageSyntax( for _, f := range pkg.Syntax { abs := pkg.Fset.File(f.Package).Name() - relative, _ := filepath.Rel(moduleRoot, abs) + relative, err := filepath.Rel(moduleRoot, abs) + if err != nil || !filepath.IsLocal(relative) { + continue + } doc := visitSyntax(pkg, pkgSymbols, f, relative)