diff --git a/builder/build.go b/builder/build.go index dabbe7544e..134bfb0113 100644 --- a/builder/build.go +++ b/builder/build.go @@ -78,17 +78,18 @@ type BuildResult struct { // key, avoiding the need for recompiling all dependencies when only the // implementation of an imported package changes. type packageAction struct { - ImportPath string - CompilerBuildID string - TinyGoVersion string - LLVMVersion string - Config *compiler.Config - CFlags []string - FileHashes map[string]string // hash of every file that's part of the package - EmbeddedFiles map[string]string // hash of all the //go:embed files in the package - Imports map[string]string // map from imported package to action ID hash - OptLevel string // LLVM optimization level (O0, O1, O2, Os, Oz) - UndefinedGlobals []string // globals that are left as external globals (no initializer) + ImportPath string + CompilerBuildID string + TinyGoVersion string + LLVMVersion string + CCompilerIdentity string + Config *compiler.Config + CFlags []string + FileHashes map[string]string // hash of every file that's part of the package + EmbeddedFiles map[string]string // hash of all the //go:embed files in the package + Imports map[string]string // map from imported package to action ID hash + OptLevel string // LLVM optimization level (O0, O1, O2, Os, Oz) + UndefinedGlobals []string // globals that are left as external globals (no initializer) } // Build performs a single package to executable Go build. It takes in a package @@ -146,53 +147,33 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe // As a side effect, this also creates the headers for the given libc, if // the libc needs them. root := goenv.Get("TINYGOROOT") + libraries, err := configuredLibraries(config) + if err != nil { + return BuildResult{}, err + } + libraryKeys, err := makeLibraryCacheKeys(config, libraries) + if err != nil { + return BuildResult{}, err + } + config.LibraryKeys = libraryKeys var libcDependencies []*compileJob - switch config.Target.Libc { - case "darwin-libSystem": + if config.Target.Libc == "darwin-libSystem" { libcJob := makeDarwinLibSystemJob(config, tmpdir) libcDependencies = append(libcDependencies, libcJob) - case "musl": - var unlock func() - libcJob, unlock, err := libMusl.load(config, tmpdir) - if err != nil { - return BuildResult{}, err - } - defer unlock() - libcDependencies = append(libcDependencies, dummyCompileJob(filepath.Join(filepath.Dir(libcJob.result), "crt1.o"))) - libcDependencies = append(libcDependencies, libcJob) - case "picolibc": - libcJob, unlock, err := libPicolibc.load(config, tmpdir) - if err != nil { - return BuildResult{}, err - } - defer unlock() - libcDependencies = append(libcDependencies, libcJob) - case "wasi-libc": - libcJob, unlock, err := libWasiLibc.load(config, tmpdir) - if err != nil { - return BuildResult{}, err - } - defer unlock() - libcDependencies = append(libcDependencies, libcJob) - case "wasmbuiltins": - libcJob, unlock, err := libWasmBuiltins.load(config, tmpdir) + } + if libraries.libc != nil { + libcJob, unlock, err := libraries.libc.load(config, tmpdir) if err != nil { return BuildResult{}, err } defer unlock() - libcDependencies = append(libcDependencies, libcJob) - case "mingw-w64": - libcJob, unlock, err := libMinGW.load(config, tmpdir) - if err != nil { - return BuildResult{}, err + if libraries.libc.crt1Source != "" { + libcDependencies = append(libcDependencies, dummyCompileJob(filepath.Join(filepath.Dir(libcJob.result), "crt1.o"))) } - defer unlock() libcDependencies = append(libcDependencies, libcJob) + } + if config.Target.Libc == "mingw-w64" { libcDependencies = append(libcDependencies, makeMinGWExtraLibs(tmpdir, config.GOARCH())...) - case "": - // no library specified, so nothing to do - default: - return BuildResult{}, fmt.Errorf("unknown libc: %s", config.Target.Libc) } optLevel, speedLevel, sizeLevel := config.OptLevel() @@ -364,6 +345,13 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe OptLevel: optLevel, UndefinedGlobals: undefinedGlobals, } + if len(pkg.CGoHeaders) != 0 { + compilerID, err := clangCompilerIdentity() + if err != nil { + return err + } + actionID.CCompilerIdentity = compilerID + } for filePath, hash := range pkg.FileHashes { actionID.FileHashes[filePath] = hex.EncodeToString(hash) } @@ -413,9 +401,6 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe } // Load bitcode of CGo headers and join the modules together. - // This may seem vulnerable to cache problems, but this is not - // the case: the Go code that was just compiled already tracks - // all C files that are read and hashes them. // These headers could be compiled in parallel but the benefit // is so small that it's probably not worth parallelizing. // Packages are compiled independently anyway. @@ -734,10 +719,9 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe } } - // Add compiler-rt dependency if needed. Usually this is a simple load from - // a cache. - if config.Target.RTLib == "compiler-rt" { - job, unlock, err := libCompilerRT.load(config, tmpdir) + // Add library dependencies needed by the linker, usually from the cache. + for _, library := range libraries.linker { + job, unlock, err := library.load(config, tmpdir) if err != nil { return result, err } @@ -745,16 +729,6 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe linkerDependencies = append(linkerDependencies, job) } - // The Boehm collector is stored in a separate C library. - if config.GC() == "boehm" { - job, unlock, err := BoehmGC.load(config, tmpdir) - if err != nil { - return BuildResult{}, err - } - defer unlock() - linkerDependencies = append(linkerDependencies, job) - } - // Add jobs to compile extra files. These files are in C or assembly and // contain things like the interrupt vector table and low level operations // such as stack switching. diff --git a/builder/cc.go b/builder/cc.go index b2cc739e3b..7f4ff7577b 100644 --- a/builder/cc.go +++ b/builder/cc.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "io" - "io/fs" "os" "path/filepath" "sort" @@ -25,37 +24,21 @@ import ( // Compiling the same file again (if nothing changed, including included header // files) the output is loaded from the build cache instead. // -// Its operation is a bit complex (more complex than Go package build caching) -// because the list of file dependencies is only known after the file is -// compiled. However, luckily compilers have a flag to write a list of file -// dependencies in Makefile syntax which can be used for caching. +// Its operation is a bit complex (more complex than Go package build caching), +// because the list of file dependencies depends on C include path resolution. +// TinyGo asks Clang for the current dependency list before looking for an object +// cache hit, then uses the hashes of those dependencies in the object key. // -// Because of this complexity, every file has in fact two cached build outputs: -// the file itself, and the list of dependencies. Its operation is as follows: -// -// depfile = hash(path, compiler, cflags, ...) -// if depfile exists: -// outfile = hash of all files and depfile name -// if outfile exists: -// # cache hit -// return outfile -// # cache miss +// dependencies = clang -M source +// outfile = hash(path, compiler, cflags, dependencies, ...) +// if outfile exists: +// # cache hit +// return outfile // tmpfile = compile file -// read dependencies (side effect of compile) -// write depfile -// outfile = hash of all files and depfile name // rename tmpfile to outfile // -// There are a few edge cases that are not handled: -// - If a file is added to an include path, that file may be included instead of -// some other file. This would be fixed by also including lookup failures in the -// dependencies file, but I'm not aware of a compiler which does that. -// - The Makefile syntax that compilers output has issues, see readDepFile for -// details. -// - A header file may be changed to add/remove an include. This invalidates the -// depfile but without invalidating its name. For this reason, the depfile is -// written on each new compilation (even when it seems unnecessary). However, it -// could in rare cases lead to a stale file fetched from the cache. +// The Makefile syntax that compilers output has issues, see readDepFile for +// details. func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands func(string, ...string)) (string, error) { // Hash input file. fileHash, err := hashFile(abspath) @@ -67,65 +50,47 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands unlock := lock(filepath.Join(goenv.Get("GOCACHE"), fileHash+".c.lock")) defer unlock() - // Create cache key for the dependencies file. - buf, err := json.Marshal(struct { - Path string - Hash string - Flags []string - LLVMVersion string - }{ - Path: abspath, - Hash: fileHash, - Flags: cflags, - LLVMVersion: llvm.Version, - }) + compilerID, err := clangCompilerIdentity() if err != nil { - panic(err) // shouldn't happen + return "", err } - depfileNameHashBuf := sha512.Sum512_224(buf) - depfileNameHash := hex.EncodeToString(depfileNameHashBuf[:]) - - // Load dependencies file, if possible. - depfileName := "dep-" + depfileNameHash + ".json" - depfileCachePath := filepath.Join(goenv.Get("GOCACHE"), depfileName) - depfileBuf, err := os.ReadFile(depfileCachePath) - var dependencies []string // sorted list of dependency paths - if err == nil { - // There is a dependency file, that's great! - // Parse it first. - err := json.Unmarshal(depfileBuf, &dependencies) - if err != nil { - return "", fmt.Errorf("could not parse dependencies JSON: %w", err) - } - // Obtain hashes of all the files listed as a dependency. - outpath, err := makeCFileCachePath(dependencies, depfileNameHash) - if err == nil { - if _, err := os.Stat(outpath); err == nil { - return outpath, nil - } else if !errors.Is(err, fs.ErrNotExist) { - return "", err - } - } - } else if !errors.Is(err, fs.ErrNotExist) { - // expected either nil or IsNotExist + dependencies, err := scanCFileDependencies(abspath, tmpdir, cflags, printCommands) + if err != nil { + return "", err + } + outpath, err := makeCFileCachePath(abspath, cFileCompileArgs(abspath, "$OBJ", cflags), compilerID, dependencies) + if err != nil { + return "", err + } + if _, err := os.Stat(outpath); err == nil { + return outpath, nil + } else if !errors.Is(err, os.ErrNotExist) { return "", err } - objTmpFile, err := os.CreateTemp(goenv.Get("GOCACHE"), "tmp-*.bc") + objTmpFile, err := compileCFile(goenv.Get("GOCACHE"), abspath, cflags, printCommands) if err != nil { return "", err } - objTmpFile.Close() - depTmpFile, err := os.CreateTemp(tmpdir, "dep-*.d") + err = os.Rename(objTmpFile, outpath) if err != nil { return "", err } + return outpath, nil +} + +func scanCFileDependencies(abspath, tmpdir string, cflags []string, printCommands func(string, ...string)) ([]string, error) { + depTmpFile, err := os.CreateTemp(tmpdir, "dep-*.d") + if err != nil { + return nil, err + } depTmpFile.Close() - flags := append([]string{}, cflags...) // copy cflags - flags = append(flags, "-MD", "-MV", "-MTdeps", "-MF", depTmpFile.Name(), "-flto=thin") // autogenerate dependencies - flags = append(flags, "-c", "-o", objTmpFile.Name(), abspath) - if strings.ToLower(filepath.Ext(abspath)) == ".s" { + defer os.Remove(depTmpFile.Name()) + + flags := append([]string{}, cflags...) + flags = append(flags, "-M", "-MV", "-MTdeps", "-MF", depTmpFile.Name(), abspath) + if isAssemblyFile(abspath) { // If this is an assembly file (.s or .S, lowercase or uppercase), then // we'll need to add -Qunused-arguments because many parameters are // relevant to C, not assembly. And with -Werror, having meaningless @@ -137,13 +102,12 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands } err = runCCompiler(flags...) if err != nil { - return "", &commandError{"failed to build", abspath, err} + return nil, &commandError{"failed to scan dependencies", abspath, err} } - // Create sorted and uniqued slice of dependencies. dependencyPaths, err := readDepFile(depTmpFile.Name()) if err != nil { - return "", err + return nil, err } dependencyPaths = append(dependencyPaths, abspath) // necessary for .s files dependencySet := make(map[string]struct{}, len(dependencyPaths)) @@ -156,50 +120,44 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands dependencySlice = append(dependencySlice, path) } sort.Strings(dependencySlice) + return dependencySlice, nil +} - // Write dependencies file. - f, err := os.CreateTemp(filepath.Dir(depfileCachePath), depfileName) - if err != nil { - return "", err +func cFileCompileArgs(abspath, objpath string, cflags []string) []string { + flags := append([]string{}, cflags...) + flags = append(flags, "-flto=thin") + flags = append(flags, "-c", "-o", objpath, abspath) + if isAssemblyFile(abspath) { + flags = append(flags, "-Qunused-arguments") } + return flags +} - buf, err = json.MarshalIndent(dependencySlice, "", "\t") - if err != nil { - panic(err) // shouldn't happen - } - _, err = f.Write(buf) - if err != nil { - return "", err - } - err = f.Close() - if err != nil { - return "", err - } - err = os.Rename(f.Name(), depfileCachePath) +func compileCFile(cacheDir, abspath string, cflags []string, printCommands func(string, ...string)) (string, error) { + objTmpFile, err := os.CreateTemp(cacheDir, "tmp-*.bc") if err != nil { return "", err } + objTmpFile.Close() - // Move temporary object file to final location. - outpath, err := makeCFileCachePath(dependencySlice, depfileNameHash) - if err != nil { - return "", err + flags := cFileCompileArgs(abspath, objTmpFile.Name(), cflags) + if printCommands != nil { + printCommands("clang", flags...) } - err = os.Rename(objTmpFile.Name(), outpath) + err = runCCompiler(flags...) if err != nil { - return "", err + return "", &commandError{"failed to build", abspath, err} } - - return outpath, nil + return objTmpFile.Name(), nil } // Create a cache path (a path in GOCACHE) to store the output of a compiler -// job. This path is based on the dep file name (which is a hash of metadata -// including compiler flags) and the hash of all input files in the paths slice. -func makeCFileCachePath(paths []string, depfileNameHash string) (string, error) { +// job. This path is based on the compiler identity, compiler flags, and the +// hash of all dependency files. +func makeCFileCachePath(path string, flags []string, compilerID string, dependencies []string) (string, error) { // Hash all input files. - fileHashes := make(map[string]string, len(paths)) - for _, path := range paths { + fileHashes := make(map[string]string, len(dependencies)) + for _, path := range dependencies { hash, err := hashFile(path) if err != nil { return "", err @@ -209,11 +167,17 @@ func makeCFileCachePath(paths []string, depfileNameHash string) (string, error) // Calculate a cache key based on the above hashes. buf, err := json.Marshal(struct { - DepfileHash string - FileHashes map[string]string + Path string + Flags []string + LLVMVersion string + CompilerIdentity string + FileHashes map[string]string }{ - DepfileHash: depfileNameHash, - FileHashes: fileHashes, + Path: path, + Flags: flags, + LLVMVersion: llvm.Version, + CompilerIdentity: compilerID, + FileHashes: fileHashes, }) if err != nil { panic(err) // shouldn't happen @@ -225,6 +189,10 @@ func makeCFileCachePath(paths []string, depfileNameHash string) (string, error) return outpath, nil } +func isAssemblyFile(path string) bool { + return strings.ToLower(filepath.Ext(path)) == ".s" +} + // hashFile hashes the given file path and returns the hash as a hex string. func hashFile(path string) (string, error) { f, err := os.Open(path) diff --git a/builder/cc_test.go b/builder/cc_test.go index 085528060e..fc9117b133 100644 --- a/builder/cc_test.go +++ b/builder/cc_test.go @@ -1,6 +1,8 @@ package builder import ( + "os" + "path/filepath" "reflect" "testing" ) @@ -31,3 +33,53 @@ func TestSplitDepFile(t *testing.T) { } } } + +func TestCFileCacheIncludePathShadowing(t *testing.T) { + t.Setenv("GOCACHEPROG", "") + + dir := t.TempDir() + + include1 := filepath.Join(dir, "include1") + include2 := filepath.Join(dir, "include2") + if err := os.Mkdir(include1, 0o777); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(include2, 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(include2, "value.h"), []byte("#define VALUE 1\n"), 0o666); err != nil { + t.Fatal(err) + } + source := filepath.Join(dir, "test.c") + if err := os.WriteFile(source, []byte("#include \"value.h\"\nint value(void) { return VALUE; }\n"), 0o666); err != nil { + t.Fatal(err) + } + + flags := []string{ + "-I", include1, + "-I", include2, + "--target=x86_64-unknown-linux-gnu", + } + first, err := compileAndCacheCFile(source, dir, flags, nil) + if err != nil { + t.Fatal(err) + } + second, err := compileAndCacheCFile(source, dir, flags, nil) + if err != nil { + t.Fatal(err) + } + if first != second { + t.Fatalf("unchanged compile did not hit cache: %s != %s", first, second) + } + + if err := os.WriteFile(filepath.Join(include1, "value.h"), []byte("#define VALUE 2\n"), 0o666); err != nil { + t.Fatal(err) + } + shadowed, err := compileAndCacheCFile(source, dir, flags, nil) + if err != nil { + t.Fatal(err) + } + if shadowed == first { + t.Fatal("include path shadowing reused stale cached object") + } +} diff --git a/builder/library.go b/builder/library.go index 1f8abce964..e4b0573b76 100644 --- a/builder/library.go +++ b/builder/library.go @@ -1,9 +1,14 @@ package builder import ( + "crypto/sha512" + "encoding/hex" + "encoding/json" "errors" + "fmt" "io/fs" "os" + "os/exec" "path/filepath" "runtime" "strings" @@ -11,6 +16,7 @@ import ( "github.com/tinygo-org/tinygo/compileopts" "github.com/tinygo-org/tinygo/goenv" + "tinygo.org/x/go-llvm" ) // Library is a container for information about a single C library, such as a @@ -37,6 +43,9 @@ type Library struct { // The source directory. sourceDir func() string + // The input directory that contains headers and other non-source inputs. + inputDir func() string + // The source files, relative to sourceDir. librarySources func(target string, libcNeedsMalloc bool) ([]string, error) @@ -44,6 +53,307 @@ type Library struct { crt1Source string } +const ( + libraryHeaderPathPlaceholder = "$HEADER" + libraryBuildDirPlaceholder = "$BUILDDIR" +) + +type librarySourceInput struct { + Path string + Hash string + CFlags []string +} + +type libraryCacheInput struct { + Name string + Target string + LibcNeedsMalloc bool + LLVMVersion string + CompilerIdentity string + ResourceDir string + SourceDir string + InputDir string + HeaderInputs map[string]string + GeneratedHeaders map[string]string + CompileArgs []string + Sources []librarySourceInput + Crt1Source string + Crt1Hash string +} + +type configuredLibrarySet struct { + libc *Library + linker []*Library +} + +func configuredLibraries(config *compileopts.Config) (configuredLibrarySet, error) { + var libraries configuredLibrarySet + switch config.Target.Libc { + case "musl": + libraries.libc = &libMusl + case "picolibc": + libraries.libc = &libPicolibc + case "wasi-libc": + libraries.libc = &libWasiLibc + case "wasmbuiltins": + libraries.libc = &libWasmBuiltins + case "mingw-w64": + libraries.libc = &libMinGW + case "darwin-libSystem", "": + // These libc configurations don't use a Library-backed cache. + default: + return configuredLibrarySet{}, fmt.Errorf("unknown libc: %s", config.Target.Libc) + } + if config.Target.RTLib == "compiler-rt" { + libraries.linker = append(libraries.linker, &libCompilerRT) + } + if config.GC() == "boehm" { + libraries.linker = append(libraries.linker, &BoehmGC) + } + return libraries, nil +} + +func (libraries configuredLibrarySet) all() []*Library { + result := make([]*Library, 0, 1+len(libraries.linker)) + if libraries.libc != nil { + result = append(result, libraries.libc) + } + result = append(result, libraries.linker...) + return result +} + +func makeLibraryCacheKeys(config *compileopts.Config, libraries configuredLibrarySet) (map[string]string, error) { + keys := make(map[string]string) + keyConfig := *config + keyConfig.LibraryKeys = keys + + setKey := func(l *Library) error { + if _, ok := keys[l.name]; ok { + return nil + } + input, err := l.cacheInput(&keyConfig) + if err != nil { + return err + } + keys[l.name] = input.key() + return nil + } + + for _, library := range libraries.all() { + if err := setKey(library); err != nil { + return nil, err + } + } + return keys, nil +} + +func (l *Library) cacheInput(config *compileopts.Config) (*libraryCacheInput, error) { + target := config.Triple() + sourceDir := l.sourceDir() + sources, err := l.librarySources(target, config.LibcNeedsMalloc()) + if err != nil { + return nil, err + } + + inputDir := sourceDir + if l.inputDir != nil { + inputDir = l.inputDir() + } + compilerID, err := clangCompilerIdentity() + if err != nil { + return nil, err + } + + input := libraryCacheInput{ + Name: l.name, + Target: target, + LibcNeedsMalloc: config.LibcNeedsMalloc(), + LLVMVersion: llvm.Version, + CompilerIdentity: compilerID, + ResourceDir: goenv.ClangResourceDir(false), + SourceDir: sourceDir, + InputDir: inputDir, + HeaderInputs: make(map[string]string), + CompileArgs: l.compileArgs(config, target, libraryHeaderPathPlaceholder, libraryBuildDirPlaceholder), + Sources: make([]librarySourceInput, 0, len(sources)), + Crt1Source: l.crt1Source, + } + for _, source := range sources { + hash, err := hashFile(filepath.Join(sourceDir, source)) + if err != nil { + return nil, err + } + sourceInput := librarySourceInput{ + Path: filepath.ToSlash(source), + Hash: hash, + } + if l.cflagsForFile != nil { + sourceInput.CFlags = l.cflagsForFile(source) + } + input.Sources = append(input.Sources, sourceInput) + } + if l.crt1Source != "" { + hash, err := hashFile(filepath.Join(sourceDir, l.crt1Source)) + if err != nil { + return nil, err + } + input.Crt1Source = filepath.ToSlash(l.crt1Source) + input.Crt1Hash = hash + } + headerInputs, err := hashLibraryHeaderInputs(inputDir) + if err != nil { + return nil, err + } + input.HeaderInputs = headerInputs + generatedHeaders, err := l.hashGeneratedHeaders(target) + if err != nil { + return nil, err + } + input.GeneratedHeaders = generatedHeaders + + return &input, nil +} + +func (input *libraryCacheInput) key() string { + data, err := json.Marshal(input) + if err != nil { + panic(err) + } + sum := sha512.Sum512_224(data) + return hex.EncodeToString(sum[:]) +} + +func (l *Library) hashGeneratedHeaders(target string) (map[string]string, error) { + if l.makeHeaders == nil { + return nil, nil + } + dir, err := os.MkdirTemp("", "tinygo-lib-headers-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + if err := l.makeHeaders(target, dir); err != nil { + return nil, err + } + return hashLibraryHeaderInputs(dir) +} + +func hashLibraryHeaderInputs(root string) (map[string]string, error) { + hashes := map[string]string{} + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".h", ".hh", ".hpp", ".inc", ".in": + default: + return nil + } + hash, err := hashFile(path) + if err != nil { + return err + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + hashes[filepath.ToSlash(rel)] = hash + return nil + }) + return hashes, err +} + +func clangCompilerIdentity() (string, error) { + if hasBuiltinTools { + return "builtin clang llvm " + llvm.Version, nil + } + name, err := LookupCommand("clang") + if err != nil { + return "", err + } + cmd := exec.Command(name, "--version") + cmd.Env = []string{} + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to identify clang: %w", err) + } + return name + "\n" + string(out), nil +} + +func (l *Library) compileArgs(config *compileopts.Config, target, headerPath, dir string) []string { + remapDir := filepath.Join(os.TempDir(), "tinygo-"+l.name) + args := append(l.cflags(target, headerPath), "-c", "-Oz", "-gdwarf-4", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+target, "-fdebug-prefix-map="+dir+"="+remapDir) + resourceDir := goenv.ClangResourceDir(false) + if resourceDir != "" { + args = append(args, "-resource-dir="+resourceDir) + } + cpu := config.CPU() + if cpu != "" { + // X86 has deprecated the -mcpu flag, so we need to use -march instead. + // However, ARM has not done this. + if strings.HasPrefix(target, "i386") || strings.HasPrefix(target, "x86_64") { + args = append(args, "-march="+cpu) + } else if strings.HasPrefix(target, "avr") { + args = append(args, "-mmcu="+cpu) + } else { + args = append(args, "-mcpu="+cpu) + } + } + if config.ABI() != "" { + args = append(args, "-mabi="+config.ABI()) + } + switch compileopts.CanonicalArchName(target) { + case "arm": + if strings.Split(target, "-")[2] == "linux" { + args = append(args, "-fno-unwind-tables", "-fno-asynchronous-unwind-tables") + } else { + args = append(args, "-fshort-enums", "-fomit-frame-pointer", "-mfloat-abi=soft", "-fno-unwind-tables", "-fno-asynchronous-unwind-tables") + } + case "avr": + // AVR defaults to C float and double both being 32-bit. This deviates + // from what most code (and certainly compiler-rt) expects. So we need + // to force the compiler to use 64-bit floating point numbers for + // double. + args = append(args, "-mdouble=64") + case "riscv32": + args = append(args, "-march="+riscvMarch(config, "rv32imac"), "-fforce-enable-int128") + case "riscv64": + args = append(args, "-march="+riscvMarch(config, "rv64gc")) + case "mips": + args = append(args, "-fno-pic") + } + if config.Target.SoftFloat { + // Use softfloat instead of floating point instructions. This is + // supported on many architectures. + args = append(args, "-msoft-float") + } else { + if strings.HasPrefix(target, "armv5") { + // On ARMv5 we need to explicitly enable hardware floating point + // instructions: Clang appears to assume the hardware doesn't have a + // FPU otherwise. + args = append(args, "-mfpu=vfpv2") + } + } + if l.needsLibc { + args = append(args, config.LibcCFlags()...) + } + return args +} + +func expandLibraryCompileArgs(args []string, headerPath, dir string) []string { + expanded := append([]string(nil), args...) + for i, arg := range expanded { + arg = strings.ReplaceAll(arg, libraryHeaderPathPlaceholder, headerPath) + arg = strings.ReplaceAll(arg, libraryBuildDirPlaceholder, dir) + expanded[i] = arg + } + return expanded +} + // load returns a compile job to build this library file for the given target // and CPU. It may return a dummy compileJob if the library build is already // cached. The path is stored as job.result but is only valid after the job has @@ -53,6 +363,18 @@ type Library struct { // As a side effect, this call creates the library header files if they didn't // exist yet. func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJob, abortLock func(), err error) { + input, err := l.cacheInput(config) + if err != nil { + return nil, nil, err + } + key := input.key() + if existingKey, ok := config.LibraryKeys[l.name]; ok { + if existingKey != key { + return nil, nil, fmt.Errorf("library cache key changed for %s", l.name) + } + } else { + return nil, nil, fmt.Errorf("library cache key missing for %s", l.name) + } outdir := config.LibraryPath(l.name) archiveFilePath := filepath.Join(outdir, "lib.a") @@ -122,7 +444,6 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ } } - remapDir := filepath.Join(os.TempDir(), "tinygo-"+l.name) dir := filepath.Join(tmpdir, "build-lib-"+l.name) err = os.Mkdir(dir, 0777) if err != nil { @@ -133,61 +454,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ // Note: -fdebug-prefix-map is necessary to make the output archive // reproducible. Otherwise the temporary directory is stored in the archive // itself, which varies each run. - args := append(l.cflags(target, headerPath), "-c", "-Oz", "-gdwarf-4", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+target, "-fdebug-prefix-map="+dir+"="+remapDir) - resourceDir := goenv.ClangResourceDir(false) - if resourceDir != "" { - args = append(args, "-resource-dir="+resourceDir) - } - cpu := config.CPU() - if cpu != "" { - // X86 has deprecated the -mcpu flag, so we need to use -march instead. - // However, ARM has not done this. - if strings.HasPrefix(target, "i386") || strings.HasPrefix(target, "x86_64") { - args = append(args, "-march="+cpu) - } else if strings.HasPrefix(target, "avr") { - args = append(args, "-mmcu="+cpu) - } else { - args = append(args, "-mcpu="+cpu) - } - } - if config.ABI() != "" { - args = append(args, "-mabi="+config.ABI()) - } - switch compileopts.CanonicalArchName(target) { - case "arm": - if strings.Split(target, "-")[2] == "linux" { - args = append(args, "-fno-unwind-tables", "-fno-asynchronous-unwind-tables") - } else { - args = append(args, "-fshort-enums", "-fomit-frame-pointer", "-mfloat-abi=soft", "-fno-unwind-tables", "-fno-asynchronous-unwind-tables") - } - case "avr": - // AVR defaults to C float and double both being 32-bit. This deviates - // from what most code (and certainly compiler-rt) expects. So we need - // to force the compiler to use 64-bit floating point numbers for - // double. - args = append(args, "-mdouble=64") - case "riscv32": - args = append(args, "-march="+riscvMarch(config, "rv32imac"), "-fforce-enable-int128") - case "riscv64": - args = append(args, "-march="+riscvMarch(config, "rv64gc")) - case "mips": - args = append(args, "-fno-pic") - } - if config.Target.SoftFloat { - // Use softfloat instead of floating point instructions. This is - // supported on many architectures. - args = append(args, "-msoft-float") - } else { - if strings.HasPrefix(target, "armv5") { - // On ARMv5 we need to explicitly enable hardware floating point - // instructions: Clang appears to assume the hardware doesn't have a - // FPU otherwise. - args = append(args, "-mfpu=vfpv2") - } - } - if l.needsLibc { - args = append(args, config.LibcCFlags()...) - } + args := expandLibraryCompileArgs(input.CompileArgs, headerPath, dir) var once sync.Once @@ -222,17 +489,14 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ }, } - sourceDir := l.sourceDir() + sourceDir := input.SourceDir // Create jobs to compile all sources. These jobs are depended upon by the // archive job above, so must be run first. - paths, err := l.librarySources(target, config.LibcNeedsMalloc()) - if err != nil { - return nil, nil, err - } - for _, path := range paths { + for _, source := range input.Sources { // Strip leading "../" parts off the path. - path := path + source := source + path := filepath.FromSlash(source.Path) cleanpath := path for strings.HasPrefix(cleanpath, "../") { cleanpath = cleanpath[3:] @@ -246,9 +510,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ run: func(*compileJob) error { var compileArgs []string compileArgs = append(compileArgs, args...) - if l.cflagsForFile != nil { - compileArgs = append(compileArgs, l.cflagsForFile(path)...) - } + compileArgs = append(compileArgs, source.CFlags...) compileArgs = append(compileArgs, "-o", objpath, srcpath) if config.Options.PrintCommands != nil { config.Options.PrintCommands("clang", compileArgs...) diff --git a/builder/musl.go b/builder/musl.go index 2ce1b19e3a..570bce7bb4 100644 --- a/builder/musl.go +++ b/builder/musl.go @@ -121,6 +121,7 @@ var libMusl = Library{ return cflags }, sourceDir: func() string { return filepath.Join(goenv.Get("TINYGOROOT"), "lib/musl/src") }, + inputDir: func() string { return filepath.Join(goenv.Get("TINYGOROOT"), "lib/musl") }, librarySources: func(target string, _ bool) ([]string, error) { arch := compileopts.MuslArchitecture(target) globs := []string{ diff --git a/builder/picolibc.go b/builder/picolibc.go index 43837fa1dc..82fa631995 100644 --- a/builder/picolibc.go +++ b/builder/picolibc.go @@ -43,6 +43,7 @@ var libPicolibc = Library{ } }, sourceDir: func() string { return filepath.Join(goenv.Get("TINYGOROOT"), "lib/picolibc/newlib") }, + inputDir: func() string { return filepath.Join(goenv.Get("TINYGOROOT"), "lib/picolibc") }, librarySources: func(target string, _ bool) ([]string, error) { sources := append([]string(nil), picolibcSources...) if !strings.HasPrefix(target, "avr") { diff --git a/compileopts/config.go b/compileopts/config.go index 673de40f95..28bd9c9dff 100644 --- a/compileopts/config.go +++ b/compileopts/config.go @@ -32,6 +32,7 @@ type Config struct { Target *TargetSpec GoMinorVersion int TestConfig TestConfig + LibraryKeys map[string]string } // Triple returns the LLVM target triple, like armv6m-unknown-unknown-eabi. @@ -291,8 +292,10 @@ func (c *Config) LibraryPath(name string) string { archname += "-" + c.Target.Libc } - // Append a version string, if this library has a version. - if v, ok := libVersions[name]; ok { + if key, ok := c.LibraryKeys[name]; ok { + archname += "-h" + key + } else if v, ok := libVersions[name]; ok { + // Append a version string, if this library has a version. archname += "-v" + strconv.Itoa(v) } @@ -374,8 +377,8 @@ func (c *Config) CFlags(libclang bool) []string { } // LibcCFlags returns the C compiler flags for the configured libc. -// It only uses flags that are part of the libc path (triple, cpu, abi, libc -// name) so it can safely be used to compile another C library. +// It only uses flags that are part of the libc path, so it can safely be used +// to compile another C library. func (c *Config) LibcCFlags() []string { switch c.Target.Libc { case "darwin-libSystem":