From ed5f7d8b7467e548dc875b11e7eda1bd2ad9b369 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 22 Sep 2026 15:35:40 +0800 Subject: [PATCH 1/4] feat(llard): reuse workspace artifacts before Kodo llard consulted Kodo on every build-cache lookup, downloading and unpacking the artifact even when it was already present in the workspace. Add a llard-local read-through cache so llard serves the workspace copy first and only falls back to Kodo on a miss. The workspace is evictable, so Kodo stays the source of truth: a remote hit is persisted into the local cache for later reads. The implementation is a deliberate copy of the client's read-through cache so llard and the client do not share an abstraction. Only cmd/llard changes; the llar CLI and shared packages are untouched. --- cmd/llard/main.go | 61 +++++++++++++++++++++++++++++----- cmd/llard/main_test.go | 75 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 9 deletions(-) diff --git a/cmd/llard/main.go b/cmd/llard/main.go index 564b8c80..fa0a5979 100644 --- a/cmd/llard/main.go +++ b/cmd/llard/main.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "io/fs" "log" "net/http" "os" @@ -16,6 +17,7 @@ import ( "syscall" "github.com/goplus/llar/internal/artifact" + "github.com/goplus/llar/internal/build" "github.com/goplus/llar/internal/build/cache" buildhttp "github.com/goplus/llar/internal/build/http" "github.com/goplus/llar/internal/formula/repo" @@ -73,15 +75,21 @@ func run() error { Bucket: cfg.bucket, Prefix: cfg.prefix, }) - buildCache := cache.NewKodo(cache.KodoConfig{ - AccessKey: cfg.accessKey, - SecretKey: cfg.secretKey, - Bucket: cfg.bucket, - PublicDomain: cfg.publicDomain, - Prefix: cfg.prefix, - WorkspaceDir: workspaceDir, - Artifacts: artifacts, - }) + // Reuse artifacts already present in the workspace before downloading them + // from Kodo. The workspace is evictable, so Kodo remains the source of + // truth and restores anything missing back into the workspace. + buildCache := readThroughCache{ + local: build.NewLocalCache(workspaceDir), + remote: cache.NewKodo(cache.KodoConfig{ + AccessKey: cfg.accessKey, + SecretKey: cfg.secretKey, + Bucket: cfg.bucket, + PublicDomain: cfg.publicDomain, + Prefix: cfg.prefix, + WorkspaceDir: workspaceDir, + Artifacts: artifacts, + }), + } handler := buildhttp.New(buildhttp.Options{ FormulaStore: formulaStore, Cache: buildCache, @@ -110,6 +118,41 @@ func run() error { return nil } +// readThroughCache reuses artifacts already present in the local workspace +// before fetching them from the remote store. The local workspace is only a +// best-effort cache: an installed artifact directory may be removed by a +// cleanup pass, so a local miss falls back to the remote store and a remote +// hit is persisted back into the local cache for later reads. +// +// This is a deliberate copy of the llar client's read-through cache: llard and +// the client evolve separately and must not share an abstraction. +type readThroughCache struct { + local cache.Cache + remote cache.Cache +} + +func (c readThroughCache) Get(ctx context.Context, key cache.Key) (cache.Entry, bool, error) { + entry, ok, err := c.local.Get(ctx, key) + if err != nil || ok { + return entry, ok, err + } + entry, ok, err = c.remote.Get(ctx, key) + if err != nil || !ok { + return entry, ok, err + } + // The remote store already restored the artifact into the shared + // workspace, so the local cache only needs to persist its entry. + entry, err = c.local.Put(ctx, key, nil, entry) + if err != nil { + return cache.Entry{}, false, err + } + return entry, true, nil +} + +func (c readThroughCache) Put(ctx context.Context, key cache.Key, output fs.FS, entry cache.Entry) (cache.Entry, error) { + return c.local.Put(ctx, key, output, entry) +} + func loadConfig() (config, error) { cfg := config{ addr: os.Getenv("LLARD_ADDR"), diff --git a/cmd/llard/main_test.go b/cmd/llard/main_test.go index c56884d0..608e1a4b 100644 --- a/cmd/llard/main_test.go +++ b/cmd/llard/main_test.go @@ -5,9 +5,15 @@ package main import ( + "context" + "io/fs" "os" "strings" "testing" + + "github.com/goplus/llar/internal/build" + "github.com/goplus/llar/internal/build/cache" + "github.com/goplus/llar/mod/module" ) func TestLoadConfig(t *testing.T) { @@ -99,6 +105,75 @@ func TestRunRequiresConfig(t *testing.T) { } } +func TestReadThroughCache_LocalHitSkipsRemote(t *testing.T) { + workspaceDir := t.TempDir() + local := build.NewLocalCache(workspaceDir) + key := cache.Key{Module: module.Version{Path: "madler/zlib", Version: "v1.3.1"}, Matrix: "amd64-linux"} + if _, err := local.Put(context.Background(), key, nil, cache.Entry{Metadata: "-local"}); err != nil { + t.Fatal(err) + } + + remote := &countingCache{} + c := readThroughCache{local: local, remote: remote} + entry, ok, err := c.Get(context.Background(), key) + if err != nil || !ok || entry.Metadata != "-local" { + t.Fatalf("Get() = %+v, %v, %v; want local hit", entry, ok, err) + } + if remote.gets != 0 { + t.Fatalf("remote Get calls = %d, want 0", remote.gets) + } +} + +func TestReadThroughCache_PersistsRemoteHit(t *testing.T) { + workspaceDir := t.TempDir() + local := build.NewLocalCache(workspaceDir) + key := cache.Key{Module: module.Version{Path: "madler/zlib", Version: "v1.3.1"}, Matrix: "amd64-linux"} + remote := &countingCache{entry: cache.Entry{Metadata: "-remote"}, hit: true} + + c := readThroughCache{local: local, remote: remote} + for i := 0; i < 2; i++ { + entry, ok, err := c.Get(context.Background(), key) + if err != nil || !ok || entry.Metadata != "-remote" { + t.Fatalf("Get() #%d = %+v, %v, %v", i+1, entry, ok, err) + } + } + if remote.gets != 1 { + t.Fatalf("remote Get calls = %d, want 1", remote.gets) + } +} + +func TestReadThroughCache_PutWritesOnlyLocal(t *testing.T) { + local := build.NewLocalCache(t.TempDir()) + remote := &countingCache{} + c := readThroughCache{local: local, remote: remote} + key := cache.Key{Module: module.Version{Path: "madler/zlib", Version: "v1.3.1"}, Matrix: "amd64-linux"} + + if _, err := c.Put(context.Background(), key, nil, cache.Entry{Metadata: "-built"}); err != nil { + t.Fatalf("Put() failed: %v", err) + } + if remote.puts != 0 { + t.Fatalf("remote Put calls = %d, want 0", remote.puts) + } +} + +type countingCache struct { + gets int + puts int + entry cache.Entry + hit bool + err error +} + +func (c *countingCache) Get(context.Context, cache.Key) (cache.Entry, bool, error) { + c.gets++ + return c.entry, c.hit, c.err +} + +func (c *countingCache) Put(context.Context, cache.Key, fs.FS, cache.Entry) (cache.Entry, error) { + c.puts++ + return cache.Entry{}, nil +} + func TestRunRejectsInvalidAddress(t *testing.T) { t.Chdir(t.TempDir()) cacheDir := t.TempDir() From 94d946182efaca452c1c38c775c1d65ab5d14754 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 22 Sep 2026 17:11:05 +0800 Subject: [PATCH 2/4] fix(llard): publish cache writes through to the remote store The llard read-through cache copied the client's Put, which only writes the local entry. On a cold build that skipped the remote Put entirely, so no artifact was uploaded or recorded and the handler failed with 'artifact not found'. Publish remotely first: on success cache the authoritative entry locally; on failure (another llard already published it) do not cache this build's copy and let the next Get restore the canonical artifact. --- cmd/llard/main.go | 12 +++++++++- cmd/llard/main_test.go | 53 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/cmd/llard/main.go b/cmd/llard/main.go index fa0a5979..1a2235e3 100644 --- a/cmd/llard/main.go +++ b/cmd/llard/main.go @@ -150,7 +150,17 @@ func (c readThroughCache) Get(ctx context.Context, key cache.Key) (cache.Entry, } func (c readThroughCache) Put(ctx context.Context, key cache.Key, output fs.FS, entry cache.Entry) (cache.Entry, error) { - return c.local.Put(ctx, key, output, entry) + // Publishing is authoritative: upload and record the artifact remotely + // first. When another llard already published it, this fails and the local + // copy must not be cached; the next Get restores the canonical artifact. + stored, err := c.remote.Put(ctx, key, output, entry) + if err != nil { + return cache.Entry{}, err + } + // Cache the authoritative entry locally. A local write failure only costs + // a future restore, so it must not fail the build. + _, _ = c.local.Put(ctx, key, output, stored) + return stored, nil } func loadConfig() (config, error) { diff --git a/cmd/llard/main_test.go b/cmd/llard/main_test.go index 608e1a4b..23fed887 100644 --- a/cmd/llard/main_test.go +++ b/cmd/llard/main_test.go @@ -6,8 +6,10 @@ package main import ( "context" + "errors" "io/fs" "os" + "reflect" "strings" "testing" @@ -142,18 +144,57 @@ func TestReadThroughCache_PersistsRemoteHit(t *testing.T) { } } -func TestReadThroughCache_PutWritesOnlyLocal(t *testing.T) { - local := build.NewLocalCache(t.TempDir()) - remote := &countingCache{} - c := readThroughCache{local: local, remote: remote} +// TestReadThroughCache_PutOrdersRemoteThenLocal pins the write order: the +// artifact is published remotely before the local entry is written. +func TestReadThroughCache_PutOrdersRemoteThenLocal(t *testing.T) { + var order []string + c := readThroughCache{ + local: &orderCache{name: "local", order: &order}, + remote: &orderCache{name: "remote", order: &order}, + } key := cache.Key{Module: module.Version{Path: "madler/zlib", Version: "v1.3.1"}, Matrix: "amd64-linux"} if _, err := c.Put(context.Background(), key, nil, cache.Entry{Metadata: "-built"}); err != nil { t.Fatalf("Put() failed: %v", err) } - if remote.puts != 0 { - t.Fatalf("remote Put calls = %d, want 0", remote.puts) + if !reflect.DeepEqual(order, []string{"remote", "local"}) { + t.Fatalf("Put order = %v, want [remote local]", order) + } +} + +// TestReadThroughCache_PutSkipsLocalWhenRemoteFails verifies that a remote +// publish failure does not leave a divergent local entry behind: the next Get +// must restore the canonical artifact from the remote store. +func TestReadThroughCache_PutSkipsLocalWhenRemoteFails(t *testing.T) { + var order []string + remoteErr := errors.New("remote put failed") + c := readThroughCache{ + local: &orderCache{name: "local", order: &order}, + remote: &orderCache{name: "remote", order: &order, err: remoteErr}, + } + key := cache.Key{Module: module.Version{Path: "madler/zlib", Version: "v1.3.1"}, Matrix: "amd64-linux"} + + if _, err := c.Put(context.Background(), key, nil, cache.Entry{Metadata: "-built"}); !errors.Is(err, remoteErr) { + t.Fatalf("Put() error = %v, want %v", err, remoteErr) } + if len(order) != 1 || order[0] != "remote" { + t.Fatalf("Put order = %v, want [remote] only", order) + } +} + +type orderCache struct { + name string + order *[]string + err error +} + +func (c *orderCache) Get(context.Context, cache.Key) (cache.Entry, bool, error) { + return cache.Entry{}, false, nil +} + +func (c *orderCache) Put(context.Context, cache.Key, fs.FS, cache.Entry) (cache.Entry, error) { + *c.order = append(*c.order, c.name) + return cache.Entry{}, c.err } type countingCache struct { From 699a00e118332c5b6dc7b3ae36b7fdf62a3ecc04 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 22 Sep 2026 17:19:02 +0800 Subject: [PATCH 3/4] fix(llard): gate local cache hits on the remote artifact record Deleting a published artifact only removes the Kodo object and its record; the worker keeps its local workspace entry. A pure local-first read-through therefore served the deleted artifact, skipped the rebuild, and the handler failed the response with 'artifact not found'. Check the remote artifact record before trusting the local entry: a missing record reports a cache miss so the module is rebuilt, while a present record still serves the local copy without downloading the artifact. --- cmd/llard/main.go | 32 ++++++++++++++++++------ cmd/llard/main_test.go | 56 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/cmd/llard/main.go b/cmd/llard/main.go index 1a2235e3..e4a2639e 100644 --- a/cmd/llard/main.go +++ b/cmd/llard/main.go @@ -79,7 +79,8 @@ func run() error { // from Kodo. The workspace is evictable, so Kodo remains the source of // truth and restores anything missing back into the workspace. buildCache := readThroughCache{ - local: build.NewLocalCache(workspaceDir), + local: build.NewLocalCache(workspaceDir), + artifacts: artifacts, remote: cache.NewKodo(cache.KodoConfig{ AccessKey: cfg.accessKey, SecretKey: cfg.secretKey, @@ -120,18 +121,35 @@ func run() error { // readThroughCache reuses artifacts already present in the local workspace // before fetching them from the remote store. The local workspace is only a -// best-effort cache: an installed artifact directory may be removed by a -// cleanup pass, so a local miss falls back to the remote store and a remote -// hit is persisted back into the local cache for later reads. +// best-effort cache: the remote artifact record is the source of truth, so a +// deleted record invalidates any local copy, a local miss falls back to the +// remote store, and a remote hit is persisted back into the local cache for +// later reads. // // This is a deliberate copy of the llar client's read-through cache: llard and -// the client evolve separately and must not share an abstraction. +// the client evolve separately and must not share an abstraction. It differs by +// gating local hits on the remote artifact record, which the client does not +// need because it never deletes published artifacts. type readThroughCache struct { - local cache.Cache - remote cache.Cache + local cache.Cache + remote cache.Cache + artifacts artifact.Store } func (c readThroughCache) Get(ctx context.Context, key cache.Key) (cache.Entry, bool, error) { + // The remote artifact record is the source of truth: when it is deleted, + // any local copy is stale and must not be used. This is a metadata-only + // lookup, so a local hit still avoids the artifact download. + if _, err := c.artifacts.Get(ctx, artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + }); err != nil { + if errors.Is(err, artifact.ErrNotFound) { + return cache.Entry{}, false, nil + } + return cache.Entry{}, false, err + } entry, ok, err := c.local.Get(ctx, key) if err != nil || ok { return entry, ok, err diff --git a/cmd/llard/main_test.go b/cmd/llard/main_test.go index 23fed887..ff721b57 100644 --- a/cmd/llard/main_test.go +++ b/cmd/llard/main_test.go @@ -13,6 +13,7 @@ import ( "strings" "testing" + "github.com/goplus/llar/internal/artifact" "github.com/goplus/llar/internal/build" "github.com/goplus/llar/internal/build/cache" "github.com/goplus/llar/mod/module" @@ -116,7 +117,7 @@ func TestReadThroughCache_LocalHitSkipsRemote(t *testing.T) { } remote := &countingCache{} - c := readThroughCache{local: local, remote: remote} + c := readThroughCache{local: local, remote: remote, artifacts: presentArtifacts(key)} entry, ok, err := c.Get(context.Background(), key) if err != nil || !ok || entry.Metadata != "-local" { t.Fatalf("Get() = %+v, %v, %v; want local hit", entry, ok, err) @@ -132,7 +133,7 @@ func TestReadThroughCache_PersistsRemoteHit(t *testing.T) { key := cache.Key{Module: module.Version{Path: "madler/zlib", Version: "v1.3.1"}, Matrix: "amd64-linux"} remote := &countingCache{entry: cache.Entry{Metadata: "-remote"}, hit: true} - c := readThroughCache{local: local, remote: remote} + c := readThroughCache{local: local, remote: remote, artifacts: presentArtifacts(key)} for i := 0; i < 2; i++ { entry, ok, err := c.Get(context.Background(), key) if err != nil || !ok || entry.Metadata != "-remote" { @@ -144,6 +145,27 @@ func TestReadThroughCache_PersistsRemoteHit(t *testing.T) { } } +// TestReadThroughCache_RecordMissingInvalidatesLocal verifies that deleting the +// authoritative artifact record invalidates a local entry: the next Get must +// miss (so the build runs again) and must not even consult the remote store. +func TestReadThroughCache_RecordMissingInvalidatesLocal(t *testing.T) { + workspaceDir := t.TempDir() + local := build.NewLocalCache(workspaceDir) + key := cache.Key{Module: module.Version{Path: "madler/zlib", Version: "v1.3.1"}, Matrix: "amd64-linux"} + if _, err := local.Put(context.Background(), key, nil, cache.Entry{Metadata: "-local"}); err != nil { + t.Fatal(err) + } + + remote := &countingCache{entry: cache.Entry{Metadata: "-remote"}, hit: true} + c := readThroughCache{local: local, remote: remote, artifacts: &fakeArtifacts{}} + if _, ok, err := c.Get(context.Background(), key); err != nil || ok { + t.Fatalf("Get() = %v, %v; want miss after record deletion", ok, err) + } + if remote.gets != 0 { + t.Fatalf("remote Get calls = %d, want 0", remote.gets) + } +} + // TestReadThroughCache_PutOrdersRemoteThenLocal pins the write order: the // artifact is published remotely before the local entry is written. func TestReadThroughCache_PutOrdersRemoteThenLocal(t *testing.T) { @@ -215,6 +237,36 @@ func (c *countingCache) Put(context.Context, cache.Key, fs.FS, cache.Entry) (cac return cache.Entry{}, nil } +func artifactRecordKey(key cache.Key) string { + return key.Module.Path + "@" + key.Module.Version + "?" + key.Matrix +} + +// presentArtifacts returns an artifact store holding the record for key. +func presentArtifacts(key cache.Key) *fakeArtifacts { + return &fakeArtifacts{record: map[string]artifact.Artifact{artifactRecordKey(key): {}}} +} + +type fakeArtifacts struct { + record map[string]artifact.Artifact + err error +} + +func (f *fakeArtifacts) Get(_ context.Context, key artifact.Key) (artifact.Artifact, error) { + if f.err != nil { + return artifact.Artifact{}, f.err + } + if a, ok := f.record[key.Module+"@"+key.Version+"?"+key.MatrixStr]; ok { + return a, nil + } + return artifact.Artifact{}, artifact.ErrNotFound +} + +func (f *fakeArtifacts) Put(context.Context, artifact.Key, artifact.Artifact) (artifact.Artifact, error) { + return artifact.Artifact{}, nil +} + +func (f *fakeArtifacts) Delete(context.Context, artifact.Key) error { return nil } + func TestRunRejectsInvalidAddress(t *testing.T) { t.Chdir(t.TempDir()) cacheDir := t.TempDir() From 07798fd369848cc79fa033a2025cc433edae6604 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 22 Sep 2026 17:25:32 +0800 Subject: [PATCH 4/4] fix(llard): drop the local install tree when the artifact record is gone A deleted remote record makes the local copy stale, but the rebuild only ran os.MkdirAll over the existing directory, so removed files could linger and be packed into the new artifact. Remove the workspace install tree on the record-missing miss so the rebuild starts clean. --- cmd/llard/main.go | 32 +++++++++++++++++++++++++++----- cmd/llard/main_test.go | 15 ++++++++++++--- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/cmd/llard/main.go b/cmd/llard/main.go index e4a2639e..7ca14761 100644 --- a/cmd/llard/main.go +++ b/cmd/llard/main.go @@ -22,6 +22,7 @@ import ( buildhttp "github.com/goplus/llar/internal/build/http" "github.com/goplus/llar/internal/formula/repo" "github.com/goplus/llar/internal/vcs" + "github.com/goplus/llar/mod/module" "github.com/joho/godotenv" ) @@ -79,8 +80,9 @@ func run() error { // from Kodo. The workspace is evictable, so Kodo remains the source of // truth and restores anything missing back into the workspace. buildCache := readThroughCache{ - local: build.NewLocalCache(workspaceDir), - artifacts: artifacts, + local: build.NewLocalCache(workspaceDir), + artifacts: artifacts, + workspaceDir: workspaceDir, remote: cache.NewKodo(cache.KodoConfig{ AccessKey: cfg.accessKey, SecretKey: cfg.secretKey, @@ -131,9 +133,10 @@ func run() error { // gating local hits on the remote artifact record, which the client does not // need because it never deletes published artifacts. type readThroughCache struct { - local cache.Cache - remote cache.Cache - artifacts artifact.Store + local cache.Cache + remote cache.Cache + artifacts artifact.Store + workspaceDir string } func (c readThroughCache) Get(ctx context.Context, key cache.Key) (cache.Entry, bool, error) { @@ -146,6 +149,11 @@ func (c readThroughCache) Get(ctx context.Context, key cache.Key) (cache.Entry, MatrixStr: key.Matrix, }); err != nil { if errors.Is(err, artifact.ErrNotFound) { + // The artifact was deleted remotely: drop the local install tree + // so the rebuild cannot mix stale files with the new build. + if err := c.removeInstallDir(key); err != nil { + return cache.Entry{}, false, err + } return cache.Entry{}, false, nil } return cache.Entry{}, false, err @@ -181,6 +189,20 @@ func (c readThroughCache) Put(ctx context.Context, key cache.Key, output fs.FS, return stored, nil } +// removeInstallDir drops the workspace install tree for key. The remote artifact +// record is gone, so the local copy is stale and a rebuild must start clean. +func (c readThroughCache) removeInstallDir(key cache.Key) error { + if c.workspaceDir == "" { + return nil + } + escaped, err := module.EscapePath(key.Module.Path) + if err != nil { + return err + } + installDir := filepath.Join(c.workspaceDir, fmt.Sprintf("%s@%s-%s", escaped, key.Module.Version, key.Matrix)) + return os.RemoveAll(installDir) +} + func loadConfig() (config, error) { cfg := config{ addr: os.Getenv("LLARD_ADDR"), diff --git a/cmd/llard/main_test.go b/cmd/llard/main_test.go index ff721b57..3e558c4b 100644 --- a/cmd/llard/main_test.go +++ b/cmd/llard/main_test.go @@ -9,6 +9,7 @@ import ( "errors" "io/fs" "os" + "path/filepath" "reflect" "strings" "testing" @@ -146,8 +147,9 @@ func TestReadThroughCache_PersistsRemoteHit(t *testing.T) { } // TestReadThroughCache_RecordMissingInvalidatesLocal verifies that deleting the -// authoritative artifact record invalidates a local entry: the next Get must -// miss (so the build runs again) and must not even consult the remote store. +// authoritative artifact record invalidates a local entry: the next Get drops +// the install tree, misses (so the build runs again), and does not consult the +// remote store. func TestReadThroughCache_RecordMissingInvalidatesLocal(t *testing.T) { workspaceDir := t.TempDir() local := build.NewLocalCache(workspaceDir) @@ -155,15 +157,22 @@ func TestReadThroughCache_RecordMissingInvalidatesLocal(t *testing.T) { if _, err := local.Put(context.Background(), key, nil, cache.Entry{Metadata: "-local"}); err != nil { t.Fatal(err) } + installDir := filepath.Join(workspaceDir, "madler", "zlib@v1.3.1-amd64-linux") + if err := os.MkdirAll(installDir, 0o755); err != nil { + t.Fatal(err) + } remote := &countingCache{entry: cache.Entry{Metadata: "-remote"}, hit: true} - c := readThroughCache{local: local, remote: remote, artifacts: &fakeArtifacts{}} + c := readThroughCache{local: local, remote: remote, artifacts: &fakeArtifacts{}, workspaceDir: workspaceDir} if _, ok, err := c.Get(context.Background(), key); err != nil || ok { t.Fatalf("Get() = %v, %v; want miss after record deletion", ok, err) } if remote.gets != 0 { t.Fatalf("remote Get calls = %d, want 0", remote.gets) } + if _, err := os.Stat(installDir); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("install dir still present after record deletion: %v", err) + } } // TestReadThroughCache_PutOrdersRemoteThenLocal pins the write order: the