@@ -21,53 +21,16 @@ import (
2121// It is used as FromStore in builtins.fetchClosure.
2222const BinaryCache = "https://cache.nixos.org"
2323
24- // isNarInfoInCache checks if the .narinfo for this package is in the `BinaryCache`.
25- // This cannot be a field on the Package struct, because that struct
26- // is constructed multiple times in a request (TODO: we could fix that).
27- var isNarInfoInCache = struct {
28- // The key is the `Package.Raw` string.
29- status map [string ]bool
30- lock sync.RWMutex
31- // re-use httpClient to re-use the connection
32- httpClient http.Client
33- }{
34- status : map [string ]bool {},
35- httpClient : http.Client {},
36- }
37-
3824// IsInBinaryCache returns true if the package is in the binary cache.
3925// ALERT: Callers in a perf-sensitive code path should call FillNarInfoCache
4026// before calling this function.
4127func (p * Package ) IsInBinaryCache () (bool , error ) {
42-
4328 if eligible , err := p .isEligibleForBinaryCache (); err != nil {
4429 return false , err
4530 } else if ! eligible {
4631 return false , nil
4732 }
48-
49- // Check if the narinfo is present in the binary cache
50- isNarInfoInCache .lock .RLock ()
51- status , statusExists := isNarInfoInCache .status [p .Raw ]
52- isNarInfoInCache .lock .RUnlock ()
53- if ! statusExists {
54- // Fallback to synchronously filling the nar info cache
55- if err := p .fillNarInfoCache (); err != nil {
56- return false , err
57- }
58-
59- // Check again
60- isNarInfoInCache .lock .RLock ()
61- status , statusExists = isNarInfoInCache .status [p .Raw ]
62- isNarInfoInCache .lock .RUnlock ()
63- if ! statusExists {
64- return false , errors .Errorf (
65- "narInfo cache miss: %v. Should be filled by now" ,
66- p .Raw ,
67- )
68- }
69- }
70- return status , nil
33+ return p .fetchNarInfoStatusOnce ()
7134}
7235
7336// FillNarInfoCache checks the remote binary cache for the narinfo of each
@@ -80,9 +43,8 @@ func FillNarInfoCache(ctx context.Context, packages ...*Package) error {
8043
8144 eligiblePackages := []* Package {}
8245 for _ , p := range packages {
83- // NOTE: isEligibleForBinaryCache also ensures the package is
84- // resolved in the lockfile, which must be done before the concurrent
85- // section in this function below.
46+ // IMPORTANT: isEligibleForBinaryCache will call resolve() which is NOT
47+ // concurrency safe. Hence, we call it outside of the go-routine.
8648 isEligible , err := p .isEligibleForBinaryCache ()
8749 // If the package is not eligible or there is an error in determining that, then skip it.
8850 if isEligible && err == nil {
@@ -103,39 +65,42 @@ func FillNarInfoCache(ctx context.Context, packages ...*Package) error {
10365
10466 group , _ := errgroup .WithContext (ctx )
10567 for _ , p := range eligiblePackages {
106- // If the package's NarInfo status is already known, skip it
107- isNarInfoInCache .lock .RLock ()
108- _ , ok := isNarInfoInCache .status [p .Raw ]
109- isNarInfoInCache .lock .RUnlock ()
110- if ok {
111- continue
112- }
11368 pkg := p // copy the loop variable since its used in a closure below
11469 group .Go (func () error {
115- err := pkg .fillNarInfoCache ()
116- if err != nil {
117- // default to false if there was an error, so we don't re-try
118- isNarInfoInCache .lock .Lock ()
119- isNarInfoInCache .status [pkg .Raw ] = false
120- isNarInfoInCache .lock .Unlock ()
121- }
70+ _ , err := pkg .fetchNarInfoStatusOnce ()
12271 return err
12372 })
12473 }
12574 return group .Wait ()
12675}
12776
128- // fillNarInfoCache fills the cache value for the narinfo of this package,
129- // assuming it is eligible for the binary cache. Callers are responsible
130- // for checking isEligibleForBinaryCache before calling this function.
131- //
132- // NOTE: this must be concurrency safe.
133- func (p * Package ) fillNarInfoCache () error {
77+ // narInfoStatusFnCache contains cached OnceValues functions that return cache
78+ // status for a package. In the future we can remove this cache by caching
79+ // package objects and ensuring packages are shared globally.
80+ var narInfoStatusFnCache = sync.Map {}
81+
82+ // fetchNarInfoStatusOnce is like fetchNarInfoStatus, but will only ever run
83+ // once and cache the result.
84+ func (p * Package ) fetchNarInfoStatusOnce () (bool , error ) {
85+ type inCacheFunc func () (bool , error )
86+ f , ok := narInfoStatusFnCache .Load (p .Raw )
87+ if ! ok {
88+ f = inCacheFunc (sync .OnceValues (p .fetchNarInfoStatus ))
89+ f , _ = narInfoStatusFnCache .LoadOrStore (p .Raw , f )
90+ }
91+ return f .(inCacheFunc )()
92+ }
93+
94+ // fetchNarInfoStatus fetches the cache status for the package. It returns
95+ // true if cache exists, false otherwise.
96+ // NOTE: This function always performs an HTTP request and should not be called
97+ // more than once per package.
98+ func (p * Package ) fetchNarInfoStatus () (bool , error ) {
13499 sysInfo , err := p .sysInfoIfExists ()
135100 if err != nil {
136- return err
101+ return false , err
137102 } else if sysInfo == nil {
138- return errors .New (
103+ return false , errors .New (
139104 "sysInfo is nil, but should not be because" +
140105 " the package is eligible for binary cache" ,
141106 )
@@ -147,20 +112,17 @@ func (p *Package) fillNarInfoCache() error {
147112 defer cancel ()
148113 req , err := http .NewRequestWithContext (ctx , http .MethodHead , reqURL , nil )
149114 if err != nil {
150- return err
115+ return false , err
151116 }
152- res , err := isNarInfoInCache . httpClient .Do (req )
117+ res , err := http . DefaultClient .Do (req )
153118 if err != nil {
154- return err
119+ return false , err
155120 }
156121 // read the body fully, and close it to ensure the connection is reused.
157122 _ , _ = io .Copy (io .Discard , res .Body )
158123 defer res .Body .Close ()
159124
160- isNarInfoInCache .lock .Lock ()
161- isNarInfoInCache .status [p .Raw ] = res .StatusCode == 200
162- isNarInfoInCache .lock .Unlock ()
163- return nil
125+ return res .StatusCode == 200 , nil
164126}
165127
166128// isEligibleForBinaryCache returns true if we have additional metadata about
0 commit comments