From 6e49376583cfaa4de48aa1e332b19f10cf91aa5a Mon Sep 17 00:00:00 2001 From: Tom Laird-McConnell Date: Mon, 25 May 2026 11:05:57 -0700 Subject: [PATCH 1/4] =?UTF-8?q?Replace=20per-file=20HEAD=20calls=20with=20?= =?UTF-8?q?a=20single=20ListBlobs=20metadata=20cache=20Previously,=20every?= =?UTF-8?q?=20FileExists(string),=20FileLength(string),=20and=20OpenInput(?= =?UTF-8?q?string,=20IOContext)=20call=20issued=20an=20individual=20GetPro?= =?UTF-8?q?perties=20(HEAD)=20request=20to=20Azure=20Blob=20Storage.=20For?= =?UTF-8?q?=20an=20index=20with=20N=20files,=20opening=20or=20refreshing?= =?UTF-8?q?=20a=20reader=20triggered=20N=20round-trips.=20This=20change=20?= =?UTF-8?q?introduces=20a=20BlobMetadata=20cache=20populated=20by=20a=20si?= =?UTF-8?q?ngle=20GetBlobsByHierarchy=20call=20(which=20returns=20properti?= =?UTF-8?q?es=20for=20all=20blobs=20in=20one=20page).=20The=20cache=20is:?= =?UTF-8?q?=20=E2=80=A2=20refreshed=20by=20ListAll(),=20which=20Lucene=20a?= =?UTF-8?q?lways=20calls=20first=20when=20opening=20or=20re-checking=20an?= =?UTF-8?q?=20index=20=E2=80=A2=20lazily=20populated=20on=20first=20access?= =?UTF-8?q?=20if=20ListAll()=20has=20not=20yet=20been=20called=20=E2=80=A2?= =?UTF-8?q?=20updated=20in-place=20after=20AzureIndexOutput.Dispose()=20co?= =?UTF-8?q?mpletes=20an=20upload,=20using=20the=20already-known=20local=20?= =?UTF-8?q?length=20=E2=80=94=20preventing=20an=20unnecessary=20re-downloa?= =?UTF-8?q?d=20when=20Lucene=20reads=20a=20file=20it=20just=20wrote=20?= =?UTF-8?q?=E2=80=A2=20invalidated=20per-entry=20on=20DeleteFile()=20Azure?= =?UTF-8?q?IndexInput=20now=20reads=20ContentLength=20from=20the=20cache?= =?UTF-8?q?=20instead=20of=20calling=20GetProperties=20on=20the=20blob=20b?= =?UTF-8?q?efore=20deciding=20whether=20to=20re-download.=20Net=20result:?= =?UTF-8?q?=20N=20HEAD=20calls=20per=20reader=20open/refresh=20replaced=20?= =?UTF-8?q?by=201=20ListBlobs=20call.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Lucene.Net.Store.Azure/AzureDirectory.cs | 137 ++++++++++++++---- .../Lucene.Net.Store.Azure/AzureIndexInput.cs | 3 +- .../AzureIndexOutput.cs | 2 + .../Lucene.Net.Store.Azure.csproj | 6 +- 4 files changed, 118 insertions(+), 30 deletions(-) diff --git a/source/Lucene.Net.Store.Azure/AzureDirectory.cs b/source/Lucene.Net.Store.Azure/AzureDirectory.cs index 3d0b32c..fdb800c 100644 --- a/source/Lucene.Net.Store.Azure/AzureDirectory.cs +++ b/source/Lucene.Net.Store.Azure/AzureDirectory.cs @@ -1,9 +1,11 @@ // License: Microsoft Public License (Ms-PL) using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; namespace Lucene.Net.Store.Azure { @@ -15,6 +17,10 @@ public class AzureDirectory : BaseDirectory private readonly Dictionary _nameCache = new Dictionary(); + // Blob metadata cache — refreshed by ListAll(); lazily populated on first access + private Dictionary _blobMetadataCache; + private readonly object _cacheLock = new object(); + /// /// Create AzureDirectory with just storagaccount string /// @@ -98,6 +104,99 @@ public AzureDirectory( public BlobContainerClient BlobContainer { get; private set; } + /// + /// Metadata cached for a single blob. + /// + internal struct BlobMetadata + { + public long ContentLength; + public string ETag; + public DateTimeOffset? LastModified; + } + + /// + /// Refreshes the in-memory blob metadata cache by issuing a single ListBlobs call. + /// + private void RefreshBlobMetadataCache() + { + var prefix = string.IsNullOrEmpty(this.subDirectory) ? null : this.subDirectory + "/"; + var newCache = new Dictionary(StringComparer.Ordinal); + + foreach (var item in BlobContainer.GetBlobsByHierarchy(delimiter: "/", prefix: prefix, traits: BlobTraits.None, states: BlobStates.None)) + { + if (!item.IsBlob) + continue; + + var shortName = item.Blob.Name.Split('/').Last(); + var props = item.Blob.Properties; + newCache[shortName] = new BlobMetadata + { + ContentLength = props.ContentLength ?? 0, + ETag = props.ETag?.ToString(), + LastModified = props.LastModified + }; + } + + _blobMetadataCache = newCache; + } + + /// + /// Ensures the blob metadata cache has been populated at least once. + /// The authoritative refresh happens in , which Lucene + /// always calls before opening files. This method only does a full list if + /// the cache has never been populated. + /// + internal void EnsureCacheUpToDate() + { + lock (_cacheLock) + { + if (_blobMetadataCache == null) + RefreshBlobMetadataCache(); + } + } + + /// + /// Returns the cached for , or null if not found. + /// + internal BlobMetadata? GetCachedMetadata(string name) + { + lock (_cacheLock) + { + if (_blobMetadataCache != null && _blobMetadataCache.TryGetValue(name, out var meta)) + return meta; + return null; + } + } + + /// + /// Removes a single entry from the metadata cache (e.g. after delete or write). + /// + private void InvalidateCacheEntry(string name) + { + lock (_cacheLock) + { + _blobMetadataCache?.Remove(name); + } + } + + /// + /// Updates (or inserts) the cached metadata for a blob after a successful upload, + /// avoiding a re-download when the file is immediately read back. + /// + internal void UpdateCacheEntry(string name, long contentLength) + { + lock (_cacheLock) + { + if (_blobMetadataCache == null) + return; + + _blobMetadataCache[name] = new BlobMetadata + { + ContentLength = contentLength + }; + } + } + public string Name { get; private set; } /// @@ -120,27 +219,21 @@ public void ClearCache() /// Returns an array of strings, one for each file in the directory. public override string[] ListAll() { - var prefix = string.IsNullOrEmpty(this.subDirectory) ? null : this.subDirectory + "/"; - - return BlobContainer.GetBlobsByHierarchy(delimiter: "/", prefix: prefix) - .Where(x => x.IsBlob) - .Select(x => x.Blob.Name.Split('/').Last()) - .ToArray(); + // Always do a fresh listing — ListAll must reflect the current state of the container. + // Refreshing also keeps the metadata cache warm for subsequent FileExists/FileLength calls. + lock (_cacheLock) + { + RefreshBlobMetadataCache(); + return _blobMetadataCache?.Keys.ToArray() ?? Array.Empty(); + } } /// Returns true if a file with the given name exists. [Obsolete("this method will be removed in 5.0")] public override bool FileExists(string name) { - // this always comes from the server - try - { - return BlobContainer.GetBlobClient(GetBlobName(name)).Exists(); - } - catch (Exception) - { - return false; - } + EnsureCacheUpToDate(); + return GetCachedMetadata(name).HasValue; } /// Removes an existing file in the directory. @@ -149,21 +242,14 @@ public override void DeleteFile(string name) var blobName = GetBlobName(name); var blob = BlobContainer.GetBlobClient(blobName); blob.DeleteIfExists(); - + InvalidateCacheEntry(name); } /// Returns the length of a file in the directory. public override long FileLength(string name) { - try - { - var blobName = GetBlobName(name); - return BlobContainer.GetBlobClient(blobName).GetProperties().Value?.ContentLength ?? 0; - } - catch - { - return 0; - } + EnsureCacheUpToDate(); + return GetCachedMetadata(name)?.ContentLength ?? 0; } public override void Sync(ICollection names) @@ -181,6 +267,7 @@ public override void Sync(ICollection names) public override IndexInput OpenInput(string name, IOContext context) { // TODO: Figure out how IOContext comes into play here. So far it doesn't -- Aviad + EnsureCacheUpToDate(); try { var blobName = GetBlobName(name); diff --git a/source/Lucene.Net.Store.Azure/AzureIndexInput.cs b/source/Lucene.Net.Store.Azure/AzureIndexInput.cs index 68c0e62..f0a2782 100644 --- a/source/Lucene.Net.Store.Azure/AzureIndexInput.cs +++ b/source/Lucene.Net.Store.Azure/AzureIndexInput.cs @@ -43,9 +43,8 @@ public AzureIndexInput(AzureDirectory azureDirectory, string name, BlobClient bl { try { - var blobProperties = blob.GetProperties(); long cachedLength = CacheDirectory.FileLength(name); - long blobLength = blobProperties?.Value?.ContentLength ?? 0; + long blobLength = _azureDirectory.GetCachedMetadata(name)?.ContentLength ?? 0; if (cachedLength != blobLength) fileNeeded = true; } diff --git a/source/Lucene.Net.Store.Azure/AzureIndexOutput.cs b/source/Lucene.Net.Store.Azure/AzureIndexOutput.cs index 57bbe25..f4e074f 100644 --- a/source/Lucene.Net.Store.Azure/AzureIndexOutput.cs +++ b/source/Lucene.Net.Store.Azure/AzureIndexOutput.cs @@ -66,6 +66,8 @@ protected override void Dispose(bool disposing) Debug.WriteLine($"{_azureDirectory.Name} PUT {_name} bytes to {blobStream.Length} in cloud"); } + _azureDirectory.UpdateCacheEntry(_name, originalLength); + #if FULLDEBUG Debug.WriteLine($"{_azureDirectory.Name} CLOSED WRITESTREAM {_name}"); #endif diff --git a/source/Lucene.Net.Store.Azure/Lucene.Net.Store.Azure.csproj b/source/Lucene.Net.Store.Azure/Lucene.Net.Store.Azure.csproj index b859112..f56003d 100644 --- a/source/Lucene.Net.Store.Azure/Lucene.Net.Store.Azure.csproj +++ b/source/Lucene.Net.Store.Azure/Lucene.Net.Store.Azure.csproj @@ -2,7 +2,7 @@ netstandard2.0 true - 4.8.5-beta019 + 4.8.5-beta021 Tom Laird-McConnell Azure blob storage for Lucene.net This project allows you to store Lucene Indexes in by Azure BlobStorage. @@ -11,8 +11,8 @@ https://github.com/tomlm/Lucene.Net.Store.Azure git MS-PL - 4.8.5.19 - 4.8.5.19 + 4.8.5.21 + 4.8.5.21 This is a release with dependency on Lucene.Net 4.8.0.beta 0017 https://raw.githubusercontent.com/tomlm/Lucene.Net.Store.Azure/master/icon.png icon.png From 658b0c41316fb09c8256c6c94bddf94126288277 Mon Sep 17 00:00:00 2001 From: Tom Laird-McConnell Date: Mon, 25 May 2026 11:10:55 -0700 Subject: [PATCH 2/4] bump dotnet test host --- .../Lucene.Net.Store.Azure.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Lucene.Net.Store.Azure.Tests/Lucene.Net.Store.Azure.Tests.csproj b/source/Lucene.Net.Store.Azure.Tests/Lucene.Net.Store.Azure.Tests.csproj index 1c380e1..f24c1bf 100644 --- a/source/Lucene.Net.Store.Azure.Tests/Lucene.Net.Store.Azure.Tests.csproj +++ b/source/Lucene.Net.Store.Azure.Tests/Lucene.Net.Store.Azure.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net10.0 false From 68ab4bce0aa3af05740f973b8ddce06940e13dd1 Mon Sep 17 00:00:00 2001 From: Tom Laird-McConnell Date: Mon, 25 May 2026 11:17:26 -0700 Subject: [PATCH 3/4] copilot code review --- .../Lucene.Net.Store.Azure/AzureDirectory.cs | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/source/Lucene.Net.Store.Azure/AzureDirectory.cs b/source/Lucene.Net.Store.Azure/AzureDirectory.cs index 84d3a89..bf4d8f1 100644 --- a/source/Lucene.Net.Store.Azure/AzureDirectory.cs +++ b/source/Lucene.Net.Store.Azure/AzureDirectory.cs @@ -1,4 +1,5 @@ // License: Microsoft Public License (Ms-PL) +using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using System; @@ -26,9 +27,9 @@ public class AzureDirectory : BaseDirectory /// /// public AzureDirectory(string storageAccount) : - this(storageAccount, - catalog:null, - cacheDirectory: null, + this(storageAccount, + catalog: null, + cacheDirectory: null, multiCasePath: false) { } @@ -44,9 +45,9 @@ public AzureDirectory( string storageAccount, string catalog, bool multiCasePath = false) - : this(storageAccount, - catalog: catalog, - cacheDirectory: null, + : this(storageAccount, + catalog: catalog, + cacheDirectory: null, multiCasePath: multiCasePath) { } @@ -62,10 +63,10 @@ public AzureDirectory( string storageAccount, string catalog, Directory cacheDirectory, - bool multiCasePath = false) - : this(new BlobServiceClient(storageAccount), - catalog: catalog, - cacheDirectory: cacheDirectory, + bool multiCasePath = false) + : this(new BlobServiceClient(storageAccount), + catalog: catalog, + cacheDirectory: cacheDirectory, multiCasePath: multiCasePath) { } @@ -119,25 +120,33 @@ internal struct BlobMetadata /// private void RefreshBlobMetadataCache() { - var prefix = string.IsNullOrEmpty(this.subDirectory) ? null : this.subDirectory + "/"; - var newCache = new Dictionary(StringComparer.Ordinal); - - foreach (var item in BlobContainer.GetBlobsByHierarchy(delimiter: "/", prefix: prefix, traits: BlobTraits.None, states: BlobStates.None)) + try { - if (!item.IsBlob) - continue; + var prefix = string.IsNullOrEmpty(this.subDirectory) ? null : this.subDirectory + "/"; + var newCache = new Dictionary(StringComparer.Ordinal); - var shortName = item.Blob.Name.Split('/').Last(); - var props = item.Blob.Properties; - newCache[shortName] = new BlobMetadata + foreach (var item in BlobContainer.GetBlobsByHierarchy(delimiter: "/", prefix: prefix, traits: BlobTraits.None, states: BlobStates.None)) { - ContentLength = props.ContentLength ?? 0, - ETag = props.ETag?.ToString(), - LastModified = props.LastModified - }; - } + if (!item.IsBlob) + continue; + + var blobName = item.Blob.Name; + var lastSlashIndex = blobName.LastIndexOf('/'); + var shortName = lastSlashIndex >= 0 ? blobName.Substring(lastSlashIndex + 1) : blobName; + var props = item.Blob.Properties; + newCache[shortName] = new BlobMetadata + { + ContentLength = props.ContentLength ?? 0, + ETag = props.ETag?.ToString(), + LastModified = props.LastModified + }; + } - _blobMetadataCache = newCache; + _blobMetadataCache = newCache; + } + catch (RequestFailedException) + { + } } /// From 8adf4ed26d70f53d1d1e0bb9204e3e6c68d41385 Mon Sep 17 00:00:00 2001 From: Tom Laird-McConnell Date: Mon, 25 May 2026 11:20:45 -0700 Subject: [PATCH 4/4] add azurite --- .github/workflows/BuildAndRunTests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/BuildAndRunTests.yml b/.github/workflows/BuildAndRunTests.yml index a3b1580..5f75263 100644 --- a/.github/workflows/BuildAndRunTests.yml +++ b/.github/workflows/BuildAndRunTests.yml @@ -25,6 +25,11 @@ jobs: with: dotnet-version: 10.0.x + - name: Install and start Azurite + run: | + npm install -g azurite + azurite --silent & + - name: Restore dependencies run: dotnet restore source/AzureDirectory.sln