diff --git a/.github/workflows/BuildAndRunTests.yml b/.github/workflows/BuildAndRunTests.yml index a3b1580..5f75263 100644 --- a/.github/workflows/BuildAndRunTests.yml +++ b/.github/workflows/BuildAndRunTests.yml @@ -25,6 +25,11 @@ jobs: with: dotnet-version: 10.0.x + - name: Install and start Azurite + run: | + npm install -g azurite + azurite --silent & + - name: Restore dependencies run: dotnet restore source/AzureDirectory.sln diff --git a/source/Lucene.Net.Store.Azure.Tests/Lucene.Net.Store.Azure.Tests.csproj b/source/Lucene.Net.Store.Azure.Tests/Lucene.Net.Store.Azure.Tests.csproj index 1c380e1..f24c1bf 100644 --- a/source/Lucene.Net.Store.Azure.Tests/Lucene.Net.Store.Azure.Tests.csproj +++ b/source/Lucene.Net.Store.Azure.Tests/Lucene.Net.Store.Azure.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net10.0 false diff --git a/source/Lucene.Net.Store.Azure/AzureDirectory.cs b/source/Lucene.Net.Store.Azure/AzureDirectory.cs index 9b5f468..bf4d8f1 100644 --- a/source/Lucene.Net.Store.Azure/AzureDirectory.cs +++ b/source/Lucene.Net.Store.Azure/AzureDirectory.cs @@ -1,9 +1,12 @@ // License: Microsoft Public License (Ms-PL) +using Azure; using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; namespace Lucene.Net.Store.Azure { @@ -15,14 +18,18 @@ public class AzureDirectory : BaseDirectory private readonly Dictionary _nameCache = new Dictionary(); + // Blob metadata cache — refreshed by ListAll(); lazily populated on first access + private Dictionary _blobMetadataCache; + private readonly object _cacheLock = new object(); + /// /// Create AzureDirectory with just storagaccount string /// /// public AzureDirectory(string storageAccount) : - this(storageAccount, - catalog:null, - cacheDirectory: null, + this(storageAccount, + catalog: null, + cacheDirectory: null, multiCasePath: false) { } @@ -38,9 +45,9 @@ public AzureDirectory( string storageAccount, string catalog, bool multiCasePath = false) - : this(storageAccount, - catalog: catalog, - cacheDirectory: null, + : this(storageAccount, + catalog: catalog, + cacheDirectory: null, multiCasePath: multiCasePath) { } @@ -56,10 +63,10 @@ public AzureDirectory( string storageAccount, string catalog, Directory cacheDirectory, - bool multiCasePath = false) - : this(new BlobServiceClient(storageAccount), - catalog: catalog, - cacheDirectory: cacheDirectory, + bool multiCasePath = false) + : this(new BlobServiceClient(storageAccount), + catalog: catalog, + cacheDirectory: cacheDirectory, multiCasePath: multiCasePath) { } @@ -98,6 +105,107 @@ public AzureDirectory( public BlobContainerClient BlobContainer { get; private set; } + /// + /// Metadata cached for a single blob. + /// + internal struct BlobMetadata + { + public long ContentLength; + public string ETag; + public DateTimeOffset? LastModified; + } + + /// + /// Refreshes the in-memory blob metadata cache by issuing a single ListBlobs call. + /// + private void RefreshBlobMetadataCache() + { + try + { + var prefix = string.IsNullOrEmpty(this.subDirectory) ? null : this.subDirectory + "/"; + var newCache = new Dictionary(StringComparer.Ordinal); + + foreach (var item in BlobContainer.GetBlobsByHierarchy(delimiter: "/", prefix: prefix, traits: BlobTraits.None, states: BlobStates.None)) + { + if (!item.IsBlob) + continue; + + var blobName = item.Blob.Name; + var lastSlashIndex = blobName.LastIndexOf('/'); + var shortName = lastSlashIndex >= 0 ? blobName.Substring(lastSlashIndex + 1) : blobName; + var props = item.Blob.Properties; + newCache[shortName] = new BlobMetadata + { + ContentLength = props.ContentLength ?? 0, + ETag = props.ETag?.ToString(), + LastModified = props.LastModified + }; + } + + _blobMetadataCache = newCache; + } + catch (RequestFailedException) + { + } + } + + /// + /// Ensures the blob metadata cache has been populated at least once. + /// The authoritative refresh happens in , which Lucene + /// always calls before opening files. This method only does a full list if + /// the cache has never been populated. + /// + internal void EnsureCacheUpToDate() + { + lock (_cacheLock) + { + if (_blobMetadataCache == null) + RefreshBlobMetadataCache(); + } + } + + /// + /// Returns the cached for , or null if not found. + /// + internal BlobMetadata? GetCachedMetadata(string name) + { + lock (_cacheLock) + { + if (_blobMetadataCache != null && _blobMetadataCache.TryGetValue(name, out var meta)) + return meta; + return null; + } + } + + /// + /// Removes a single entry from the metadata cache (e.g. after delete or write). + /// + private void InvalidateCacheEntry(string name) + { + lock (_cacheLock) + { + _blobMetadataCache?.Remove(name); + } + } + + /// + /// Updates (or inserts) the cached metadata for a blob after a successful upload, + /// avoiding a re-download when the file is immediately read back. + /// + internal void UpdateCacheEntry(string name, long contentLength) + { + lock (_cacheLock) + { + if (_blobMetadataCache == null) + return; + + _blobMetadataCache[name] = new BlobMetadata + { + ContentLength = contentLength + }; + } + } + public string Name { get; private set; } /// @@ -120,27 +228,21 @@ public void ClearCache() /// Returns an array of strings, one for each file in the directory. public override string[] ListAll() { - var prefix = string.IsNullOrEmpty(this.subDirectory) ? null : this.subDirectory + "/"; - - return BlobContainer.GetBlobsByHierarchy(delimiter: "/", prefix: prefix) - .Where(x => x.IsBlob) - .Select(x => x.Blob.Name.Split('/').Last()) - .ToArray(); + // Always do a fresh listing — ListAll must reflect the current state of the container. + // Refreshing also keeps the metadata cache warm for subsequent FileExists/FileLength calls. + lock (_cacheLock) + { + RefreshBlobMetadataCache(); + return _blobMetadataCache?.Keys.ToArray() ?? Array.Empty(); + } } /// Returns true if a file with the given name exists. [Obsolete("this method will be removed in 5.0")] public override bool FileExists(string name) { - // this always comes from the server - try - { - return BlobContainer.GetBlobClient(GetBlobName(name)).Exists(); - } - catch (Exception) - { - return false; - } + EnsureCacheUpToDate(); + return GetCachedMetadata(name).HasValue; } /// Removes an existing file in the directory. @@ -149,21 +251,14 @@ public override void DeleteFile(string name) var blobName = GetBlobName(name); var blob = BlobContainer.GetBlobClient(blobName); blob.DeleteIfExists(); - + InvalidateCacheEntry(name); } /// Returns the length of a file in the directory. public override long FileLength(string name) { - try - { - var blobName = GetBlobName(name); - return BlobContainer.GetBlobClient(blobName).GetProperties().Value?.ContentLength ?? 0; - } - catch - { - return 0; - } + EnsureCacheUpToDate(); + return GetCachedMetadata(name)?.ContentLength ?? 0; } public override void Sync(ICollection names) @@ -181,6 +276,7 @@ public override void Sync(ICollection names) public override IndexInput OpenInput(string name, IOContext context) { // TODO: Figure out how IOContext comes into play here. So far it doesn't -- Aviad + EnsureCacheUpToDate(); try { var blobName = GetBlobName(name); diff --git a/source/Lucene.Net.Store.Azure/AzureIndexInput.cs b/source/Lucene.Net.Store.Azure/AzureIndexInput.cs index 68c0e62..f0a2782 100644 --- a/source/Lucene.Net.Store.Azure/AzureIndexInput.cs +++ b/source/Lucene.Net.Store.Azure/AzureIndexInput.cs @@ -43,9 +43,8 @@ public AzureIndexInput(AzureDirectory azureDirectory, string name, BlobClient bl { try { - var blobProperties = blob.GetProperties(); long cachedLength = CacheDirectory.FileLength(name); - long blobLength = blobProperties?.Value?.ContentLength ?? 0; + long blobLength = _azureDirectory.GetCachedMetadata(name)?.ContentLength ?? 0; if (cachedLength != blobLength) fileNeeded = true; } diff --git a/source/Lucene.Net.Store.Azure/AzureIndexOutput.cs b/source/Lucene.Net.Store.Azure/AzureIndexOutput.cs index 57bbe25..f4e074f 100644 --- a/source/Lucene.Net.Store.Azure/AzureIndexOutput.cs +++ b/source/Lucene.Net.Store.Azure/AzureIndexOutput.cs @@ -66,6 +66,8 @@ protected override void Dispose(bool disposing) Debug.WriteLine($"{_azureDirectory.Name} PUT {_name} bytes to {blobStream.Length} in cloud"); } + _azureDirectory.UpdateCacheEntry(_name, originalLength); + #if FULLDEBUG Debug.WriteLine($"{_azureDirectory.Name} CLOSED WRITESTREAM {_name}"); #endif diff --git a/source/Lucene.Net.Store.Azure/Lucene.Net.Store.Azure.csproj b/source/Lucene.Net.Store.Azure/Lucene.Net.Store.Azure.csproj index c94d89c..f56003d 100644 --- a/source/Lucene.Net.Store.Azure/Lucene.Net.Store.Azure.csproj +++ b/source/Lucene.Net.Store.Azure/Lucene.Net.Store.Azure.csproj @@ -2,7 +2,7 @@ netstandard2.0 true - 4.8.5-beta020 + 4.8.5-beta021 Tom Laird-McConnell Azure blob storage for Lucene.net This project allows you to store Lucene Indexes in by Azure BlobStorage. @@ -11,8 +11,8 @@ https://github.com/tomlm/Lucene.Net.Store.Azure git MS-PL - 4.8.5.20 - 4.8.5.20 + 4.8.5.21 + 4.8.5.21 This is a release with dependency on Lucene.Net 4.8.0.beta 0017 https://raw.githubusercontent.com/tomlm/Lucene.Net.Store.Azure/master/icon.png icon.png