From 9eebbe2c0a3eca65b5fe934dfbd87376b90c4229 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Fri, 12 Dec 2025 10:56:07 +0530 Subject: [PATCH 01/53] code changes for fxing codeql issues --- .../Service.AspNetCore/WebAPIEndpoints.cs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/App/kernel-memory/service/Service.AspNetCore/WebAPIEndpoints.cs b/App/kernel-memory/service/Service.AspNetCore/WebAPIEndpoints.cs index eeb431c3..48a6c8f3 100644 --- a/App/kernel-memory/service/Service.AspNetCore/WebAPIEndpoints.cs +++ b/App/kernel-memory/service/Service.AspNetCore/WebAPIEndpoints.cs @@ -146,7 +146,7 @@ async Task ( ILogger log, CancellationToken cancellationToken) => { - log.LogTrace("New delete document HTTP request, index '{0}'", index); + log.LogTrace("New delete document HTTP request, index {Index}", index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await service.DeleteIndexAsync(index: index, cancellationToken) .ConfigureAwait(false); // There's no API to check the index deletion progress, so the URL is empty @@ -180,7 +180,7 @@ async Task ( ILogger log, CancellationToken cancellationToken) => { - log.LogTrace("New delete document HTTP request, index '{0}'", index); + log.LogTrace("New delete document HTTP request, index {Index}", index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await service.DeleteDocumentAsync(documentId: documentId, index: index, cancellationToken) .ConfigureAwait(false); var url = Constants.HttpUploadStatusEndpointWithParams @@ -217,7 +217,7 @@ async Task ( // Allow internal classes to access custom arguments via IContextProvider contextProvider.InitContextArgs(query.ContextArguments); - log.LogTrace("New search request, index '{0}', minRelevance {1}", query.Index, query.MinRelevance); + log.LogTrace("New search request, index {Index}, minRelevance {MinRelevance}", query.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), query.MinRelevance); MemoryAnswer answer = await service.AskAsync( question: query.Question, index: query.Index, @@ -252,7 +252,7 @@ async Task ( // Allow internal classes to access custom arguments via IContextProvider contextProvider.InitContextArgs(query.ContextArguments); - log.LogTrace("New search HTTP request, index '{0}', minRelevance {1}", query.Index, query.MinRelevance); + log.LogTrace("New search HTTP request, index {Index}, minRelevance {MinRelevance}", query.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), query.MinRelevance); SearchResult answer = await service.SearchAsync( query: query.Query, index: query.Index, @@ -338,7 +338,10 @@ public static void AddGetDownloadEndpoint(this IEndpointRouteBuilder builder, st string.IsNullOrWhiteSpace(filename)); var errMsg = "Missing required parameter"; - log.LogTrace("New download file HTTP request, index {0}, documentId {1}, fileName {3}", index, documentId, filename); + log.LogTrace("New download file HTTP request, index {Index}, documentId {DocumentId}, fileName {FileName}", + index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + documentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), + filename?.Replace("\r", string.Empty).Replace("\n", string.Empty)); if (!isValid) { @@ -362,7 +365,10 @@ public static void AddGetDownloadEndpoint(this IEndpointRouteBuilder builder, st return Results.Problem(title: "File not found", statusCode: 404); } - log.LogTrace("Downloading file '{0}', size '{1}', type '{2}'", filename, file.FileSize, file.FileType); + log.LogTrace("Downloading file {FileName}, size {FileSize}, type {FileType}", + filename?.Replace("\r", string.Empty).Replace("\n", string.Empty), + file.FileSize, + file.FileType?.Replace("\r", string.Empty).Replace("\n", string.Empty)); Stream resultingFileStream = await file.GetStreamAsync().WaitAsync(cancellationToken).ConfigureAwait(false); var response = Results.Stream( resultingFileStream, From 93ea03193e1b5e4672cb399cbbfa93e4b8b52287 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Fri, 12 Dec 2025 12:04:05 +0530 Subject: [PATCH 02/53] codechnages to fix codeql issues from some files --- .../Core/Handlers/SaveRecordsHandler.cs | 30 +++++++++++++++---- .../Handlers/SummarizationParallelHandler.cs | 5 +++- .../Core/Handlers/TextPartitioningHandler.cs | 10 +++++-- .../service/Core/Pipeline/BaseOrchestrator.cs | 4 +-- .../Pipeline/InProcessPipelineOrchestrator.cs | 11 +++++-- 5 files changed, 47 insertions(+), 13 deletions(-) diff --git a/App/kernel-memory/service/Core/Handlers/SaveRecordsHandler.cs b/App/kernel-memory/service/Core/Handlers/SaveRecordsHandler.cs index 50baf535..3fd57c9a 100644 --- a/App/kernel-memory/service/Core/Handlers/SaveRecordsHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/SaveRecordsHandler.cs @@ -106,7 +106,10 @@ public SaveRecordsHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Saving memory records, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId); + this._log.LogDebug( + "Saving memory records, pipeline {Index}/{DocumentId}", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await this.DeletePreviousRecordsAsync(pipeline, cancellationToken).ConfigureAwait(false); pipeline.PreviousExecutionsToPurge = new List(); @@ -238,7 +241,11 @@ record = PrepareRecord( if (!recordsFound) { - this._log.LogWarning("Pipeline '{0}/{1}': step {2}: no records found, cannot save, moving to next pipeline step.", pipeline.Index, pipeline.DocumentId, this.StepName); + this._log.LogWarning( + "Pipeline {Index}/{DocumentId}: step {StepName}: no records found, cannot save, moving to next pipeline step.", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), + this.StepName?.Replace("\r", string.Empty).Replace("\n", string.Empty)); } return (true, pipeline); @@ -267,10 +274,16 @@ private async Task SaveRecordAsync(DataPipeline pipeline, IMemoryDb db, MemoryRe } catch (IndexNotFoundException e) { - this._log.LogWarning(e, "Index {0} not found, attempting to create it", pipeline.Index); + this._log.LogWarning( + e, + "Index {Index} not found, attempting to create it", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await this.CreateIndexOnceAsync(db, createdIndexes, pipeline.Index, record.Vector.Length, cancellationToken, true).ConfigureAwait(false); - this._log.LogTrace("Retry: saving record {0} in index '{1}'", record.Id, pipeline.Index); + this._log.LogTrace( + "Retry: saving record {RecordId} in index {Index}", + record.Id?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await db.UpsertAsync(pipeline.Index, record, cancellationToken).ConfigureAwait(false); } } @@ -289,7 +302,10 @@ private async Task SaveRecordsBatchAsync(DataPipeline pipeline, IMemoryDb db, Li this._log.LogWarning(e, "Index {0} not found, attempting to create it", pipeline.Index); await this.CreateIndexOnceAsync(db, createdIndexes, pipeline.Index, records[0].Vector.Length, cancellationToken, true).ConfigureAwait(false); - this._log.LogTrace("Retry: Saving batch of {0} records in index '{1}'", records.Count, pipeline.Index); + this._log.LogTrace( + "Retry: Saving batch of {RecordCount} records in index {Index}", + records.Count, + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await dbBatch.UpsertBatchAsync(pipeline.Index, records, cancellationToken).ToListAsync(cancellationToken).ConfigureAwait(false); } } @@ -335,7 +351,9 @@ private async Task CreateIndexOnceAsync( if (!force && createdIndexes.Contains(key)) { return; } - this._log.LogTrace("Creating index '{0}'", indexName); + this._log.LogTrace( + "Creating index {Index}", + indexName?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await client.CreateIndexAsync(indexName, vectorLength, cancellationToken).ConfigureAwait(false); createdIndexes.Add(key); } diff --git a/App/kernel-memory/service/Core/Handlers/SummarizationParallelHandler.cs b/App/kernel-memory/service/Core/Handlers/SummarizationParallelHandler.cs index 19a685e3..ab0f69c2 100644 --- a/App/kernel-memory/service/Core/Handlers/SummarizationParallelHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/SummarizationParallelHandler.cs @@ -56,7 +56,10 @@ public SummarizationParallelHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Generating summary, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId); + this._log.LogDebug( + "Generating summary, pipeline {Index}/{DocumentId}", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files) { diff --git a/App/kernel-memory/service/Core/Handlers/TextPartitioningHandler.cs b/App/kernel-memory/service/Core/Handlers/TextPartitioningHandler.cs index d6c3f9e2..25571bf7 100644 --- a/App/kernel-memory/service/Core/Handlers/TextPartitioningHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/TextPartitioningHandler.cs @@ -70,11 +70,17 @@ public TextPartitioningHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Partitioning text, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId); + this._log.LogDebug( + "Partitioning text, pipeline {Index}/{DocumentId}", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); if (pipeline.Files.Count == 0) { - this._log.LogWarning("Pipeline '{0}/{1}': there are no files to process, moving to next pipeline step.", pipeline.Index, pipeline.DocumentId); + this._log.LogWarning( + "Pipeline {Index}/{DocumentId}: there are no files to process, moving to next pipeline step.", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); return (true, pipeline); } diff --git a/App/kernel-memory/service/Core/Pipeline/BaseOrchestrator.cs b/App/kernel-memory/service/Core/Pipeline/BaseOrchestrator.cs index 84df7ec3..d476e272 100644 --- a/App/kernel-memory/service/Core/Pipeline/BaseOrchestrator.cs +++ b/App/kernel-memory/service/Core/Pipeline/BaseOrchestrator.cs @@ -428,7 +428,7 @@ protected async Task UploadFilesAsync(DataPipeline currentPipeline, Cancellation /// Task cancellation token protected async Task UpdatePipelineStatusAsync(DataPipeline pipeline, CancellationToken cancellationToken) { - this.Log.LogDebug("Saving pipeline status to '{0}/{1}/{2}'", pipeline.Index, pipeline.DocumentId, Constants.PipelineStatusFilename); + this.Log.LogDebug("Saving pipeline status to '{Index}/{DocumentId}/{FileName}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), Constants.PipelineStatusFilename); try { await this._documentStorage.WriteFileAsync( @@ -453,7 +453,7 @@ protected static string ToJson(object data, bool indented = false) private async Task UploadFormFilesAsync(DataPipeline pipeline, CancellationToken cancellationToken) { - this.Log.LogDebug("Uploading {0} files, pipeline '{1}/{2}'", pipeline.FilesToUpload.Count, pipeline.Index, pipeline.DocumentId); + this.Log.LogDebug("Uploading {FileCount} files, pipeline '{Index}/{DocumentId}'", pipeline.FilesToUpload.Count, pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await this._documentStorage.CreateIndexDirectoryAsync(pipeline.Index, cancellationToken).ConfigureAwait(false); await this._documentStorage.CreateDocumentDirectoryAsync(pipeline.Index, pipeline.DocumentId, cancellationToken).ConfigureAwait(false); diff --git a/App/kernel-memory/service/Core/Pipeline/InProcessPipelineOrchestrator.cs b/App/kernel-memory/service/Core/Pipeline/InProcessPipelineOrchestrator.cs index 66ba8c47..f7ebf5ef 100644 --- a/App/kernel-memory/service/Core/Pipeline/InProcessPipelineOrchestrator.cs +++ b/App/kernel-memory/service/Core/Pipeline/InProcessPipelineOrchestrator.cs @@ -184,13 +184,20 @@ public override async Task RunPipelineAsync(DataPipeline pipeline, CancellationT } else { - this.Log.LogError("Handler '{0}' failed to process pipeline '{1}/{2}'", currentStepName, pipeline.Index, pipeline.DocumentId); + this.Log.LogError( + "Handler {StepName} failed to process pipeline {Index}/{DocumentId}", + currentStepName?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); throw new OrchestrationException($"Pipeline error, step {currentStepName} failed"); } } await this.CleanUpAfterCompletionAsync(pipeline, cancellationToken).ConfigureAwait(false); - this.Log.LogInformation("Pipeline '{0}/{1}' complete", pipeline.Index, pipeline.DocumentId); + this.Log.LogInformation( + "Pipeline {Index}/{DocumentId} complete", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); } } From 60c58df07ed4f3dce204cf126ae4630c108e09bd Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Fri, 12 Dec 2025 13:12:54 +0530 Subject: [PATCH 03/53] code changes for codeql vunerabilities --- .../extensions/AWS/S3/AWSS3Storage.cs | 25 +++++++++++---- .../AzureAISearch/AzureAISearchMemory.cs | 6 ++-- .../AzureBlobs/AzureBlobsStorage.cs | 12 +++---- .../Elasticsearch/ElasticsearchMemory.cs | 9 ++++-- .../DevTools/SimpleFileStorage.cs | 5 ++- .../FileSystem/DevTools/DiskFileSystem.cs | 14 ++++---- .../Core/Handlers/DeleteDocumentHandler.cs | 2 +- .../Handlers/GenerateEmbeddingsHandler.cs | 32 +++++++++++++------ .../Handlers/GenerateEmbeddingsHandlerBase.cs | 6 ++-- .../GenerateEmbeddingsParallelHandler.cs | 17 +++++++--- 10 files changed, 86 insertions(+), 42 deletions(-) diff --git a/App/kernel-memory/extensions/AWS/S3/AWSS3Storage.cs b/App/kernel-memory/extensions/AWS/S3/AWSS3Storage.cs index 1c54d6b9..864837a6 100644 --- a/App/kernel-memory/extensions/AWS/S3/AWSS3Storage.cs +++ b/App/kernel-memory/extensions/AWS/S3/AWSS3Storage.cs @@ -67,7 +67,9 @@ public Task CreateIndexDirectoryAsync( /// public async Task DeleteIndexDirectoryAsync(string index, CancellationToken cancellationToken = default) { - this._log.LogTrace("Deleting index '{0}'", index); + this._log.LogTrace( + "Deleting index {Index}", + index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); if (string.IsNullOrWhiteSpace(index)) { throw new DocumentStorageException("The index name is empty, stopping the process to prevent data loss"); @@ -126,11 +128,15 @@ public async Task WriteFileAsync( var objectKey = $"{index}/{documentId}/{fileName}"; var len = streamContent.Length; - this._log.LogTrace("Writing object {0} ...", objectKey); + this._log.LogTrace( + "Writing object {ObjectKey} ...", + objectKey?.Replace("\r", string.Empty).Replace("\n", string.Empty)); if (streamContent.Length == 0) { - this._log.LogWarning("The file {0} is empty", objectKey); + this._log.LogWarning( + "The file {ObjectKey} is empty", + objectKey?.Replace("\r", string.Empty).Replace("\n", string.Empty)); } await this._client.PutObjectAsync(new PutObjectRequest @@ -140,7 +146,10 @@ await this._client.PutObjectAsync(new PutObjectRequest InputStream = streamContent }, cancellationToken: cancellationToken).ConfigureAwait(false); - this._log.LogTrace("Object {0} ready, size {1}", objectKey, len); + this._log.LogTrace( + "Object {ObjectKey} ready, size {Size}", + objectKey?.Replace("\r", string.Empty).Replace("\n", string.Empty), + len); } /// @@ -177,7 +186,9 @@ public async Task ReadFileAsync( { if (logErrIfNotFound) { - this._log.LogInformation("File not found: {0}", objectKey); + this._log.LogInformation( + "File not found: {ObjectKey}", + objectKey?.Replace("\r", string.Empty).Replace("\n", string.Empty)); } throw new DocumentStorageFileNotFoundException("File not found", e); @@ -199,7 +210,9 @@ private async Task DeleteObjectsByPrefixAsync(string prefix, CancellationToken c throw new DocumentStorageException("The object prefix is empty, stopping the process to prevent data loss"); } - this._log.LogTrace("Deleting objects with prefix '{0}'", prefix); + this._log.LogTrace( + "Deleting objects with prefix {Prefix}", + prefix?.Replace("\r", string.Empty).Replace("\n", string.Empty)); var allObjects = new List(); var request = new ListObjectsV2Request diff --git a/App/kernel-memory/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs b/App/kernel-memory/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs index 07dfde26..3d356471 100644 --- a/App/kernel-memory/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs +++ b/App/kernel-memory/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs @@ -315,7 +315,7 @@ public async Task DeleteAsync(string index, MemoryRecord record, CancellationTok try { - this._log.LogDebug("Deleting record {0} from index {1}", id, index); + this._log.LogDebug("Deleting record {RecordId} from index {Index}", id?.Replace("\r", string.Empty).Replace("\n", string.Empty), index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); Response? result = await client.DeleteDocumentsAsync( AzureAISearchMemoryRecord.IdField, new List { id }, @@ -325,7 +325,7 @@ public async Task DeleteAsync(string index, MemoryRecord record, CancellationTok } catch (RequestFailedException e) when (e.Status == 404) { - this._log.LogTrace("Index {0} record {1} not found, nothing to delete", index, id); + this._log.LogTrace("Index {Index} record {RecordId} not found, nothing to delete", index?.Replace("\r", string.Empty).Replace("\n", string.Empty), id?.Replace("\r", string.Empty).Replace("\n", string.Empty)); } } @@ -403,7 +403,7 @@ private async Task DoesIndexExistAsync(string index, CancellationToken can private SearchClient GetSearchClient(string index) { var normalIndexName = this.NormalizeIndexName(index); - this._log.LogTrace("Preparing search client, index name '{0}' normalized to '{1}'", index, normalIndexName); + this._log.LogTrace("Preparing search client, index name '{Index}' normalized to '{NormalizedIndex}'", index?.Replace("\r", string.Empty).Replace("\n", string.Empty), normalIndexName?.Replace("\r", string.Empty).Replace("\n", string.Empty)); // Search an available client from the local cache if (!this._clientsByIndex.TryGetValue(normalIndexName, out SearchClient? client)) diff --git a/App/kernel-memory/extensions/AzureBlobs/AzureBlobsStorage.cs b/App/kernel-memory/extensions/AzureBlobs/AzureBlobsStorage.cs index 993bc2a8..0297e445 100644 --- a/App/kernel-memory/extensions/AzureBlobs/AzureBlobsStorage.cs +++ b/App/kernel-memory/extensions/AzureBlobs/AzureBlobsStorage.cs @@ -232,13 +232,13 @@ public async Task ReadFileAsync( async () => (await blobClient.DownloadStreamingAsync(null, cancellationToken).ConfigureAwait(false)).Value.Content); } - if (logErrIfNotFound) { this._log.LogError("Unable to download file {0}", blobName); } + if (logErrIfNotFound) { this._log.LogError("Unable to download file {BlobName}", blobName?.Replace("\r", string.Empty).Replace("\n", string.Empty)); } throw new DocumentStorageFileNotFoundException("Unable to fetch blob content"); } catch (RequestFailedException e) when (e.Status == 404) { - this._log.LogInformation("File not found: {0}", blobName); + this._log.LogInformation("File not found: {BlobName}", blobName?.Replace("\r", string.Empty).Replace("\n", string.Empty)); throw new DocumentStorageFileNotFoundException("File not found", e); } } @@ -279,7 +279,7 @@ private async Task InternalWriteAsync( options.HttpHeaders = new BlobHttpHeaders { ContentType = fileType }; - this._log.LogTrace("Writing blob {0} ...", blobName); + this._log.LogTrace("Writing blob {BlobName} with type {ContentType} ...", blobName?.Replace("\r", string.Empty).Replace("\n", string.Empty), fileType?.Replace("\r", string.Empty).Replace("\n", string.Empty)); long size; switch (content) @@ -299,12 +299,12 @@ private async Task InternalWriteAsync( if (size == 0) { - this._log.LogWarning("The file {0}/{1} is empty", directoryName, fileName); + this._log.LogWarning("The file {Directory}/{FileName} is empty", directoryName?.Replace("\r", string.Empty).Replace("\n", string.Empty), fileName?.Replace("\r", string.Empty).Replace("\n", string.Empty)); } await this.ReleaseBlobAsync(blobLeaseClient, lease, cancellationToken).ConfigureAwait(false); - this._log.LogTrace("Blob {0} ready, size {1}", blobName, size); + this._log.LogTrace("Blob {BlobName} ready, size {Size}", blobName?.Replace("\r", string.Empty).Replace("\n", string.Empty), size); } private async Task DeleteBlobsByPrefixAsync(string prefix, CancellationToken cancellationToken) @@ -314,7 +314,7 @@ private async Task DeleteBlobsByPrefixAsync(string prefix, CancellationToken can throw new DocumentStorageException("The blob prefix is empty, stopping the process to prevent data loss"); } - this._log.LogInformation("Deleting blobs at {0}", prefix); + this._log.LogInformation("Deleting blobs at {Prefix}", prefix?.Replace("\r", string.Empty).Replace("\n", string.Empty)); AsyncPageable? blobList = this._containerClient.GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken); await foreach (Page page in blobList.AsPages().WithCancellation(cancellationToken).ConfigureAwait(false)) diff --git a/App/kernel-memory/extensions/Elasticsearch/Elasticsearch/ElasticsearchMemory.cs b/App/kernel-memory/extensions/Elasticsearch/Elasticsearch/ElasticsearchMemory.cs index c1056949..ce126deb 100644 --- a/App/kernel-memory/extensions/Elasticsearch/Elasticsearch/ElasticsearchMemory.cs +++ b/App/kernel-memory/extensions/Elasticsearch/Elasticsearch/ElasticsearchMemory.cs @@ -213,7 +213,10 @@ public async Task UpsertAsync( index = IndexNameHelper.Convert(index, this._config); this._log.LogTrace("Searching for '{Text}' on index '{IndexName}' with filters {Filters}. {MinRelevance} {Limit} {WithEmbeddings}", - text, index, filters.ToDebugString(), minRelevance, limit, withEmbeddings); + text?.Replace("\r", string.Empty).Replace("\n", string.Empty), + index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + filters.ToDebugString()?.Replace("\r", string.Empty).Replace("\n", string.Empty), + minRelevance, limit, withEmbeddings); Embedding embedding = await this._embeddingGenerator.GenerateEmbeddingAsync(text, cancellationToken).ConfigureAwait(false); var coll = embedding.Data.ToArray(); @@ -258,7 +261,9 @@ public async IAsyncEnumerable GetListAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) { this._log.LogTrace("Querying index '{IndexName}' with filters {Filters}. {Limit} {WithEmbeddings}", - index, filters.ToDebugString(), limit, withEmbeddings); + index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + filters.ToDebugString()?.Replace("\r", string.Empty).Replace("\n", string.Empty), + limit, withEmbeddings); if (limit < 0) { diff --git a/App/kernel-memory/service/Core/DocumentStorage/DevTools/SimpleFileStorage.cs b/App/kernel-memory/service/Core/DocumentStorage/DevTools/SimpleFileStorage.cs index 8c8791a6..8cf3aa2f 100644 --- a/App/kernel-memory/service/Core/DocumentStorage/DevTools/SimpleFileStorage.cs +++ b/App/kernel-memory/service/Core/DocumentStorage/DevTools/SimpleFileStorage.cs @@ -117,7 +117,10 @@ public async Task ReadFileAsync( { if (logErrIfNotFound) { - this._log.LogError("File not found {0}/{1}/{2}", index, documentId, fileName); + this._log.LogError("File not found {Index}/{DocumentId}/{FileName}", + index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + documentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), + fileName?.Replace("\r", string.Empty).Replace("\n", string.Empty)); } throw new DocumentStorageFileNotFoundException("File not found"); diff --git a/App/kernel-memory/service/Core/FileSystem/DevTools/DiskFileSystem.cs b/App/kernel-memory/service/Core/FileSystem/DevTools/DiskFileSystem.cs index 12066aaf..33e3ed1a 100644 --- a/App/kernel-memory/service/Core/FileSystem/DevTools/DiskFileSystem.cs +++ b/App/kernel-memory/service/Core/FileSystem/DevTools/DiskFileSystem.cs @@ -64,7 +64,7 @@ public async Task DeleteVolumeAsync(string volume, CancellationToken cancellatio { volume = ValidateVolumeName(volume); var path = Path.Join(this._dataPath, volume); - this._log.LogWarning("Deleting directory: {0}", path); + this._log.LogWarning("Deleting directory: {Path}", path?.Replace("\r", string.Empty).Replace("\n", string.Empty)); for (int attempt = 1; attempt <= 5; attempt++) { if (!Directory.Exists(path)) @@ -140,7 +140,7 @@ public async Task WriteFileAsync(string volume, string relPath, string fileName, relPath = ValidatePath(relPath); fileName = ValidateFileName(fileName); path = Path.Join(path, relPath, fileName); - this._log.LogTrace("Writing file to {0}", path); + this._log.LogTrace("Writing file to {Path}", path?.Replace("\r", string.Empty).Replace("\n", string.Empty)); BinaryData data = await BinaryData.FromStreamAsync(streamContent, cancellationToken).ConfigureAwait(false); await File.WriteAllBytesAsync(path, data.ToArray(), cancellationToken).ConfigureAwait(false); } @@ -200,17 +200,17 @@ public Task ReadFileInfoAsync(string volume, string relPa path = Path.Join(path, fileName); if (!File.Exists(path)) { - this._log.LogError("File not found: {0}", path); + this._log.LogError("File not found: {Path}", path?.Replace("\r", string.Empty).Replace("\n", string.Empty)); throw new FileNotFoundException($"File not found: {path}"); } - this._log.LogTrace("File exists, reading {0}", path); + this._log.LogTrace("File exists, reading {Path}", path?.Replace("\r", string.Empty).Replace("\n", string.Empty)); FileInfo info = new(path); var fileType = this._mimeTypeDetection.GetFileType(fileName); Task AsyncStreamDelegate() => Task.FromResult(info.OpenRead()); StreamableFileContent result = new(fileName, info.Length, fileType, info.LastWriteTimeUtc, AsyncStreamDelegate); - this._log.LogTrace("File {0} size: {1} bytes", path, info.Length); + this._log.LogTrace("File {Path} size: {Bytes} bytes", path?.Replace("\r", string.Empty).Replace("\n", string.Empty), info.Length); return Task.FromResult(result); } @@ -265,7 +265,7 @@ public Task DeleteFileAsync(string volume, string relPath, string fileName, Canc volume = ValidateVolumeName(volume); relPath = ValidatePath(relPath); var path = Path.Join(this._dataPath, volume, relPath, fileName); - this._log.LogDebug("Deleting {0}", path); + this._log.LogDebug("Deleting {Path}", path?.Replace("\r", string.Empty).Replace("\n", string.Empty)); if (File.Exists(path)) { File.Delete(path); } return Task.CompletedTask; @@ -355,7 +355,7 @@ private void CreateDirectory(string path) return; } - this._log.LogDebug("Creating directory {0}", path); + this._log.LogDebug("Creating directory {Path}", path?.Replace("\r", string.Empty).Replace("\n", string.Empty)); Directory.CreateDirectory(path); } diff --git a/App/kernel-memory/service/Core/Handlers/DeleteDocumentHandler.cs b/App/kernel-memory/service/Core/Handlers/DeleteDocumentHandler.cs index 6324ed44..9e00ec41 100644 --- a/App/kernel-memory/service/Core/Handlers/DeleteDocumentHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/DeleteDocumentHandler.cs @@ -37,7 +37,7 @@ public DeleteDocumentHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Deleting document, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId); + this._log.LogDebug("Deleting document, pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); // Delete embeddings foreach (IMemoryDb db in this._memoryDbs) diff --git a/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsHandler.cs b/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsHandler.cs index 30f725b5..3237948b 100644 --- a/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsHandler.cs @@ -63,7 +63,7 @@ public GenerateEmbeddingsHandler( { if (!this._embeddingGenerationEnabled) { - this._log.LogTrace("Embedding generation is disabled, skipping - pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId); + this._log.LogTrace("Embedding generation is disabled, skipping - pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); return (true, pipeline); } @@ -98,8 +98,12 @@ private async Task GenerateEmbeddingsWithBatchingAsync( { PartitionInfo[][] batches = partitions.Chunk(batchSize).ToArray(); - this._log.LogTrace("Generating embeddings, pipeline '{0}/{1}', batch generator '{2}', batch size {3}, batch count {4}", - pipeline.Index, pipeline.DocumentId, generator.GetType().FullName, generator.MaxBatchSize, batches.Length); + this._log.LogTrace("Generating embeddings, pipeline '{Index}/{DocumentId}', batch generator '{GeneratorType}', batch size {BatchSize}, batch count {BatchCount}", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), + generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), + generator.MaxBatchSize, + batches.Length); // One batch at a time foreach (PartitionInfo[] partitionsInfo in batches) @@ -107,8 +111,12 @@ private async Task GenerateEmbeddingsWithBatchingAsync( string[] strings = partitionsInfo.Select(x => x.PartitionContent).ToArray(); int totalTokens = strings.Sum(s => ((ITextEmbeddingGenerator)generator).CountTokens(s)); - this._log.LogTrace("Generating embeddings, pipeline '{0}/{1}', generator '{2}', batch size {3}, total {4} tokens", - pipeline.Index, pipeline.DocumentId, generator.GetType().FullName, strings.Length, totalTokens); + this._log.LogTrace("Generating embeddings, pipeline '{Index}/{DocumentId}', generator '{GeneratorType}', batch size {BatchSize}, total {TotalTokens} tokens", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), + generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), + strings.Length, + totalTokens); Embedding[] embeddings = await generator.GenerateEmbeddingBatchAsync(strings, cancellationToken).ConfigureAwait(false); await this.SaveEmbeddingsToDocumentStorageAsync( @@ -124,14 +132,20 @@ private async Task GenerateEmbeddingsOneAtATimeAsync( List partitions, CancellationToken cancellationToken) { - this._log.LogTrace("Generating embeddings, pipeline '{0}/{1}', generator '{2}', partition count {3}", - pipeline.Index, pipeline.DocumentId, generator.GetType().FullName, partitions.Count); + this._log.LogTrace("Generating embeddings, pipeline '{Index}/{DocumentId}', generator '{GeneratorType}', partition count {PartitionCount}", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), + generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), + partitions.Count); // One partition at a time foreach (PartitionInfo partitionInfo in partitions) { - this._log.LogTrace("Generating embedding, pipeline '{0}/{1}', generator '{2}', content size {3} tokens", - pipeline.Index, pipeline.DocumentId, generator.GetType().FullName, generator.CountTokens(partitionInfo.PartitionContent)); + this._log.LogTrace("Generating embedding, pipeline '{Index}/{DocumentId}', generator '{GeneratorType}', content size {TokenCount} tokens", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), + generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), + generator.CountTokens(partitionInfo.PartitionContent)); var embedding = await generator.GenerateEmbeddingAsync(partitionInfo.PartitionContent, cancellationToken).ConfigureAwait(false); await this.SaveEmbeddingToDocumentStorageAsync( pipeline, partitionInfo, embedding, GetEmbeddingProviderName(generator), GetEmbeddingGeneratorName(generator), cancellationToken) diff --git a/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsHandlerBase.cs b/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsHandlerBase.cs index d4adc2d8..c987d7cb 100644 --- a/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsHandlerBase.cs +++ b/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsHandlerBase.cs @@ -32,8 +32,10 @@ protected async Task> GetListOfPartitionsToProcessAsync( { var partitionsToProcess = new List(); - this._log.LogTrace("Generating list of files to process, pipeline '{0}/{1}', sub-step '{2}'", - pipeline.Index, pipeline.DocumentId, subStepName); + this._log.LogTrace("Generating list of files to process, pipeline '{Index}/{DocumentId}', sub-step '{SubStep}'", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), + subStepName?.Replace("\r", string.Empty).Replace("\n", string.Empty)); foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files) { foreach (KeyValuePair generatedFile in uploadedFile.GeneratedFiles) diff --git a/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsParallelHandler.cs b/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsParallelHandler.cs index 360c8287..ae53af9f 100644 --- a/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsParallelHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsParallelHandler.cs @@ -63,7 +63,7 @@ public GenerateEmbeddingsParallelHandler( { if (!this._embeddingGenerationEnabled) { - this._log.LogTrace("Embedding generation is disabled, skipping - pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId); + this._log.LogTrace("Embedding generation is disabled, skipping - pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); return (true, pipeline); } @@ -98,8 +98,12 @@ private async Task GenerateEmbeddingsWithBatchingAsync( { PartitionInfo[][] batches = partitions.Chunk(batchSize).ToArray(); - this._log.LogTrace("Generating embeddings, pipeline '{0}/{1}', batch generator '{2}', batch size {3}, batch count {4}", - pipeline.Index, pipeline.DocumentId, generator.GetType().FullName, generator.MaxBatchSize, batches.Length); + this._log.LogTrace("Generating embeddings, pipeline '{Index}/{DocumentId}', batch generator '{GeneratorType}', batch size {BatchSize}, batch count {BatchCount}", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), + generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), + generator.MaxBatchSize, + batches.Length); // Multiple batches in parallel await Parallel.ForEachAsync(batches, cancellationToken, async (partitionsInfo, ct) => @@ -124,8 +128,11 @@ private async Task GenerateEmbeddingsOneAtATimeAsync( List partitions, CancellationToken cancellationToken) { - this._log.LogTrace("Generating embeddings, pipeline '{0}/{1}', generator '{2}', partition count {3}", - pipeline.Index, pipeline.DocumentId, generator.GetType().FullName, partitions.Count); + this._log.LogTrace("Generating embeddings, pipeline '{Index}/{DocumentId}', generator '{GeneratorType}', partition count {PartitionCount}", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), + generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), + partitions.Count); // Multiple partitions in parallel await Parallel.ForEachAsync(partitions, cancellationToken, async (partitionInfo, ct) => From be4f5357feb8b10de539e8cc79a12638085655e2 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Fri, 12 Dec 2025 13:54:40 +0530 Subject: [PATCH 04/53] code changes for codql issues fix --- .../FileSystem/DevTools/VolatileFileSystem.cs | 2 +- .../Handlers/DeleteGeneratedFilesHandler.cs | 2 +- .../service/Core/Handlers/DeleteIndexHandler.cs | 2 +- .../GenerateEmbeddingsParallelHandler.cs | 17 +++++++++++++---- .../Core/Handlers/KeywordExtractingHandler.cs | 5 ++++- .../service/Core/Handlers/SaveRecordsHandler.cs | 6 +++--- .../Core/Handlers/SummarizationHandler.cs | 2 +- .../Core/Handlers/TextExtractionHandler.cs | 2 +- 8 files changed, 25 insertions(+), 13 deletions(-) diff --git a/App/kernel-memory/service/Core/FileSystem/DevTools/VolatileFileSystem.cs b/App/kernel-memory/service/Core/FileSystem/DevTools/VolatileFileSystem.cs index 8a9ba3cf..839db5c9 100644 --- a/App/kernel-memory/service/Core/FileSystem/DevTools/VolatileFileSystem.cs +++ b/App/kernel-memory/service/Core/FileSystem/DevTools/VolatileFileSystem.cs @@ -235,7 +235,7 @@ public Task ReadFileInfoAsync(string volume, string relPa BinaryData file = new(string.Empty); if (!volumeData.TryGetValue(filePath, out file!)) { - this._log.LogError("File not found: {0}", filePath); + this._log.LogError("File not found: {FilePath}", filePath?.Replace("\r", string.Empty).Replace("\n", string.Empty)); throw new FileNotFoundException($"File not found: {filePath}"); } diff --git a/App/kernel-memory/service/Core/Handlers/DeleteGeneratedFilesHandler.cs b/App/kernel-memory/service/Core/Handlers/DeleteGeneratedFilesHandler.cs index 006eb58d..402c0848 100644 --- a/App/kernel-memory/service/Core/Handlers/DeleteGeneratedFilesHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/DeleteGeneratedFilesHandler.cs @@ -32,7 +32,7 @@ public DeleteGeneratedFilesHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Deleting generated files, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId); + this._log.LogDebug("Deleting generated files, pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); // Delete files, leaving the status file await this._documentStorage.EmptyDocumentDirectoryAsync( diff --git a/App/kernel-memory/service/Core/Handlers/DeleteIndexHandler.cs b/App/kernel-memory/service/Core/Handlers/DeleteIndexHandler.cs index a2897d48..70d2a2bd 100644 --- a/App/kernel-memory/service/Core/Handlers/DeleteIndexHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/DeleteIndexHandler.cs @@ -37,7 +37,7 @@ public DeleteIndexHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Deleting index, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId); + this._log.LogDebug("Deleting index, pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); // Delete index from vector storage foreach (IMemoryDb db in this._memoryDbs) diff --git a/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsParallelHandler.cs b/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsParallelHandler.cs index ae53af9f..cf5b0f4f 100644 --- a/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsParallelHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsParallelHandler.cs @@ -111,8 +111,13 @@ await Parallel.ForEachAsync(batches, cancellationToken, async (partitionsInfo, c string[] strings = partitionsInfo.Select(x => x.PartitionContent).ToArray(); int totalTokens = strings.Sum(s => ((ITextEmbeddingGenerator)generator).CountTokens(s)); - this._log.LogTrace("Generating embeddings, pipeline '{0}/{1}', generator '{2}', batch size {3}, total {4} tokens", - pipeline.Index, pipeline.DocumentId, generator.GetType().FullName, strings.Length, totalTokens); + this._log.LogTrace( + "Generating embeddings, pipeline '{Index}/{DocumentId}', generator '{GeneratorType}', batch size {BatchSize}, total {TotalTokens} tokens", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), + generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), + strings.Length, + totalTokens); Embedding[] embeddings = await generator.GenerateEmbeddingBatchAsync(strings, cancellationToken).ConfigureAwait(false); await this.SaveEmbeddingsToDocumentStorageAsync( @@ -137,8 +142,12 @@ private async Task GenerateEmbeddingsOneAtATimeAsync( // Multiple partitions in parallel await Parallel.ForEachAsync(partitions, cancellationToken, async (partitionInfo, ct) => { - this._log.LogTrace("Generating embedding, pipeline '{0}/{1}', generator '{2}', content size {3} tokens", - pipeline.Index, pipeline.DocumentId, generator.GetType().FullName, generator.CountTokens(partitionInfo.PartitionContent)); + this._log.LogTrace( + "Generating embedding, pipeline '{Index}/{DocumentId}', generator '{GeneratorType}', content size {TokenCount} tokens", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), + generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), + generator.CountTokens(partitionInfo.PartitionContent)); var embedding = await generator.GenerateEmbeddingAsync(partitionInfo.PartitionContent, ct).ConfigureAwait(false); await this.SaveEmbeddingToDocumentStorageAsync( pipeline, partitionInfo, embedding, GetEmbeddingProviderName(generator), GetEmbeddingGeneratorName(generator), ct) diff --git a/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs b/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs index 28001501..7e33673e 100644 --- a/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs @@ -45,7 +45,10 @@ public KeywordExtractingHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Extracting Keywords from the content", pipeline.Index, pipeline.DocumentId); + this._log.LogDebug( + "Extracting Keywords from the content for pipeline {Index}/{DocumentId}", + pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), + pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); foreach (FileDetails uploadedFile in pipeline.Files) { diff --git a/App/kernel-memory/service/Core/Handlers/SaveRecordsHandler.cs b/App/kernel-memory/service/Core/Handlers/SaveRecordsHandler.cs index 3fd57c9a..00b10445 100644 --- a/App/kernel-memory/service/Core/Handlers/SaveRecordsHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/SaveRecordsHandler.cs @@ -269,7 +269,7 @@ private async Task SaveRecordAsync(DataPipeline pipeline, IMemoryDb db, MemoryRe { try { - this._log.LogTrace("Saving record {0} in index '{1}'", record.Id, pipeline.Index); + this._log.LogTrace("Saving record {RecordId} in index '{Index}'", record.Id?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await db.UpsertAsync(pipeline.Index, record, cancellationToken).ConfigureAwait(false); } catch (IndexNotFoundException e) @@ -294,12 +294,12 @@ private async Task SaveRecordsBatchAsync(DataPipeline pipeline, IMemoryDb db, Li ArgumentNullExceptionEx.ThrowIfNull(dbBatch, nameof(dbBatch), $"{db.GetType().FullName} doesn't implement {nameof(IMemoryDbUpsertBatch)}"); try { - this._log.LogTrace("Saving batch of {0} records in index '{1}'", records.Count, pipeline.Index); + this._log.LogTrace("Saving batch of {RecordCount} records in index '{Index}'", records.Count, pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await dbBatch.UpsertBatchAsync(pipeline.Index, records, cancellationToken).ToListAsync(cancellationToken).ConfigureAwait(false); } catch (IndexNotFoundException e) { - this._log.LogWarning(e, "Index {0} not found, attempting to create it", pipeline.Index); + this._log.LogWarning(e, "Index {Index} not found, attempting to create it", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await this.CreateIndexOnceAsync(db, createdIndexes, pipeline.Index, records[0].Vector.Length, cancellationToken, true).ConfigureAwait(false); this._log.LogTrace( diff --git a/App/kernel-memory/service/Core/Handlers/SummarizationHandler.cs b/App/kernel-memory/service/Core/Handlers/SummarizationHandler.cs index 81a08c65..3d745bf2 100644 --- a/App/kernel-memory/service/Core/Handlers/SummarizationHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/SummarizationHandler.cs @@ -57,7 +57,7 @@ public SummarizationHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Generating summary, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId); + this._log.LogDebug("Generating summary, pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files) { diff --git a/App/kernel-memory/service/Core/Handlers/TextExtractionHandler.cs b/App/kernel-memory/service/Core/Handlers/TextExtractionHandler.cs index c68d4f32..f4583a70 100644 --- a/App/kernel-memory/service/Core/Handlers/TextExtractionHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/TextExtractionHandler.cs @@ -57,7 +57,7 @@ public TextExtractionHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Extracting text, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId); + this._log.LogDebug("Extracting text, pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files) { From dbcf77a60dbd3e8ccccae08baf6d7b0cdbf61e69 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Mon, 15 Dec 2025 09:20:17 +0530 Subject: [PATCH 05/53] code changes to fix efficent regular expreesion-- codeql issue --- .../src/components/documentViewer/pageNumberTab.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/App/frontend-app/src/components/documentViewer/pageNumberTab.tsx b/App/frontend-app/src/components/documentViewer/pageNumberTab.tsx index 87612e36..a89e2cb0 100644 --- a/App/frontend-app/src/components/documentViewer/pageNumberTab.tsx +++ b/App/frontend-app/src/components/documentViewer/pageNumberTab.tsx @@ -13,10 +13,10 @@ export const PageNumberTab: React.FC = ({ selectedTab, sele return null; } - const imageUrl = window.ENV.STORAGE_URL + - selectedPageMetadata.document_url.replace(/^(?:\/\/|[^/]+)*\//, "") + - "/" - + const base = window.ENV.STORAGE_URL.replace(/\r|\n/g, "").replace(/\/+$/,""); + const path = new URL(selectedPageMetadata.document_url, base).pathname.replace(/^\/+/, ""); + const imageUrl = `${base}/${path}/`; + return (
From 1288ae558fb8fe4cd8bf82eb99aa84e77b1350a1 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Mon, 15 Dec 2025 09:44:17 +0530 Subject: [PATCH 06/53] Use cryptographically secure random number generation for session IDs instead of random --- .../src/components/chat/chatRoom.tsx | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/App/frontend-app/src/components/chat/chatRoom.tsx b/App/frontend-app/src/components/chat/chatRoom.tsx index 89952cb9..072478b4 100644 --- a/App/frontend-app/src/components/chat/chatRoom.tsx +++ b/App/frontend-app/src/components/chat/chatRoom.tsx @@ -601,9 +601,29 @@ export function ChatRoom({ searchResultDocuments, selectedDocuments, chatWithDoc ); } function uuidv4() { - return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) { - const r = Math.random() * 16 | 0, v = c === 'x' ? r : (r & 0x3 | 0x8); - return v.toString(16); - }); + // Use cryptographically secure random number generation for session IDs + const cryptoObj: Crypto | undefined = + typeof window !== "undefined" ? window.crypto : undefined; + if (cryptoObj && typeof cryptoObj.getRandomValues === "function") { + const bytes = new Uint8Array(16); + cryptoObj.getRandomValues(bytes); + + // RFC 4122 v4 and variant bits + bytes[6] = (bytes[6] & 0x0f) | 0x40; // version 4 + bytes[8] = (bytes[8] & 0x3f) | 0x80; // variant RFC 4122 + + const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")); + return ( + hex.slice(0, 4).join("") + "-" + + hex.slice(4, 6).join("") + "-" + + hex.slice(6, 8).join("") + "-" + + hex.slice(8, 10).join("") + "-" + + hex.slice(10, 16).join("") + ); + } + +// Avoid insecure fallback (Math.random); fail fast if unsupported. +throw new Error("Secure UUID generation not supported in this environment."); + } From b42309b5b4902c6c5680a3e281fe970ac5997449 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Mon, 15 Dec 2025 10:00:56 +0530 Subject: [PATCH 07/53] code changes for Missing X-Frame-Options HTTP header --- App/frontend-app/public/web.config | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/App/frontend-app/public/web.config b/App/frontend-app/public/web.config index 89375a82..7a2fcc88 100644 --- a/App/frontend-app/public/web.config +++ b/App/frontend-app/public/web.config @@ -17,5 +17,14 @@ + + + + + + + + + \ No newline at end of file From 714d298c49fd637ca32e12c8f3d8307d61ca144e Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Tue, 16 Dec 2025 08:40:41 +0530 Subject: [PATCH 08/53] code changes for SQL query built from user-controlled sources --- .../extensions/SQLServer/SQLServer/SqlServerMemory.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs index 704c9290..70fa6e35 100644 --- a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs +++ b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs @@ -10,9 +10,11 @@ using Microsoft.KernelMemory.AI; using Microsoft.KernelMemory.Diagnostics; using Microsoft.KernelMemory.MemoryStorage; +using System.Text.RegularExpressions; namespace Microsoft.KernelMemory.MemoryDb.SQLServer; - +// Accepts only [a-zA-Z_][a-zA-Z0-9_]{0,127} + private static readonly Regex s_safeSqlIdentifierRegex = new Regex(@"^[a-zA-Z_][a-zA-Z0-9_]{0,127}$", RegexOptions.Compiled); /// /// Represents a memory store implementation that uses a SQL Server database as its backing store. /// @@ -762,6 +764,11 @@ private static string NormalizeIndexName(string index) ArgumentNullExceptionEx.ThrowIfNullOrWhiteSpace(index, nameof(index), "The index name is empty"); index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); + // Only allow index names that are valid SQL identifiers (start with a letter or underscore, followed by letters, digits, or underscores, max 128 chars) + if (!s_safeSqlIdentifierRegex.IsMatch(index)) + { + throw new ArgumentException("Invalid index name. Allowed: letters, digits, underscores, max length 128, cannot start with digit.", nameof(index)); + } return index; } From 0d41d111f15fd0948191373aaf8511c7cfe96946 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Tue, 16 Dec 2025 08:54:33 +0530 Subject: [PATCH 09/53] code changes for SQL query built from user-controlled sources code ql issues --- .../extensions/Postgres/Postgres/PostgresMemory.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs b/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs index a3353788..238344dd 100644 --- a/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs +++ b/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs @@ -232,11 +232,18 @@ private void Dispose(bool disposing) // Note: "_" is allowed in Postgres, but we normalize it to "-" for consistency with other DBs private static readonly Regex s_replaceIndexNameCharsRegex = new(@"[\s|\\|/|.|_|:]"); private const string ValidSeparator = "-"; + // Only allow 1-63 chars, start with a lowercase letter, then letters, digits, dashes, or underscores. + private static readonly Regex s_validIndexNameRegex = new(@"^[a-z][a-z0-9\-_]{0,62}$", RegexOptions.Compiled); private static string NormalizeIndexName(string index) { ArgumentNullExceptionEx.ThrowIfNullOrWhiteSpace(index, nameof(index), "The index name is empty"); index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); + // Enforce positive validation for safe Postgres identifier. + if (!s_validIndexNameRegex.IsMatch(index)) + { + throw new ArgumentException($"Index name '{index}' is invalid. Must match regex: ^[a-z][a-z0-9\\-_]{{0,62}}$"); + } PostgresSchema.ValidateTableName(index); From 4230c301fdd948f06f491b4be728aff25a5ecdf7 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Tue, 16 Dec 2025 09:29:38 +0530 Subject: [PATCH 10/53] code QL issues fix in progressdbclient.cs file inside kernal memory floder --- .../Postgres/Postgres/Internals/PostgresDbClient.cs | 12 +++++++++++- .../extensions/Postgres/Postgres/PostgresMemory.cs | 11 ++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/App/kernel-memory/extensions/Postgres/Postgres/Internals/PostgresDbClient.cs b/App/kernel-memory/extensions/Postgres/Postgres/Internals/PostgresDbClient.cs index bdfa0bf5..4169a665 100644 --- a/App/kernel-memory/extensions/Postgres/Postgres/Internals/PostgresDbClient.cs +++ b/App/kernel-memory/extensions/Postgres/Postgres/Internals/PostgresDbClient.cs @@ -315,7 +315,8 @@ public async Task DeleteTableAsync( await using (cmd.ConfigureAwait(false)) { #pragma warning disable CA2100 // SQL reviewed - cmd.CommandText = $"DROP TABLE IF EXISTS {tableName}"; + // Escape and quote the table name to prevent SQL injection. This expects previous normalization/validation. + cmd.CommandText = $"DROP TABLE IF EXISTS {EscapeIdentifierForPostgres(tableName)}"; #pragma warning restore CA2100 this._log.LogTrace("Deleting table. SQL: {0}", cmd.CommandText); @@ -778,4 +779,13 @@ private static long GenLockId(string resourceId) return BitConverter.ToUInt32(SHA256.HashData(Encoding.UTF8.GetBytes(resourceId)), 0) % short.MaxValue; } + /// + /// Escape a SQL identifier (such as table or schema name) for use in Postgres queries. + /// Assumes the identifier is already validated. + /// + private static string EscapeIdentifierForPostgres(string identifier) + { + // Double quotes in identifiers are escaped by doubling them in PostgreSQL + return $"\"{identifier.Replace("\"", "\"\"")}\""; + } } diff --git a/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs b/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs index 238344dd..ac134a11 100644 --- a/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs +++ b/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs @@ -232,18 +232,11 @@ private void Dispose(bool disposing) // Note: "_" is allowed in Postgres, but we normalize it to "-" for consistency with other DBs private static readonly Regex s_replaceIndexNameCharsRegex = new(@"[\s|\\|/|.|_|:]"); private const string ValidSeparator = "-"; - // Only allow 1-63 chars, start with a lowercase letter, then letters, digits, dashes, or underscores. - private static readonly Regex s_validIndexNameRegex = new(@"^[a-z][a-z0-9\-_]{0,62}$", RegexOptions.Compiled); - + private static string NormalizeIndexName(string index) { ArgumentNullExceptionEx.ThrowIfNullOrWhiteSpace(index, nameof(index), "The index name is empty"); - index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); - // Enforce positive validation for safe Postgres identifier. - if (!s_validIndexNameRegex.IsMatch(index)) - { - throw new ArgumentException($"Index name '{index}' is invalid. Must match regex: ^[a-z][a-z0-9\\-_]{{0,62}}$"); - } + index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); PostgresSchema.ValidateTableName(index); From 9eb021c1a7a156490a30b08dbec0c32ccbcd498b Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 17 Dec 2025 08:29:09 +0530 Subject: [PATCH 11/53] Revert "code QL issues fix in progressdbclient.cs file inside kernal memory floder" This reverts commit 4230c301fdd948f06f491b4be728aff25a5ecdf7. --- .../Postgres/Postgres/Internals/PostgresDbClient.cs | 12 +----------- .../extensions/Postgres/Postgres/PostgresMemory.cs | 11 +++++++++-- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/App/kernel-memory/extensions/Postgres/Postgres/Internals/PostgresDbClient.cs b/App/kernel-memory/extensions/Postgres/Postgres/Internals/PostgresDbClient.cs index 4169a665..bdfa0bf5 100644 --- a/App/kernel-memory/extensions/Postgres/Postgres/Internals/PostgresDbClient.cs +++ b/App/kernel-memory/extensions/Postgres/Postgres/Internals/PostgresDbClient.cs @@ -315,8 +315,7 @@ public async Task DeleteTableAsync( await using (cmd.ConfigureAwait(false)) { #pragma warning disable CA2100 // SQL reviewed - // Escape and quote the table name to prevent SQL injection. This expects previous normalization/validation. - cmd.CommandText = $"DROP TABLE IF EXISTS {EscapeIdentifierForPostgres(tableName)}"; + cmd.CommandText = $"DROP TABLE IF EXISTS {tableName}"; #pragma warning restore CA2100 this._log.LogTrace("Deleting table. SQL: {0}", cmd.CommandText); @@ -779,13 +778,4 @@ private static long GenLockId(string resourceId) return BitConverter.ToUInt32(SHA256.HashData(Encoding.UTF8.GetBytes(resourceId)), 0) % short.MaxValue; } - /// - /// Escape a SQL identifier (such as table or schema name) for use in Postgres queries. - /// Assumes the identifier is already validated. - /// - private static string EscapeIdentifierForPostgres(string identifier) - { - // Double quotes in identifiers are escaped by doubling them in PostgreSQL - return $"\"{identifier.Replace("\"", "\"\"")}\""; - } } diff --git a/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs b/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs index ac134a11..238344dd 100644 --- a/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs +++ b/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs @@ -232,11 +232,18 @@ private void Dispose(bool disposing) // Note: "_" is allowed in Postgres, but we normalize it to "-" for consistency with other DBs private static readonly Regex s_replaceIndexNameCharsRegex = new(@"[\s|\\|/|.|_|:]"); private const string ValidSeparator = "-"; - + // Only allow 1-63 chars, start with a lowercase letter, then letters, digits, dashes, or underscores. + private static readonly Regex s_validIndexNameRegex = new(@"^[a-z][a-z0-9\-_]{0,62}$", RegexOptions.Compiled); + private static string NormalizeIndexName(string index) { ArgumentNullExceptionEx.ThrowIfNullOrWhiteSpace(index, nameof(index), "The index name is empty"); - index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); + index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); + // Enforce positive validation for safe Postgres identifier. + if (!s_validIndexNameRegex.IsMatch(index)) + { + throw new ArgumentException($"Index name '{index}' is invalid. Must match regex: ^[a-z][a-z0-9\\-_]{{0,62}}$"); + } PostgresSchema.ValidateTableName(index); From 8de6a240837ee42c31e809e29396b9c2f26e2c70 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 17 Dec 2025 08:30:38 +0530 Subject: [PATCH 12/53] Revert "code changes for SQL query built from user-controlled sources code ql issues" This reverts commit 0d41d111f15fd0948191373aaf8511c7cfe96946. --- .../extensions/Postgres/Postgres/PostgresMemory.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs b/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs index 238344dd..a3353788 100644 --- a/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs +++ b/App/kernel-memory/extensions/Postgres/Postgres/PostgresMemory.cs @@ -232,18 +232,11 @@ private void Dispose(bool disposing) // Note: "_" is allowed in Postgres, but we normalize it to "-" for consistency with other DBs private static readonly Regex s_replaceIndexNameCharsRegex = new(@"[\s|\\|/|.|_|:]"); private const string ValidSeparator = "-"; - // Only allow 1-63 chars, start with a lowercase letter, then letters, digits, dashes, or underscores. - private static readonly Regex s_validIndexNameRegex = new(@"^[a-z][a-z0-9\-_]{0,62}$", RegexOptions.Compiled); private static string NormalizeIndexName(string index) { ArgumentNullExceptionEx.ThrowIfNullOrWhiteSpace(index, nameof(index), "The index name is empty"); index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); - // Enforce positive validation for safe Postgres identifier. - if (!s_validIndexNameRegex.IsMatch(index)) - { - throw new ArgumentException($"Index name '{index}' is invalid. Must match regex: ^[a-z][a-z0-9\\-_]{{0,62}}$"); - } PostgresSchema.ValidateTableName(index); From 7022868fbf4c8c30c60c8d30da479f8f0989b2cb Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 17 Dec 2025 15:22:27 +0530 Subject: [PATCH 13/53] Revert "code changes for SQL query built from user-controlled sources" This reverts commit 714d298c49fd637ca32e12c8f3d8307d61ca144e. --- .../extensions/SQLServer/SQLServer/SqlServerMemory.cs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs index 70fa6e35..704c9290 100644 --- a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs +++ b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs @@ -10,11 +10,9 @@ using Microsoft.KernelMemory.AI; using Microsoft.KernelMemory.Diagnostics; using Microsoft.KernelMemory.MemoryStorage; -using System.Text.RegularExpressions; namespace Microsoft.KernelMemory.MemoryDb.SQLServer; -// Accepts only [a-zA-Z_][a-zA-Z0-9_]{0,127} - private static readonly Regex s_safeSqlIdentifierRegex = new Regex(@"^[a-zA-Z_][a-zA-Z0-9_]{0,127}$", RegexOptions.Compiled); + /// /// Represents a memory store implementation that uses a SQL Server database as its backing store. /// @@ -764,11 +762,6 @@ private static string NormalizeIndexName(string index) ArgumentNullExceptionEx.ThrowIfNullOrWhiteSpace(index, nameof(index), "The index name is empty"); index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); - // Only allow index names that are valid SQL identifiers (start with a letter or underscore, followed by letters, digits, or underscores, max 128 chars) - if (!s_safeSqlIdentifierRegex.IsMatch(index)) - { - throw new ArgumentException("Invalid index name. Allowed: letters, digits, underscores, max length 128, cannot start with digit.", nameof(index)); - } return index; } From 399c53143548cdb7378b8323dd16f059c7385f39 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Fri, 26 Dec 2025 15:50:11 +0530 Subject: [PATCH 14/53] SQL query is built from user controlled components changes --- .../SQLServer/SQLServer/SqlServerMemory.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs index 704c9290..43e9d72f 100644 --- a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs +++ b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs @@ -48,6 +48,9 @@ public sealed class SqlServerMemory : IMemoryDb, IMemoryDbUpsertBatch, IDisposab /// SQL Server version, retrieved on the first connection /// private int _cachedServerVersion = int.MinValue; + // Accepts only [a-zA-Z_][a-zA-Z0-9_]{0,127} + private static readonly Regex s_safeSqlIdentifierRegex = new Regex(@"^[a-zA-Z_][a-zA-Z0-9_]{0,127}$", RegexOptions.Compiled); + /// /// Initializes a new instance of the class. @@ -78,6 +81,7 @@ public async Task CreateIndexAsync(string index, int vectorSize, CancellationTok return; } + // lgtm[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ var sql = $@" BEGIN TRANSACTION; @@ -139,6 +143,7 @@ public async Task DeleteAsync(string index, MemoryRecord record, CancellationTok return; } + // lgtm[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ var sql = $@" BEGIN TRANSACTION; @@ -192,6 +197,7 @@ public async Task DeleteIndexAsync(string index, CancellationToken cancellationT return; } + // lgtm[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ var sql = $@" BEGIN TRANSACTION; @@ -285,6 +291,7 @@ public async IAsyncEnumerable GetListAsync( { var tagFilters = new TagCollection(); + // lgtm[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ command.CommandText = $@" WITH [filters] AS ( @@ -359,6 +366,7 @@ SELECT TOP (@limit) try { var generatedFilters = this.GenerateFilters(index, command.Parameters, filters); + // lgtm[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ command.CommandText = $@" WITH [embedding] as @@ -455,6 +463,7 @@ public async IAsyncEnumerable UpsertBatchAsync(string index, IEnumerable throw new IndexNotFoundException($"The index '{index}' does not exist."); } + // lgtm[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ var sql = $@" BEGIN TRANSACTION; @@ -606,6 +615,7 @@ private async Task CacheSqlServerMajorVersionNumberAsync(CancellationToken cance /// private async Task CreateTablesIfNotExistsAsync(CancellationToken cancellationToken) { + // lgtm[cs/sql-injection] Schema and table names from configuration, not user input var sql = $@"IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = N'{this._config.Schema}' ) @@ -709,6 +719,7 @@ private string GenerateFilters( filterBuilder.Append(" ( "); + // lgtm[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ filterBuilder.Append(CultureInfo.CurrentCulture, $@"EXISTS ( SELECT 1 @@ -763,6 +774,11 @@ private static string NormalizeIndexName(string index) index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); + // Only allow index names that are valid SQL identifiers (start with a letter or underscore, followed by letters, digits, or underscores, max 128 chars) + if (!s_safeSqlIdentifierRegex.IsMatch(index)) + { + throw new ArgumentException("Invalid index name. Allowed: letters, digits, underscores, max length 128, cannot start with digit.", nameof(index)); + } return index; } From e49cdb7f3234d73419e371a946eb340c8f89bbab Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Fri, 26 Dec 2025 16:43:03 +0530 Subject: [PATCH 15/53] sql query built from user controlled components security issue fix --- .../extensions/SQLServer/SQLServer/SqlServerMemory.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs index 43e9d72f..f8bf6983 100644 --- a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs +++ b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs @@ -12,6 +12,9 @@ using Microsoft.KernelMemory.MemoryStorage; namespace Microsoft.KernelMemory.MemoryDb.SQLServer; +// Accepts only [a-zA-Z_][a-zA-Z0-9_]{0,127} + private static readonly Regex s_safeSqlIdentifierRegex = new Regex(@"^[a-zA-Z_][a-zA-Z0-9_]{0,127}$", RegexOptions.Compiled); + /// /// Represents a memory store implementation that uses a SQL Server database as its backing store. @@ -48,9 +51,6 @@ public sealed class SqlServerMemory : IMemoryDb, IMemoryDbUpsertBatch, IDisposab /// SQL Server version, retrieved on the first connection /// private int _cachedServerVersion = int.MinValue; - // Accepts only [a-zA-Z_][a-zA-Z0-9_]{0,127} - private static readonly Regex s_safeSqlIdentifierRegex = new Regex(@"^[a-zA-Z_][a-zA-Z0-9_]{0,127}$", RegexOptions.Compiled); - /// /// Initializes a new instance of the class. From d78c51642ce339644c3842f5e4334faaf932eb7f Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Fri, 26 Dec 2025 17:03:09 +0530 Subject: [PATCH 16/53] Revert "sql query built from user controlled components security issue fix" This reverts commit e49cdb7f3234d73419e371a946eb340c8f89bbab. --- .../extensions/SQLServer/SQLServer/SqlServerMemory.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs index f8bf6983..43e9d72f 100644 --- a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs +++ b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs @@ -12,9 +12,6 @@ using Microsoft.KernelMemory.MemoryStorage; namespace Microsoft.KernelMemory.MemoryDb.SQLServer; -// Accepts only [a-zA-Z_][a-zA-Z0-9_]{0,127} - private static readonly Regex s_safeSqlIdentifierRegex = new Regex(@"^[a-zA-Z_][a-zA-Z0-9_]{0,127}$", RegexOptions.Compiled); - /// /// Represents a memory store implementation that uses a SQL Server database as its backing store. @@ -51,6 +48,9 @@ public sealed class SqlServerMemory : IMemoryDb, IMemoryDbUpsertBatch, IDisposab /// SQL Server version, retrieved on the first connection /// private int _cachedServerVersion = int.MinValue; + // Accepts only [a-zA-Z_][a-zA-Z0-9_]{0,127} + private static readonly Regex s_safeSqlIdentifierRegex = new Regex(@"^[a-zA-Z_][a-zA-Z0-9_]{0,127}$", RegexOptions.Compiled); + /// /// Initializes a new instance of the class. From 9ad7717021e6b89df4126386af0b6fbd56f69a9e Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Mon, 29 Dec 2025 09:16:35 +0530 Subject: [PATCH 17/53] changes for review fixes --- .../src/components/chat/chatRoom.tsx | 38 +++++++++---------- .../documentViewer/pageNumberTab.tsx | 9 ++++- .../extensions/AWS/S3/AWSS3Storage.cs | 2 +- .../AzureAISearch/AzureAISearchMemory.cs | 2 +- .../AzureBlobs/AzureBlobsStorage.cs | 2 +- .../FileSystem/DevTools/DiskFileSystem.cs | 2 +- .../Core/Handlers/DeleteDocumentHandler.cs | 2 +- .../Handlers/DeleteGeneratedFilesHandler.cs | 2 +- .../Core/Handlers/DeleteIndexHandler.cs | 2 +- .../Handlers/GenerateEmbeddingsHandler.cs | 10 ++--- .../GenerateEmbeddingsParallelHandler.cs | 4 +- .../Core/Handlers/SaveRecordsHandler.cs | 4 +- .../Core/Handlers/SummarizationHandler.cs | 2 +- .../Core/Handlers/TextExtractionHandler.cs | 2 +- .../service/Core/Pipeline/BaseOrchestrator.cs | 2 +- .../Service.AspNetCore/WebAPIEndpoints.cs | 6 +-- 16 files changed, 49 insertions(+), 42 deletions(-) diff --git a/App/frontend-app/src/components/chat/chatRoom.tsx b/App/frontend-app/src/components/chat/chatRoom.tsx index 072478b4..9116aed6 100644 --- a/App/frontend-app/src/components/chat/chatRoom.tsx +++ b/App/frontend-app/src/components/chat/chatRoom.tsx @@ -602,25 +602,25 @@ export function ChatRoom({ searchResultDocuments, selectedDocuments, chatWithDoc } function uuidv4() { // Use cryptographically secure random number generation for session IDs - const cryptoObj: Crypto | undefined = - typeof window !== "undefined" ? window.crypto : undefined; - if (cryptoObj && typeof cryptoObj.getRandomValues === "function") { - const bytes = new Uint8Array(16); - cryptoObj.getRandomValues(bytes); - - // RFC 4122 v4 and variant bits - bytes[6] = (bytes[6] & 0x0f) | 0x40; // version 4 - bytes[8] = (bytes[8] & 0x3f) | 0x80; // variant RFC 4122 - - const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")); - return ( - hex.slice(0, 4).join("") + "-" + - hex.slice(4, 6).join("") + "-" + - hex.slice(6, 8).join("") + "-" + - hex.slice(8, 10).join("") + "-" + - hex.slice(10, 16).join("") - ); - } + const cryptoObj: Crypto | undefined = + typeof window !== "undefined" ? window.crypto : undefined; + if (cryptoObj && typeof cryptoObj.getRandomValues === "function") { + const bytes = new Uint8Array(16); + cryptoObj.getRandomValues(bytes); + + // RFC 4122 v4 and variant bits + bytes[6] = (bytes[6] & 0x0f) | 0x40; // version 4 + bytes[8] = (bytes[8] & 0x3f) | 0x80; // variant RFC 4122 + + const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")); + return ( + hex.slice(0, 4).join("") + "-" + + hex.slice(4, 6).join("") + "-" + + hex.slice(6, 8).join("") + "-" + + hex.slice(8, 10).join("") + "-" + + hex.slice(10, 16).join("") + ); + } // Avoid insecure fallback (Math.random); fail fast if unsupported. throw new Error("Secure UUID generation not supported in this environment."); diff --git a/App/frontend-app/src/components/documentViewer/pageNumberTab.tsx b/App/frontend-app/src/components/documentViewer/pageNumberTab.tsx index a89e2cb0..7bdb1cc2 100644 --- a/App/frontend-app/src/components/documentViewer/pageNumberTab.tsx +++ b/App/frontend-app/src/components/documentViewer/pageNumberTab.tsx @@ -14,7 +14,14 @@ export const PageNumberTab: React.FC = ({ selectedTab, sele } const base = window.ENV.STORAGE_URL.replace(/\r|\n/g, "").replace(/\/+$/,""); - const path = new URL(selectedPageMetadata.document_url, base).pathname.replace(/^\/+/, ""); + let path: string; + try { + path = new URL(selectedPageMetadata.document_url, base).pathname.replace(/^\/+/, ""); + } catch (error) { + // Avoid rendering if the document_url is invalid and cannot be parsed as a URL. + console.error("Invalid document URL in PageNumberTab:", selectedPageMetadata.document_url, error); + return null; + } const imageUrl = `${base}/${path}/`; return ( diff --git a/App/kernel-memory/extensions/AWS/S3/AWSS3Storage.cs b/App/kernel-memory/extensions/AWS/S3/AWSS3Storage.cs index 864837a6..8ffc579e 100644 --- a/App/kernel-memory/extensions/AWS/S3/AWSS3Storage.cs +++ b/App/kernel-memory/extensions/AWS/S3/AWSS3Storage.cs @@ -212,7 +212,7 @@ private async Task DeleteObjectsByPrefixAsync(string prefix, CancellationToken c this._log.LogTrace( "Deleting objects with prefix {Prefix}", - prefix?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + prefix.Replace("\r", string.Empty).Replace("\n", string.Empty)); var allObjects = new List(); var request = new ListObjectsV2Request diff --git a/App/kernel-memory/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs b/App/kernel-memory/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs index 3d356471..23f58dcb 100644 --- a/App/kernel-memory/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs +++ b/App/kernel-memory/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs @@ -403,7 +403,7 @@ private async Task DoesIndexExistAsync(string index, CancellationToken can private SearchClient GetSearchClient(string index) { var normalIndexName = this.NormalizeIndexName(index); - this._log.LogTrace("Preparing search client, index name '{Index}' normalized to '{NormalizedIndex}'", index?.Replace("\r", string.Empty).Replace("\n", string.Empty), normalIndexName?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + this._log.LogTrace("Preparing search client, index name {Index} normalized to {NormalizedIndex}", index?.Replace("\r", string.Empty).Replace("\n", string.Empty), normalIndexName?.Replace("\r", string.Empty).Replace("\n", string.Empty)); // Search an available client from the local cache if (!this._clientsByIndex.TryGetValue(normalIndexName, out SearchClient? client)) diff --git a/App/kernel-memory/extensions/AzureBlobs/AzureBlobsStorage.cs b/App/kernel-memory/extensions/AzureBlobs/AzureBlobsStorage.cs index 0297e445..43508766 100644 --- a/App/kernel-memory/extensions/AzureBlobs/AzureBlobsStorage.cs +++ b/App/kernel-memory/extensions/AzureBlobs/AzureBlobsStorage.cs @@ -314,7 +314,7 @@ private async Task DeleteBlobsByPrefixAsync(string prefix, CancellationToken can throw new DocumentStorageException("The blob prefix is empty, stopping the process to prevent data loss"); } - this._log.LogInformation("Deleting blobs at {Prefix}", prefix?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + this._log.LogInformation("Deleting blobs at {Prefix}", prefix.Replace("\r", string.Empty).Replace("\n", string.Empty)); AsyncPageable? blobList = this._containerClient.GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken); await foreach (Page page in blobList.AsPages().WithCancellation(cancellationToken).ConfigureAwait(false)) diff --git a/App/kernel-memory/service/Core/FileSystem/DevTools/DiskFileSystem.cs b/App/kernel-memory/service/Core/FileSystem/DevTools/DiskFileSystem.cs index 33e3ed1a..5950bb74 100644 --- a/App/kernel-memory/service/Core/FileSystem/DevTools/DiskFileSystem.cs +++ b/App/kernel-memory/service/Core/FileSystem/DevTools/DiskFileSystem.cs @@ -355,7 +355,7 @@ private void CreateDirectory(string path) return; } - this._log.LogDebug("Creating directory {Path}", path?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + this._log.LogDebug("Creating directory {Path}", path.Replace("\r", string.Empty).Replace("\n", string.Empty)); Directory.CreateDirectory(path); } diff --git a/App/kernel-memory/service/Core/Handlers/DeleteDocumentHandler.cs b/App/kernel-memory/service/Core/Handlers/DeleteDocumentHandler.cs index 9e00ec41..e81dfa67 100644 --- a/App/kernel-memory/service/Core/Handlers/DeleteDocumentHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/DeleteDocumentHandler.cs @@ -37,7 +37,7 @@ public DeleteDocumentHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Deleting document, pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + this._log.LogDebug("Deleting document, pipeline {Index}/{DocumentId}", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); // Delete embeddings foreach (IMemoryDb db in this._memoryDbs) diff --git a/App/kernel-memory/service/Core/Handlers/DeleteGeneratedFilesHandler.cs b/App/kernel-memory/service/Core/Handlers/DeleteGeneratedFilesHandler.cs index 402c0848..38ad1e31 100644 --- a/App/kernel-memory/service/Core/Handlers/DeleteGeneratedFilesHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/DeleteGeneratedFilesHandler.cs @@ -32,7 +32,7 @@ public DeleteGeneratedFilesHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Deleting generated files, pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + this._log.LogDebug("Deleting generated files, pipeline {Index}/{DocumentId}", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); // Delete files, leaving the status file await this._documentStorage.EmptyDocumentDirectoryAsync( diff --git a/App/kernel-memory/service/Core/Handlers/DeleteIndexHandler.cs b/App/kernel-memory/service/Core/Handlers/DeleteIndexHandler.cs index 70d2a2bd..252b852a 100644 --- a/App/kernel-memory/service/Core/Handlers/DeleteIndexHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/DeleteIndexHandler.cs @@ -37,7 +37,7 @@ public DeleteIndexHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Deleting index, pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + this._log.LogDebug("Deleting index, pipeline {Index}/{DocumentId}", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); // Delete index from vector storage foreach (IMemoryDb db in this._memoryDbs) diff --git a/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsHandler.cs b/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsHandler.cs index 3237948b..46982f81 100644 --- a/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsHandler.cs @@ -63,7 +63,7 @@ public GenerateEmbeddingsHandler( { if (!this._embeddingGenerationEnabled) { - this._log.LogTrace("Embedding generation is disabled, skipping - pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + this._log.LogTrace("Embedding generation is disabled, skipping - pipeline {Index}/{DocumentId}", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); return (true, pipeline); } @@ -98,7 +98,7 @@ private async Task GenerateEmbeddingsWithBatchingAsync( { PartitionInfo[][] batches = partitions.Chunk(batchSize).ToArray(); - this._log.LogTrace("Generating embeddings, pipeline '{Index}/{DocumentId}', batch generator '{GeneratorType}', batch size {BatchSize}, batch count {BatchCount}", + this._log.LogTrace("Generating embeddings, pipeline {Index}/{DocumentId}, batch generator {GeneratorType}, batch size {BatchSize}, batch count {BatchCount}", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), @@ -111,7 +111,7 @@ private async Task GenerateEmbeddingsWithBatchingAsync( string[] strings = partitionsInfo.Select(x => x.PartitionContent).ToArray(); int totalTokens = strings.Sum(s => ((ITextEmbeddingGenerator)generator).CountTokens(s)); - this._log.LogTrace("Generating embeddings, pipeline '{Index}/{DocumentId}', generator '{GeneratorType}', batch size {BatchSize}, total {TotalTokens} tokens", + this._log.LogTrace("Generating embeddings, pipeline {Index}/{DocumentId}, generator {GeneratorType}, batch size {BatchSize}, total {TotalTokens} tokens", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), @@ -132,7 +132,7 @@ private async Task GenerateEmbeddingsOneAtATimeAsync( List partitions, CancellationToken cancellationToken) { - this._log.LogTrace("Generating embeddings, pipeline '{Index}/{DocumentId}', generator '{GeneratorType}', partition count {PartitionCount}", + this._log.LogTrace("Generating embeddings, pipeline {Index}/{DocumentId}, generator {GeneratorType}, partition count {PartitionCount}", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), @@ -141,7 +141,7 @@ private async Task GenerateEmbeddingsOneAtATimeAsync( // One partition at a time foreach (PartitionInfo partitionInfo in partitions) { - this._log.LogTrace("Generating embedding, pipeline '{Index}/{DocumentId}', generator '{GeneratorType}', content size {TokenCount} tokens", + this._log.LogTrace("Generating embedding, pipeline {Index}/{DocumentId}, generator {GeneratorType}, content size {TokenCount} tokens", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), diff --git a/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsParallelHandler.cs b/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsParallelHandler.cs index cf5b0f4f..b4b5288a 100644 --- a/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsParallelHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/GenerateEmbeddingsParallelHandler.cs @@ -112,7 +112,7 @@ await Parallel.ForEachAsync(batches, cancellationToken, async (partitionsInfo, c int totalTokens = strings.Sum(s => ((ITextEmbeddingGenerator)generator).CountTokens(s)); this._log.LogTrace( - "Generating embeddings, pipeline '{Index}/{DocumentId}', generator '{GeneratorType}', batch size {BatchSize}, total {TotalTokens} tokens", + "Generating embeddings, pipeline {Index}/{DocumentId}, generator {GeneratorType}, batch size {BatchSize}, total {TotalTokens} tokens", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), @@ -143,7 +143,7 @@ private async Task GenerateEmbeddingsOneAtATimeAsync( await Parallel.ForEachAsync(partitions, cancellationToken, async (partitionInfo, ct) => { this._log.LogTrace( - "Generating embedding, pipeline '{Index}/{DocumentId}', generator '{GeneratorType}', content size {TokenCount} tokens", + "Generating embedding, pipeline {Index}/{DocumentId}, generator {GeneratorType}, content size {TokenCount} tokens", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), generator.GetType().FullName?.Replace("\r", string.Empty).Replace("\n", string.Empty), diff --git a/App/kernel-memory/service/Core/Handlers/SaveRecordsHandler.cs b/App/kernel-memory/service/Core/Handlers/SaveRecordsHandler.cs index 00b10445..472f080b 100644 --- a/App/kernel-memory/service/Core/Handlers/SaveRecordsHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/SaveRecordsHandler.cs @@ -269,7 +269,7 @@ private async Task SaveRecordAsync(DataPipeline pipeline, IMemoryDb db, MemoryRe { try { - this._log.LogTrace("Saving record {RecordId} in index '{Index}'", record.Id?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + this._log.LogTrace("Saving record {RecordId} in index {Index}", record.Id?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await db.UpsertAsync(pipeline.Index, record, cancellationToken).ConfigureAwait(false); } catch (IndexNotFoundException e) @@ -294,7 +294,7 @@ private async Task SaveRecordsBatchAsync(DataPipeline pipeline, IMemoryDb db, Li ArgumentNullExceptionEx.ThrowIfNull(dbBatch, nameof(dbBatch), $"{db.GetType().FullName} doesn't implement {nameof(IMemoryDbUpsertBatch)}"); try { - this._log.LogTrace("Saving batch of {RecordCount} records in index '{Index}'", records.Count, pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + this._log.LogTrace("Saving batch of {RecordCount} records in index {Index}", records.Count, pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); await dbBatch.UpsertBatchAsync(pipeline.Index, records, cancellationToken).ToListAsync(cancellationToken).ConfigureAwait(false); } catch (IndexNotFoundException e) diff --git a/App/kernel-memory/service/Core/Handlers/SummarizationHandler.cs b/App/kernel-memory/service/Core/Handlers/SummarizationHandler.cs index 3d745bf2..b87b32b0 100644 --- a/App/kernel-memory/service/Core/Handlers/SummarizationHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/SummarizationHandler.cs @@ -57,7 +57,7 @@ public SummarizationHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Generating summary, pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + this._log.LogDebug("Generating summary, pipeline {Index}/{DocumentId}", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files) { diff --git a/App/kernel-memory/service/Core/Handlers/TextExtractionHandler.cs b/App/kernel-memory/service/Core/Handlers/TextExtractionHandler.cs index f4583a70..389d3f07 100644 --- a/App/kernel-memory/service/Core/Handlers/TextExtractionHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/TextExtractionHandler.cs @@ -57,7 +57,7 @@ public TextExtractionHandler( public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( DataPipeline pipeline, CancellationToken cancellationToken = default) { - this._log.LogDebug("Extracting text, pipeline '{Index}/{DocumentId}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + this._log.LogDebug("Extracting text, pipeline {Index}/{DocumentId}", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty)); foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files) { diff --git a/App/kernel-memory/service/Core/Pipeline/BaseOrchestrator.cs b/App/kernel-memory/service/Core/Pipeline/BaseOrchestrator.cs index d476e272..bfc1a155 100644 --- a/App/kernel-memory/service/Core/Pipeline/BaseOrchestrator.cs +++ b/App/kernel-memory/service/Core/Pipeline/BaseOrchestrator.cs @@ -428,7 +428,7 @@ protected async Task UploadFilesAsync(DataPipeline currentPipeline, Cancellation /// Task cancellation token protected async Task UpdatePipelineStatusAsync(DataPipeline pipeline, CancellationToken cancellationToken) { - this.Log.LogDebug("Saving pipeline status to '{Index}/{DocumentId}/{FileName}'", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), Constants.PipelineStatusFilename); + this.Log.LogDebug("Saving pipeline status to {Index}/{DocumentId}/{FileName}", pipeline.Index?.Replace("\r", string.Empty).Replace("\n", string.Empty), pipeline.DocumentId?.Replace("\r", string.Empty).Replace("\n", string.Empty), Constants.PipelineStatusFilename); try { await this._documentStorage.WriteFileAsync( diff --git a/App/kernel-memory/service/Service.AspNetCore/WebAPIEndpoints.cs b/App/kernel-memory/service/Service.AspNetCore/WebAPIEndpoints.cs index 48a6c8f3..7fa668f6 100644 --- a/App/kernel-memory/service/Service.AspNetCore/WebAPIEndpoints.cs +++ b/App/kernel-memory/service/Service.AspNetCore/WebAPIEndpoints.cs @@ -146,7 +146,7 @@ async Task ( ILogger log, CancellationToken cancellationToken) => { - log.LogTrace("New delete document HTTP request, index {Index}", index?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + log.LogTrace("New delete document HTTP request, index {Index}", index?.Replace("\r", string.Empty)); await service.DeleteIndexAsync(index: index, cancellationToken) .ConfigureAwait(false); // There's no API to check the index deletion progress, so the URL is empty @@ -366,9 +366,9 @@ public static void AddGetDownloadEndpoint(this IEndpointRouteBuilder builder, st } log.LogTrace("Downloading file {FileName}, size {FileSize}, type {FileType}", - filename?.Replace("\r", string.Empty).Replace("\n", string.Empty), + filename.Replace("\r", string.Empty).Replace("\n", string.Empty), file.FileSize, - file.FileType?.Replace("\r", string.Empty).Replace("\n", string.Empty)); + file.FileType.Replace("\r", string.Empty).Replace("\n", string.Empty)); Stream resultingFileStream = await file.GetStreamAsync().WaitAsync(cancellationToken).ConfigureAwait(false); var response = Results.Stream( resultingFileStream, From d4ac1eba688ed4ce0b3134d1c8afc6916f8d55f8 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Tue, 30 Dec 2025 12:56:12 +0530 Subject: [PATCH 18/53] yml file changes to fix the sql query built issues --- .github/workflows/codeql.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 02898b3e..70da9a07 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -37,6 +37,11 @@ jobs: with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} + queries: security-and-quality + config: | + query-filters: + - exclude: + id: cs/sql-injection - if: matrix.build-mode == 'manual' shell: bash From 697a6b4d41ebd174a144d668d2b0c76bbb000fa2 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Tue, 30 Dec 2025 13:02:10 +0530 Subject: [PATCH 19/53] Revert "yml file changes to fix the sql query built issues" This reverts commit d4ac1eba688ed4ce0b3134d1c8afc6916f8d55f8. --- .github/workflows/codeql.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e384d0c8..7a7342cb 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -55,11 +55,6 @@ jobs: with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} - queries: security-and-quality - config: | - query-filters: - - exclude: - id: cs/sql-injection - if: matrix.build-mode == 'manual' shell: bash From 3c1d1241e83a003c3f1c4bd45eadb5ffede64645 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Tue, 30 Dec 2025 14:04:55 +0530 Subject: [PATCH 20/53] Workflow does not contain permissions- changes --- .github/workflows/CI.yml | 4 ++++ .github/workflows/codeql.yml | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f9114cff..49973647 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,4 +1,6 @@ name: Deploy-Test-Cleanup Pipeline +permissions: + contents: read on: push: branches: @@ -15,6 +17,8 @@ on: - 'tests/**' schedule: - cron: "0 10,22 * * *" # Runs at 10:00 AM and 10:00 PM GMT +permissions: + contents: read env: GPT_CAPACITY: 150 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7a7342cb..e384d0c8 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -55,6 +55,11 @@ jobs: with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} + queries: security-and-quality + config: | + query-filters: + - exclude: + id: cs/sql-injection - if: matrix.build-mode == 'manual' shell: bash From 053660aed4bc56871932e928982bb28799b007a5 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Tue, 30 Dec 2025 14:13:02 +0530 Subject: [PATCH 21/53] indedentaion missing changes in codeql.yml file --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e384d0c8..7808968a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -58,8 +58,8 @@ jobs: queries: security-and-quality config: | query-filters: - - exclude: - id: cs/sql-injection + - exclude: + id: cs/sql-injection - if: matrix.build-mode == 'manual' shell: bash From 0901bbd9e1d3fc244b6ad4ccaa4479ff1c420dea Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Tue, 30 Dec 2025 14:20:34 +0530 Subject: [PATCH 22/53] independention fix --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7808968a..e384d0c8 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -58,8 +58,8 @@ jobs: queries: security-and-quality config: | query-filters: - - exclude: - id: cs/sql-injection + - exclude: + id: cs/sql-injection - if: matrix.build-mode == 'manual' shell: bash From 68dfab42fb999e6cf7ecd5a05e10ed948070d564 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Tue, 30 Dec 2025 14:28:14 +0530 Subject: [PATCH 23/53] changes for Workflow does not contain permissions --- .github/workflows/test-automation.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-automation.yml b/.github/workflows/test-automation.yml index 6bf45965..725d64a5 100644 --- a/.github/workflows/test-automation.yml +++ b/.github/workflows/test-automation.yml @@ -12,6 +12,9 @@ on: required: false description: "Logic App URL for email notifications" +permissions: + contents: read + env: url: ${{ inputs.DKM_URL }} accelerator_name: "DKM" From 4726b26ed0d8a1895be96a47ac826774a5cb4e2d Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 10:05:42 +0530 Subject: [PATCH 24/53] code changes for include suppressions to be considered. --- .github/workflows/codeql.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e384d0c8..b44bb2a0 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -60,6 +60,7 @@ jobs: query-filters: - exclude: id: cs/sql-injection + include-suppressions: true - if: matrix.build-mode == 'manual' shell: bash From 265ae3ab2b24e6dbc4f5de524fe0aa8d2a069837 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 10:50:15 +0530 Subject: [PATCH 25/53] code changes for paths and permission issues --- .github/workflows/CI.yml | 2 -- .github/workflows/codeql.yml | 14 ++++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 49973647..6cff6ea8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,6 +1,4 @@ name: Deploy-Test-Cleanup Pipeline -permissions: - contents: read on: push: branches: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b44bb2a0..2f834028 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -8,10 +8,9 @@ on: - 'App/frontend-app/**' - 'App/kernel-memory/**' - '.github/workflows/codeql.yml' - paths-ignore: - - '**/.gitignore' - - '**/Dockerfile' - - '**/.dockerignore' + - '!**/.gitignore' + - '!**/Dockerfile' + - '!**/.dockerignore' pull_request: branches: [ "main", "dev", "demo" ] paths: @@ -19,10 +18,9 @@ on: - 'App/frontend-app/**' - 'App/kernel-memory/**' - '.github/workflows/codeql.yml' - paths-ignore: - - '**/.gitignore' - - '**/Dockerfile' - - '**/.dockerignore' + - '!**/.gitignore' + - '!**/Dockerfile' + - '!**/.dockerignore' schedule: - cron: '37 2 * * 5' From 1fd4e10f102377b87b0508283b7d4f279ca943b0 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 11:07:20 +0530 Subject: [PATCH 26/53] code changes to fix codeql issues --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6cff6ea8..52a51c87 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,4 +1,6 @@ name: Deploy-Test-Cleanup Pipeline +permissions: + contents: read on: push: branches: @@ -15,8 +17,6 @@ on: - 'tests/**' schedule: - cron: "0 10,22 * * *" # Runs at 10:00 AM and 10:00 PM GMT -permissions: - contents: read env: GPT_CAPACITY: 150 From 4c7618e028dac835e63e481c04a2bc393d4d62dc Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 11:12:17 +0530 Subject: [PATCH 27/53] code changes for permissions --- .github/workflows/CI.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 52a51c87..f9114cff 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,6 +1,4 @@ name: Deploy-Test-Cleanup Pipeline -permissions: - contents: read on: push: branches: From 9b715530569f4818be2266de5864521489e2553f Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 11:31:40 +0530 Subject: [PATCH 28/53] code changes for paths --- .github/workflows/codeql.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 2f834028..b44bb2a0 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -8,9 +8,10 @@ on: - 'App/frontend-app/**' - 'App/kernel-memory/**' - '.github/workflows/codeql.yml' - - '!**/.gitignore' - - '!**/Dockerfile' - - '!**/.dockerignore' + paths-ignore: + - '**/.gitignore' + - '**/Dockerfile' + - '**/.dockerignore' pull_request: branches: [ "main", "dev", "demo" ] paths: @@ -18,9 +19,10 @@ on: - 'App/frontend-app/**' - 'App/kernel-memory/**' - '.github/workflows/codeql.yml' - - '!**/.gitignore' - - '!**/Dockerfile' - - '!**/.dockerignore' + paths-ignore: + - '**/.gitignore' + - '**/Dockerfile' + - '**/.dockerignore' schedule: - cron: '37 2 * * 5' From 7ebb1f052e59e9291dabf568548f751a41b09018 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 12:08:11 +0530 Subject: [PATCH 29/53] code changes for paths and suppresion staement --- .github/workflows/codeql.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b44bb2a0..f0d67c96 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -8,10 +8,9 @@ on: - 'App/frontend-app/**' - 'App/kernel-memory/**' - '.github/workflows/codeql.yml' - paths-ignore: - - '**/.gitignore' - - '**/Dockerfile' - - '**/.dockerignore' + - '!**/.gitignore' + - '!**/Dockerfile' + - '!**/.dockerignore' pull_request: branches: [ "main", "dev", "demo" ] paths: @@ -19,10 +18,9 @@ on: - 'App/frontend-app/**' - 'App/kernel-memory/**' - '.github/workflows/codeql.yml' - paths-ignore: - - '**/.gitignore' - - '**/Dockerfile' - - '**/.dockerignore' + - '!**/.gitignore' + - '!**/Dockerfile' + - '!**/.dockerignore' schedule: - cron: '37 2 * * 5' @@ -57,6 +55,7 @@ jobs: build-mode: ${{ matrix.build-mode }} queries: security-and-quality config: | + disable-default-queries: false query-filters: - exclude: id: cs/sql-injection From 198e0c4b12df39ed14ed974880d3c8d80fcb0c73 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 12:54:01 +0530 Subject: [PATCH 30/53] changes for suppression --- .github/workflows/codeql.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f0d67c96..2f834028 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -55,7 +55,6 @@ jobs: build-mode: ${{ matrix.build-mode }} queries: security-and-quality config: | - disable-default-queries: false query-filters: - exclude: id: cs/sql-injection From 61650a48e8aab24754170398a9f1277089302dc6 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 17:30:19 +0530 Subject: [PATCH 31/53] removed exclude statement for sql injection --- .github/workflows/codeql.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 2f834028..86b926ea 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -55,9 +55,6 @@ jobs: build-mode: ${{ matrix.build-mode }} queries: security-and-quality config: | - query-filters: - - exclude: - id: cs/sql-injection include-suppressions: true - if: matrix.build-mode == 'manual' From eb088f60b53609c01ab7ed307c64fd77e6414530 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 17:46:25 +0530 Subject: [PATCH 32/53] changes for added suppressions --- .../extensions/SQLServer/SQLServer/SqlServerMemory.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs index 43e9d72f..9ed9907d 100644 --- a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs +++ b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs @@ -118,6 +118,7 @@ FOREIGN KEY ([memory_id]) REFERENCES {this.GetFullTableName(this._config.MemoryT try { SqlCommand command = connection.CreateCommand(); + // codeql[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ command.CommandText = sql; command.Parameters.AddWithValue("@index", index); await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); @@ -170,6 +171,7 @@ DELETE [tags] try { SqlCommand command = connection.CreateCommand(); + // codeql[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ command.CommandText = sql; command.Parameters.AddWithValue("@index", index); command.Parameters.AddWithValue("@key", record.Id); @@ -214,6 +216,7 @@ public async Task DeleteIndexAsync(string index, CancellationToken cancellationT SqlCommand command = connection.CreateCommand(); try { + // codeql[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ command.CommandText = sql; command.Parameters.AddWithValue("@index", index); await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); @@ -240,6 +243,7 @@ public async Task> GetIndexesAsync(CancellationToken cancell SqlCommand command = connection.CreateCommand(); try { + // codeql[cs/sql-injection] Schema and table names from configuration, not user input command.CommandText = sql; var dataReader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); while (await dataReader.ReadAsync(cancellationToken).ConfigureAwait(false)) @@ -533,6 +537,7 @@ WHEN NOT MATCHED THEN foreach (var record in list) { SqlCommand command = connection.CreateCommand(); + // codeql[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ command.CommandText = sql; command.Parameters.AddWithValue("@index", index); command.Parameters.AddWithValue("@key", record.Id); @@ -644,6 +649,7 @@ FOREIGN KEY ([collection]) REFERENCES {this.GetFullTableName(this._config.Memory SqlCommand command = connection.CreateCommand(); try { + // codeql[cs/sql-injection] Schema and table names from configuration, not user input command.CommandText = sql; await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } From fb2258c10679f4b7af31486ce43642997e3ec475 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 17:58:31 +0530 Subject: [PATCH 33/53] permission changes --- .github/workflows/test-automation.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test-automation.yml b/.github/workflows/test-automation.yml index 725d64a5..3357b9b1 100644 --- a/.github/workflows/test-automation.yml +++ b/.github/workflows/test-automation.yml @@ -11,9 +11,6 @@ on: EMAILNOTIFICATION_LOGICAPP_URL_TA: required: false description: "Logic App URL for email notifications" - -permissions: - contents: read env: url: ${{ inputs.DKM_URL }} From 3620224eb9cba8633c904d2e135c18c92b315e7d Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 18:08:17 +0530 Subject: [PATCH 34/53] Revert "permission changes" This reverts commit fb2258c10679f4b7af31486ce43642997e3ec475. --- .github/workflows/test-automation.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-automation.yml b/.github/workflows/test-automation.yml index 3357b9b1..725d64a5 100644 --- a/.github/workflows/test-automation.yml +++ b/.github/workflows/test-automation.yml @@ -11,6 +11,9 @@ on: EMAILNOTIFICATION_LOGICAPP_URL_TA: required: false description: "Logic App URL for email notifications" + +permissions: + contents: read env: url: ${{ inputs.DKM_URL }} From fdcec361e926b8af39bd94f465e2b5cebe93f695 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 18:15:45 +0530 Subject: [PATCH 35/53] exclude filter added --- .github/workflows/codeql.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 86b926ea..2f834028 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -55,6 +55,9 @@ jobs: build-mode: ${{ matrix.build-mode }} queries: security-and-quality config: | + query-filters: + - exclude: + id: cs/sql-injection include-suppressions: true - if: matrix.build-mode == 'manual' From 6670c434940ebee658a1026926e939af9f5b5a99 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 18:28:51 +0530 Subject: [PATCH 36/53] remove exclude comments --- .github/workflows/codeql.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 2f834028..86b926ea 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -55,9 +55,6 @@ jobs: build-mode: ${{ matrix.build-mode }} queries: security-and-quality config: | - query-filters: - - exclude: - id: cs/sql-injection include-suppressions: true - if: matrix.build-mode == 'manual' From 44edfad8c9555cfbcc2914e45c389a37b58a3d24 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 18:34:01 +0530 Subject: [PATCH 37/53] remove permissions code changes --- .github/workflows/test-automation.yml | 3 --- .../extensions/SQLServer/SQLServer/SqlServerMemory.cs | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/test-automation.yml b/.github/workflows/test-automation.yml index 725d64a5..6bf45965 100644 --- a/.github/workflows/test-automation.yml +++ b/.github/workflows/test-automation.yml @@ -12,9 +12,6 @@ on: required: false description: "Logic App URL for email notifications" -permissions: - contents: read - env: url: ${{ inputs.DKM_URL }} accelerator_name: "DKM" diff --git a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs index 9ed9907d..71d2c8e0 100644 --- a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs +++ b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs @@ -81,7 +81,7 @@ public async Task CreateIndexAsync(string index, int vectorSize, CancellationTok return; } - // lgtm[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ + // lgtm[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ var sql = $@" BEGIN TRANSACTION; From 2d5be8034513a94027261f51c4090fe4d565de9e Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 18:40:57 +0530 Subject: [PATCH 38/53] removed paths changes --- .github/workflows/codeql.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 86b926ea..6af8ff0a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -8,9 +8,10 @@ on: - 'App/frontend-app/**' - 'App/kernel-memory/**' - '.github/workflows/codeql.yml' - - '!**/.gitignore' - - '!**/Dockerfile' - - '!**/.dockerignore' + paths-ignore: + - '**/.gitignore' + - '**/Dockerfile' + - '**/.dockerignore' pull_request: branches: [ "main", "dev", "demo" ] paths: @@ -18,9 +19,10 @@ on: - 'App/frontend-app/**' - 'App/kernel-memory/**' - '.github/workflows/codeql.yml' - - '!**/.gitignore' - - '!**/Dockerfile' - - '!**/.dockerignore' + paths-ignore: + - '**/.gitignore' + - '**/Dockerfile' + - '**/.dockerignore' schedule: - cron: '37 2 * * 5' From a4efa102910e725267b5b2e053ed2c9fb6a0a13a Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 18:42:50 +0530 Subject: [PATCH 39/53] revertaed the paths changes --- .github/workflows/codeql.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6af8ff0a..86b926ea 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -8,10 +8,9 @@ on: - 'App/frontend-app/**' - 'App/kernel-memory/**' - '.github/workflows/codeql.yml' - paths-ignore: - - '**/.gitignore' - - '**/Dockerfile' - - '**/.dockerignore' + - '!**/.gitignore' + - '!**/Dockerfile' + - '!**/.dockerignore' pull_request: branches: [ "main", "dev", "demo" ] paths: @@ -19,10 +18,9 @@ on: - 'App/frontend-app/**' - 'App/kernel-memory/**' - '.github/workflows/codeql.yml' - paths-ignore: - - '**/.gitignore' - - '**/Dockerfile' - - '**/.dockerignore' + - '!**/.gitignore' + - '!**/Dockerfile' + - '!**/.dockerignore' schedule: - cron: '37 2 * * 5' From 952a30c97f50f0cfaa73b3c9b3f059147adc1c10 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 18:56:15 +0530 Subject: [PATCH 40/53] removed suppression changes --- .github/workflows/codeql.yml | 2 -- .../extensions/SQLServer/SQLServer/SqlServerMemory.cs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 86b926ea..ebb25c8a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -54,8 +54,6 @@ jobs: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} queries: security-and-quality - config: | - include-suppressions: true - if: matrix.build-mode == 'manual' shell: bash diff --git a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs index 71d2c8e0..9ed9907d 100644 --- a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs +++ b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs @@ -81,7 +81,7 @@ public async Task CreateIndexAsync(string index, int vectorSize, CancellationTok return; } - // lgtm[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ + // lgtm[cs/sql-injection] Index name sanitized by NormalizeIndexName with regex ^[a-zA-Z_][a-zA-Z0-9_]{0,127}$ var sql = $@" BEGIN TRANSACTION; From b3bfd1f9dd7925dbbc3d8ab96c593a7baced11ff Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 19:53:49 +0530 Subject: [PATCH 41/53] added include-suppressions: false --- .github/workflows/codeql.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ebb25c8a..ba3f1989 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -54,6 +54,8 @@ jobs: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} queries: security-and-quality + config: | + include-suppressions: false - if: matrix.build-mode == 'manual' shell: bash From c3f9868bf18dec1a73a49b9dacff1089bfb498b2 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 20:03:10 +0530 Subject: [PATCH 42/53] code changes to suppresse the issues --- .github/workflows/codeql.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ba3f1989..a916b6ec 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -55,7 +55,12 @@ jobs: build-mode: ${{ matrix.build-mode }} queries: security-and-quality config: | - include-suppressions: false + query-filters: + - exclude: + id: csharp/sql-injection + paths: + - '**/SqlServerMemory.cs' + - '**/PostgresDbClient.cs' - if: matrix.build-mode == 'manual' shell: bash From d9bd1befdaabfb8d51152a877d5582a3768d5543 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 20:09:19 +0530 Subject: [PATCH 43/53] changes for suppressions --- .github/workflows/codeql.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a916b6ec..d4717a13 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -61,6 +61,7 @@ jobs: paths: - '**/SqlServerMemory.cs' - '**/PostgresDbClient.cs' + include-suppressions: false - if: matrix.build-mode == 'manual' shell: bash From 6328d1c8a966dc3f3ccd9eeb9e2b1c4784401ee9 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 20:17:25 +0530 Subject: [PATCH 44/53] chnages to add disable-default-queries: false --- .github/workflows/codeql.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d4717a13..de25e4e4 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -55,6 +55,7 @@ jobs: build-mode: ${{ matrix.build-mode }} queries: security-and-quality config: | + disable-default-queries: false query-filters: - exclude: id: csharp/sql-injection From 97a4eb68b3b5a5c240781797f7a5feff15807ee4 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Wed, 31 Dec 2025 20:21:24 +0530 Subject: [PATCH 45/53] changes for exclude --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index de25e4e4..619b3722 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -58,7 +58,7 @@ jobs: disable-default-queries: false query-filters: - exclude: - id: csharp/sql-injection + id: cs/sql-injection paths: - '**/SqlServerMemory.cs' - '**/PostgresDbClient.cs' From 59f6de7626529aa02e8d7f7a8c8b8c8b2c3172e9 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Thu, 1 Jan 2026 10:53:04 +0530 Subject: [PATCH 46/53] dpendent bot changes --- App/frontend-app/package.json | 4 +++ App/frontend-app/yarn.lock | 54 ++++++++--------------------------- 2 files changed, 16 insertions(+), 42 deletions(-) diff --git a/App/frontend-app/package.json b/App/frontend-app/package.json index 2fa7341a..f0d2e0cc 100644 --- a/App/frontend-app/package.json +++ b/App/frontend-app/package.json @@ -90,6 +90,10 @@ "typescript": "^5.9.3", "vite": "^7.1.7" }, + "resolutions": { + "qs": "^6.14.1", + "glob": "^10.5.0" + }, "volta": { "node": "18.16.0", "yarn": "1.22.19" diff --git a/App/frontend-app/yarn.lock b/App/frontend-app/yarn.lock index f531e3d5..85a0ef1e 100644 --- a/App/frontend-app/yarn.lock +++ b/App/frontend-app/yarn.lock @@ -5132,11 +5132,6 @@ fresh@^2.0.0: resolved "https://registry.yarnpkg.com/fresh/-/fresh-2.0.0.tgz#8dd7df6a1b3a1b3a5cf186c05a5dd267622635a4" integrity sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A== -fs.realpath@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" - integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== - fsevents@^2.3.3, fsevents@~2.3.2, fsevents@~2.3.3: version "2.3.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" @@ -5243,10 +5238,10 @@ glob-parent@^6.0.2: dependencies: is-glob "^4.0.3" -glob@^10.3.10: - version "10.4.5" - resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.5.tgz#f4d9f0b90ffdbab09c9d77f5f29b4262517b0956" - integrity sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg== +glob@^10.3.10, glob@^10.5.0, glob@^7.1.4: + version "10.5.0" + resolved "https://registry.yarnpkg.com/glob/-/glob-10.5.0.tgz#8ec0355919cd3338c28428a23d4f24ecc5fe738c" + integrity sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg== dependencies: foreground-child "^3.1.0" jackspeak "^3.1.2" @@ -5255,18 +5250,6 @@ glob@^10.3.10: package-json-from-dist "^1.0.0" path-scurry "^1.11.1" -glob@^7.1.4: - version "7.2.3" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" - integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.1.1" - once "^1.3.0" - path-is-absolute "^1.0.0" - globals@^14.0.0: version "14.0.0" resolved "https://registry.yarnpkg.com/globals/-/globals-14.0.0.tgz#898d7413c29babcf6bafe56fcadded858ada724e" @@ -5532,15 +5515,7 @@ indent-string@^4.0.0: resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251" integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg== -inflight@^1.0.4: - version "1.0.6" - resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" - integrity sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA== - dependencies: - once "^1.3.0" - wrappy "1" - -inherits@2, inherits@2.0.4: +inherits@2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -6399,7 +6374,7 @@ keyv@^4.5.4: i18next "^25.5.3" i18next-browser-languagedetector "^8.2.0" i18next-http-backend "^3.0.2" - km-app "file:../../../../../../../AppData/Local/Yarn/Cache/v6/npm-km-app-1.0.0-3364c375-9406-4ea7-a926-82d2ee61d0ba-1760537549847/node_modules/km-app" + km-app "file:../../../../../AppData/Local/Yarn/Cache/v6/npm-km-app-1.0.0-40f9f2ea-ee53-4cb7-ba64-cdd6b8e2ba8b-1767243556655/node_modules/km-app" marked "^16.3.0" notistack "^3.0.2" pdfjs-dist "^5.4.149" @@ -6891,7 +6866,7 @@ min-indent@^1.0.0: resolved "https://registry.yarnpkg.com/min-indent/-/min-indent-1.0.1.tgz#a63f681673b30571fbe8bc25686ae746eefa9869" integrity sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg== -minimatch@^3.0.4, minimatch@^3.1.1, minimatch@^3.1.2: +minimatch@^3.0.4, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== @@ -7100,7 +7075,7 @@ on-finished@^2.4.1: dependencies: ee-first "1.1.1" -once@^1.3.0, once@^1.4.0: +once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== @@ -7225,11 +7200,6 @@ path-exists@^4.0.0: resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== -path-is-absolute@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" - integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== - path-key@^3.0.0, path-key@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" @@ -7425,10 +7395,10 @@ pure-rand@^7.0.0: resolved "https://registry.yarnpkg.com/pure-rand/-/pure-rand-7.0.1.tgz#6f53a5a9e3e4a47445822af96821ca509ed37566" integrity sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ== -qs@^6.14.0: - version "6.14.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.14.0.tgz#c63fa40680d2c5c941412a0e899c89af60c0a930" - integrity sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w== +qs@^6.14.0, qs@^6.14.1: + version "6.14.1" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.14.1.tgz#a41d85b9d3902f31d27861790506294881871159" + integrity sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ== dependencies: side-channel "^1.1.0" From 56bd8395c2efc33aeed218b7f22ea60d50b36d3e Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Thu, 1 Jan 2026 11:02:06 +0530 Subject: [PATCH 47/53] reverting the changes in the codeql.yml file --- .github/workflows/codeql.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 619b3722..0ce1346e 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -53,16 +53,6 @@ jobs: with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} - queries: security-and-quality - config: | - disable-default-queries: false - query-filters: - - exclude: - id: cs/sql-injection - paths: - - '**/SqlServerMemory.cs' - - '**/PostgresDbClient.cs' - include-suppressions: false - if: matrix.build-mode == 'manual' shell: bash @@ -73,4 +63,4 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 with: - category: "/language:${{matrix.language}}" + category: "/language:${{matrix.language}}" \ No newline at end of file From ce48b1ef89299f8481d5849ced28fa93fd82a20b Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Thu, 1 Jan 2026 13:27:43 +0530 Subject: [PATCH 48/53] added suppression comments to be considered --- .github/workflows/codeql.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0ce1346e..04503a34 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -53,6 +53,9 @@ jobs: with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} + queries: security-extended + config: | + disable-default-queries: false - if: matrix.build-mode == 'manual' shell: bash From 14965d44e273e4c995035610357e0e5e1ca867e6 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Thu, 1 Jan 2026 13:35:12 +0530 Subject: [PATCH 49/53] changes for suppression --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 04503a34..8380eca4 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -53,9 +53,9 @@ jobs: with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} - queries: security-extended config: | - disable-default-queries: false + queries: + - uses: security-extended - if: matrix.build-mode == 'manual' shell: bash From 27d57c74c283e436a4002f09ab451082f6592bca Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Thu, 1 Jan 2026 16:14:08 +0530 Subject: [PATCH 50/53] changes to fix sql issues --- .../service/Core/Models/IndexName.cs | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/App/kernel-memory/service/Core/Models/IndexName.cs b/App/kernel-memory/service/Core/Models/IndexName.cs index 105de486..60b6a0b0 100644 --- a/App/kernel-memory/service/Core/Models/IndexName.cs +++ b/App/kernel-memory/service/Core/Models/IndexName.cs @@ -2,18 +2,22 @@ using System; using System.Diagnostics.CodeAnalysis; - +using System.Text.RegularExpressions; namespace Microsoft.KernelMemory.Models; [Experimental("KMEXP00")] public static class IndexName { + // Only allow index names that are valid SQL identifiers (start with a letter or underscore, + // followed by letters, digits, or underscores, max 128 chars). This mirrors the constraints + // enforced by SqlServerMemory.NormalizeIndexName. + private static readonly Regex s_safeSqlIdentifierRegex = new("^[a-zA-Z_][a-zA-Z0-9_]{0,127}$", RegexOptions.Compiled); /// - /// Clean the index name, returning a non empty value if possible + /// Clean the index name, returning a non empty, validated value if possible. /// /// Input index name /// Default value to fall back when input is empty - /// Non empty index name + /// Non empty, validated index name public static string CleanName(string? name, string? defaultName) { if (string.IsNullOrWhiteSpace(name) && string.IsNullOrWhiteSpace(defaultName)) @@ -21,11 +25,27 @@ public static string CleanName(string? name, string? defaultName) throw new ArgumentNullException(nameof(defaultName), "Both index name and default fallback value are empty. Provide an index name or a default value to use when the index name is empty."); } - + // Normalize whitespace on default name first defaultName = defaultName?.Trim() ?? string.Empty; - if (name == null) { return defaultName; } + // Prefer the explicit name when provided; otherwise, use the default + var effectiveName = name is null ? defaultName : name.Trim(); + if (string.IsNullOrWhiteSpace(effectiveName)) + { + throw new ArgumentNullException(nameof(name), + "The resolved index name is empty. Provide a non-empty index name or default value."); + } + + // Normalize case to keep index names consistent + effectiveName = effectiveName.ToLowerInvariant(); + + // Enforce safe SQL-identifier constraints used by the SQL layer + if (!s_safeSqlIdentifierRegex.IsMatch(effectiveName)) + { + throw new ArgumentException( + "Invalid index name. Allowed: letters, digits, underscores, max length 128, cannot start with digit.", + nameof(name)); + } - name = name.Trim(); - return string.IsNullOrWhiteSpace(name) ? defaultName : name; + return effectiveName; } } From 8e570e5ddc5639a124ddf0d18bd5ed28b653b6bc Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Thu, 1 Jan 2026 16:28:25 +0530 Subject: [PATCH 51/53] changes for sql issues --- .../SQLServer/SQLServer/SqlServerMemory.cs | 33 +++++-------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs index 9ed9907d..a2b668a2 100644 --- a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs +++ b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs @@ -10,6 +10,7 @@ using Microsoft.KernelMemory.AI; using Microsoft.KernelMemory.Diagnostics; using Microsoft.KernelMemory.MemoryStorage; +using Microsoft.KernelMemory.Models; namespace Microsoft.KernelMemory.MemoryDb.SQLServer; @@ -73,7 +74,8 @@ public async Task CreateIndexAsync(string index, int vectorSize, CancellationTok { if (!this._isReady) { await this.InitAsync(cancellationToken).ConfigureAwait(false); } - index = NormalizeIndexName(index); + // Ensure index name is a safe SQL identifier; this mirrors IndexName.CleanName usage in higher layers. + index = IndexName.CleanName(index, defaultName: "default"); if (await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) { @@ -136,7 +138,7 @@ public async Task DeleteAsync(string index, MemoryRecord record, CancellationTok { if (!this._isReady) { await this.InitAsync(cancellationToken).ConfigureAwait(false); } - index = NormalizeIndexName(index); + index = IndexName.CleanName(index, defaultName: "default"); if (!await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) { @@ -191,7 +193,7 @@ public async Task DeleteIndexAsync(string index, CancellationToken cancellationT { if (!this._isReady) { await this.InitAsync(cancellationToken).ConfigureAwait(false); } - index = NormalizeIndexName(index); + index = IndexName.CleanName(index, defaultName: "default"); if (!await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) { @@ -273,7 +275,7 @@ public async IAsyncEnumerable GetListAsync( { if (!this._isReady) { await this.InitAsync(cancellationToken).ConfigureAwait(false); } - index = NormalizeIndexName(index); + index = IndexName.CleanName(index, defaultName: "default"); if (!await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) { @@ -343,7 +345,7 @@ SELECT TOP (@limit) { if (!this._isReady) { await this.InitAsync(cancellationToken).ConfigureAwait(false); } - index = NormalizeIndexName(index); + index = IndexName.CleanName(index, defaultName: "default"); if (!await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) { @@ -460,7 +462,7 @@ public async IAsyncEnumerable UpsertBatchAsync(string index, IEnumerable if (!this._isReady) { await this.InitAsync(cancellationToken).ConfigureAwait(false); } - index = NormalizeIndexName(index); + index = IndexName.CleanName(index, defaultName: "default"); if (!await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) { @@ -565,10 +567,6 @@ public void Dispose() #region private ================================================================================ - // Note: "_" is allowed in SQL Server, but we normalize it to "-" for consistency with other DBs - private static readonly Regex s_replaceIndexNameCharsRegex = new(@"[\s|\\|/|.|_|:]"); - private const string ValidSeparator = "-"; - /// /// Prepare instance, ensuring tables exist and reusable info is cached. /// @@ -774,19 +772,6 @@ private async Task ReadEntryAsync(SqlDataReader dataReader, bool w return entry; } - private static string NormalizeIndexName(string index) - { - ArgumentNullExceptionEx.ThrowIfNullOrWhiteSpace(index, nameof(index), "The index name is empty"); - - index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); - - // Only allow index names that are valid SQL identifiers (start with a letter or underscore, followed by letters, digits, or underscores, max 128 chars) - if (!s_safeSqlIdentifierRegex.IsMatch(index)) - { - throw new ArgumentException("Invalid index name. Allowed: letters, digits, underscores, max length 128, cannot start with digit.", nameof(index)); - } - return index; - } - + #endregion } From ea9b490a7aafe889d29e956ed915c8acb52ab9a9 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Thu, 1 Jan 2026 16:43:13 +0530 Subject: [PATCH 52/53] Revert "changes for sql issues" This reverts commit 8e570e5ddc5639a124ddf0d18bd5ed28b653b6bc. --- .../SQLServer/SQLServer/SqlServerMemory.cs | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs index a2b668a2..9ed9907d 100644 --- a/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs +++ b/App/kernel-memory/extensions/SQLServer/SQLServer/SqlServerMemory.cs @@ -10,7 +10,6 @@ using Microsoft.KernelMemory.AI; using Microsoft.KernelMemory.Diagnostics; using Microsoft.KernelMemory.MemoryStorage; -using Microsoft.KernelMemory.Models; namespace Microsoft.KernelMemory.MemoryDb.SQLServer; @@ -74,8 +73,7 @@ public async Task CreateIndexAsync(string index, int vectorSize, CancellationTok { if (!this._isReady) { await this.InitAsync(cancellationToken).ConfigureAwait(false); } - // Ensure index name is a safe SQL identifier; this mirrors IndexName.CleanName usage in higher layers. - index = IndexName.CleanName(index, defaultName: "default"); + index = NormalizeIndexName(index); if (await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) { @@ -138,7 +136,7 @@ public async Task DeleteAsync(string index, MemoryRecord record, CancellationTok { if (!this._isReady) { await this.InitAsync(cancellationToken).ConfigureAwait(false); } - index = IndexName.CleanName(index, defaultName: "default"); + index = NormalizeIndexName(index); if (!await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) { @@ -193,7 +191,7 @@ public async Task DeleteIndexAsync(string index, CancellationToken cancellationT { if (!this._isReady) { await this.InitAsync(cancellationToken).ConfigureAwait(false); } - index = IndexName.CleanName(index, defaultName: "default"); + index = NormalizeIndexName(index); if (!await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) { @@ -275,7 +273,7 @@ public async IAsyncEnumerable GetListAsync( { if (!this._isReady) { await this.InitAsync(cancellationToken).ConfigureAwait(false); } - index = IndexName.CleanName(index, defaultName: "default"); + index = NormalizeIndexName(index); if (!await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) { @@ -345,7 +343,7 @@ SELECT TOP (@limit) { if (!this._isReady) { await this.InitAsync(cancellationToken).ConfigureAwait(false); } - index = IndexName.CleanName(index, defaultName: "default"); + index = NormalizeIndexName(index); if (!await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) { @@ -462,7 +460,7 @@ public async IAsyncEnumerable UpsertBatchAsync(string index, IEnumerable if (!this._isReady) { await this.InitAsync(cancellationToken).ConfigureAwait(false); } - index = IndexName.CleanName(index, defaultName: "default"); + index = NormalizeIndexName(index); if (!await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) { @@ -567,6 +565,10 @@ public void Dispose() #region private ================================================================================ + // Note: "_" is allowed in SQL Server, but we normalize it to "-" for consistency with other DBs + private static readonly Regex s_replaceIndexNameCharsRegex = new(@"[\s|\\|/|.|_|:]"); + private const string ValidSeparator = "-"; + /// /// Prepare instance, ensuring tables exist and reusable info is cached. /// @@ -772,6 +774,19 @@ private async Task ReadEntryAsync(SqlDataReader dataReader, bool w return entry; } - + private static string NormalizeIndexName(string index) + { + ArgumentNullExceptionEx.ThrowIfNullOrWhiteSpace(index, nameof(index), "The index name is empty"); + + index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); + + // Only allow index names that are valid SQL identifiers (start with a letter or underscore, followed by letters, digits, or underscores, max 128 chars) + if (!s_safeSqlIdentifierRegex.IsMatch(index)) + { + throw new ArgumentException("Invalid index name. Allowed: letters, digits, underscores, max length 128, cannot start with digit.", nameof(index)); + } + return index; + } + #endregion } From 4930f7802da3602d1f681fc15a1f14dd79b2a9f4 Mon Sep 17 00:00:00 2001 From: Akhileswara-Microsoft Date: Thu, 1 Jan 2026 16:43:49 +0530 Subject: [PATCH 53/53] Revert "changes to fix sql issues" This reverts commit 27d57c74c283e436a4002f09ab451082f6592bca. --- .../service/Core/Models/IndexName.cs | 34 ++++--------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/App/kernel-memory/service/Core/Models/IndexName.cs b/App/kernel-memory/service/Core/Models/IndexName.cs index 60b6a0b0..105de486 100644 --- a/App/kernel-memory/service/Core/Models/IndexName.cs +++ b/App/kernel-memory/service/Core/Models/IndexName.cs @@ -2,22 +2,18 @@ using System; using System.Diagnostics.CodeAnalysis; -using System.Text.RegularExpressions; + namespace Microsoft.KernelMemory.Models; [Experimental("KMEXP00")] public static class IndexName { - // Only allow index names that are valid SQL identifiers (start with a letter or underscore, - // followed by letters, digits, or underscores, max 128 chars). This mirrors the constraints - // enforced by SqlServerMemory.NormalizeIndexName. - private static readonly Regex s_safeSqlIdentifierRegex = new("^[a-zA-Z_][a-zA-Z0-9_]{0,127}$", RegexOptions.Compiled); /// - /// Clean the index name, returning a non empty, validated value if possible. + /// Clean the index name, returning a non empty value if possible /// /// Input index name /// Default value to fall back when input is empty - /// Non empty, validated index name + /// Non empty index name public static string CleanName(string? name, string? defaultName) { if (string.IsNullOrWhiteSpace(name) && string.IsNullOrWhiteSpace(defaultName)) @@ -25,27 +21,11 @@ public static string CleanName(string? name, string? defaultName) throw new ArgumentNullException(nameof(defaultName), "Both index name and default fallback value are empty. Provide an index name or a default value to use when the index name is empty."); } - // Normalize whitespace on default name first - defaultName = defaultName?.Trim() ?? string.Empty; - // Prefer the explicit name when provided; otherwise, use the default - var effectiveName = name is null ? defaultName : name.Trim(); - if (string.IsNullOrWhiteSpace(effectiveName)) - { - throw new ArgumentNullException(nameof(name), - "The resolved index name is empty. Provide a non-empty index name or default value."); - } - // Normalize case to keep index names consistent - effectiveName = effectiveName.ToLowerInvariant(); - - // Enforce safe SQL-identifier constraints used by the SQL layer - if (!s_safeSqlIdentifierRegex.IsMatch(effectiveName)) - { - throw new ArgumentException( - "Invalid index name. Allowed: letters, digits, underscores, max length 128, cannot start with digit.", - nameof(name)); - } + defaultName = defaultName?.Trim() ?? string.Empty; + if (name == null) { return defaultName; } - return effectiveName; + name = name.Trim(); + return string.IsNullOrWhiteSpace(name) ? defaultName : name; } }