Filter files array to include only files with tokens in backup snapshots

tools/blobstorage-backupdata/BackupJob.cs

@@ -39,7 +39,7 @@ public async Task ProcessJob()
         var existingIndex = await GetIndex();
         var changesIndex = new SortedDictionary<string, List<string>>();
         var database = MongoClient.GetDatabase("clearlydefined", null);
-        var collection = database.GetCollection<BsonDocument>("definitions-trimmed");
+        var collection = database.GetCollection<BsonDocument>("definitions-paged");
 
         // lambda to enable lazy evaluation, avoiding the `IndexOutOfRangeException` exception on empty `existingIndex`
         var beginningDateFilter = () =>
@@ -71,6 +71,21 @@ public async Task ProcessJob()
         await SaveData(cursor, existingIndex, changesIndex);
     }
 
+    private JObject FilterFilesWithoutToken(JObject definition)
+    {
+        if (definition["files"] is not JArray filesArray)
+        {
+            return definition;
+        }
+
+        var originalCount = filesArray.Count;
+        var filteredFiles = new JArray(filesArray.Where(file => file["token"] != null));
+
+        definition["files"] = filteredFiles;
+
+        return definition;
+    }
+
     private async Task SaveData(
         IAsyncCursor<BsonDocument> cursor,
         string[] existingIndex,
@@ -104,6 +119,10 @@ await Parallel.ForEachAsync(cursor.Current, async (document, _) =>
                         {
                             throw new Exception("Failed to deserialize the document.");
                         }
+
+                        // Filter files from files array without token
+                        jObject = FilterFilesWithoutToken(jObject);
+
                         var blobName = jObject.GetBlobName();
                         if (string.IsNullOrWhiteSpace(blobName))
                         {
@@ -206,4 +225,4 @@ public string RenderFilter(FilterDefinition<BsonDocument> filter, IMongoCollecti
     {
         return filter.Render(collection.DocumentSerializer, collection.Settings.SerializerRegistry).ToString();
     }
-}
+}

tools/blobstorage-backupdata/BackupJobTests.cs

@@ -173,7 +173,7 @@ public void SetUp()
         var mockMongoCollection = new Mock<IMongoCollection<BsonDocument>>();
 
         mockMongoClient.Setup(x => x.GetDatabase("clearlydefined", null)).Returns(mockDatabase.Object);
-        mockDatabase.Setup(x => x.GetCollection<BsonDocument>("definitions-trimmed", null)).Returns(mockMongoCollection.Object);
+        mockDatabase.Setup(x => x.GetCollection<BsonDocument>("definitions-paged", null)).Returns(mockMongoCollection.Object);
         mockMongoCollection
         .Setup(x => x.FindAsync(It.IsAny<FilterDefinition<BsonDocument>>(), It.IsAny<FindOptions<BsonDocument>>(), default))
         .Callback((FilterDefinition<BsonDocument> filter, FindOptions<BsonDocument, BsonDocument> options, CancellationToken token) => {
@@ -303,4 +303,145 @@ public void TestSaveData_ShouldExcludeCurrentHour() {
             .Select(x => BsonDocument.Parse(x.Value))
             .Should().HaveCount(1);
     }
-}
+    [Test]
+    public void TestFilterFilesWithoutToken_RemovesFilesWithoutToken()
+    {
+        var backupJob = new BackupJob(
+            mockBlobContainerClient.Object,
+            mockMongoClient.Object,
+            DateTime.UtcNow,
+            loggerFactory,
+            new MockFilterRenderer()
+        );
+
+        var data = new Dictionary<string, string?>
+        {
+            { indexPath, "" },
+            { "changes/2023-01-01-00", null },
+            { "npm/npmjs/-/test-package/1.0.0.json", null },
+        };
+        SetupMockBlobClient(mockBlobContainerClient, data);
+
+        var definitionWithFiles = """
+            {
+                "_id": "npm/npmjs/-/test-package/1.0.0",
+                "_meta": {"updated": "2023-01-01T00:00:00Z"},
+                "described": {"files": 5},
+                "files": [
+                    {
+                        "path": "package/LICENSE",
+                        "license": "MIT",
+                        "token": "abc123token"
+                    },
+                    {
+                        "path": "package/README.md",
+                        "license": "MIT",
+                        "token": "def456token"
+                    },
+                    {
+                        "path": "package/index.js"
+
+                    },
+                    {
+                        "path": "package/test.js"
+                    }
+                ]
+            }
+            """;
+
+        var bsonDefinitions = new List<BsonDocument> { BsonDocument.Parse(definitionWithFiles) };
+        mockCursor.Initialize(new List<List<BsonDocument>> { bsonDefinitions });
+
+        backupJob.ProcessJob().Wait();
+
+        // Verify the uploaded definition
+        var uploadedDefinition = JObject.Parse(data["npm/npmjs/-/test-package/1.0.0.json"]!);
+        var filesArray = uploadedDefinition["files"] as JArray;
+
+        // Should only contain files with token
+        filesArray.Should().NotBeNull();
+        filesArray.Should().HaveCount(2);
+        filesArray!.All(f => f["token"] != null).Should().BeTrue();
+
+        // Verify the paths of remaining files
+        var remainingPaths = filesArray.Select(f => f["path"]?.ToString()).ToList();
+        remainingPaths.Should().BeEquivalentTo(new[] { "package/LICENSE", "package/README.md" });
+    }
+
+    [Test]
+    public void TestFilterFilesWithoutToken_HandlesNoFilesArray()
+    {
+        var backupJob = new BackupJob(
+            mockBlobContainerClient.Object,
+            mockMongoClient.Object,
+            DateTime.UtcNow,
+            loggerFactory,
+            new MockFilterRenderer()
+        );
+
+        var data = new Dictionary<string, string?>
+        {
+            { indexPath, "" },
+            { "changes/2023-01-01-00", null },
+            { "npm/npmjs/-/no-files/1.0.0.json", null },
+        };
+        SetupMockBlobClient(mockBlobContainerClient, data);
+
+        var definitionWithoutFiles = """
+            {
+                "_id": "npm/npmjs/-/no-files/1.0.0",
+                "_meta": {"updated": "2023-01-01T00:00:00Z"},
+                "described": {"releaseDate": "2023-01-01"}
+            }
+            """;
+
+        var bsonDefinitions = new List<BsonDocument> { BsonDocument.Parse(definitionWithoutFiles) };
+        mockCursor.Initialize(new List<List<BsonDocument>> { bsonDefinitions });
+
+        Assert.DoesNotThrow(() => backupJob.ProcessJob().Wait());
+
+        var uploadedDefinition = JObject.Parse(data["npm/npmjs/-/no-files/1.0.0.json"]!);
+        uploadedDefinition["files"].Should().BeNull();
+    }
+
+    [Test]
+    public void TestFilterFilesWithoutToken_HandlesEmptyFilesArray()
+    {
+        var backupJob = new BackupJob(
+            mockBlobContainerClient.Object,
+            mockMongoClient.Object,
+            DateTime.UtcNow,
+            loggerFactory,
+            new MockFilterRenderer()
+        );
+
+        var data = new Dictionary<string, string?>
+        {
+            { indexPath, "" },
+            { "changes/2023-01-01-00", null },
+            { "npm/npmjs/-/empty-files/1.0.0.json", null },
+        };
+        SetupMockBlobClient(mockBlobContainerClient, data);
+
+        var definitionWithEmptyFiles = """
+            {
+                "_id": "npm/npmjs/-/empty-files/1.0.0",
+                "_meta": {"updated": "2023-01-01T00:00:00Z"},
+                "described": {"files": 0},
+                "files": []
+            }
+            """;
+
+        var bsonDefinitions = new List<BsonDocument>
+        {
+            BsonDocument.Parse(definitionWithEmptyFiles),
+        };
+        mockCursor.Initialize(new List<List<BsonDocument>> { bsonDefinitions });
+
+        backupJob.ProcessJob().Wait();
+
+        var uploadedDefinition = JObject.Parse(data["npm/npmjs/-/empty-files/1.0.0.json"]!);
+        var filesArray = uploadedDefinition["files"] as JArray;
+        filesArray.Should().NotBeNull().And.BeEmpty();
+    }
+}