Filter files array to include only files with tokens in backup snapshots

[elaine-mattos]

This PR modifies the backup job to include files with tokens in the production-snapshot blob storage, enabling downstream tools to fetch license files via the ClearlyDefined attachments endpoint.

What's changed

• Changed data source from the definitions-trimmed to the definitions-paged collection in MongoDB;
• Added FilterFilesWithoutToken function to filter the files array, keeping only the elements that have a token property.

Why this change?

This enables external tools to:

• Identify which files have available attachments;
• Fetch license files using the token via /attachments/{token};
• Build compliance tooling on top of the alrady available ClearlyDefined data.

Testing

• Added unit tests;
• Manual local tests were also successful:

Before:

The `files` array was missing entirely from backup definitions

{
  "_id": "npm/npmjs/-/react-native-navigation-bar-color/2.0.2",
  "described": {
    "hashes": {
      "sha1": "2e14ca79a656d81d5b4b7433ae43966ef5206444",
      "sha256": "45ab1ca267c9ce8ae4913c73146c43e855156a529f8eab012731394b55cfb495"
    },
    "files": 16,
    "releaseDate": "2022-12-28",
    "urls": {
      "registry": "https://npmjs.com/package/react-native-navigation-bar-color",
      "version": "https://npmjs.com/package/react-native-navigation-bar-color/v/2.0.2",
      "download": "https://registry.npmjs.com/react-native-navigation-bar-color/-/react-native-navigation-bar-color-2.0.2.tgz"
    },
    "projectWebsite": "https://github.com/thebylito/react-native-navigation-bar-color#readme",
    "issueTracker": "https://github.com/thebylito/react-native-navigation-bar-color/issues",
    "tools": [
      "clearlydefined/1.3.4"
    ],
    "toolScore": {
      "total": 30,
      "date": 30,
      "source": 0
    },
    "score": {
      "total": 30,
      "date": 30,
      "source": 0
    }
  },
  "licensed": {
    "declared": "MIT",
    "toolScore": {
      "total": 60,
      "declared": 30,
      "discovered": 0,
      "consistency": 15,
      "spdx": 15,
      "texts": 0
    },
    "facets": {
      "core": {
        "attribution": {
          "unknown": 16
        },
        "discovered": {
          "unknown": 16
        },
        "files": 16
      }
    },
    "score": {
      "total": 60,
      "declared": 30,
      "discovered": 0,
      "consistency": 15,
      "spdx": 15,
      "texts": 0
    }
  },
  "_meta": {
    "schemaVersion": "1.6.1",
    "updated": "2023-01-17T15:16:09.306Z"
  },
  "scores": {
    "effective": 45,
    "tool": 45
  }
}

After

The `files` array now includes only files with tokens

{
 "_id": "npm/npmjs/-/react-native-navigation-bar-color/2.0.2",
 "described": {
   "hashes": {
     "sha1": "2e14ca79a656d81d5b4b7433ae43966ef5206444",
     "sha256": "45ab1ca267c9ce8ae4913c73146c43e855156a529f8eab012731394b55cfb495"
   },
   "files": 16,
   "releaseDate": "2022-12-28",
   "urls": {
     "registry": "https://npmjs.com/package/react-native-navigation-bar-color",
     "version": "https://npmjs.com/package/react-native-navigation-bar-color/v/2.0.2",
     "download": "https://registry.npmjs.com/react-native-navigation-bar-color/-/react-native-navigation-bar-color-2.0.2.tgz"
   },
   "projectWebsite": "https://github.com/thebylito/react-native-navigation-bar-color#readme",
   "issueTracker": "https://github.com/thebylito/react-native-navigation-bar-color/issues",    
   "tools": [
     "clearlydefined/1.3.4",
     "reuse/5.2.2",
     "scancode/32.5.2"
   ],
   "toolScore": {
     "total": 30,
     "date": 30,
     "source": 0
   },
   "score": {
     "total": 30,
     "date": 30,
     "source": 0
   }
 },
 "licensed": {
   "declared": "MIT",
   "toolScore": {
     "total": 77,
     "declared": 30,
     "discovered": 2,
     "consistency": 15,
     "spdx": 15,
     "texts": 15
   },
   "facets": {
     "core": {
       "attribution": {
         "unknown": 15,
         "parties": [
           "Copyright (c) 2018 thebylito"
         ]
       },
       "discovered": {
         "unknown": 13,
         "expressions": [
           "MIT"
         ]
       },
       "files": 16
     }
   },
   "score": {
     "total": 77,
     "declared": 30,
     "discovered": 2,
     "consistency": 15,
     "spdx": 15,
     "texts": 15
   }
 },
 "files": [
   {
     "path": "package/LICENSE",
     "license": "MIT",
     "natures": [
       "license"
     ],
     "attributions": [
       "Copyright (c) 2018 thebylito"
     ],
     "hashes": {
       "sha1": "4968b8f57a3f4362a5bee61a4e0f32a3d6f16c94",
       "sha256": "4bcebe9a76f1fbdef1ca52e59f8a97d45444ccdf6816cf4e9ce19af60b9ad6a0"
     },
     "token": "4bcebe9a76f1fbdef1ca52e59f8a97d45444ccdf6816cf4e9ce19af60b9ad6a0"
   },
   {
     "path": "package/package.json",
     "license": "MIT",
     "hashes": {
       "sha1": "83529975468e7f6337a4802db48bb6c2b6049fdf",
       "sha256": "a3e740dbee7af78a67bc3dd75bc78f9f1a679fdfa1c70bac5a226082e70404b3"
     },
     "token": "a3e740dbee7af78a67bc3dd75bc78f9f1a679fdfa1c70bac5a226082e70404b3"
   }
 ],
 "_meta": {
   "schemaVersion": "1.7.0",
   "updated": "2025-11-11T16:42:32.022Z"
 },
 "scores": {
   "effective": 53,
   "tool": 53
 }
}

Refers to Issue