[Hub apps] Move file uploads to hub-storage (#1543)

Author: flanakin

src/templates/finops-hub/modules/dataFactory.bicep

@@ -822,15 +822,15 @@ resource dataset_ftkReleaseFile 'Microsoft.DataFactory/factories/datasets@2018-0
 // TODO: Create apps_PublishEvent pipeline { event, properties }
 
 module trigger_ExportManifestAdded 'hub-event-trigger.bicep' = {
-  name: 'trigger_ExportManifestAdded'
+  name: 'Microsoft.FinOpsHubs.Core_ExportManifestAddedTrigger'
   dependsOn: [
     stopTriggers
   ]
   params: {
     dataFactoryName: dataFactory.name
     triggerName: exportManifestAddedTriggerName
 
-    // TODO: Replace pipeline with event: 'Microsoft.FinOpsToolkit.CostManagement.ExportManifestAdded'
+    // TODO: Replace pipeline with event: 'Microsoft.CostManagement.Exports.ManifestAdded'
     pipelineName: pipeline_ExecuteExportsETL.name
     pipelineParameters: {
       folderPath: '@triggerBody().folderPath'
@@ -843,16 +843,16 @@ module trigger_ExportManifestAdded 'hub-event-trigger.bicep' = {
   }
 }
 
-module trigger_IngestionManifestAdded 'hub-event-trigger.bicep' = {
-  name: 'trigger_IngestionManifestAdded'
+module trigger_IngestionManifestAdded 'hub-event-trigger.bicep' = if (deployDataExplorer) {
+  name: 'Microsoft.FinOpsHubs.Core_IngestionManifestAddedTrigger'
   dependsOn: [
     stopTriggers
   ]
   params: {
     dataFactoryName: dataFactory.name
     triggerName: ingestionManifestAddedTriggerName
 
-    // TODO: Replace pipeline with event: 'Microsoft.FinOpsToolkit.Hubs.IngestionManifestAdded'
+    // TODO: Replace pipeline with event: 'Microsoft.FinOpsHubs.Core.IngestionManifestAdded'
     pipelineName: pipeline_ExecuteIngestionETL.name
     pipelineParameters: {
       folderPath: '@triggerBody().folderPath'
@@ -865,15 +865,15 @@ module trigger_IngestionManifestAdded 'hub-event-trigger.bicep' = {
 }
 
 module trigger_SettingsUpdated 'hub-event-trigger.bicep' = {
-  name: 'trigger_SettingsUpdated'
+  name: 'Microsoft.FinOpsHubs.Core_SettingsUpdatedTrigger'
   dependsOn: [
     stopTriggers
   ]
   params: {
     dataFactoryName: dataFactory.name
     triggerName: updateConfigTriggerName
 
-    // TODO: Replace pipeline with event: 'Microsoft.FinOpsToolkit.Hubs.SettingsUpdated'
+    // TODO: Replace pipeline with event: 'Microsoft.FinOpsHubs.Core.SettingsUpdated'
     pipelineName: pipeline_ConfigureExports.name
     pipelineParameters: {}
 

src/templates/finops-hub/modules/hub-storage.bicep

@@ -5,11 +5,47 @@
 // Parameters
 //==============================================================================
 
+@description('Required. Name of the storage container to create or update.')
+param container string
+
+@description('Optional. Dictionary of key/value pairs for the files to upload to the specified container. The key is the target path under the container and the value is the contents of the file. Default: {} (no files to upload).')
+param files object = {}
+
+//------------------------------------------------------------------------------
+// Hub context
+//------------------------------------------------------------------------------
+
 @description('Required. Name of the publisher-specific storage account to create or update.')
 param storageAccountName string
 
-@description('Required. Name of the storage container to create or update.')
-param container string
+@description('Optional. Azure location where all resources should be created. See https://aka.ms/azureregions. Default: (resource group location).')
+param location string = resourceGroup().location
+
+@description('Optional. Tags to apply to all resources.')
+param tags object = {}
+
+@description('Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources.')
+param tagsByResource object = {}
+
+@description('Optional. The name of the managed identity to use for uploading files.')
+param blobManagerIdentityName string = ''
+
+@description('Required. Indicates whether public access should be enabled.')
+param enablePublicAccess bool
+
+@description('Optional. The name of the storage account used for deployment scripts.')
+param scriptStorageAccountName string = ''
+
+@description('Optional. Resource ID of the virtual network for running deployment scripts.')
+param scriptSubnetId string = ''
+
+
+//==============================================================================
+// Variables
+//==============================================================================
+
+var fileCount = length(items(files))
+var hasFiles = fileCount > 0
 
 
 //==============================================================================
@@ -23,7 +59,7 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' existing
   resource blobService 'blobServices@2022-09-01' = {
     name: 'default'
 
-    resource configContainer 'containers@2022-09-01' = {
+    resource targetContainer 'containers@2022-09-01' = {
       name: container
       properties: {
         publicAccess: 'None'
@@ -33,11 +69,74 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' existing
   }
 }
 
-// TODO: Upload files
 // TODO: Enforce retention
 
+//------------------------------------------------------------------------------
+// Upload schema file to storage
+//------------------------------------------------------------------------------
+
+resource scriptStorageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' existing =  if (hasFiles && !enablePublicAccess) {
+  name: scriptStorageAccountName
+}
+
+resource blobManagerIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (hasFiles) {
+  name: blobManagerIdentityName
+}
+
+resource uploadFiles 'Microsoft.Resources/deploymentScripts@2023-08-01' = if (hasFiles) {
+  name: '${storageAccountName}_uploadFiles'
+  kind: 'AzurePowerShell'
+  // cSpell:ignore chinaeast
+  // chinaeast2 is the only region in China that supports deployment scripts
+  location: startsWith(location, 'china') ? 'chinaeast2' : location
+  tags: union(tags, tagsByResource[?'Microsoft.Resources/deploymentScripts'] ?? {})
+  identity: {
+    type: 'UserAssigned'
+    userAssignedIdentities: {
+      '${blobManagerIdentity.id}': {}
+    }
+  }
+  dependsOn: []
+  properties: union(enablePublicAccess ? {} : {
+    storageAccountSettings: {
+      storageAccountName: scriptStorageAccount.name
+    }
+    containerSettings: {
+      containerGroupName: '${scriptStorageAccount.name}cg'
+      subnetIds: [
+        {
+          id: scriptSubnetId
+        }
+      ]
+    }
+  }, {
+    azPowerShellVersion: '9.0'
+    retentionInterval: 'PT1H'
+    environmentVariables: [
+      {
+        name: 'storageAccountName'
+        value: storageAccount.name
+      }
+      {
+        name: 'containerName'
+        value: container
+      }
+      {
+        name: 'files'
+        value: string(files)
+      }
+    ]
+    scriptContent: loadTextContent('./scripts/Upload-StorageFile.ps1')
+  })
+}
+
+
 //==============================================================================
 // Outputs
 //==============================================================================
 
-output containerName string = storageAccount::blobService::configContainer.name
+@description('The name of the storage container.')
+output containerName string = storageAccount::blobService::targetContainer.name
+
+@description('The number of files uploaded to the storage container.')
+output filesUploaded int = fileCount

src/templates/finops-hub/modules/hub.bicep

@@ -248,8 +248,8 @@ module appRegistration 'hub-app.bicep' = {
   name: 'pid-${telemetryId}_${telemetryString}_${uniqueString(deployment().name, location)}'
   params: {
     hubName: hubName
-    publisher: 'FinOps hubs'
-    namespace: 'Microsoft.FinOpsToolkit.Hubs'
+    publisher: 'Microsoft FinOps hubs'
+    namespace: 'Microsoft.FinOpsHubs'
     appName: 'Core'
     displayName: 'FinOps hub core'
     appVersion: finOpsToolkitVersion

src/templates/finops-hub/modules/scripts/Copy-FileToAzureBlob.ps1

@@ -168,14 +168,3 @@ $text | Out-File $filePath
 # Upload new/updated settings
 Write-Output "Uploading settings.json file..."
 Set-AzStorageBlobContent @storageContext -File $filePath -Force | Out-Null
-
-# Save focusSchemaFile file to storage
-$schemaFiles = $env:schemaFiles | ConvertFrom-Json -Depth 10
-Write-Output "Uploading ${$schemaFiles.PSObject.Properties.Count} schema files..."
-$schemaFiles.PSObject.Properties | ForEach-Object {
-    $fileName = "$($_.Name).json"
-    $tempPath = "./$fileName"
-    Write-Output "  Uploading $($_.Name).json..."
-    $_.Value | Out-File $tempPath
-    Set-AzStorageBlobContent @storageContext -File $tempPath -Blob "schemas/$fileName" -Force | Out-Null
-}

src/templates/finops-hub/modules/scripts/Upload-StorageFile.ps1

@@ -0,0 +1,19 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Get storage context
+$storageContext = @{
+    Context   = New-AzStorageContext -StorageAccountName $env:storageAccountName -UseConnectedAccount
+    Container = $env:containerName
+}
+
+# Uploading files
+$files = $env:files | ConvertFrom-Json -Depth 10
+Write-Output "Uploading ${$files.PSObject.Properties.Count} files..."
+$files.PSObject.Properties | ForEach-Object {
+    $filePath = $_.Name
+    $tempPath = "./$($filePath -replace "/", "_")"
+    Write-Output "  Uploading $filePath..."
+    $_.Value | Out-File $tempPath
+    Set-AzStorageBlobContent @storageContext -File $tempPath -Blob $filePath -Force | Out-Null
+}

src/templates/finops-hub/modules/storage.bicep

@@ -68,19 +68,19 @@ var storageAccountSuffix = uniqueSuffix
 var storageAccountName = '${take(safeHubName, 24 - length(storageAccountSuffix))}${storageAccountSuffix}'
 var scriptStorageAccountName = '${take(safeHubName, 16 - length(storageAccountSuffix))}script${storageAccountSuffix}'
 var schemaFiles = {
-  // cSpell:ignore focuscost, pricesheet, reservationdetails, reservationrecommendations, reservationtransactions
-  'actualcost_c360-2025-04': loadTextContent('../schemas/actualcost_c360-2025-04.json')
-  'amortizedcost_c360-2025-04': loadTextContent('../schemas/amortizedcost_c360-2025-04.json')
-  'focuscost_1.0r2': loadTextContent('../schemas/focuscost_1.0r2.json')
-  'focuscost_1.0': loadTextContent('../schemas/focuscost_1.0.json')
-  'focuscost_1.0-preview(v1)': loadTextContent('../schemas/focuscost_1.0-preview(v1).json')
-  'pricesheet_2023-05-01_ea': loadTextContent('../schemas/pricesheet_2023-05-01_ea.json')
-  'pricesheet_2023-05-01_mca': loadTextContent('../schemas/pricesheet_2023-05-01_mca.json')
-  'reservationdetails_2023-03-01': loadTextContent('../schemas/reservationdetails_2023-03-01.json')
-  'reservationrecommendations_2023-05-01_ea': loadTextContent('../schemas/reservationrecommendations_2023-05-01_ea.json')
-  'reservationrecommendations_2023-05-01_mca': loadTextContent('../schemas/reservationrecommendations_2023-05-01_mca.json')
-  'reservationtransactions_2023-05-01_ea': loadTextContent('../schemas/reservationtransactions_2023-05-01_ea.json')
-  'reservationtransactions_2023-05-01_mca': loadTextContent('../schemas/reservationtransactions_2023-05-01_mca.json')
+  // cSpell:ignore actualcost, amortizedcost, focuscost, pricesheet, reservationdetails, reservationrecommendations, reservationtransactions
+  'schemas/actualcost_c360-2025-04.json': loadTextContent('../schemas/actualcost_c360-2025-04.json')
+  'schemas/amortizedcost_c360-2025-04.json': loadTextContent('../schemas/amortizedcost_c360-2025-04.json')
+  'schemas/focuscost_1.0r2.json': loadTextContent('../schemas/focuscost_1.0r2.json')
+  'schemas/focuscost_1.0.json': loadTextContent('../schemas/focuscost_1.0.json')
+  'schemas/focuscost_1.0-preview(v1).json': loadTextContent('../schemas/focuscost_1.0-preview(v1).json')
+  'schemas/pricesheet_2023-05-01_ea.json': loadTextContent('../schemas/pricesheet_2023-05-01_ea.json')
+  'schemas/pricesheet_2023-05-01_mca.json': loadTextContent('../schemas/pricesheet_2023-05-01_mca.json')
+  'schemas/reservationdetails_2023-03-01.json': loadTextContent('../schemas/reservationdetails_2023-03-01.json')
+  'schemas/reservationrecommendations_2023-05-01_ea.json': loadTextContent('../schemas/reservationrecommendations_2023-05-01_ea.json')
+  'schemas/reservationrecommendations_2023-05-01_mca.json': loadTextContent('../schemas/reservationrecommendations_2023-05-01_mca.json')
+  'schemas/reservationtransactions_2023-05-01_ea.json': loadTextContent('../schemas/reservationtransactions_2023-05-01_ea.json')
+  'schemas/reservationtransactions_2023-05-01_mca.json': loadTextContent('../schemas/reservationtransactions_2023-05-01_mca.json')
 }
 
 // Roles needed to upload files
@@ -347,27 +347,46 @@ resource blobService 'Microsoft.Storage/storageAccounts/blobServices@2022-09-01'
   name: 'default'
 }
 
+// TODO: Move to core module
 module configContainer 'hub-storage.bicep' = {
-  name: 'configContainer'
+  name: 'Microsoft.FinOpsHubs.Core_SchemaFiles'
   params: {
-    storageAccountName: storageAccount.name
     container: 'config'
+    files: schemaFiles
+
+    // Hub context
+    storageAccountName: storageAccount.name
+    location: location
+    tags: tags
+    tagsByResource: tagsByResource
+    blobManagerIdentityName: identity.name
+    enablePublicAccess: enablePublicAccess
+    scriptStorageAccountName: scriptStorageAccount.name
+    scriptSubnetId: scriptSubnetId
   }
 }
 
+// TODO: Move to separate CM exports module
 module exportContainer 'hub-storage.bicep' = {
-  name: 'exportContainer'
+  name: 'Microsoft.CostManagement.Exports_ExportContainer'
   params: {
-    storageAccountName: storageAccount.name
     container: 'msexports'
+    
+    // Hub context
+    storageAccountName: storageAccount.name
+    enablePublicAccess: enablePublicAccess
   }
 }
 
+// TODO: Move to core module
 module ingestionContainer 'hub-storage.bicep' = {
-  name: 'ingestionContainer'
+  name: 'Microsoft.FinOpsHubs.Core_IngestionContainer'
   params: {
-    storageAccountName: storageAccount.name
     container: 'ingestion'
+    
+    // Hub context
+    storageAccountName: storageAccount.name
+    enablePublicAccess: enablePublicAccess
   }
 }
 
@@ -463,10 +482,6 @@ resource uploadSettings 'Microsoft.Resources/deploymentScripts@2023-08-01' = {
         name: 'containerName'
         value: 'config'
       }
-      {
-        name: 'schemaFiles'
-        value: string(schemaFiles)
-      }
     ]
     scriptContent: loadTextContent('./scripts/Copy-FileToAzureBlob.ps1')
   })