Azure Optimization Engine August release (#866)

Co-authored-by: Helder Pinto <[email protected]> Co-authored-by: Arthur Clares <[email protected]>
microsoft · Aug 9, 2024 · e487501 · e487501
1 parent ae9a544
commit e487501
Show file tree

Hide file tree

Showing 47 changed files with 858 additions and 385 deletions.
diff --git a/.all-contributorsrc b/.all-contributorsrc
@@ -159,6 +159,8 @@
       "avatar_url": "https://avatars.githubusercontent.com/u/10661605?v=4",
       "profile": "https://aka.ms/helderpinto",
       "contributions": [
+        "code",
+        "review",
         "doc",
         "bug"
       ]

diff --git a/.github/workflows/aoe-cd-dev.yml b/.github/workflows/aoe-cd-dev.yml
@@ -5,6 +5,7 @@ on:
     branches:
       - dev
     paths:
+      - 'docs/deploy/optimization-engine/**'
       - 'src/optimization-engine/**'       
 permissions:
       id-token: write
@@ -15,8 +16,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-      AOE_SQL_ADMIN: ${{ secrets.AOE_SQL_ADMIN }}
-      AOE_SQL_PASSWD: ${{ secrets.AOE_SQL_PASSWD }}
       AOE_LOCATION: ${{ secrets.AOE_LOCATION }}
       AOE_NAMEPREFIX: ${{ secrets.AOE_NAMEPREFIX }}
     steps:
@@ -48,14 +47,12 @@ jobs:
             "NamePrefix": "'"$AOE_NAMEPREFIX"'",
             "WorkspaceReuse": "n",
             "DeployWorkbooks": "y",
-            "SqlAdmin": "'"$AOE_SQL_ADMIN"'",
-            "SqlPass": "'"$AOE_SQL_PASSWD"'",
             "TargetLocation": "'"$AOE_LOCATION"'",
             "DeployBenefitsUsageDependencies": "n"
           }' > ./src/optimization-engine/deploymentSettings.json          
       - name: Testing PowerShell script call
         shell: pwsh
         run: |
           Set-Location ./src/optimization-engine
-          ./Deploy-AzureOptimizationEngine.ps1 -SilentDeploymentSettingsPath ./deploymentSettings.json -TemplateUri "https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref_name }}/src/optimization-engine/azuredeploy.bicep"
+          ./Deploy-AzureOptimizationEngine.ps1 -SilentDeploymentSettingsPath ./deploymentSettings.json -TemplateUri "https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref_name }}/src/optimization-engine/azuredeploy.bicep" -SqlAdminPrincipalType "Group" -SqlAdminPrincipalName ${{ secrets.AOE_SQL_ADMIN_PRINCIPAL_NAME }} -SqlAdminPrincipalObjectId ${{ secrets.AOE_SQL_ADMIN_PRINCIPAL_ID }}
       - run: echo "🍏 This job's status is ${{ job.status }}."
diff --git a/.github/workflows/aoe-cd-prod.yml b/.github/workflows/aoe-cd-prod.yml
@@ -5,6 +5,7 @@ on:
     branches:
       - main
     paths:
+      - 'docs/deploy/optimization-engine/**'
       - 'src/optimization-engine/**'       
 permissions:
       id-token: write
@@ -15,8 +16,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-      AOE_SQL_ADMIN: ${{ secrets.AOE_SQL_ADMIN }}
-      AOE_SQL_PASSWD: ${{ secrets.AOE_SQL_PASSWD }}
       AOE_LOCATION: ${{ secrets.AOE_LOCATION }}
       AOE_NAMEPREFIX: ${{ secrets.AOE_NAMEPREFIX }}
     steps:
@@ -48,14 +47,12 @@ jobs:
             "NamePrefix": "'"$AOE_NAMEPREFIX"'",
             "WorkspaceReuse": "n",
             "DeployWorkbooks": "y",
-            "SqlAdmin": "'"$AOE_SQL_ADMIN"'",
-            "SqlPass": "'"$AOE_SQL_PASSWD"'",
             "TargetLocation": "'"$AOE_LOCATION"'",
             "DeployBenefitsUsageDependencies": "n"
           }' > ./src/optimization-engine/deploymentSettings.json          
       - name: Testing PowerShell script call
         shell: pwsh
         run: |
           Set-Location ./src/optimization-engine
-          ./Deploy-AzureOptimizationEngine.ps1 -SilentDeploymentSettingsPath ./deploymentSettings.json -TemplateUri "https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref_name }}/src/optimization-engine/azuredeploy.bicep"
+          ./Deploy-AzureOptimizationEngine.ps1 -SilentDeploymentSettingsPath ./deploymentSettings.json -TemplateUri "https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref_name }}/src/optimization-engine/azuredeploy.bicep" -SqlAdminPrincipalType "Group" -SqlAdminPrincipalName ${{ secrets.AOE_SQL_ADMIN_PRINCIPAL_NAME }} -SqlAdminPrincipalObjectId ${{ secrets.AOE_SQL_ADMIN_PRINCIPAL_ID }}
       - run: echo "🍏 This job's status is ${{ job.status }}."
diff --git a/.github/workflows/aoe-cd-test.yml b/.github/workflows/aoe-cd-test.yml
@@ -5,6 +5,7 @@ on:
     branches:
       - features/aoe
     paths:
+      - 'docs/deploy/optimization-engine/**'
       - 'src/optimization-engine/**'       
 permissions:
       id-token: write
@@ -15,8 +16,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-      AOE_SQL_ADMIN: ${{ secrets.AOE_SQL_ADMIN }}
-      AOE_SQL_PASSWD: ${{ secrets.AOE_SQL_PASSWD }}
       AOE_LOCATION: ${{ secrets.AOE_LOCATION }}
       AOE_NAMEPREFIX: ${{ secrets.AOE_NAMEPREFIX }}
     steps:
@@ -48,14 +47,12 @@ jobs:
             "NamePrefix": "'"$AOE_NAMEPREFIX"'",
             "WorkspaceReuse": "n",
             "DeployWorkbooks": "y",
-            "SqlAdmin": "'"$AOE_SQL_ADMIN"'",
-            "SqlPass": "'"$AOE_SQL_PASSWD"'",
             "TargetLocation": "'"$AOE_LOCATION"'",
             "DeployBenefitsUsageDependencies": "n"
           }' > ./src/optimization-engine/deploymentSettings.json          
       - name: Testing PowerShell script call
         shell: pwsh
         run: |
           Set-Location ./src/optimization-engine
-          ./Deploy-AzureOptimizationEngine.ps1 -SilentDeploymentSettingsPath ./deploymentSettings.json -TemplateUri "https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref_name }}/src/optimization-engine/azuredeploy.bicep"
+          ./Deploy-AzureOptimizationEngine.ps1 -SilentDeploymentSettingsPath ./deploymentSettings.json -TemplateUri "https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref_name }}/src/optimization-engine/azuredeploy.bicep" -SqlAdminPrincipalType "Group" -SqlAdminPrincipalName ${{ secrets.AOE_SQL_ADMIN_PRINCIPAL_NAME }} -SqlAdminPrincipalObjectId ${{ secrets.AOE_SQL_ADMIN_PRINCIPAL_ID }}
       - run: echo "🍏 This job's status is ${{ job.status }}."
diff --git a/README.md b/README.md
@@ -43,7 +43,7 @@ There are many ways to participate. From reporting bugs and requesting features
     <tr>
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/arjenhuitema"><img src="https://avatars.githubusercontent.com/u/15944031?v=4?s=100" width="100px;" alt="Arjen Huitema"/><br /><sub><b>Arjen Huitema</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=arjenhuitema" title="Code">💻</a></td>
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/bandersmsft"><img src="https://avatars.githubusercontent.com/u/9596428?v=4?s=100" width="100px;" alt="Bill Anderson"/><br /><sub><b>Bill Anderson</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=bandersmsft" title="Documentation">📖</a></td>
-      <td align="center" valign="top" width="16.66%"><a href="https://aka.ms/helderpinto"><img src="https://avatars.githubusercontent.com/u/10661605?v=4?s=100" width="100px;" alt="Hélder Pinto"/><br /><sub><b>Hélder Pinto</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=helderpinto" title="Documentation">📖</a> <a href="https://github.com/microsoft/finops-toolkit/issues?q=author%3Ahelderpinto" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="16.66%"><a href="https://aka.ms/helderpinto"><img src="https://avatars.githubusercontent.com/u/10661605?v=4?s=100" width="100px;" alt="Hélder Pinto"/><br /><sub><b>Hélder Pinto</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=helderpinto" title="Code">💻</a> <a href="https://github.com/microsoft/finops-toolkit/pulls?q=is%3Apr+reviewed-by%3Ahelderpinto" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/microsoft/finops-toolkit/commits?author=helderpinto" title="Documentation">📖</a> <a href="https://github.com/microsoft/finops-toolkit/issues?q=author%3Ahelderpinto" title="Bug reports">🐛</a></td>
       <td align="center" valign="top" width="16.66%"><a href="https://aka.ms/yuanzhang9"><img src="https://avatars.githubusercontent.com/u/114724932?v=4?s=100" width="100px;" alt="Yuan Zhang"/><br /><sub><b>Yuan Zhang</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=yuanzhang9" title="Code">💻</a></td>
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/ymehdimsft"><img src="https://avatars.githubusercontent.com/u/134303029?v=4?s=100" width="100px;" alt="ymehdimsft"/><br /><sub><b>ymehdimsft</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=ymehdimsft" title="Code">💻</a></td>
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/sri-"><img src="https://avatars.githubusercontent.com/u/4493254?v=4?s=100" width="100px;" alt="srilatha inavolu"/><br /><sub><b>srilatha inavolu</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=sri-" title="Code">💻</a> <a href="https://github.com/microsoft/finops-toolkit/pulls?q=is%3Apr+reviewed-by%3Asri-" title="Reviewed Pull Requests">👀</a></td>

diff --git a/docs/README.md b/docs/README.md
@@ -126,7 +126,7 @@ All the main changes are tracked in the changelog. For additional details, refer
     <tr>
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/arjenhuitema"><img src="https://avatars.githubusercontent.com/u/15944031?v=4?s=100" width="100px;" alt="Arjen Huitema"/><br /><sub><b>Arjen Huitema</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=arjenhuitema" title="Code">💻</a></td>
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/bandersmsft"><img src="https://avatars.githubusercontent.com/u/9596428?v=4?s=100" width="100px;" alt="Bill Anderson"/><br /><sub><b>Bill Anderson</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=bandersmsft" title="Documentation">📖</a></td>
-      <td align="center" valign="top" width="16.66%"><a href="https://aka.ms/helderpinto"><img src="https://avatars.githubusercontent.com/u/10661605?v=4?s=100" width="100px;" alt="Hélder Pinto"/><br /><sub><b>Hélder Pinto</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=helderpinto" title="Documentation">📖</a> <a href="https://github.com/microsoft/finops-toolkit/issues?q=author%3Ahelderpinto" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="16.66%"><a href="https://aka.ms/helderpinto"><img src="https://avatars.githubusercontent.com/u/10661605?v=4?s=100" width="100px;" alt="Hélder Pinto"/><br /><sub><b>Hélder Pinto</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=helderpinto" title="Code">💻</a> <a href="https://github.com/microsoft/finops-toolkit/pulls?q=is%3Apr+reviewed-by%3Ahelderpinto" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/microsoft/finops-toolkit/commits?author=helderpinto" title="Documentation">📖</a> <a href="https://github.com/microsoft/finops-toolkit/issues?q=author%3Ahelderpinto" title="Bug reports">🐛</a></td>
       <td align="center" valign="top" width="16.66%"><a href="https://aka.ms/yuanzhang9"><img src="https://avatars.githubusercontent.com/u/114724932?v=4?s=100" width="100px;" alt="Yuan Zhang"/><br /><sub><b>Yuan Zhang</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=yuanzhang9" title="Code">💻</a></td>
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/ymehdimsft"><img src="https://avatars.githubusercontent.com/u/134303029?v=4?s=100" width="100px;" alt="ymehdimsft"/><br /><sub><b>ymehdimsft</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=ymehdimsft" title="Code">💻</a></td>
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/sri-"><img src="https://avatars.githubusercontent.com/u/4493254?v=4?s=100" width="100px;" alt="srilatha inavolu"/><br /><sub><b>srilatha inavolu</b></sub></a><br /><a href="https://github.com/microsoft/finops-toolkit/commits?author=sri-" title="Code">💻</a> <a href="https://github.com/microsoft/finops-toolkit/pulls?q=is%3Apr+reviewed-by%3Asri-" title="Reviewed Pull Requests">👀</a></td>

diff --git a/docs/_optimize/optimization-engine/README.md b/docs/_optimize/optimization-engine/README.md
@@ -1,6 +1,6 @@
 ---
 layout: default
-title: Optimization Engine
+title: Optimization engine
 has_children: true
 nav_order: 40
 description: 'The Azure Optimization Engine (AOE) is an extensible solution designed to generate optimization recommendations for your Azure environment.'
@@ -105,7 +105,7 @@ Once deployed and after all the initial ingestion and recommendations generation
 
 * A supported Azure subscription (see the [FAQ](./faq.md))
 * A user account with Owner permissions over the chosen subscription, so that the Automation Managed Identity is granted the required privileges over the subscription (Reader) and deployment resource group (Contributor)
-* Azure Powershell 6.6.0+
+* Azure Powershell 9.0.0+
 * (Optional, for Identity and RBAC governance) Microsoft.Graph.Authentication and Microsoft.Graph.Identity.DirectoryManagement PowerShell modules (version 2.4.0+)
 * (Optional, for Identity and RBAC governance) A user account with at least Privileged Role Administrator permissions over the Microsoft Entra tenant, so that the Managed Identity is granted the required privileges over Microsoft Entra ID (Global Reader)
 * (Optional, for Azure commitments insights) A user account with administrative privileges over the Enterprise Agreement (Enterprise Enrollment Administrator) or the Microsoft Customer Agreement (Billing Profile Owner), so that the Managed Identity is granted the required privileges over your consumption agreement

diff --git a/docs/_optimize/optimization-engine/configuring-workspaces.md b/docs/_optimize/optimization-engine/configuring-workspaces.md
@@ -1,6 +1,6 @@
 ---
 layout: default
-parent: Optimization Engine
+parent: Optimization engine
 title: Configuring workspaces
 nav_order: 30
 description: 'Include the VM performance logs available in your Log Analytics workspaces to get deeper insights and more accurate results.'

diff --git a/docs/_optimize/optimization-engine/customize.md b/docs/_optimize/optimization-engine/customize.md
@@ -1,6 +1,6 @@
 ---
 layout: default
-parent: Optimization Engine
+parent: Optimization engine
 title: Customizations
 nav_order: 20
 description: 'Customize the Azure Optimization Engine settings according to your organization requirements.'
@@ -29,6 +29,17 @@ By default, the Azure Automation Managed Identity is assigned the Reader role on
 
 In the context of augmented VM right-size recommendations, you may have your VMs reporting to multiple workspaces. If you need to include other workspaces - besides the main one AOE is using - in the recommendations scope, you just have to add their workspace IDs to the `AzureOptimization_RightSizeAdditionalPerfWorkspaces` variable (see more details in [Configuring workspaces](./configuring-workspaces.md)).
 
+If you are a multi-tenant customer, you can extend the reach of AOE to a tenant other than the one where it was deployed. To achieve this, you must ensure the following pre-requisites:
+
+* Create a service principal (App registration) and a secret in the secondary tenant.
+* Grant the required permissions to the service principal in the secondary tenant, namely **Reader** in Azure subscriptions/management groups and **Global Reader** in Entra ID.
+* Create an [Automation credential](https://learn.microsoft.com/azure/automation/shared-resources/credentials?tabs=azure-powershell#create-a-new-credential-asset) in the AOE's Automation Account, with the service principal's client ID as username and the secret as password.
+* Execute the `Register-MultitenantAutomationSchedules.ps1` script (available in the [AOE root folder](https://aka.ms/AzureOptimizationEngine/code)) in the context of the subscription where AOE was deployed. This script will create new job schedules for each of the export runbooks and configure them to query the secondary tenant. You just have to call the script following the syntax below:
+
+```powershell
+./Register-MultitenantAutomationSchedules.ps1 -AutomationAccountName <AOE automation account> -ResourceGroupName <AOE resource group> -TargetSchedulesSuffix <suffix to append to every new job schedules, e.g., Tenant2> -TargetTenantId <secondary tenant GUID> -TargetTenantCredentialName <name of the Automation credential created in the previous step> [-TargetSchedulesOffsetMinutes <offset in minutes relative to original schedules, defaults to 0>] [-TargetAzureEnvironment <AzureUSGovernment|AzureGermanCloud|AzureCloud>] [-ExcludedRunbooks <An array of runbook names to exclude from the process>] [-IncludedRunbooks <An array of runbook names to include in the process>]
+```
+
 <br>
 
 ## ⏰ Adjust schedules
@@ -113,4 +124,4 @@ Variable | Description
 `AzureOptimization_RecommendationsMaxAgeInDays` | The maximum age (in days) for a recommendation to be kept in the SQL database. Default: 365.
 `AzureOptimization_RetailPricesCurrencyCode` | The currency code (e.g., EUR, USD, etc.) used to collect the Reservations retail prices.
 `AzureOptimization_PriceSheetMeterCategories` | The comma-separated meter categories used for Pricesheet filtering, in order to avoid ingesting unnecessary data. Defaults to "Virtual Machines,Storage"
-`AzureOptimization_ConsumptionScope` | The scope of the consumption exports: `Subscription` (default) or `BillingAccount`. See [more details](./setup-options.md#-enabling-azure-commitments-workbooks).
+`AzureOptimization_ConsumptionScope` | The scope of the consumption exports: `Subscription` (default), `BillingProfile` (MCA only) or `BillingAccount` (for MCA, requires adding the Billing Account Reader role to the AOE managed identity). See [more details](./setup-options.md#-enabling-azure-commitments-workbooks).