[Asset Inventory][Azure] Add missing resources and ECS fields require…

…d for GA (#2954)

* add Azure RoleDefinition

* add Azure Entra Groups

* add Azure Entra Users

* update Azure Service Principal category

* fix missing tenant info in Azure

* add ECS to Roles

* add ECS Group fields to Asset definition

* start filling Azure ECS

* add missing Blob Containers and fetch Labels/Tags where possible

* fix existing unit tests for Azure Inventory

* reclassify Blob Container Service

* extract VM properties

* fix expected value

* align principal with cloud assets sheet

* make linters happier

* update all missing ECS fields

* fix test cases

* make test cases race-condition-proof

* remove redundant todo

* add helper func test

* extend test case

* make linter happy

* skip asset.go in coverage

* cover resource-graph file

* add test for BlobContainers

* update ASSETS.md

* use mapstructure.Decode

* unpack properties to struct and test it

* fixes for linters

* gci

.github/workflows/ci-pull_request.yml

@@ -94,7 +94,7 @@ jobs:
         run: |
           go install gotest.tools/gotestsum
           GOOS=linux TEST_DIRECTORY=./... gotestsum --format pkgname -- -race -coverpkg=./... -coverprofile=cover.out.tmp
-          cat cover.out.tmp | grep -v "mock_.*.go" | grep -v "elastic/cloudbeat/deploy" > cover.out # remove mock files and deploy dir
+          cat cover.out.tmp | grep -v "mock_.*.go" | grep -v "elastic/cloudbeat/deploy" | grep -v "internal/inventory/asset.go" > cover.out # remove mock files and deploy dir
 
       - name: Upload coverage artifact
         uses: actions/upload-artifact@v4

internal/flavors/assetinventory/strategy.go

@@ -93,7 +93,7 @@ func (s *strategy) initAzureFetchers(_ context.Context) ([]inventory.AssetFetche
 		return nil, fmt.Errorf("failed to initialize azure msgraph provider: %w", err)
 	}
 
-	return azurefetcher.New(s.logger, provider, msgraphProvider), nil
+	return azurefetcher.New(s.logger, s.cfg.CloudConfig.Azure.Credentials.TenantID, provider, msgraphProvider), nil
 }
 
 func (s *strategy) initGcpFetchers(ctx context.Context) ([]inventory.AssetFetcher, error) {

internal/inventory/ASSETS.md

@@ -109,16 +109,18 @@ Storage Bucket: 100% (1/1)
 
 ## AZURE Resources
 
-**Progress: 42% (23/54)**
+**Progress: 44% (24/54)**
 Access Management: 100% (3/3)
 Container Registry: 100% (1/1)
 Database: 100% (4/4)
 File System Service: 100% (2/2)
+Group: 100% (1/1)
 Host: 100% (1/1)
-Identity: 9% (1/11)
+Identity: 0% (0/9)
 Infrastructure: 8% (2/23)
 Messaging Service: 100% (2/2)
 Private Endpoint: 100% (1/1)
+Service Account: 100% (1/1)
 Service Usage Technology: 100% (2/2)
 Snapshot: 100% (1/1)
 Storage Bucket: 100% (1/1)
@@ -139,14 +141,13 @@ Web Service: 100% (1/1)
 | Database | Azure Storage Table |  | Yes ✅ |
 | File System Service | Azure Storage File Service |  | Yes ✅ |
 | File System Service | Azure Storage File Share |  | Yes ✅ |
+| Group | Azure AD Group | Azure Microsoft Entra ID Group | Yes ✅ |
 | Host | Azure Virtual Machine | Azure Virtual Machine | Yes ✅ |
 | Identity | Access Key |  | No ❌ |
 | Identity | API Gateway Client Certificate |  | No ❌ |
 | Identity | Azure AD Application |  | No ❌ |
-| Identity | Azure AD Group |  | No ❌ |
 | Identity | Azure AD Service Principal |  | No ❌ |
 | Identity | Azure AD User |  | No ❌ |
-| Identity | Azure Principal | Azure Principal | Yes ✅ |
 | Identity | Azure Role |  | No ❌ |
 | Identity | Azure Role Assignment |  | No ❌ |
 | Identity | Azure Server AD Administrator |  | No ❌ |
@@ -177,6 +178,7 @@ Web Service: 100% (1/1)
 | Messaging Service | Azure Storage Queue | Azure Storage Queue | Yes ✅ |
 | Messaging Service | Azure Storage Queue Service | Azure Storage Queue Service | Yes ✅ |
 | Private Endpoint | Azure Storage Account | Azure Storage Account | Yes ✅ |
+| Service Account | Azure Principal | Azure Principal | Yes ✅ |
 | Service Usage Technology | Azure Storage Blob Service | Azure Storage Blob Service | Yes ✅ |
 | Service Usage Technology | Azure Storage Table Service |  | Yes ✅ |
 | Snapshot | Azure Snapshot | Azure Snapshot | Yes ✅ |

internal/inventory/asset.go

@@ -17,7 +17,10 @@
 
 package inventory
 
-import "github.com/samber/lo"
+import (
+	"github.com/mitchellh/mapstructure"
+	"github.com/samber/lo"
+)
 
 // AssetCategory is used to build the document index.
 type AssetCategory string
@@ -32,6 +35,7 @@ const (
 	CategoryFileSystemService      AssetCategory = "File System Service"
 	CategoryFirewall               AssetCategory = "Firewall"
 	CategoryGateway                AssetCategory = "Gateway"
+	CategoryGroup                  AssetCategory = "Group"
 	CategoryHost                   AssetCategory = "Host"
 	CategoryIdentity               AssetCategory = "Identity"
 	CategoryInfrastructure         AssetCategory = "Infrastructure"
@@ -99,10 +103,13 @@ var (
 	AssetClassificationAzureCosmosDBSQLDatabase  = AssetClassification{CategoryInfrastructure, "Azure Cosmos DB SQL Database"}
 	AssetClassificationAzureDisk                 = AssetClassification{CategoryVolume, "Azure Disk"}
 	AssetClassificationAzureElasticPool          = AssetClassification{CategoryDatabase, "Azure Elastic Pool"}
+	AssetClassificationAzureEntraGroup           = AssetClassification{CategoryGroup, "Azure Microsoft Entra ID Group"}
+	AssetClassificationAzureEntraUser            = AssetClassification{CategoryIdentity, "Azure Microsoft Entra ID User"}
 	AssetClassificationAzureResourceGroup        = AssetClassification{CategoryAccessManagement, "Azure Resource Group"}
+	AssetClassificationAzureRoleDefinition       = AssetClassification{CategoryAccessManagement, "Azure RoleDefinition"}
 	AssetClassificationAzureSQLDatabase          = AssetClassification{CategoryDatabase, "Azure SQL Database"}
 	AssetClassificationAzureSQLServer            = AssetClassification{CategoryDatabase, "Azure SQL Server"}
-	AssetClassificationAzureServicePrincipal     = AssetClassification{CategoryIdentity, "Azure Principal"}
+	AssetClassificationAzureServicePrincipal     = AssetClassification{CategoryServiceAccount, "Azure Principal"}
 	AssetClassificationAzureSnapshot             = AssetClassification{CategorySnapshot, "Azure Snapshot"}
 	AssetClassificationAzureStorageAccount       = AssetClassification{CategoryPrivateEndpoint, "Azure Storage Account"}
 	AssetClassificationAzureStorageBlobContainer = AssetClassification{CategoryStorageBucket, "Azure Storage Blob Container"}
@@ -138,14 +145,15 @@ var (
 type AssetEvent struct {
 	Entity        Entity
 	Event         Event
-	Network       *Network
-	URL           *URL
-	Organization  *Organization
 	Cloud         *Cloud
-	Fass          *Fass
-	Orchestrator  *Orchestrator
 	Container     *Container
+	Fass          *Fass
+	Group         *Group
 	Host          *Host
+	Network       *Network
+	Orchestrator  *Orchestrator
+	Organization  *Organization
+	URL           *URL
 	User          *User
 	Labels        map[string]string
 	Tags          []string
@@ -200,6 +208,12 @@ type Cloud struct {
 	ProjectName      string `json:"project.name,omitempty"`
 }
 
+type Group struct {
+	ID     string `json:"id,omitempty"`
+	Name   string `json:"name,omitempty"`
+	Domain string `json:"domain,omitempty"`
+}
+
 type Host struct {
 	ID           string   `json:"id,omitempty"`
 	Name         string   `json:"name,omitempty"`
@@ -281,13 +295,16 @@ func WithLabels(labels map[string]string) AssetEnricher {
 	}
 }
 
-func WithTags(tags []string) AssetEnricher {
+func WithLabelsFromAny(labels map[string]any) AssetEnricher {
 	return func(a *AssetEvent) {
-		if len(tags) == 0 {
+		if len(labels) == 0 {
 			return
 		}
-
-		a.Tags = tags
+		output := map[string]string{}
+		if err := mapstructure.Decode(labels, &output); err != nil {
+			return
+		}
+		a.Labels = output
 	}
 }
 
@@ -303,12 +320,27 @@ func WithCloud(cloud Cloud) AssetEnricher {
 	}
 }
 
+func WithGroup(group Group) AssetEnricher {
+	return func(a *AssetEvent) {
+		a.Group = &group
+	}
+}
+
 func WithHost(host Host) AssetEnricher {
 	return func(a *AssetEvent) {
 		a.Host = &host
 	}
 }
 
+func WithTags(tags []string) AssetEnricher {
+	return func(a *AssetEvent) {
+		if len(tags) == 0 {
+			return
+		}
+		a.Tags = tags
+	}
+}
+
 func WithUser(user User) AssetEnricher {
 	return func(a *AssetEvent) {
 		a.User = &user

internal/inventory/azurefetcher/azurefetchers.go

@@ -24,11 +24,11 @@ import (
 	"github.com/elastic/cloudbeat/internal/resources/providers/msgraph"
 )
 
-func New(logger *clog.Logger, provider azurelib.ProviderAPI, msgraphProvider msgraph.ProviderAPI) []inventory.AssetFetcher {
+func New(logger *clog.Logger, tenantID string, provider azurelib.ProviderAPI, msgraphProvider msgraph.ProviderAPI) []inventory.AssetFetcher {
 	return []inventory.AssetFetcher{
-		newAccountFetcher(logger, provider),
-		newActiveDirectoryFetcher(logger, msgraphProvider),
-		newResourceGraphFetcher(logger, provider),
-		newStorageFetcher(logger, provider),
+		newAccountFetcher(logger, tenantID, provider),
+		newActiveDirectoryFetcher(logger, tenantID, msgraphProvider),
+		newResourceGraphFetcher(logger, tenantID, provider),
+		newStorageFetcher(logger, tenantID, provider),
 	}
 }

internal/inventory/azurefetcher/fetcher_account.go

@@ -27,6 +27,7 @@ import (
 
 type accountFetcher struct {
 	logger   *clog.Logger
+	tenantID string //nolint:unused
 	provider accountProvider
 }
 
@@ -38,9 +39,10 @@ type (
 	}
 )
 
-func newAccountFetcher(logger *clog.Logger, provider accountProvider) inventory.AssetFetcher {
+func newAccountFetcher(logger *clog.Logger, tenantID string, provider accountProvider) inventory.AssetFetcher {
 	return &accountFetcher{
 		logger:   logger,
+		tenantID: tenantID,
 		provider: provider,
 	}
 }
@@ -80,6 +82,10 @@ func (f *accountFetcher) fetch(ctx context.Context, resourceName string, functio
 				AccountID:   item.TenantId,
 				ServiceName: "Azure",
 			}),
+			inventory.WithLabelsFromAny(item.Tags),
+			inventory.WithOrganization(inventory.Organization{
+				ID: item.TenantId,
+			}),
 		)
 	}
 }

internal/inventory/azurefetcher/fetcher_account_test.go

@@ -48,6 +48,9 @@ func TestAccountFetcher_Fetch_Tenants(t *testing.T) {
 				AccountID:   "<tenant UUID>",
 				ServiceName: "Azure",
 			}),
+			inventory.WithOrganization(inventory.Organization{
+				ID: "<tenant UUID>",
+			}),
 		),
 	}
 
@@ -56,7 +59,7 @@ func TestAccountFetcher_Fetch_Tenants(t *testing.T) {
 	provider := newMockAccountProvider(t)
 	provider.EXPECT().ListTenants(mock.Anything).Return(azureAssets, nil)
 	provider.EXPECT().ListSubscriptions(mock.Anything).Return(nil, nil)
-	fetcher := newAccountFetcher(logger, provider)
+	fetcher := newAccountFetcher(logger, "<tenant UUID>", provider)
 	// test & compare
 	testutil.CollectResourcesAndMatch(t, fetcher, expected)
 }
@@ -81,6 +84,9 @@ func TestAccountFetcher_Fetch_Subscriptions(t *testing.T) {
 				AccountID:   "<sub UUID>",
 				ServiceName: "Azure",
 			}),
+			inventory.WithOrganization(inventory.Organization{
+				ID: "<sub UUID>",
+			}),
 		),
 	}
 
@@ -89,7 +95,7 @@ func TestAccountFetcher_Fetch_Subscriptions(t *testing.T) {
 	provider := newMockAccountProvider(t)
 	provider.EXPECT().ListTenants(mock.Anything).Return(nil, nil)
 	provider.EXPECT().ListSubscriptions(mock.Anything).Return(azureAssets, nil)
-	fetcher := newAccountFetcher(logger, provider)
+	fetcher := newAccountFetcher(logger, "<tenant UUID>", provider)
 	// test & compare
 	testutil.CollectResourcesAndMatch(t, fetcher, expected)
 }