first attempt to add cross account permissions to aws

flake.nix

@@ -0,0 +1,48 @@
+{
+  description = "cilium dev environment";
+
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs";
+    nixpkgs-tparse.url = "github:NixOS/nixpkgs/e518d4ad2bcad74f98fec028cf21ce5b1e5020dd"; #revision for tparse
+    nixpkgs-ginkgo.url = "github:NixOS/nixpkgs/89f196fe781c53cb50fef61d3063fa5e8d61b6e5"; #revision for ginkgo
+  };
+
+  outputs = { self, nixpkgs, nixpkgs-tparse, nixpkgs-ginkgo }:
+  let
+      allSystems = [
+        "x86_64-linux" # 64-bit Intel/AMD Linux
+        "aarch64-linux" # 64-bit ARM Linux for my pbp
+        "x86_64-darwin" # 64-bit Intel macOS
+        "aarch64-darwin" # 64-bit ARM macOS
+      ];
+
+      # Helper to provide system-specific attributes
+      forAllSystems = f: nixpkgs.lib.genAttrs allSystems (system: f {
+        pkgs = import nixpkgs { inherit system; };
+        pkgs-tp = import nixpkgs-tparse { inherit system; };
+        pkgs-g = import nixpkgs-ginkgo { inherit system; };
+      });
+  in {
+    # Development env package required.
+    devShells = forAllSystems ({ pkgs, pkgs-tp, pkgs-g }: {
+        default = pkgs.mkShell {
+          packages = with pkgs; [
+            neovim
+            go # Go 1.24.4
+            gotools
+            llvmPackages_18.clangUseLLVM
+            pkgs-g.ginkgo
+            golangci-lint
+            docker
+            docker-compose
+            python313Packages.pip
+            kubernetes-helm
+            kind
+            kubectl
+            cilium-cli
+            pkgs-tp.tparse
+          ];
+        };
+    });
+  };
+}

operator/option/config.go

@@ -132,6 +132,14 @@ const (
 	// and DescribeSecurityGroups. Set to 0 to let AWS determine the optimal page size.
 	AWSMaxResultsPerCall = "aws-max-results-per-call"
 
+	// AWSCrossAccountRoleARN is the ARN of an IAM role in the VPC-owner (network) account.
+	// When not empty, Cilium assumes this role for all ENI lifecycle operations (create, delete,
+	// assign/unassign IPs) ENI ops will be completed by this, separate connection and 
+	// the local instance profile will be used for all instance-level operations
+	// (attach, describe instances). Required when pod subnets live in a different AWS account
+	// from the EC2 instances and are not shared via RAM.
+	AWSCrossAccountRoleARN = "aws-cross-account-role"
+
 	// Azure options
 
 	// AzureSubscriptionID is the subscription ID to use when accessing the Azure API

operator/pkg/ipam/aws.go

@@ -42,6 +42,7 @@ type AWSConfig struct {
 	AWSUsePrimaryAddress         bool
 	EC2APIEndpoint               string
 	AWSMaxResultsPerCall         int32
+	AWSCrossAccountRoleARN       string
 }
 
 var awsDefaultConfig = AWSConfig{
@@ -54,6 +55,7 @@ var awsDefaultConfig = AWSConfig{
 	AWSUsePrimaryAddress:         false,
 	EC2APIEndpoint:               "",
 	AWSMaxResultsPerCall:         0,
+	AWSCrossAccountRoleARN:       "",
 }
 
 func (cfg AWSConfig) Flags(flags *pflag.FlagSet) {
@@ -69,6 +71,8 @@ func (cfg AWSConfig) Flags(flags *pflag.FlagSet) {
 	flags.Bool(operatorOption.AWSUsePrimaryAddress, awsDefaultConfig.AWSUsePrimaryAddress, "Allows for using primary address of the ENI for allocations on the node")
 	flags.String(operatorOption.EC2APIEndpoint, awsDefaultConfig.EC2APIEndpoint, "AWS API endpoint for the EC2 service")
 	flags.Int32(operatorOption.AWSMaxResultsPerCall, awsDefaultConfig.AWSMaxResultsPerCall, "Maximum results per AWS API call for DescribeNetworkInterfaces and DescribeSecurityGroups. Set to 0 to let AWS determine optimal page size (default). If set to 0 and AWS returns OperationNotPermitted errors, automatically switches to 1000 for all future requests")
+	flags.String(operatorOption.AWSCrossAccountRoleARN, awsDefaultConfig.AWSCrossAccountRoleARN,
+		"IAM role ARN to assume in the VPC-owner's account for cross-account ENI management")
 }
 
 type awsParams struct {
@@ -101,6 +105,7 @@ func startAWSAllocator(p awsParams) {
 		AWSUsePrimaryAddress:         p.AwsCfg.AWSUsePrimaryAddress,
 		EC2APIEndpoint:               p.AwsCfg.EC2APIEndpoint,
 		AWSMaxResultsPerCall:         p.AwsCfg.AWSMaxResultsPerCall,
+		AWSCrossAccountRoleARN:       p.AwsCfg.AWSCrossAccountRoleARN,
 		ParallelAllocWorkers:         p.Cfg.ParallelAllocWorkers,
 	}
 

pkg/aws/ec2/ec2.go

@@ -16,9 +16,11 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	ec2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	"github.com/aws/aws-sdk-go-v2/service/sts"
 	"github.com/aws/smithy-go"
 
 	"github.com/cilium/cilium/pkg/api/helpers"
@@ -45,11 +47,12 @@ const (
 	// OperationNotPermittedStr indicates the request returned too many results without sufficient filtering or pagination
 	OperationNotPermittedStr = "OperationNotPermitted"
 
-	AssignPrivateIpAddresses        = "AssignPrivateIpAddresses"
-	AssociateAddress                = "AssociateAddress"
-	AttachNetworkInterface          = "AttachNetworkInterface"
-	CreateNetworkInterface          = "CreateNetworkInterface"
-	DeleteNetworkInterface          = "DeleteNetworkInterface"
+	AssignPrivateIpAddresses             = "AssignPrivateIpAddresses"
+	AssociateAddress                     = "AssociateAddress"
+	AttachNetworkInterface               = "AttachNetworkInterface"
+	CreateNetworkInterface               = "CreateNetworkInterface"
+	CreateNetworkInterfacePermission     = "CreateNetworkInterfacePermission"
+	DeleteNetworkInterface               = "DeleteNetworkInterface"
 	DescribeAddresses               = "DescribeAddresses"
 	DescribeInstances               = "DescribeInstances"
 	DescribeInstanceTypes           = "DescribeInstanceTypes"
@@ -170,6 +173,29 @@ func NewConfig(ctx context.Context) (aws.Config, error) {
 	return cfg, nil
 }
 
+// NewCrossAccountConfig returns an aws.Config that assumes the given IAM role ARN.
+// The base config (with region and retry settings) is reused; maybe this will need a change?
+func NewCrossAccountConfig(ctx context.Context, baseConfig aws.Config, roleARN string) (aws.Config, error) {
+	stsClient := sts.NewFromConfig(baseConfig)
+	creds := stscreds.NewAssumeRoleProvider(stsClient, roleARN)
+	cfg := baseConfig.Copy()
+	cfg.Credentials = aws.NewCredentialsCache(creds)
+	// make a call with it confirm the creds work rather than waiting for object's first call
+	if _, err := cfg.Credentials.Retrieve(ctx); err != nil {
+		return aws.Config{}, fmt.Errorf("unable to assume cross-account role %s: %w", roleARN, err)
+	}
+	return cfg, nil
+}
+
+// GetLocalAccountID returns the AWS account ID of the instance running this process.
+func GetLocalAccountID(ctx context.Context, cfg aws.Config) (string, error) {
+	doc, err := imds.NewFromConfig(cfg).GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
+	if err != nil {
+		return "", fmt.Errorf("unable to retrieve instance identity document: %w", err)
+	}
+	return doc.AccountID, nil
+}
+
 // NewSubnetsFilters transforms a map of tags and values and a slice of subnets
 // into a slice of ec2.Filter adequate to filter AWS subnets.
 func NewSubnetsFilters(tags map[string]string, ids []string) []ec2_types.Filter {
@@ -940,6 +966,24 @@ func (c *Client) UnassignENIPrefixes(ctx context.Context, eniID string, prefixes
 	return err
 }
 
+// CreateNetworkInterfacePermission grants INSTANCE-ATTACH permission on the given ENI to a difft AWS account.
+// This is required before an instance in accountID can attach an ENI owned by a different account.
+func (c *Client) CreateNetworkInterfacePermission(ctx context.Context, eniID string, accountID string) error {
+	input := &ec2.CreateNetworkInterfacePermissionInput{
+		NetworkInterfaceId: aws.String(eniID),
+		AwsAccountId:       aws.String(accountID),
+		Permission:         ec2_types.InterfacePermissionTypeInstanceAttach,
+	}
+
+	// wrap this in a limiter
+	c.limiter.Limit(ctx, CreateNetworkInterfacePermission)
+	// track how long it takes
+	sinceStart := spanstat.Start()
+	_, err := c.ec2Client.CreateNetworkInterfacePermission(ctx, input)
+	c.metricsAPI.ObserveAPICall(CreateNetworkInterfacePermission, deriveStatus(err), sinceStart.Seconds())
+	return err
+}
+
 // AssociateEIP tries to find an Elastic IP Address with the given tags and associates it with the given instance
 func (c *Client) AssociateEIP(ctx context.Context, eniID string, eipTags ipamTypes.Tags) (string, error) {
 	if len(eipTags) == 0 {

pkg/aws/ec2/mock/mock.go

@@ -798,3 +798,7 @@ func (e *API) GetSecurityGroups(ctx context.Context, vpcID string) (types.Securi
 func (e *API) GetInstanceTypes(ctx context.Context) ([]ec2_types.InstanceTypeInfo, error) {
 	return e.instanceTypes, nil
 }
+
+func (e *API) CreateNetworkInterfacePermission(_ context.Context, _ string, _ string) error {
+	return nil
+}

pkg/aws/eni/crossaccount.go

@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package eni
+
+import (
+	"context"
+	"log/slog"
+	ec2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	eniTypes "github.com/cilium/cilium/pkg/aws/eni/types"
+	"github.com/cilium/cilium/pkg/aws/types"
+	ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
+	"github.com/cilium/cilium/pkg/logging/logfields"
+)
+
+// CrossAccountEC2Client splits EC2 API calls between two accounts:
+//   - remote: ownwer of the VPC and pod subnets. This handles all ENI lifecycle operations
+//   - local: owner of the EC2 instances. This handles all instance-level operations including attachments
+//
+// After CreateNetworkInterface succeeds on the remote client, a
+// CreateNetworkInterfacePermission call grants the local account INSTANCE-ATTACH
+// access so that AttachNetworkInterface can be called from the local account.
+type CrossAccountEC2Client struct {
+	logger         *slog.Logger
+	local          EC2API
+	remote         EC2API
+	localAccountID string
+}
+
+// NewCrossAccountEC2Client constructs a CrossAccountEC2Client.
+// the localAccountID is needed to setup every CreateNetworkInterfacePermission
+func NewCrossAccountEC2Client(logger *slog.Logger, local, remote EC2API, localAccountID string) *CrossAccountEC2Client {
+	return &CrossAccountEC2Client{
+		logger:         logger,
+		local:          local,
+		remote:         remote,
+		localAccountID: localAccountID,
+	}
+}
+
+// *********************************************************************************
+// --- VPC / Subnet / ENI-owner accouint operations → remote (network account) ---
+// *********************************************************************************
+
+func (c *CrossAccountEC2Client) GetSubnets(ctx context.Context, vpcID string) (ipamTypes.SubnetMap, error) {
+	return c.remote.GetSubnets(ctx, vpcID)
+}
+
+func (c *CrossAccountEC2Client) GetVpcs(ctx context.Context, vpcID string) (ipamTypes.VirtualNetworkMap, error) {
+	return c.remote.GetVpcs(ctx, vpcID)
+}
+
+func (c *CrossAccountEC2Client) GetRouteTables(ctx context.Context, vpcID string) (ipamTypes.RouteTableMap, error) {
+	return c.remote.GetRouteTables(ctx, vpcID)
+}
+
+//TODO: not sure if I need this
+func (c *CrossAccountEC2Client) GetSecurityGroups(ctx context.Context, vpcID string) (types.SecurityGroupMap, error) {
+	return c.remote.GetSecurityGroups(ctx, vpcID)
+}
+
+func (c *CrossAccountEC2Client) GetDetachedNetworkInterfaces(ctx context.Context, tags ipamTypes.Tags, maxResults int32) ([]string, error) {
+	return c.remote.GetDetachedNetworkInterfaces(ctx, tags, maxResults)
+}
+
+// CreateNetworkInterface creates the ENI in the remote account's subnet, then
+// immediately grants the local account INSTANCE-ATTACH permission so that
+// AttachNetworkInterface (local) can succeed.
+func (c *CrossAccountEC2Client) CreateNetworkInterface(ctx context.Context, toAllocate int32, subnetID, desc string, groups []string, allocatePrefixes bool) (string, *eniTypes.ENI, error) {
+	eniID, eni, err := c.remote.CreateNetworkInterface(ctx, toAllocate, subnetID, desc, groups, allocatePrefixes)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if permErr := c.remote.CreateNetworkInterfacePermission(ctx, eniID, c.localAccountID); permErr != nil {
+		// Permission grant call failed. Delete the orphaned eni and rethrow
+		c.logger.Warn(
+			"Failed to grant cross-account ENI attach permission. Deleting orphaned ENI",
+			logfields.ENI, eniID,
+			logfields.Error, permErr,
+		)
+		if delErr := c.remote.DeleteNetworkInterface(ctx, eniID); delErr != nil {
+			//TODO: maybe make a bigger deal of this
+			c.logger.Warn("Failed to delete orphaned ENI",
+				logfields.ENI, eniID,
+				logfields.Error, delErr,
+			)
+		}
+		return "", nil, permErr
+	}
+
+	return eniID, eni, nil
+}
+
+func (c *CrossAccountEC2Client) CreateNetworkInterfacePermission(ctx context.Context, eniID string, accountID string) error {
+	return c.remote.CreateNetworkInterfacePermission(ctx, eniID, accountID)
+}
+
+func (c *CrossAccountEC2Client) DeleteNetworkInterface(ctx context.Context, eniID string) error {
+	return c.remote.DeleteNetworkInterface(ctx, eniID)
+}
+
+func (c *CrossAccountEC2Client) AssignPrivateIpAddresses(ctx context.Context, eniID string, addresses int32) ([]string, error) {
+	return c.remote.AssignPrivateIpAddresses(ctx, eniID, addresses)
+}
+
+func (c *CrossAccountEC2Client) UnassignPrivateIpAddresses(ctx context.Context, eniID string, addresses []string) error {
+	return c.remote.UnassignPrivateIpAddresses(ctx, eniID, addresses)
+}
+
+func (c *CrossAccountEC2Client) AssignENIPrefixes(ctx context.Context, eniID string, prefixes int32) error {
+	return c.remote.AssignENIPrefixes(ctx, eniID, prefixes)
+}
+
+func (c *CrossAccountEC2Client) UnassignENIPrefixes(ctx context.Context, eniID string, prefixes []string) error {
+	return c.remote.UnassignENIPrefixes(ctx, eniID, prefixes)
+}
+
+
+// *********************************************************************************
+// --- Instance-owner operations → local ---
+// *********************************************************************************
+
+func (c *CrossAccountEC2Client) GetInstance(ctx context.Context, vpcs ipamTypes.VirtualNetworkMap, subnets ipamTypes.SubnetMap, instanceID string) (*ipamTypes.Instance, error) {
+	return c.local.GetInstance(ctx, vpcs, subnets, instanceID)
+}
+
+func (c *CrossAccountEC2Client) GetInstances(ctx context.Context, vpcs ipamTypes.VirtualNetworkMap, subnets ipamTypes.SubnetMap) (*ipamTypes.InstanceMap, error) {
+	return c.local.GetInstances(ctx, vpcs, subnets)
+}
+
+// Needed so we can get max limits by type
+func (c *CrossAccountEC2Client) GetInstanceTypes(ctx context.Context) ([]ec2_types.InstanceTypeInfo, error) {
+	return c.local.GetInstanceTypes(ctx)
+}
+
+func (c *CrossAccountEC2Client) AttachNetworkInterface(ctx context.Context, index int32, instanceID, eniID string) (string, error) {
+	return c.local.AttachNetworkInterface(ctx, index, instanceID, eniID)
+}
+
+func (c *CrossAccountEC2Client) ModifyNetworkInterface(ctx context.Context, eniID, attachmentID string, deleteOnTermination bool) error {
+	return c.local.ModifyNetworkInterface(ctx, eniID, attachmentID, deleteOnTermination)
+}
+
+func (c *CrossAccountEC2Client) AssociateEIP(ctx context.Context, eniID string, eipTags ipamTypes.Tags) (string, error) {
+	return c.local.AssociateEIP(ctx, eniID, eipTags)
+}