Add support for custom vnets

api/v1alpha2/types.go

@@ -90,17 +90,21 @@ type Network struct {
 
 // NetworkSpec encapsulates all things related to Azure network.
 type NetworkSpec struct {
-	// Vnet configuration.
+	// Vnet is the configuration for the Azure virtual network.
 	// +optional
 	Vnet VnetSpec `json:"vnet,omitempty"`
 
-	// Subnets configuration.
+	// Subnets is the configuration for the control-plane subnet and the node subnet.
 	// +optional
 	Subnets Subnets `json:"subnets,omitempty"`
 }
 
 // VnetSpec configures an Azure virtual network.
 type VnetSpec struct {
+	// ResourceGroup is the name of the resource group of the existing virtual network
+	// or the resource group where a managed virtual network should be created.
+	ResourceGroup string `json:"resourceGroup,omitempty"`
+
 	// ID is the identifier of the virtual network this provider should use to create resources.
 	ID string `json:"id,omitempty"`
 
@@ -114,9 +118,9 @@ type VnetSpec struct {
 	Tags Tags `json:"tags,omitempty"`
 }
 
-// IsManaged returns true if the vnet is unmanaged.
+// IsManaged returns true if the vnet is managed.
 func (v *VnetSpec) IsManaged(clusterName string) bool {
-	return v.ID != "" && !v.Tags.HasOwned(clusterName)
+	return v.ID == "" || v.Tags.HasOwned(clusterName)
 }
 
 // Subnets is a slice of Subnet.
@@ -147,9 +151,9 @@ var (
 
 // SecurityGroup defines an Azure security group.
 type SecurityGroup struct {
-	ID           string       `json:"id"`
-	Name         string       `json:"name"`
-	IngressRules IngressRules `json:"ingressRule"`
+	ID           string       `json:"id,omitempty"`
+	Name         string       `json:"name,omitempty"`
+	IngressRules IngressRules `json:"ingressRule,omitempty"`
 	Tags         Tags         `json:"tags,omitempty"`
 }
 
@@ -433,22 +437,37 @@ type ManagedDisk struct {
 	StorageAccountType string `json:"storageAccountType"`
 }
 
+// SubnetRole defines the unique role of a subnet.
+type SubnetRole string
+
+var (
+	// SubnetNode defines a Kubernetes workload node role
+	SubnetNode = SubnetRole(Node)
+
+	// SubnetControlPlane defines a Kubernetes control plane node role
+	SubnetControlPlane = SubnetRole(ControlPlane)
+)
+
 // SubnetSpec configures an Azure subnet.
 type SubnetSpec struct {
+	// Role defines the subnet role (eg. Node, ControlPlane)
+	Role SubnetRole `json:"role,omitempty"`
+
 	// ID defines a unique identifier to reference this resource.
 	ID string `json:"id,omitempty"`
 
 	// Name defines a name for the subnet resource.
 	Name string `json:"name"`
 
-	// VnetID defines the ID of the virtual network this subnet should be built in.
-	VnetID string `json:"vnetId"`
-
 	// CidrBlock is the CIDR block to be used when the provider creates a managed Vnet.
 	CidrBlock string `json:"cidrBlock,omitempty"`
 
+	// InternalLBIPAddress is the IP address that will be used as the internal LB private IP.
+	// For the control plane subnet only.
+	InternalLBIPAddress string `json:"internalLBIPAddress,omitempty"`
+
 	// SecurityGroup defines the NSG (network security group) that should be attached to this subnet.
-	SecurityGroup SecurityGroup `json:"securityGroup"`
+	SecurityGroup SecurityGroup `json:"securityGroup,omitempty"`
 }
 
 const (

cloud/errors.go

@@ -20,7 +20,7 @@ import (
 	"github.com/Azure/go-autorest/autorest"
 )
 
-// ResourceNotFound parses the error to check if its a resource not found
+// ResourceNotFound parses the error to check if it's a resource not found
 func ResourceNotFound(err error) bool {
 	if derr, ok := err.(autorest.DetailedError); ok && derr.StatusCode == 404 {
 		return true

cloud/scope/cluster.go

@@ -99,11 +99,42 @@ func (s *ClusterScope) Subnets() infrav1.Subnets {
 	return s.AzureCluster.Spec.NetworkSpec.Subnets
 }
 
+// ControlPlaneSubnet returns the cluster control plane subnet.
+func (s *ClusterScope) ControlPlaneSubnet() *infrav1.SubnetSpec {
+	for _, sn := range s.AzureCluster.Spec.NetworkSpec.Subnets {
+		if sn.Role == infrav1.SubnetControlPlane {
+			return sn
+		}
+	}
+	if len(s.AzureCluster.Spec.NetworkSpec.Subnets) > 0 {
+		return s.AzureCluster.Spec.NetworkSpec.Subnets[0]
+	}
+	return nil
+}
+
+// NodeSubnet returns the cluster node subnet.
+func (s *ClusterScope) NodeSubnet() *infrav1.SubnetSpec {
+	for _, sn := range s.AzureCluster.Spec.NetworkSpec.Subnets {
+		if sn.Role == infrav1.SubnetNode {
+			return sn
+		}
+	}
+	if len(s.AzureCluster.Spec.NetworkSpec.Subnets) > 1 {
+		return s.AzureCluster.Spec.NetworkSpec.Subnets[1]
+	}
+	return nil
+}
+
 // SecurityGroups returns the cluster security groups as a map, it creates the map if empty.
 func (s *ClusterScope) SecurityGroups() map[infrav1.SecurityGroupRole]infrav1.SecurityGroup {
 	return s.AzureCluster.Status.Network.SecurityGroups
 }
 
+// ResourceGroup returns the cluster resource group.
+func (s *ClusterScope) ResourceGroup() string {
+	return s.AzureCluster.Spec.ResourceGroup
+}
+
 // Name returns the cluster name.
 func (s *ClusterScope) Name() string {
 	return s.Cluster.Name

cloud/services/disks/disks.go

@@ -46,13 +46,13 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error {
 		return errors.New("Invalid disk specification")
 	}
 	klog.V(2).Infof("deleting disk %s", diskSpec.Name)
-	err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, diskSpec.Name)
+	err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), diskSpec.Name)
 	if err != nil && azure.ResourceNotFound(err) {
 		// already deleted
 		return nil
 	}
 	if err != nil {
-		return errors.Wrapf(err, "failed to delete disk %s in resource group %s", diskSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup)
+		return errors.Wrapf(err, "failed to delete disk %s in resource group %s", diskSpec.Name, s.Scope.ResourceGroup())
 	}
 
 	klog.V(2).Infof("successfully deleted disk %s", diskSpec.Name)

cloud/services/groups/groups.go

@@ -30,7 +30,7 @@ import (
 
 // Get provides information about a resource group.
 func (s *Service) Get(ctx context.Context, spec interface{}) (resources.Group, error) {
-	return s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup)
+	return s.Client.Get(ctx, s.Scope.ResourceGroup())
 }
 
 // Reconcile gets/creates/updates a resource group.
@@ -39,19 +39,19 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error {
 		// resource group already exists, skip creation
 		return nil
 	}
-	klog.V(2).Infof("creating resource group %s", s.Scope.AzureCluster.Spec.ResourceGroup)
+	klog.V(2).Infof("creating resource group %s", s.Scope.ResourceGroup())
 	group := resources.Group{
 		Location: to.StringPtr(s.Scope.Location()),
 		Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{
 			ClusterName: s.Scope.Name(),
 			Lifecycle:   infrav1.ResourceLifecycleOwned,
-			Name:        to.StringPtr(s.Scope.AzureCluster.Spec.ResourceGroup),
+			Name:        to.StringPtr(s.Scope.ResourceGroup()),
 			Role:        to.StringPtr(infrav1.CommonRoleTagValue),
 			Additional:  s.Scope.AdditionalTags(),
 		})),
 	}
-	_, err := s.Client.CreateOrUpdate(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, group)
-	klog.V(2).Infof("successfully created resource group %s", s.Scope.AzureCluster.Spec.ResourceGroup)
+	_, err := s.Client.CreateOrUpdate(ctx, s.Scope.ResourceGroup(), group)
+	klog.V(2).Infof("successfully created resource group %s", s.Scope.ResourceGroup())
 	return err
 }
 
@@ -66,17 +66,17 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error {
 		s.Scope.V(4).Info("Skipping resource group deletion in unmanaged mode")
 		return nil
 	}
-	klog.V(2).Infof("deleting resource group %s", s.Scope.AzureCluster.Spec.ResourceGroup)
-	err = s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup)
+	klog.V(2).Infof("deleting resource group %s", s.Scope.ResourceGroup())
+	err = s.Client.Delete(ctx, s.Scope.ResourceGroup())
 	if err != nil && azure.ResourceNotFound(err) {
 		// already deleted
 		return nil
 	}
 	if err != nil {
-		return errors.Wrapf(err, "failed to delete resource group %s", s.Scope.AzureCluster.Spec.ResourceGroup)
+		return errors.Wrapf(err, "failed to delete resource group %s", s.Scope.ResourceGroup())
 	}
 
-	klog.V(2).Infof("successfully deleted resource group %s", s.Scope.AzureCluster.Spec.ResourceGroup)
+	klog.V(2).Infof("successfully deleted resource group %s", s.Scope.ResourceGroup())
 	return nil
 }
 

cloud/services/internalloadbalancers/internalloadbalancers.go

@@ -31,24 +31,18 @@ import (
 type Spec struct {
 	Name       string
 	SubnetName string
+	SubnetCidr string
 	VnetName   string
 	IPAddress  string
 }
 
 // Get provides information about an internal load balancer.
-func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error) {
+func (s *Service) Get(ctx context.Context, spec interface{}) (network.LoadBalancer, error) {
 	internalLBSpec, ok := spec.(*Spec)
 	if !ok {
 		return network.LoadBalancer{}, errors.New("invalid internal load balancer specification")
 	}
-	//lbName := fmt.Sprintf("%s-api-internallb", s.Scope.Cluster.Name)
-	lb, err := s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, internalLBSpec.Name)
-	if err != nil && azure.ResourceNotFound(err) {
-		return nil, errors.Wrapf(err, "load balancer %s not found", internalLBSpec.Name)
-	} else if err != nil {
-		return lb, err
-	}
-	return lb, nil
+	return s.Client.Get(ctx, s.Scope.ResourceGroup(), internalLBSpec.Name)
 }
 
 // Reconcile gets/creates/updates an internal load balancer.
@@ -61,20 +55,38 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error {
 	probeName := "tcpHTTPSProbe"
 	frontEndIPConfigName := "controlplane-internal-lbFrontEnd"
 	backEndAddressPoolName := "controlplane-internal-backEndPool"
-	idPrefix := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers", s.Scope.SubscriptionID, s.Scope.AzureCluster.Spec.ResourceGroup)
+	idPrefix := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers", s.Scope.SubscriptionID, s.Scope.ResourceGroup())
 	lbName := internalLBSpec.Name
+	var privateIP string
+
+	internalLB, err := s.Get(ctx, internalLBSpec)
+	if err == nil {
+		ipConfigs := internalLB.LoadBalancerPropertiesFormat.FrontendIPConfigurations
+		if ipConfigs != nil && len(*ipConfigs) > 0 {
+			privateIP = to.String((*ipConfigs)[0].FrontendIPConfigurationPropertiesFormat.PrivateIPAddress)
+		}
+	} else if azure.ResourceNotFound(err) {
+		klog.V(2).Infof("internalLB %s not found in RG %s", internalLBSpec.Name, s.Scope.ResourceGroup())
+		privateIP, err = s.getAvailablePrivateIP(ctx, s.Scope.Vnet().ResourceGroup, internalLBSpec.VnetName, internalLBSpec.SubnetCidr, internalLBSpec.IPAddress)
+		if err != nil {
+			return err
+		}
+		klog.V(2).Infof("setting internal load balancer IP to %s", privateIP)
+	} else {
+		return errors.Wrap(err, "failed to look for existing internal LB")
+	}
 
 	klog.V(2).Infof("getting subnet %s", internalLBSpec.SubnetName)
-	subnet, err := s.SubnetsClient.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, internalLBSpec.VnetName, internalLBSpec.SubnetName)
+	subnet, err := s.SubnetsClient.Get(ctx, s.Scope.Vnet().ResourceGroup, internalLBSpec.VnetName, internalLBSpec.SubnetName)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to get subnet")
 	}
 
 	klog.V(2).Infof("successfully got subnet %s", internalLBSpec.SubnetName)
 
 	// https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-standard-availability-zones#zone-redundant-by-default
 	err = s.Client.CreateOrUpdate(ctx,
-		s.Scope.AzureCluster.Spec.ResourceGroup,
+		s.Scope.ResourceGroup(),
 		lbName,
 		network.LoadBalancer{
 			Sku:      &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard},
@@ -86,7 +98,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error {
 						FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
 							PrivateIPAllocationMethod: network.Static,
 							Subnet:                    &subnet,
-							PrivateIPAddress:          to.StringPtr(internalLBSpec.IPAddress),
+							PrivateIPAddress:          to.StringPtr(privateIP),
 						},
 					},
 				},
@@ -146,14 +158,38 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error {
 		return errors.New("invalid internal load balancer specification")
 	}
 	klog.V(2).Infof("deleting internal load balancer %s", internalLBSpec.Name)
-	err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, internalLBSpec.Name)
+	err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), internalLBSpec.Name)
 	if err != nil && azure.ResourceNotFound(err) {
 		// already deleted
 		return nil
 	}
 	if err != nil {
-		return errors.Wrapf(err, "failed to delete internal load balancer %s in resource group %s", internalLBSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup)
+		return errors.Wrapf(err, "failed to delete internal load balancer %s in resource group %s", internalLBSpec.Name, s.Scope.ResourceGroup())
 	}
 	klog.V(2).Infof("successfully deleted internal load balancer %s", internalLBSpec.Name)
 	return nil
 }
+
+// getAvailablePrivateIP checks if the desired private IP address is available in a virtual network.
+// If the IP address is taken or empty, it will make an attempt to find an available IP in the same subnet
+func (s *Service) getAvailablePrivateIP(ctx context.Context, resourceGroup, vnetName, subnetCIDR, PreferredIPAddress string) (string, error) {
+	ip := PreferredIPAddress
+	if ip == "" {
+		ip = azure.DefaultInternalLBIPAddress
+		if subnetCIDR != azure.DefaultControlPlaneSubnetCIDR {
+			// If the user provided a custom subnet CIDR without providing a private IP, try finding an available IP in the subnet space
+			ip = subnetCIDR[0:7] + "0"
+		}
+	}
+	result, err := s.VirtualNetworksClient.CheckIPAddressAvailability(ctx, resourceGroup, vnetName, ip)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to check IP availability")
+	}
+	if !to.Bool(result.Available) {
+		if len(to.StringSlice(result.AvailableIPAddresses)) == 0 {
+			return "", errors.Errorf("IP %s is not available in vnet %s and there were no other available IPs found", ip, vnetName)
+		}
+		ip = to.StringSlice(result.AvailableIPAddresses)[0]
+	}
+	return ip, nil
+}

cloud/services/internalloadbalancers/service.go

@@ -19,20 +19,23 @@ package internalloadbalancers
 import (
 	"sigs.k8s.io/cluster-api-provider-azure/cloud/scope"
 	"sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets"
+	"sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualnetworks"
 )
 
 // Service provides operations on azure resources
 type Service struct {
 	Scope *scope.ClusterScope
 	Client
-	SubnetsClient subnets.Client
+	SubnetsClient         subnets.Client
+	VirtualNetworksClient virtualnetworks.Client
 }
 
 // NewService creates a new service.
 func NewService(scope *scope.ClusterScope) *Service {
 	return &Service{
-		Scope:         scope,
-		Client:        NewClient(scope.SubscriptionID, scope.Authorizer),
-		SubnetsClient: subnets.NewClient(scope.SubscriptionID, scope.Authorizer),
+		Scope:                 scope,
+		Client:                NewClient(scope.SubscriptionID, scope.Authorizer),
+		SubnetsClient:         subnets.NewClient(scope.SubscriptionID, scope.Authorizer),
+		VirtualNetworksClient: virtualnetworks.NewClient(scope.SubscriptionID, scope.Authorizer),
 	}
 }