From ef8b3aa16f5142a857a230b3fe930aeb0318f847 Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Tue, 5 Nov 2019 09:50:42 -0800 Subject: [PATCH] Add support for custom vnets --- api/v1alpha2/types.go | 41 ++- cloud/errors.go | 2 +- cloud/scope/cluster.go | 31 +++ cloud/services/disks/disks.go | 4 +- cloud/services/groups/groups.go | 18 +- .../internalloadbalancers.go | 68 +++-- .../services/internalloadbalancers/service.go | 11 +- .../networkinterfaces/networkinterfaces.go | 18 +- cloud/services/publicips/publicips.go | 8 +- .../publicloadbalancers.go | 12 +- cloud/services/routetables/routetables.go | 18 +- .../services/securitygroups/securitygroups.go | 14 +- cloud/services/subnets/subnets.go | 75 ++++-- .../virtualmachineextensions.go | 8 +- .../virtualmachines/virtualmachines.go | 14 +- .../virtualmachines/virtualmachines_test.go | 10 +- cloud/services/virtualnetworks/client.go | 6 + .../virtualnetworks_mock.go | 15 ++ .../virtualnetworks/virtualnetworks.go | 65 +++-- .../virtualnetworks/virtualnetworks_test.go | 255 ++++++++++++++++++ ...ucture.cluster.x-k8s.io_azureclusters.yaml | 32 +-- controllers/azurecluster_reconciler.go | 132 ++++++--- controllers/azuremachine_reconciler.go | 6 +- controllers/azuremachine_tags.go | 4 +- docs/topics/custom-vnet.md | 84 ++++++ 25 files changed, 765 insertions(+), 186 deletions(-) create mode 100644 cloud/services/virtualnetworks/virtualnetworks_test.go create mode 100644 docs/topics/custom-vnet.md diff --git a/api/v1alpha2/types.go b/api/v1alpha2/types.go index 2fbb8e2ef67..cde1f02c850 100644 --- a/api/v1alpha2/types.go +++ b/api/v1alpha2/types.go @@ -90,17 +90,21 @@ type Network struct { // NetworkSpec encapsulates all things related to Azure network. type NetworkSpec struct { - // Vnet configuration. + // Vnet is the configuration for the Azure virtual network. // +optional Vnet VnetSpec `json:"vnet,omitempty"` - // Subnets configuration. + // Subnets is the configuration for the control-plane subnet and the node subnet. // +optional Subnets Subnets `json:"subnets,omitempty"` } // VnetSpec configures an Azure virtual network. type VnetSpec struct { + // ResourceGroup is the name of the resource group of the existing virtual network + // or the resource group where a managed virtual network should be created. + ResourceGroup string `json:"resourceGroup,omitempty"` + // ID is the identifier of the virtual network this provider should use to create resources. ID string `json:"id,omitempty"` @@ -114,9 +118,9 @@ type VnetSpec struct { Tags Tags `json:"tags,omitempty"` } -// IsManaged returns true if the vnet is unmanaged. +// IsManaged returns true if the vnet is managed. func (v *VnetSpec) IsManaged(clusterName string) bool { - return v.ID != "" && !v.Tags.HasOwned(clusterName) + return v.ID == "" || v.Tags.HasOwned(clusterName) } // Subnets is a slice of Subnet. @@ -147,9 +151,9 @@ var ( // SecurityGroup defines an Azure security group. type SecurityGroup struct { - ID string `json:"id"` - Name string `json:"name"` - IngressRules IngressRules `json:"ingressRule"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + IngressRules IngressRules `json:"ingressRule,omitempty"` Tags Tags `json:"tags,omitempty"` } @@ -433,22 +437,37 @@ type ManagedDisk struct { StorageAccountType string `json:"storageAccountType"` } +// SubnetRole defines the unique role of a subnet. +type SubnetRole string + +var ( + // SubnetNode defines a Kubernetes workload node role + SubnetNode = SubnetRole(Node) + + // SubnetControlPlane defines a Kubernetes control plane node role + SubnetControlPlane = SubnetRole(ControlPlane) +) + // SubnetSpec configures an Azure subnet. type SubnetSpec struct { + // Role defines the subnet role (eg. Node, ControlPlane) + Role SubnetRole `json:"role,omitempty"` + // ID defines a unique identifier to reference this resource. ID string `json:"id,omitempty"` // Name defines a name for the subnet resource. Name string `json:"name"` - // VnetID defines the ID of the virtual network this subnet should be built in. - VnetID string `json:"vnetId"` - // CidrBlock is the CIDR block to be used when the provider creates a managed Vnet. CidrBlock string `json:"cidrBlock,omitempty"` + // InternalLBIPAddress is the IP address that will be used as the internal LB private IP. + // For the control plane subnet only. + InternalLBIPAddress string `json:"internalLBIPAddress,omitempty"` + // SecurityGroup defines the NSG (network security group) that should be attached to this subnet. - SecurityGroup SecurityGroup `json:"securityGroup"` + SecurityGroup SecurityGroup `json:"securityGroup,omitempty"` } const ( diff --git a/cloud/errors.go b/cloud/errors.go index 4e48981615e..ab8fd5dea3b 100644 --- a/cloud/errors.go +++ b/cloud/errors.go @@ -20,7 +20,7 @@ import ( "github.com/Azure/go-autorest/autorest" ) -// ResourceNotFound parses the error to check if its a resource not found +// ResourceNotFound parses the error to check if it's a resource not found func ResourceNotFound(err error) bool { if derr, ok := err.(autorest.DetailedError); ok && derr.StatusCode == 404 { return true diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index 3b1fd54c1e4..386e8033532 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -99,11 +99,42 @@ func (s *ClusterScope) Subnets() infrav1.Subnets { return s.AzureCluster.Spec.NetworkSpec.Subnets } +// ControlPlaneSubnet returns the cluster control plane subnet. +func (s *ClusterScope) ControlPlaneSubnet() *infrav1.SubnetSpec { + for _, sn := range s.AzureCluster.Spec.NetworkSpec.Subnets { + if sn.Role == infrav1.SubnetControlPlane { + return sn + } + } + if len(s.AzureCluster.Spec.NetworkSpec.Subnets) > 0 { + return s.AzureCluster.Spec.NetworkSpec.Subnets[0] + } + return nil +} + +// NodeSubnet returns the cluster node subnet. +func (s *ClusterScope) NodeSubnet() *infrav1.SubnetSpec { + for _, sn := range s.AzureCluster.Spec.NetworkSpec.Subnets { + if sn.Role == infrav1.SubnetNode { + return sn + } + } + if len(s.AzureCluster.Spec.NetworkSpec.Subnets) > 1 { + return s.AzureCluster.Spec.NetworkSpec.Subnets[1] + } + return nil +} + // SecurityGroups returns the cluster security groups as a map, it creates the map if empty. func (s *ClusterScope) SecurityGroups() map[infrav1.SecurityGroupRole]infrav1.SecurityGroup { return s.AzureCluster.Status.Network.SecurityGroups } +// ResourceGroup returns the cluster resource group. +func (s *ClusterScope) ResourceGroup() string { + return s.AzureCluster.Spec.ResourceGroup +} + // Name returns the cluster name. func (s *ClusterScope) Name() string { return s.Cluster.Name diff --git a/cloud/services/disks/disks.go b/cloud/services/disks/disks.go index db9de48de81..863e63c9e03 100644 --- a/cloud/services/disks/disks.go +++ b/cloud/services/disks/disks.go @@ -46,13 +46,13 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error { return errors.New("Invalid disk specification") } klog.V(2).Infof("deleting disk %s", diskSpec.Name) - err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, diskSpec.Name) + err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), diskSpec.Name) if err != nil && azure.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete disk %s in resource group %s", diskSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to delete disk %s in resource group %s", diskSpec.Name, s.Scope.ResourceGroup()) } klog.V(2).Infof("successfully deleted disk %s", diskSpec.Name) diff --git a/cloud/services/groups/groups.go b/cloud/services/groups/groups.go index 27388a53d81..3a8e06ad370 100644 --- a/cloud/services/groups/groups.go +++ b/cloud/services/groups/groups.go @@ -30,7 +30,7 @@ import ( // Get provides information about a resource group. func (s *Service) Get(ctx context.Context, spec interface{}) (resources.Group, error) { - return s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup) + return s.Client.Get(ctx, s.Scope.ResourceGroup()) } // Reconcile gets/creates/updates a resource group. @@ -39,19 +39,19 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { // resource group already exists, skip creation return nil } - klog.V(2).Infof("creating resource group %s", s.Scope.AzureCluster.Spec.ResourceGroup) + klog.V(2).Infof("creating resource group %s", s.Scope.ResourceGroup()) group := resources.Group{ Location: to.StringPtr(s.Scope.Location()), Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ ClusterName: s.Scope.Name(), Lifecycle: infrav1.ResourceLifecycleOwned, - Name: to.StringPtr(s.Scope.AzureCluster.Spec.ResourceGroup), + Name: to.StringPtr(s.Scope.ResourceGroup()), Role: to.StringPtr(infrav1.CommonRoleTagValue), Additional: s.Scope.AdditionalTags(), })), } - _, err := s.Client.CreateOrUpdate(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, group) - klog.V(2).Infof("successfully created resource group %s", s.Scope.AzureCluster.Spec.ResourceGroup) + _, err := s.Client.CreateOrUpdate(ctx, s.Scope.ResourceGroup(), group) + klog.V(2).Infof("successfully created resource group %s", s.Scope.ResourceGroup()) return err } @@ -66,17 +66,17 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error { s.Scope.V(4).Info("Skipping resource group deletion in unmanaged mode") return nil } - klog.V(2).Infof("deleting resource group %s", s.Scope.AzureCluster.Spec.ResourceGroup) - err = s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup) + klog.V(2).Infof("deleting resource group %s", s.Scope.ResourceGroup()) + err = s.Client.Delete(ctx, s.Scope.ResourceGroup()) if err != nil && azure.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete resource group %s", s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to delete resource group %s", s.Scope.ResourceGroup()) } - klog.V(2).Infof("successfully deleted resource group %s", s.Scope.AzureCluster.Spec.ResourceGroup) + klog.V(2).Infof("successfully deleted resource group %s", s.Scope.ResourceGroup()) return nil } diff --git a/cloud/services/internalloadbalancers/internalloadbalancers.go b/cloud/services/internalloadbalancers/internalloadbalancers.go index 7a5c1f18ca6..d64456354fa 100644 --- a/cloud/services/internalloadbalancers/internalloadbalancers.go +++ b/cloud/services/internalloadbalancers/internalloadbalancers.go @@ -31,24 +31,18 @@ import ( type Spec struct { Name string SubnetName string + SubnetCidr string VnetName string IPAddress string } // Get provides information about an internal load balancer. -func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error) { +func (s *Service) Get(ctx context.Context, spec interface{}) (network.LoadBalancer, error) { internalLBSpec, ok := spec.(*Spec) if !ok { return network.LoadBalancer{}, errors.New("invalid internal load balancer specification") } - //lbName := fmt.Sprintf("%s-api-internallb", s.Scope.Cluster.Name) - lb, err := s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, internalLBSpec.Name) - if err != nil && azure.ResourceNotFound(err) { - return nil, errors.Wrapf(err, "load balancer %s not found", internalLBSpec.Name) - } else if err != nil { - return lb, err - } - return lb, nil + return s.Client.Get(ctx, s.Scope.ResourceGroup(), internalLBSpec.Name) } // Reconcile gets/creates/updates an internal load balancer. @@ -61,20 +55,38 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { probeName := "tcpHTTPSProbe" frontEndIPConfigName := "controlplane-internal-lbFrontEnd" backEndAddressPoolName := "controlplane-internal-backEndPool" - idPrefix := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers", s.Scope.SubscriptionID, s.Scope.AzureCluster.Spec.ResourceGroup) + idPrefix := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers", s.Scope.SubscriptionID, s.Scope.ResourceGroup()) lbName := internalLBSpec.Name + var privateIP string + + internalLB, err := s.Get(ctx, internalLBSpec) + if err == nil { + ipConfigs := internalLB.LoadBalancerPropertiesFormat.FrontendIPConfigurations + if ipConfigs != nil && len(*ipConfigs) > 0 { + privateIP = to.String((*ipConfigs)[0].FrontendIPConfigurationPropertiesFormat.PrivateIPAddress) + } + } else if azure.ResourceNotFound(err) { + klog.V(2).Infof("internalLB %s not found in RG %s", internalLBSpec.Name, s.Scope.ResourceGroup()) + privateIP, err = s.getAvailablePrivateIP(ctx, s.Scope.Vnet().ResourceGroup, internalLBSpec.VnetName, internalLBSpec.SubnetCidr, internalLBSpec.IPAddress) + if err != nil { + return err + } + klog.V(2).Infof("setting internal load balancer IP to %s", privateIP) + } else { + return errors.Wrap(err, "failed to look for existing internal LB") + } klog.V(2).Infof("getting subnet %s", internalLBSpec.SubnetName) - subnet, err := s.SubnetsClient.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, internalLBSpec.VnetName, internalLBSpec.SubnetName) + subnet, err := s.SubnetsClient.Get(ctx, s.Scope.Vnet().ResourceGroup, internalLBSpec.VnetName, internalLBSpec.SubnetName) if err != nil { - return err + return errors.Wrap(err, "failed to get subnet") } klog.V(2).Infof("successfully got subnet %s", internalLBSpec.SubnetName) // https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-standard-availability-zones#zone-redundant-by-default err = s.Client.CreateOrUpdate(ctx, - s.Scope.AzureCluster.Spec.ResourceGroup, + s.Scope.ResourceGroup(), lbName, network.LoadBalancer{ Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, @@ -86,7 +98,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ PrivateIPAllocationMethod: network.Static, Subnet: &subnet, - PrivateIPAddress: to.StringPtr(internalLBSpec.IPAddress), + PrivateIPAddress: to.StringPtr(privateIP), }, }, }, @@ -146,14 +158,38 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error { return errors.New("invalid internal load balancer specification") } klog.V(2).Infof("deleting internal load balancer %s", internalLBSpec.Name) - err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, internalLBSpec.Name) + err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), internalLBSpec.Name) if err != nil && azure.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete internal load balancer %s in resource group %s", internalLBSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to delete internal load balancer %s in resource group %s", internalLBSpec.Name, s.Scope.ResourceGroup()) } klog.V(2).Infof("successfully deleted internal load balancer %s", internalLBSpec.Name) return nil } + +// getAvailablePrivateIP checks if the desired private IP address is available in a virtual network. +// If the IP address is taken or empty, it will make an attempt to find an available IP in the same subnet +func (s *Service) getAvailablePrivateIP(ctx context.Context, resourceGroup, vnetName, subnetCIDR, PreferredIPAddress string) (string, error) { + ip := PreferredIPAddress + if ip == "" { + ip = azure.DefaultInternalLBIPAddress + if subnetCIDR != azure.DefaultControlPlaneSubnetCIDR { + // If the user provided a custom subnet CIDR without providing a private IP, try finding an available IP in the subnet space + ip = subnetCIDR[0:7] + "0" + } + } + result, err := s.VirtualNetworksClient.CheckIPAddressAvailability(ctx, resourceGroup, vnetName, ip) + if err != nil { + return "", errors.Wrap(err, "failed to check IP availability") + } + if !to.Bool(result.Available) { + if len(to.StringSlice(result.AvailableIPAddresses)) == 0 { + return "", errors.Errorf("IP %s is not available in vnet %s and there were no other available IPs found", ip, vnetName) + } + ip = to.StringSlice(result.AvailableIPAddresses)[0] + } + return ip, nil +} diff --git a/cloud/services/internalloadbalancers/service.go b/cloud/services/internalloadbalancers/service.go index f69bb982a85..6dad2d83526 100644 --- a/cloud/services/internalloadbalancers/service.go +++ b/cloud/services/internalloadbalancers/service.go @@ -19,20 +19,23 @@ package internalloadbalancers import ( "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualnetworks" ) // Service provides operations on azure resources type Service struct { Scope *scope.ClusterScope Client - SubnetsClient subnets.Client + SubnetsClient subnets.Client + VirtualNetworksClient virtualnetworks.Client } // NewService creates a new service. func NewService(scope *scope.ClusterScope) *Service { return &Service{ - Scope: scope, - Client: NewClient(scope.SubscriptionID, scope.Authorizer), - SubnetsClient: subnets.NewClient(scope.SubscriptionID, scope.Authorizer), + Scope: scope, + Client: NewClient(scope.SubscriptionID, scope.Authorizer), + SubnetsClient: subnets.NewClient(scope.SubscriptionID, scope.Authorizer), + VirtualNetworksClient: virtualnetworks.NewClient(scope.SubscriptionID, scope.Authorizer), } } diff --git a/cloud/services/networkinterfaces/networkinterfaces.go b/cloud/services/networkinterfaces/networkinterfaces.go index 9b8a02e73f5..b3e8943ce6e 100644 --- a/cloud/services/networkinterfaces/networkinterfaces.go +++ b/cloud/services/networkinterfaces/networkinterfaces.go @@ -44,7 +44,7 @@ func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error if !ok { return network.Interface{}, errors.New("invalid network interface specification") } - nic, err := s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, nicSpec.Name) + nic, err := s.Client.Get(ctx, s.Scope.ResourceGroup(), nicSpec.Name) if err != nil && azure.ResourceNotFound(err) { return nil, errors.Wrapf(err, "network interface %s not found", nicSpec.Name) } else if err != nil { @@ -62,7 +62,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { nicConfig := &network.InterfaceIPConfigurationPropertiesFormat{} - subnet, err := s.SubnetsClient.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, nicSpec.VnetName, nicSpec.SubnetName) + subnet, err := s.SubnetsClient.Get(ctx, s.Scope.Vnet().ResourceGroup, nicSpec.VnetName, nicSpec.SubnetName) if err != nil { return err } @@ -76,7 +76,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { backendAddressPools := []network.BackendAddressPool{} if nicSpec.PublicLoadBalancerName != "" { - lb, lberr := s.LoadBalancersClient.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, nicSpec.PublicLoadBalancerName) + lb, lberr := s.LoadBalancersClient.Get(ctx, s.Scope.ResourceGroup(), nicSpec.PublicLoadBalancerName) if lberr != nil { return lberr } @@ -93,7 +93,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { } } if nicSpec.InternalLoadBalancerName != "" { - internalLB, ilberr := s.LoadBalancersClient.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, nicSpec.InternalLoadBalancerName) + internalLB, ilberr := s.LoadBalancersClient.Get(ctx, s.Scope.ResourceGroup(), nicSpec.InternalLoadBalancerName) if ilberr != nil { return ilberr } @@ -106,7 +106,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { nicConfig.LoadBalancerBackendAddressPools = &backendAddressPools if nicSpec.PublicIPName != "" { - publicIP, err := s.PublicIPsClient.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, nicSpec.PublicIPName) + publicIP, err := s.PublicIPsClient.Get(ctx, s.Scope.ResourceGroup(), nicSpec.PublicIPName) if err != nil { return errors.Wrap(err, "failed to get publicIP") } @@ -114,7 +114,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { } err = s.Client.CreateOrUpdate(ctx, - s.Scope.AzureCluster.Spec.ResourceGroup, + s.Scope.ResourceGroup(), nicSpec.Name, network.Interface{ Location: to.StringPtr(s.Scope.Location()), @@ -129,7 +129,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { }) if err != nil { - return errors.Wrapf(err, "failed to create network interface %s in resource group %s", nicSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to create network interface %s in resource group %s", nicSpec.Name, s.Scope.ResourceGroup()) } klog.V(2).Infof("successfully created network interface %s", nicSpec.Name) @@ -143,13 +143,13 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error { return errors.New("invalid network interface Specification") } klog.V(2).Infof("deleting nic %s", nicSpec.Name) - err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, nicSpec.Name) + err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), nicSpec.Name) if err != nil && azure.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete network interface %s in resource group %s", nicSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to delete network interface %s in resource group %s", nicSpec.Name, s.Scope.ResourceGroup()) } klog.V(2).Infof("successfully deleted nic %s", nicSpec.Name) diff --git a/cloud/services/publicips/publicips.go b/cloud/services/publicips/publicips.go index 6fb6ae4fd4a..64b9fd65de2 100644 --- a/cloud/services/publicips/publicips.go +++ b/cloud/services/publicips/publicips.go @@ -38,7 +38,7 @@ func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error if !ok { return network.PublicIPAddress{}, errors.New("Invalid PublicIP Specification") } - publicIP, err := s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, publicIPSpec.Name) + publicIP, err := s.Client.Get(ctx, s.Scope.ResourceGroup(), publicIPSpec.Name) if err != nil && azure.ResourceNotFound(err) { return nil, errors.Wrapf(err, "publicip %s not found", publicIPSpec.Name) } else if err != nil { @@ -59,7 +59,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { // https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-standard-availability-zones#zone-redundant-by-default err := s.Client.CreateOrUpdate( ctx, - s.Scope.AzureCluster.Spec.ResourceGroup, + s.Scope.ResourceGroup(), ipName, network.PublicIPAddress{ Sku: &network.PublicIPAddressSku{Name: network.PublicIPAddressSkuNameStandard}, @@ -91,13 +91,13 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error { return errors.New("Invalid PublicIP Specification") } klog.V(2).Infof("deleting public ip %s", publicIPSpec.Name) - err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, publicIPSpec.Name) + err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), publicIPSpec.Name) if err != nil && azure.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete public ip %s in resource group %s", publicIPSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to delete public ip %s in resource group %s", publicIPSpec.Name, s.Scope.ResourceGroup()) } klog.V(2).Infof("deleted public ip %s", publicIPSpec.Name) diff --git a/cloud/services/publicloadbalancers/publicloadbalancers.go b/cloud/services/publicloadbalancers/publicloadbalancers.go index 767088a9ea2..60ae89943bb 100644 --- a/cloud/services/publicloadbalancers/publicloadbalancers.go +++ b/cloud/services/publicloadbalancers/publicloadbalancers.go @@ -41,7 +41,7 @@ func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error if !ok { return network.LoadBalancer{}, errors.New("invalid public loadbalancer specification") } - lb, err := s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, publicLBSpec.Name) + lb, err := s.Client.Get(ctx, s.Scope.ResourceGroup(), publicLBSpec.Name) if err != nil && azure.ResourceNotFound(err) { return nil, errors.Wrapf(err, "load balancer %s not found", publicLBSpec.Name) } else if err != nil { @@ -59,12 +59,12 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { probeName := "tcpHTTPSProbe" frontEndIPConfigName := "controlplane-lbFrontEnd" backEndAddressPoolName := "controlplane-backEndPool" - idPrefix := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers", s.Scope.SubscriptionID, s.Scope.AzureCluster.Spec.ResourceGroup) + idPrefix := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers", s.Scope.SubscriptionID, s.Scope.ResourceGroup()) lbName := publicLBSpec.Name klog.V(2).Infof("creating public load balancer %s", lbName) klog.V(2).Infof("getting public ip %s", publicLBSpec.PublicIPName) - publicIP, err := s.PublicIPsClient.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, publicLBSpec.PublicIPName) + publicIP, err := s.PublicIPsClient.Get(ctx, s.Scope.ResourceGroup(), publicLBSpec.PublicIPName) if err != nil { return err } @@ -73,7 +73,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { // https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-standard-availability-zones#zone-redundant-by-default err = s.Client.CreateOrUpdate(ctx, - s.Scope.AzureCluster.Spec.ResourceGroup, + s.Scope.ResourceGroup(), lbName, network.LoadBalancer{ Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ @@ -191,13 +191,13 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error { return errors.New("invalid public loadbalancer specification") } klog.V(2).Infof("deleting public load balancer %s", publicLBSpec.Name) - err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, publicLBSpec.Name) + err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), publicLBSpec.Name) if err != nil && azure.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete public load balancer %s in resource group %s", publicLBSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to delete public load balancer %s in resource group %s", publicLBSpec.Name, s.Scope.ResourceGroup()) } klog.V(2).Infof("deleted public load balancer %s", publicLBSpec.Name) diff --git a/cloud/services/routetables/routetables.go b/cloud/services/routetables/routetables.go index 5011f70d009..65546564f77 100644 --- a/cloud/services/routetables/routetables.go +++ b/cloud/services/routetables/routetables.go @@ -37,7 +37,7 @@ func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error if !ok { return network.RouteTable{}, errors.New("Invalid Route Table Specification") } - routeTable, err := s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, routeTableSpec.Name) + routeTable, err := s.Client.Get(ctx, s.Scope.ResourceGroup(), routeTableSpec.Name) if err != nil && azure.ResourceNotFound(err) { return nil, errors.Wrapf(err, "route table %s not found", routeTableSpec.Name) } else if err != nil { @@ -48,6 +48,10 @@ func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error // Reconcile gets/creates/updates a route table. func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { + if !s.Scope.Vnet().IsManaged(s.Scope.Name()) { + s.Scope.V(4).Info("Skipping route tables reconcile in custom vnet mode") + return nil + } routeTableSpec, ok := spec.(*Spec) if !ok { return errors.New("Invalid Route Table Specification") @@ -55,7 +59,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { klog.V(2).Infof("creating route table %s", routeTableSpec.Name) err := s.Client.CreateOrUpdate( ctx, - s.Scope.AzureCluster.Spec.ResourceGroup, + s.Scope.ResourceGroup(), routeTableSpec.Name, network.RouteTable{ Location: to.StringPtr(s.Scope.Location()), @@ -63,7 +67,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { }, ) if err != nil { - return errors.Wrapf(err, "failed to create route table %s in resource group %s", routeTableSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to create route table %s in resource group %s", routeTableSpec.Name, s.Scope.ResourceGroup()) } klog.V(2).Infof("successfully created route table %s", routeTableSpec.Name) @@ -72,18 +76,22 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { // Delete deletes the route table with the provided name. func (s *Service) Delete(ctx context.Context, spec interface{}) error { + if !s.Scope.Vnet().IsManaged(s.Scope.Name()) { + s.Scope.V(4).Info("Skipping route table deletion in custom vnet mode") + return nil + } routeTableSpec, ok := spec.(*Spec) if !ok { return errors.New("Invalid Route Table Specification") } klog.V(2).Infof("deleting route table %s", routeTableSpec.Name) - err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, routeTableSpec.Name) + err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), routeTableSpec.Name) if err != nil && azure.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete route table %s in resource group %s", routeTableSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to delete route table %s in resource group %s", routeTableSpec.Name, s.Scope.ResourceGroup()) } klog.V(2).Infof("successfully deleted route table %s", routeTableSpec.Name) diff --git a/cloud/services/securitygroups/securitygroups.go b/cloud/services/securitygroups/securitygroups.go index bdaad01b955..534d252b882 100644 --- a/cloud/services/securitygroups/securitygroups.go +++ b/cloud/services/securitygroups/securitygroups.go @@ -39,7 +39,7 @@ func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error if !ok { return network.SecurityGroup{}, errors.New("invalid security groups specification") } - securityGroup, err := s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, nsgSpec.Name) + securityGroup, err := s.Client.Get(ctx, s.Scope.ResourceGroup(), nsgSpec.Name) if err != nil && azure.ResourceNotFound(err) { return nil, errors.Wrapf(err, "security group %s not found", nsgSpec.Name) } else if err != nil { @@ -50,6 +50,10 @@ func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error // Reconcile gets/creates/updates a network security group. func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { + if !s.Scope.Vnet().IsManaged(s.Scope.Name()) { + s.Scope.V(4).Info("Skipping network security group reconcile in custom vnet mode") + return nil + } nsgSpec, ok := spec.(*Spec) if !ok { return errors.New("invalid security groups specification") @@ -92,7 +96,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { klog.V(2).Infof("creating security group %s", nsgSpec.Name) err := s.Client.CreateOrUpdate( ctx, - s.Scope.AzureCluster.Spec.ResourceGroup, + s.Scope.ResourceGroup(), nsgSpec.Name, network.SecurityGroup{ Location: to.StringPtr(s.Scope.Location()), @@ -102,7 +106,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { }, ) if err != nil { - return errors.Wrapf(err, "failed to create security group %s in resource group %s", nsgSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to create security group %s in resource group %s", nsgSpec.Name, s.Scope.ResourceGroup()) } klog.V(2).Infof("created security group %s", nsgSpec.Name) @@ -116,13 +120,13 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error { return errors.New("invalid security groups specification") } klog.V(2).Infof("deleting security group %s", nsgSpec.Name) - err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, nsgSpec.Name) + err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), nsgSpec.Name) if err != nil && azure.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete security group %s in resource group %s", nsgSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to delete security group %s in resource group %s", nsgSpec.Name, s.Scope.ResourceGroup()) } klog.V(2).Infof("deleted security group %s", nsgSpec.Name) diff --git a/cloud/services/subnets/subnets.go b/cloud/services/subnets/subnets.go index 855be5191e8..375a4c6117a 100644 --- a/cloud/services/subnets/subnets.go +++ b/cloud/services/subnets/subnets.go @@ -18,36 +18,54 @@ package subnets import ( "context" + "fmt" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha2" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" ) // Spec input specification for Get/CreateOrUpdate/Delete calls type Spec struct { - Name string - CIDR string - VnetName string - RouteTableName string - SecurityGroupName string + Name string + CIDR string + VnetName string + RouteTableName string + SecurityGroupName string + Role infrav1.SubnetRole + InternalLBIPAddress string } // Get provides information about a subnet. -func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error) { +func (s *Service) Get(ctx context.Context, spec interface{}) (*infrav1.SubnetSpec, error) { subnetSpec, ok := spec.(*Spec) if !ok { - return network.Subnet{}, errors.New("Invalid Subnet Specification") + return nil, errors.New("Invalid Subnet Specification") } - subnet, err := s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, subnetSpec.VnetName, subnetSpec.Name) - if err != nil && azure.ResourceNotFound(err) { - return nil, errors.Wrapf(err, "subnet %s not found", subnetSpec.Name) - } else if err != nil { - return subnet, err + subnet, err := s.Client.Get(ctx, s.Scope.Vnet().ResourceGroup, subnetSpec.VnetName, subnetSpec.Name) + if err != nil { + return nil, err } - return subnet, nil + var sg infrav1.SecurityGroup + if subnet.SubnetPropertiesFormat != nil && subnet.SubnetPropertiesFormat.NetworkSecurityGroup != nil { + sg = infrav1.SecurityGroup{ + Name: to.String(subnet.SubnetPropertiesFormat.NetworkSecurityGroup.Name), + ID: to.String(subnet.SubnetPropertiesFormat.NetworkSecurityGroup.ID), + Tags: converters.MapToTags(subnet.SubnetPropertiesFormat.NetworkSecurityGroup.Tags), + } + } + return &infrav1.SubnetSpec{ + Role: subnetSpec.Role, + InternalLBIPAddress: subnetSpec.InternalLBIPAddress, + Name: to.String(subnet.Name), + ID: to.String(subnet.ID), + CidrBlock: to.String(subnet.SubnetPropertiesFormat.AddressPrefix), + SecurityGroup: sg, + }, nil } // Reconcile gets/creates/updates a subnet. @@ -56,12 +74,27 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { if !ok { return errors.New("Invalid Subnet Specification") } + if subnet, err := s.Get(ctx, subnetSpec); err == nil { + // TODO: add validation on existing subnet + // subnet already exists, skip creation + if subnetSpec.Role == infrav1.SubnetControlPlane { + subnet.DeepCopyInto(s.Scope.ControlPlaneSubnet()) + } else if subnetSpec.Role == infrav1.SubnetNode { + subnet.DeepCopyInto(s.Scope.NodeSubnet()) + } + return nil + } + if !s.Scope.Vnet().IsManaged(s.Scope.Name()) { + // if vnet is unmanaged, we expect all subnets to be created as well + return fmt.Errorf("vnet was provided but subnet %s is missing", subnetSpec.Name) + } + subnetProperties := network.SubnetPropertiesFormat{ AddressPrefix: to.StringPtr(subnetSpec.CIDR), } if subnetSpec.RouteTableName != "" { klog.V(2).Infof("getting route table %s", subnetSpec.RouteTableName) - rt, err := s.RouteTablesClient.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, subnetSpec.RouteTableName) + rt, err := s.RouteTablesClient.Get(ctx, s.Scope.ResourceGroup(), subnetSpec.RouteTableName) if err != nil { return err } @@ -70,7 +103,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { } klog.V(2).Infof("getting nsg %s", subnetSpec.SecurityGroupName) - nsg, err := s.SecurityGroupsClient.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, subnetSpec.SecurityGroupName) + nsg, err := s.SecurityGroupsClient.Get(ctx, s.Scope.ResourceGroup(), subnetSpec.SecurityGroupName) if err != nil { return err } @@ -80,7 +113,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { klog.V(2).Infof("creating subnet %s in vnet %s", subnetSpec.Name, subnetSpec.VnetName) err = s.Client.CreateOrUpdate( ctx, - s.Scope.AzureCluster.Spec.ResourceGroup, + s.Scope.Vnet().ResourceGroup, subnetSpec.VnetName, subnetSpec.Name, network.Subnet{ @@ -89,7 +122,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { }, ) if err != nil { - return errors.Wrapf(err, "failed to create subnet %s in resource group %s", subnetSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to create subnet %s in resource group %s", subnetSpec.Name, s.Scope.Vnet().ResourceGroup) } klog.V(2).Infof("successfully created subnet %s in vnet %s", subnetSpec.Name, subnetSpec.VnetName) @@ -98,8 +131,8 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { // Delete deletes the subnet with the provided name. func (s *Service) Delete(ctx context.Context, spec interface{}) error { - if s.Scope.Vnet().IsManaged(s.Scope.Name()) { - s.Scope.V(4).Info("Skipping subnets deletion in unmanaged mode") + if !s.Scope.Vnet().IsManaged(s.Scope.Name()) { + s.Scope.V(4).Info("Skipping subnets deletion in custom vnet mode") return nil } subnetSpec, ok := spec.(*Spec) @@ -107,13 +140,13 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error { return errors.New("Invalid Subnet Specification") } klog.V(2).Infof("deleting subnet %s in vnet %s", subnetSpec.Name, subnetSpec.VnetName) - err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, subnetSpec.VnetName, subnetSpec.Name) + err := s.Client.Delete(ctx, s.Scope.Vnet().ResourceGroup, subnetSpec.VnetName, subnetSpec.Name) if err != nil && azure.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete subnet %s in resource group %s", subnetSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to delete subnet %s in resource group %s", subnetSpec.Name, s.Scope.Vnet().ResourceGroup) } klog.V(2).Infof("successfully deleted subnet %s in vnet %s", subnetSpec.Name, subnetSpec.VnetName) diff --git a/cloud/services/virtualmachineextensions/virtualmachineextensions.go b/cloud/services/virtualmachineextensions/virtualmachineextensions.go index 880a6052dc0..26b82a7120c 100644 --- a/cloud/services/virtualmachineextensions/virtualmachineextensions.go +++ b/cloud/services/virtualmachineextensions/virtualmachineextensions.go @@ -39,7 +39,7 @@ func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error if !ok { return compute.VirtualMachineExtension{}, errors.New("invalid vm specification") } - vmExt, err := s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, vmExtSpec.VMName, vmExtSpec.Name) + vmExt, err := s.Client.Get(ctx, s.Scope.ResourceGroup(), vmExtSpec.VMName, vmExtSpec.Name) if err != nil && azure.ResourceNotFound(err) { return nil, errors.Wrapf(err, "vm extension %s not found", vmExtSpec.Name) } else if err != nil { @@ -59,7 +59,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { err := s.Client.CreateOrUpdate( ctx, - s.Scope.AzureCluster.Spec.ResourceGroup, + s.Scope.ResourceGroup(), vmExtSpec.VMName, vmExtSpec.Name, compute.VirtualMachineExtension{ @@ -94,13 +94,13 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error { return errors.New("Invalid VNET Specification") } klog.V(2).Infof("deleting vm extension %s ", vmExtSpec.Name) - err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, vmExtSpec.VMName, vmExtSpec.Name) + err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), vmExtSpec.VMName, vmExtSpec.Name) if err != nil && azure.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete vm extension %s in resource group %s", vmExtSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to delete vm extension %s in resource group %s", vmExtSpec.Name, s.Scope.ResourceGroup()) } klog.V(2).Infof("successfully deleted vm %s ", vmExtSpec.Name) diff --git a/cloud/services/virtualmachines/virtualmachines.go b/cloud/services/virtualmachines/virtualmachines.go index 55379c3dea4..97673e4897c 100644 --- a/cloud/services/virtualmachines/virtualmachines.go +++ b/cloud/services/virtualmachines/virtualmachines.go @@ -53,7 +53,7 @@ func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error if !ok { return compute.VirtualMachine{}, errors.New("invalid vm specification") } - vm, err := s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, vmSpec.Name) + vm, err := s.Client.Get(ctx, s.Scope.ResourceGroup(), vmSpec.Name) if err != nil && azure.ResourceNotFound(err) { return nil, errors.Wrapf(err, "vm %s not found", vmSpec.Name) } else if err != nil { @@ -88,7 +88,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { } klog.V(2).Infof("getting nic %s", vmSpec.NICName) - nic, err := s.InterfacesClient.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, vmSpec.NICName) + nic, err := s.InterfacesClient.Get(ctx, s.Scope.ResourceGroup(), vmSpec.NICName) if err != nil { return err } @@ -172,7 +172,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { err = s.Client.CreateOrUpdate( ctx, - s.Scope.AzureCluster.Spec.ResourceGroup, + s.Scope.ResourceGroup(), vmSpec.Name, virtualMachine) if err != nil { @@ -190,13 +190,13 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error { return errors.New("invalid vm Specification") } klog.V(2).Infof("deleting vm %s ", vmSpec.Name) - err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, vmSpec.Name) + err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), vmSpec.Name) if err != nil && azure.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete vm %s in resource group %s", vmSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to delete vm %s in resource group %s", vmSpec.Name, s.Scope.ResourceGroup()) } klog.V(2).Infof("successfully deleted vm %s ", vmSpec.Name) @@ -221,7 +221,7 @@ func (s *Service) getAddresses(ctx context.Context, vm compute.VirtualMachine) ( nicName := getResourceNameByID(to.String(nicRef.ID)) // Fetch nic and append its addresses - nic, err := s.InterfacesClient.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, nicName) + nic, err := s.InterfacesClient.Get(ctx, s.Scope.ResourceGroup(), nicName) if err != nil { return addresses, err } @@ -259,7 +259,7 @@ func (s *Service) getAddresses(ctx context.Context, vm compute.VirtualMachine) ( // getPublicIPAddress will fetch a public ip address resource by name and return a nodeaddresss representation func (s *Service) getPublicIPAddress(ctx context.Context, publicIPAddressName string) (corev1.NodeAddress, error) { retAddress := corev1.NodeAddress{} - publicIP, err := s.PublicIPsClient.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, publicIPAddressName) + publicIP, err := s.PublicIPsClient.Get(ctx, s.Scope.ResourceGroup(), publicIPAddressName) if err != nil { return retAddress, err } diff --git a/cloud/services/virtualmachines/virtualmachines_test.go b/cloud/services/virtualmachines/virtualmachines_test.go index 0acb6d2fbbb..9db3d166760 100644 --- a/cloud/services/virtualmachines/virtualmachines_test.go +++ b/cloud/services/virtualmachines/virtualmachines_test.go @@ -139,10 +139,6 @@ func TestCreateVM(t *testing.T) { }, } - azureCluster := tc.azureCluster - - machine := &tc.machine - azureMachine := &infrav1.AzureMachine{ ObjectMeta: metav1.ObjectMeta{ Name: "azure-test1", @@ -156,18 +152,18 @@ func TestCreateVM(t *testing.T) { }, } - client := fake.NewFakeClient(secret, cluster, machine) + client := fake.NewFakeClient(secret, cluster, &tc.machine) machineScope, err := scope.NewMachineScope(scope.MachineScopeParams{ Client: client, Cluster: cluster, - Machine: machine, + Machine: &tc.machine, AzureClients: scope.AzureClients{ SubscriptionID: "123", Authorizer: autorest.NullAuthorizer{}, }, AzureMachine: azureMachine, - AzureCluster: azureCluster, + AzureCluster: tc.azureCluster, }) if err != nil { t.Fatalf("Failed to create test context: %v", err) diff --git a/cloud/services/virtualnetworks/client.go b/cloud/services/virtualnetworks/client.go index 37f62b0f3ce..353f881a428 100644 --- a/cloud/services/virtualnetworks/client.go +++ b/cloud/services/virtualnetworks/client.go @@ -29,6 +29,7 @@ type Client interface { Get(context.Context, string, string) (network.VirtualNetwork, error) CreateOrUpdate(context.Context, string, string, network.VirtualNetwork) error Delete(context.Context, string, string) error + CheckIPAddressAvailability(context.Context, string, string, string) (network.IPAddressAvailabilityResult, error) } // AzureClient contains the Azure go-sdk Client @@ -84,3 +85,8 @@ func (ac *AzureClient) Delete(ctx context.Context, resourceGroupName, vnetName s _, err = future.Result(ac.virtualnetworks) return err } + +// CheckIPAddressAvailability checks whether a private IP address is available for use. +func (ac *AzureClient) CheckIPAddressAvailability(ctx context.Context, resourceGroupName, vnetName, ip string) (network.IPAddressAvailabilityResult, error) { + return ac.virtualnetworks.CheckIPAddressAvailability(ctx, resourceGroupName, vnetName, ip) +} diff --git a/cloud/services/virtualnetworks/mock_virtualnetworks/virtualnetworks_mock.go b/cloud/services/virtualnetworks/mock_virtualnetworks/virtualnetworks_mock.go index a944cb525ca..03af2131674 100644 --- a/cloud/services/virtualnetworks/mock_virtualnetworks/virtualnetworks_mock.go +++ b/cloud/services/virtualnetworks/mock_virtualnetworks/virtualnetworks_mock.go @@ -92,3 +92,18 @@ func (mr *MockClientMockRecorder) Delete(arg0, arg1, arg2 interface{}) *gomock.C mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockClient)(nil).Delete), arg0, arg1, arg2) } + +// CheckIPAddressAvailability mocks base method +func (m *MockClient) CheckIPAddressAvailability(arg0 context.Context, arg1, arg2, arg3 string) (network.IPAddressAvailabilityResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckIPAddressAvailability", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(network.IPAddressAvailabilityResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CheckIPAddressAvailability indicates an expected call of CheckIPAddressAvailability +func (mr *MockClientMockRecorder) CheckIPAddressAvailability(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckIPAddressAvailability", reflect.TypeOf((*MockClient)(nil).CheckIPAddressAvailability), arg0, arg1, arg2, arg3) +} diff --git a/cloud/services/virtualnetworks/virtualnetworks.go b/cloud/services/virtualnetworks/virtualnetworks.go index c4f93129290..4d0065dde69 100644 --- a/cloud/services/virtualnetworks/virtualnetworks.go +++ b/cloud/services/virtualnetworks/virtualnetworks.go @@ -18,7 +18,6 @@ package virtualnetworks import ( "context" - "fmt" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" "github.com/Azure/go-autorest/autorest/to" @@ -31,23 +30,38 @@ import ( // Spec input specification for Get/CreateOrUpdate/Delete calls type Spec struct { - Name string - CIDR string + ResourceGroup string + Name string + CIDR string } // Get provides information about a virtual network. -func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error) { +func (s *Service) Get(ctx context.Context, spec interface{}) (*infrav1.VnetSpec, error) { vnetSpec, ok := spec.(*Spec) if !ok { - return network.VirtualNetwork{}, errors.New("Invalid VNET Specification") + return nil, errors.New("Invalid VNET Specification") } - vnet, err := s.Client.Get(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, vnetSpec.Name) - if err != nil && azure.ResourceNotFound(err) { - return nil, errors.Wrapf(err, "vnet %s not found", vnetSpec.Name) - } else if err != nil { - return vnet, err + vnet, err := s.Client.Get(ctx, vnetSpec.ResourceGroup, vnetSpec.Name) + if err != nil { + if azure.ResourceNotFound(err) { + return nil, err + } + return nil, errors.Wrapf(err, "failed to get vnet %s", vnetSpec.Name) } - return vnet, nil + cidr := "" + if vnet.VirtualNetworkPropertiesFormat != nil && vnet.VirtualNetworkPropertiesFormat.AddressSpace != nil { + prefixes := to.StringSlice(vnet.VirtualNetworkPropertiesFormat.AddressSpace.AddressPrefixes) + if prefixes != nil && len(prefixes) > 0 { + cidr = prefixes[0] + } + } + return &infrav1.VnetSpec{ + ResourceGroup: vnetSpec.ResourceGroup, + ID: to.String(vnet.ID), + Name: to.String(vnet.Name), + CidrBlock: cidr, + Tags: converters.MapToTags(vnet.Tags), + }, nil } // Reconcile gets/creates/updates a virtual network. @@ -65,17 +79,26 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { return errors.New("Invalid VNET Specification") } - if _, err := s.Get(ctx, vnetSpec); err == nil { - // vnet already exists, cannot update since its immutable + vnet, err := s.Get(ctx, vnetSpec) + if !azure.ResourceNotFound(err) { + if err != nil { + return errors.Wrap(err, "failed to get vnet") + } + + if !vnet.IsManaged(s.Scope.Name()) { + s.Scope.V(2).Info("Working on custom vnet", "vnet-id", vnet.ID) + } + // vnet already exists, cannot update since it's immutable + // TODO: ensure tags & other managed vnet attributes + vnet.DeepCopyInto(s.Scope.Vnet()) return nil } - klog.V(2).Infof("creating vnet %s ", vnetSpec.Name) - vnet := network.VirtualNetwork{ + vnetProperties := network.VirtualNetwork{ Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ ClusterName: s.Scope.Name(), Lifecycle: infrav1.ResourceLifecycleOwned, - Name: to.StringPtr(fmt.Sprintf("%s-vnet", s.Scope.Name())), + Name: to.StringPtr(vnetSpec.Name), Role: to.StringPtr(infrav1.CommonRoleTagValue), Additional: s.Scope.AdditionalTags(), })), @@ -86,7 +109,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { }, }, } - err := s.Client.CreateOrUpdate(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, vnetSpec.Name, vnet) + err = s.Client.CreateOrUpdate(ctx, vnetSpec.ResourceGroup, vnetSpec.Name, vnetProperties) if err != nil { return err } @@ -97,18 +120,22 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { // Delete deletes the virtual network with the provided name. func (s *Service) Delete(ctx context.Context, spec interface{}) error { + if !s.Scope.Vnet().IsManaged(s.Scope.Name()) { + s.Scope.V(4).Info("Skipping vnet deletion in custom vnet mode") + return nil + } vnetSpec, ok := spec.(*Spec) if !ok { return errors.New("Invalid VNET Specification") } klog.V(2).Infof("deleting vnet %s ", vnetSpec.Name) - err := s.Client.Delete(ctx, s.Scope.AzureCluster.Spec.ResourceGroup, vnetSpec.Name) + err := s.Client.Delete(ctx, vnetSpec.ResourceGroup, vnetSpec.Name) if err != nil && azure.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete vnet %s in resource group %s", vnetSpec.Name, s.Scope.AzureCluster.Spec.ResourceGroup) + return errors.Wrapf(err, "failed to delete vnet %s in resource group %s", vnetSpec.Name, vnetSpec.ResourceGroup) } klog.V(2).Infof("successfully deleted vnet %s ", vnetSpec.Name) diff --git a/cloud/services/virtualnetworks/virtualnetworks_test.go b/cloud/services/virtualnetworks/virtualnetworks_test.go new file mode 100644 index 00000000000..cee585fc7a4 --- /dev/null +++ b/cloud/services/virtualnetworks/virtualnetworks_test.go @@ -0,0 +1,255 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package virtualnetworks + +import ( + "context" + "encoding/json" + "net/http" + "reflect" + "testing" + + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/to" + "github.com/golang/mock/gomock" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha2" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualnetworks/mock_virtualnetworks" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha2" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestReconcileVnet(t *testing.T) { + testcases := []struct { + name string + input *infrav1.VnetSpec + output *infrav1.VnetSpec + expect func(m *mock_virtualnetworks.MockClientMockRecorder) + }{ + { + name: "managed vnet exists", + input: &infrav1.VnetSpec{ResourceGroup: "my-rg", Name: "vnet-exists"}, + output: &infrav1.VnetSpec{ResourceGroup: "my-rg", ID: "azure/fake/id", Name: "vnet-exists", CidrBlock: "10.0.0.0/8", Tags: infrav1.Tags{ + "Name": "vnet-exists", + "sigs.k8s.io_cluster-api-provider-azure_cluster_test-cluster": "owned", + "sigs.k8s.io_cluster-api-provider-azure_role": "common", + }}, + expect: func(m *mock_virtualnetworks.MockClientMockRecorder) { + m.Get(context.TODO(), "my-rg", "vnet-exists"). + Return(network.VirtualNetwork{ + ID: to.StringPtr("azure/fake/id"), + Name: to.StringPtr("vnet-exists"), + VirtualNetworkPropertiesFormat: &network.VirtualNetworkPropertiesFormat{ + AddressSpace: &network.AddressSpace{ + AddressPrefixes: to.StringSlicePtr([]string{"10.0.0.0/8"}), + }, + }, + Tags: map[string]*string{ + "Name": to.StringPtr("vnet-exists"), + "sigs.k8s.io_cluster-api-provider-azure_cluster_test-cluster": to.StringPtr("owned"), + "sigs.k8s.io_cluster-api-provider-azure_role": to.StringPtr("common"), + }, + }, nil) + }, + }, + { + name: "managed vnet does not exist", + input: &infrav1.VnetSpec{ResourceGroup: "my-rg", Name: "vnet-new", CidrBlock: "10.0.0.0/8"}, + output: &infrav1.VnetSpec{ResourceGroup: "my-rg", Name: "vnet-new", CidrBlock: "10.0.0.0/8"}, + expect: func(m *mock_virtualnetworks.MockClientMockRecorder) { + m.Get(context.TODO(), "my-rg", "vnet-new"). + Return(network.VirtualNetwork{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not found")) + + m.CreateOrUpdate(context.TODO(), "my-rg", "vnet-new", gomock.AssignableToTypeOf(network.VirtualNetwork{})) + }, + }, + { + name: "unmanaged vnet exists", + input: &infrav1.VnetSpec{ResourceGroup: "custom-vnet-rg", Name: "custom-vnet", CidrBlock: "10.0.0.0/16"}, + output: &infrav1.VnetSpec{ResourceGroup: "custom-vnet-rg", ID: "azure/custom-vnet/id", Name: "custom-vnet", CidrBlock: "10.0.0.0/16", Tags: infrav1.Tags{"Name": "my-custom-vnet"}}, + expect: func(m *mock_virtualnetworks.MockClientMockRecorder) { + m.Get(context.TODO(), "custom-vnet-rg", "custom-vnet"). + Return(network.VirtualNetwork{ + ID: to.StringPtr("azure/custom-vnet/id"), + Name: to.StringPtr("custom-vnet"), + VirtualNetworkPropertiesFormat: &network.VirtualNetworkPropertiesFormat{ + AddressSpace: &network.AddressSpace{ + AddressPrefixes: to.StringSlicePtr([]string{"10.0.0.0/16"}), + }, + }, + Tags: map[string]*string{ + "Name": to.StringPtr("my-custom-vnet"), + }, + }, nil) + }, + }, + { + name: "custom vnet not found", + input: &infrav1.VnetSpec{ResourceGroup: "custom-vnet-rg", Name: "custom-vnet", CidrBlock: "10.0.0.0/16"}, + output: &infrav1.VnetSpec{ResourceGroup: "custom-vnet-rg", Name: "custom-vnet", CidrBlock: "10.0.0.0/16"}, + expect: func(m *mock_virtualnetworks.MockClientMockRecorder) { + m.Get(context.TODO(), "custom-vnet-rg", "custom-vnet"). + Return(network.VirtualNetwork{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not found")) + + m.CreateOrUpdate(context.TODO(), "custom-vnet-rg", "custom-vnet", gomock.AssignableToTypeOf(network.VirtualNetwork{})) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockCtrl := gomock.NewController(t) + vnetMock := mock_virtualnetworks.NewMockClient(mockCtrl) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cluster"}, + } + + client := fake.NewFakeClient(cluster) + + tc.expect(vnetMock.EXPECT()) + + clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + AzureClients: scope.AzureClients{ + SubscriptionID: "123", + Authorizer: autorest.NullAuthorizer{}, + }, + Client: client, + Cluster: cluster, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + Location: "test-location", + NetworkSpec: infrav1.NetworkSpec{ + Vnet: *tc.input, + }, + }, + }, + }) + if err != nil { + t.Fatalf("Failed to create test context: %v", err) + } + + s := &Service{ + Scope: clusterScope, + Client: vnetMock, + } + + vnetSpec := &Spec{ + Name: clusterScope.Vnet().Name, + ResourceGroup: clusterScope.Vnet().ResourceGroup, + CIDR: clusterScope.Vnet().CidrBlock, + } + if err := s.Reconcile(context.TODO(), vnetSpec); err != nil { + t.Fatalf("got an unexpected error: %v", err) + } + + if !reflect.DeepEqual(clusterScope.Vnet(), tc.output) { + expected, _ := json.MarshalIndent(tc.output, "", "\t") + actual, _ := json.MarshalIndent(clusterScope.Vnet(), "", "\t") + t.Errorf("Expected %s, got %s", string(expected), string(actual)) + } + }) + } +} + +func TestDeleteVnet(t *testing.T) { + testcases := []struct { + name string + input *infrav1.VnetSpec + expect func(m *mock_virtualnetworks.MockClientMockRecorder) + }{ + { + name: "managed vnet exists", + input: &infrav1.VnetSpec{ResourceGroup: "my-rg", Name: "vnet-exists", ID: "azure/vnet/id", Tags: infrav1.Tags{ + "Name": "vnet-exists", + "sigs.k8s.io_cluster-api-provider-azure_cluster_test-cluster": "owned", + "sigs.k8s.io_cluster-api-provider-azure_role": "common", + }}, + expect: func(m *mock_virtualnetworks.MockClientMockRecorder) { + m.Delete(context.TODO(), "my-rg", "vnet-exists") + }, + }, + { + name: "managed vnet already deleted", + input: &infrav1.VnetSpec{ResourceGroup: "my-rg", Name: "vnet-exists", ID: "azure/vnet/id", Tags: infrav1.Tags{ + "Name": "vnet-exists", + "sigs.k8s.io_cluster-api-provider-azure_cluster_test-cluster": "owned", + "sigs.k8s.io_cluster-api-provider-azure_role": "common", + }}, + expect: func(m *mock_virtualnetworks.MockClientMockRecorder) { + m.Delete(context.TODO(), "my-rg", "vnet-exists"). + Return(autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not found")) + }, + }, + { + name: "unmanaged vnet", + input: &infrav1.VnetSpec{ResourceGroup: "my-rg", Name: "my-vnet", ID: "azure/custom-vnet/id"}, + expect: func(m *mock_virtualnetworks.MockClientMockRecorder) {}, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockCtrl := gomock.NewController(t) + vnetMock := mock_virtualnetworks.NewMockClient(mockCtrl) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cluster"}, + } + + client := fake.NewFakeClient(cluster) + + tc.expect(vnetMock.EXPECT()) + + clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + AzureClients: scope.AzureClients{ + SubscriptionID: "123", + Authorizer: autorest.NullAuthorizer{}, + }, + Client: client, + Cluster: cluster, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + Location: "test-location", + NetworkSpec: infrav1.NetworkSpec{ + Vnet: *tc.input, + }, + }, + }, + }) + if err != nil { + t.Fatalf("Failed to create test context: %v", err) + } + + s := &Service{ + Scope: clusterScope, + Client: vnetMock, + } + + vnetSpec := &Spec{ + Name: clusterScope.Vnet().Name, + ResourceGroup: clusterScope.Vnet().ResourceGroup, + CIDR: clusterScope.Vnet().CidrBlock, + } + if err := s.Delete(context.TODO(), vnetSpec); err != nil { + t.Fatalf("got an unexpected error: %v", err) + } + }) + } +} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml index 38f02c2d12f..30e658a3239 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml @@ -49,7 +49,8 @@ spec: description: NetworkSpec encapsulates all things related to Azure network. properties: subnets: - description: Subnets configuration. + description: Subnets is the configuration for the control-plane + subnet and the node subnet. items: description: SubnetSpec configures an Azure subnet. properties: @@ -61,9 +62,17 @@ spec: description: ID defines a unique identifier to reference this resource. type: string + internalLBIPAddress: + description: InternalLBIPAddress is the IP address that will + be used as the internal LB private IP. For the control plane + subnet only. + type: string name: description: Name defines a name for the subnet resource. type: string + role: + description: Role defines the subnet role (eg. Node, ControlPlane) + type: string securityGroup: description: SecurityGroup defines the NSG (network security group) that should be attached to this subnet. @@ -121,23 +130,13 @@ spec: type: string description: Tags defines a map of tags. type: object - required: - - id - - ingressRule - - name type: object - vnetId: - description: VnetID defines the ID of the virtual network - this subnet should be built in. - type: string required: - name - - securityGroup - - vnetId type: object type: array vnet: - description: Vnet configuration. + description: Vnet is the configuration for the Azure virtual network. properties: cidrBlock: description: CidrBlock is the CIDR block to be used when the @@ -150,6 +149,11 @@ spec: name: description: Name defines a name for the virtual network resource. type: string + resourceGroup: + description: ResourceGroup is the name of the resource group + of the existing virtual network or the resource group where + a managed virtual network should be created. + type: string tags: additionalProperties: type: string @@ -368,10 +372,6 @@ spec: type: string description: Tags defines a map of tags. type: object - required: - - id - - ingressRule - - name type: object description: SecurityGroups is a map from the role/kind of the security group to its unique name, if any. diff --git a/controllers/azurecluster_reconciler.go b/controllers/azurecluster_reconciler.go index 397f53b22d2..8190d84af11 100644 --- a/controllers/azurecluster_reconciler.go +++ b/controllers/azurecluster_reconciler.go @@ -22,6 +22,7 @@ import ( "github.com/pkg/errors" "k8s.io/klog" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha2" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/groups" @@ -71,23 +72,46 @@ func (r *azureClusterReconciler) Reconcile() error { return errors.Wrapf(err, "failed to reconcile resource group for cluster %s", r.scope.Name()) } + if r.scope.Vnet().ResourceGroup == "" { + r.scope.Vnet().ResourceGroup = r.scope.ResourceGroup() + } + if r.scope.Vnet().Name == "" { + r.scope.Vnet().Name = azure.GenerateVnetName(r.scope.Name()) + } + if r.scope.Vnet().CidrBlock == "" { + r.scope.Vnet().CidrBlock = azure.DefaultVnetCIDR + } + + if len(r.scope.Subnets()) == 0 { + r.scope.AzureCluster.Spec.NetworkSpec.Subnets = infrav1.Subnets{&infrav1.SubnetSpec{}, &infrav1.SubnetSpec{}} + } + vnetSpec := &virtualnetworks.Spec{ - Name: azure.GenerateVnetName(r.scope.Name()), - CIDR: azure.DefaultVnetCIDR, + ResourceGroup: r.scope.Vnet().ResourceGroup, + Name: r.scope.Vnet().Name, + CIDR: r.scope.Vnet().CidrBlock, } if err := r.vnetSvc.Reconcile(r.scope.Context, vnetSpec); err != nil { return errors.Wrapf(err, "failed to reconcile virtual network for cluster %s", r.scope.Name()) } + sgName := azure.GenerateControlPlaneSecurityGroupName(r.scope.Name()) + if r.scope.ControlPlaneSubnet() != nil && r.scope.ControlPlaneSubnet().SecurityGroup.Name != "" { + sgName = r.scope.ControlPlaneSubnet().SecurityGroup.Name + } sgSpec := &securitygroups.Spec{ - Name: azure.GenerateControlPlaneSecurityGroupName(r.scope.Name()), + Name: sgName, IsControlPlane: true, } if err := r.securityGroupSvc.Reconcile(r.scope.Context, sgSpec); err != nil { return errors.Wrapf(err, "failed to reconcile control plane network security group for cluster %s", r.scope.Name()) } + sgName = azure.GenerateNodeSecurityGroupName(r.scope.Name()) + if r.scope.NodeSubnet() != nil && r.scope.NodeSubnet().SecurityGroup.Name != "" { + sgName = r.scope.NodeSubnet().SecurityGroup.Name + } sgSpec = &securitygroups.Spec{ - Name: azure.GenerateNodeSecurityGroupName(r.scope.Name()), + Name: sgName, IsControlPlane: false, } if err := r.securityGroupSvc.Reconcile(r.scope.Context, sgSpec); err != nil { @@ -101,22 +125,60 @@ func (r *azureClusterReconciler) Reconcile() error { return errors.Wrapf(err, "failed to reconcile node route table for cluster %s", r.scope.Name()) } + cpSubnet := r.scope.ControlPlaneSubnet() + if cpSubnet == nil { + cpSubnet = &infrav1.SubnetSpec{} + } + if cpSubnet.Role == "" { + cpSubnet.Role = infrav1.SubnetControlPlane + } + if cpSubnet.Name == "" { + cpSubnet.Name = azure.GenerateControlPlaneSubnetName(r.scope.Name()) + } + if cpSubnet.CidrBlock == "" { + cpSubnet.CidrBlock = azure.DefaultControlPlaneSubnetCIDR + } + if cpSubnet.SecurityGroup.Name == "" { + cpSubnet.SecurityGroup.Name = azure.GenerateControlPlaneSecurityGroupName(r.scope.Name()) + } + subnetSpec := &subnets.Spec{ - Name: azure.GenerateControlPlaneSubnetName(r.scope.Name()), - CIDR: azure.DefaultControlPlaneSubnetCIDR, - VnetName: azure.GenerateVnetName(r.scope.Name()), - SecurityGroupName: azure.GenerateControlPlaneSecurityGroupName(r.scope.Name()), + Name: cpSubnet.Name, + CIDR: cpSubnet.CidrBlock, + VnetName: r.scope.Vnet().Name, + SecurityGroupName: cpSubnet.SecurityGroup.Name, + RouteTableName: azure.GenerateNodeRouteTableName(r.scope.Name()), + Role: cpSubnet.Role, + InternalLBIPAddress: cpSubnet.InternalLBIPAddress, } if err := r.subnetsSvc.Reconcile(r.scope.Context, subnetSpec); err != nil { return errors.Wrapf(err, "failed to reconcile control plane subnet for cluster %s", r.scope.Name()) } + nodeSubnet := r.scope.NodeSubnet() + if nodeSubnet == nil { + nodeSubnet = &infrav1.SubnetSpec{} + } + if nodeSubnet.Role == "" { + nodeSubnet.Role = infrav1.SubnetNode + } + if nodeSubnet.Name == "" { + nodeSubnet.Name = azure.GenerateNodeSubnetName(r.scope.Name()) + } + if nodeSubnet.CidrBlock == "" { + nodeSubnet.CidrBlock = azure.DefaultNodeSubnetCIDR + } + if nodeSubnet.SecurityGroup.Name == "" { + nodeSubnet.SecurityGroup.Name = azure.GenerateNodeSecurityGroupName(r.scope.Name()) + } + subnetSpec = &subnets.Spec{ - Name: azure.GenerateNodeSubnetName(r.scope.Name()), - CIDR: azure.DefaultNodeSubnetCIDR, - VnetName: azure.GenerateVnetName(r.scope.Name()), - SecurityGroupName: azure.GenerateNodeSecurityGroupName(r.scope.Name()), + Name: nodeSubnet.Name, + CIDR: nodeSubnet.CidrBlock, + VnetName: r.scope.Vnet().Name, + SecurityGroupName: nodeSubnet.SecurityGroup.Name, RouteTableName: azure.GenerateNodeRouteTableName(r.scope.Name()), + Role: nodeSubnet.Role, } if err := r.subnetsSvc.Reconcile(r.scope.Context, subnetSpec); err != nil { return errors.Wrapf(err, "failed to reconcile node subnet for cluster %s", r.scope.Name()) @@ -124,9 +186,10 @@ func (r *azureClusterReconciler) Reconcile() error { internalLBSpec := &internalloadbalancers.Spec{ Name: azure.GenerateInternalLBName(r.scope.Name()), - SubnetName: azure.GenerateControlPlaneSubnetName(r.scope.Name()), - VnetName: azure.GenerateVnetName(r.scope.Name()), - IPAddress: azure.DefaultInternalLBIPAddress, + SubnetName: r.scope.ControlPlaneSubnet().Name, + SubnetCidr: r.scope.ControlPlaneSubnet().CidrBlock, + VnetName: r.scope.Vnet().Name, + IPAddress: r.scope.ControlPlaneSubnet().InternalLBIPAddress, } if err := r.internalLBSvc.Reconcile(r.scope.Context, internalLBSpec); err != nil { return errors.Wrapf(err, "failed to reconcile control plane internal load balancer for cluster %s", r.scope.Name()) @@ -152,6 +215,13 @@ func (r *azureClusterReconciler) Reconcile() error { // Delete reconciles all the services in pre determined order func (r *azureClusterReconciler) Delete() error { + if r.scope.Vnet().ResourceGroup == "" { + r.scope.Vnet().ResourceGroup = r.scope.ResourceGroup() + } + if r.scope.Vnet().Name == "" { + r.scope.Vnet().Name = azure.GenerateVnetName(r.scope.Name()) + } + if err := r.deleteLB(); err != nil { return errors.Wrap(err, "failed to delete load balancer") } @@ -174,11 +244,12 @@ func (r *azureClusterReconciler) Delete() error { } vnetSpec := &virtualnetworks.Spec{ - Name: azure.GenerateVnetName(r.scope.Name()), + ResourceGroup: r.scope.Vnet().ResourceGroup, + Name: r.scope.Vnet().Name, } if err := r.vnetSvc.Delete(r.scope.Context, vnetSpec); err != nil { if !azure.ResourceNotFound(err) { - return errors.Wrapf(err, "failed to delete virtual network %s for cluster %s", azure.GenerateVnetName(r.scope.Name()), r.scope.Name()) + return errors.Wrapf(err, "failed to delete virtual network %s for cluster %s", r.scope.Vnet().Name, r.scope.Name()) } } @@ -222,26 +293,17 @@ func (r *azureClusterReconciler) deleteLB() error { } func (r *azureClusterReconciler) deleteSubnets() error { - subnetSpec := &subnets.Spec{ - Name: azure.GenerateNodeSubnetName(r.scope.Name()), - VnetName: azure.GenerateVnetName(r.scope.Name()), - } - if err := r.subnetsSvc.Delete(r.scope.Context, subnetSpec); err != nil { - if !azure.ResourceNotFound(err) { - return errors.Wrapf(err, "failed to delete %s subnet for cluster %s", azure.GenerateNodeSubnetName(r.scope.Name()), r.scope.Name()) + for _, s := range r.scope.Subnets() { + subnetSpec := &subnets.Spec{ + Name: s.Name, + VnetName: r.scope.Vnet().Name, } - } - - subnetSpec = &subnets.Spec{ - Name: azure.GenerateControlPlaneSubnetName(r.scope.Name()), - VnetName: azure.GenerateVnetName(r.scope.Name()), - } - if err := r.subnetsSvc.Delete(r.scope.Context, subnetSpec); err != nil { - if !azure.ResourceNotFound(err) { - return errors.Wrapf(err, "failed to delete %s subnet for cluster %s", azure.GenerateControlPlaneSubnetName(r.scope.Name()), r.scope.Name()) + if err := r.subnetsSvc.Delete(r.scope.Context, subnetSpec); err != nil { + if !azure.ResourceNotFound(err) { + return errors.Wrapf(err, "failed to delete %s subnet for cluster %s", s.Name, r.scope.Name()) + } } } - return nil } @@ -270,7 +332,7 @@ func (r *azureClusterReconciler) deleteNSG() error { func (r *azureClusterReconciler) createOrUpdateNetworkAPIServerIP() { if r.scope.Network().APIServerIP.Name == "" { h := fnv.New32a() - h.Write([]byte(fmt.Sprintf("%s/%s/%s", r.scope.SubscriptionID, r.scope.AzureCluster.Spec.ResourceGroup, r.scope.Name()))) + h.Write([]byte(fmt.Sprintf("%s/%s/%s", r.scope.SubscriptionID, r.scope.ResourceGroup(), r.scope.Name()))) r.scope.Network().APIServerIP.Name = azure.GeneratePublicIPName(r.scope.Name(), fmt.Sprintf("%x", h.Sum32())) } diff --git a/controllers/azuremachine_reconciler.go b/controllers/azuremachine_reconciler.go index 9b9636bc14b..936c4485b9a 100644 --- a/controllers/azuremachine_reconciler.go +++ b/controllers/azuremachine_reconciler.go @@ -222,7 +222,7 @@ func (s *azureMachineService) reconcilePublicIP(publicIPName string) error { func (s *azureMachineService) reconcileNetworkInterface(nicName string) error { networkInterfaceSpec := &networkinterfaces.Spec{ Name: nicName, - VnetName: azure.GenerateVnetName(s.clusterScope.Name()), + VnetName: s.clusterScope.Vnet().Name, } if s.machineScope.AzureMachine.Spec.AllocatePublicIP == true { @@ -236,7 +236,7 @@ func (s *azureMachineService) reconcileNetworkInterface(nicName string) error { switch role := s.machineScope.Role(); role { case infrav1.Node: - networkInterfaceSpec.SubnetName = azure.GenerateNodeSubnetName(s.clusterScope.Name()) + networkInterfaceSpec.SubnetName = s.clusterScope.NodeSubnet().Name case infrav1.ControlPlane: // TODO: Come up with a better way to determine the control plane NAT rule natRuleString := strings.TrimPrefix(nicName, fmt.Sprintf("%s-controlplane-", s.clusterScope.Name())) @@ -247,7 +247,7 @@ func (s *azureMachineService) reconcileNetworkInterface(nicName string) error { } networkInterfaceSpec.NatRule = natRule - networkInterfaceSpec.SubnetName = azure.GenerateControlPlaneSubnetName(s.clusterScope.Name()) + networkInterfaceSpec.SubnetName = s.clusterScope.ControlPlaneSubnet().Name networkInterfaceSpec.PublicLoadBalancerName = azure.GeneratePublicLBName(s.clusterScope.Name()) networkInterfaceSpec.InternalLoadBalancerName = azure.GenerateInternalLBName(s.clusterScope.Name()) default: diff --git a/controllers/azuremachine_tags.go b/controllers/azuremachine_tags.go index 3d75b8c5405..e373e1df082 100644 --- a/controllers/azuremachine_tags.go +++ b/controllers/azuremachine_tags.go @@ -46,7 +46,7 @@ func (r *AzureMachineReconciler) reconcileTags(machineScope *scope.MachineScope, Name: machineScope.Name(), } svc := virtualmachines.NewService(clusterScope, machineScope) - vm, err := svc.Client.Get(clusterScope.Context, clusterScope.AzureCluster.Spec.ResourceGroup, machineScope.Name()) + vm, err := svc.Client.Get(clusterScope.Context, clusterScope.ResourceGroup(), machineScope.Name()) if err != nil { return errors.Wrapf(err, "failed to query AzureMachine VM") } @@ -63,7 +63,7 @@ func (r *AzureMachineReconciler) reconcileTags(machineScope *scope.MachineScope, err = svc.Client.CreateOrUpdate( clusterScope.Context, - clusterScope.AzureCluster.Spec.ResourceGroup, + clusterScope.ResourceGroup(), vmSpec.Name, vm) if err != nil { diff --git a/docs/topics/custom-vnet.md b/docs/topics/custom-vnet.md new file mode 100644 index 00000000000..184c65da46d --- /dev/null +++ b/docs/topics/custom-vnet.md @@ -0,0 +1,84 @@ +# Custom Vnets + +## Pre-existing vnet and subnets + +To deploy a cluster using a pre-existing vnet, modify the `AzureCluster` spec to include the name and resource group of the existing vnet as follows, as well as the control plane and node subnets as follows: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: AzureCluster +metadata: + name: cluster-byo-vnet + namespace: default +spec: + location: southcentralus + networkSpec: + vnet: + resourceGroup: custom-vnet + name: my-vnet + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + resourceGroup: cluster-byo-vnet + ``` + +When providing a vnet, it is required to also provide the two subnets that should be used for control planes and nodes. The internal load balancer private IP can be optionally provided in the control plane subnet spec as follows: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: AzureCluster +metadata: + name: cluster-byo-vnet + namespace: default +spec: + location: southcentralus + networkSpec: + vnet: + resourceGroup: custom-vnet + name: my-vnet + subnets: + - name: control-plane-subnet + role: control-plane + internalLBIPAddress: "10.0.1.6" + - name: node-subnet + role: node + resourceGroup: cluster-byo-vnet +``` + +If provided, the private IP should be a valid IP within the control plane subnet address space. If no IP is provided, the internal load balancer reconciler will select a free IP within the subnet range at creation. + +If providing an existing vnet and subnets with existing network security groups, make sure that the control plane security group allows inbound to port 6443, as port 6443 is used by kubeadm to bootstrap the control planes. Alternatively, you can [provide a custom control plane endpoint](https://github.com/kubernetes-sigs/cluster-api-bootstrap-provider-kubeadm#kubeadmconfig-objects) in the `KubeadmConfig` spec. + +The pre-existing vnet can be in the same resource group or a different resource group in the same subscription as the target cluster. When deleting the `AzureCluster`, the vnet and resource group will only be deleted if they are "managed" by capz, ie. they were created during cluster deployment. Pre-existing vnets and resource groups will *not* be deleted. + +## Custom Network Spec + +It is also possible to customize the vnet to be created without providing an already existing vnet. To do so, simply modify the `AzureCluster` `NetworkSpec` as desired. Here is an illustrative example of a cluster with a customized vnet address space (CIDR) and customized subnets: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: AzureCluster +metadata: + name: cluster-example + namespace: default +spec: + location: southcentralus + networkSpec: + vnet: + name: my-vnet + cidrBlock: 10.0.0.0/16 + subnets: + - name: my-subnet-cp + role: control-plane + cidrBlock: 10.0.1.0/24 + - name: my-subnet-node + role: node + cidrBlock: 10.0.2.0/24 + resourceGroup: cluster-example + ``` + +If no CIDR block is provided, `10.0.0.0/8` will be used by default, with default internal LB private IP `10.0.0.100`. + +Whenever using custom vnet and subnet names and/or a different vnet resource group, please make sure to update the `azure.json` content part of each control plane's `KubeadmConfig` accordingly before creating the control plane machines.