Support job deployment into a vnet. #158 (#159)

Author: mgreisen

cli/raft.py

@@ -125,6 +125,34 @@ def validate(defaults):
             service_cli.upload_utils(
                 utils_file_share, args.get('custom_tools_path'))
             service_cli.restart()
+        elif service_action == 'config-vnet':
+            vnetName = args.get('vnetName')
+            if vnetName is None:
+                ArgumentRequired('--vnetName')
+
+            subnetName = args.get('vnetSubnetName')
+            if subnetName is None:
+                ArgumentRequired('--vnetSubnetName')
+
+            vnetResourceGroup = args.get('vnetResourceGroup')
+            if vnetResourceGroup is None:
+                ArgumentRequired('--vnetResourceGroup')
+
+            vnetRegion = args.get('vnetRegion')
+            if vnetRegion is None:
+                ArgumentRequired('--vnetRegion')
+
+            service_cli.config_vnet(vnetName,
+                                    subnetName,
+                                    vnetResourceGroup,
+                                    vnetRegion)
+
+        elif service_action == 'clear-vnet':
+            vnetResourceGroup = args.get('vnetResourceGroup')
+            if vnetResourceGroup is None:
+                ArgumentRequired('--vnetResourceGroup')
+
+            service_cli.clear_vnet(vnetResourceGroup)
         else:
             raise Exception(f'Unhandled service argument: {service_action}')
 
@@ -335,7 +363,8 @@ def main():
         formatter_class=argparse.RawTextHelpFormatter)
     service_parser.add_argument(
         'service-action',
-        choices=['deploy', 'restart', 'info', 'upload-tools', 'update'],
+        choices=['deploy', 'restart', 'info', 'upload-tools',
+                 'update', 'config-vnet', 'clear-vnet'],
         help=textwrap.dedent('''\
 deploy       - Deploys the service
 
@@ -346,8 +375,19 @@ def main():
 
 upload-tools - Uploads the tools definitions to the service
 
-update - Uploads the tools definitions to the service and
-         restarts service to get latest service components
+update       - Uploads the tools definitions to the service and
+               restarts service to get latest service components
+
+config-vnet  - Defines the VNET that Azure Container Instances are deployed
+               into. The --vnetName, --vnetSubnetName, --vnetResourceGroup
+               and --vnetRegion parameters are required. Note that the subnet
+               must be created with Microsoft.Storage service endpoint and
+               it must be delegated to
+               Microsoft.ContainerInstance/containerGroups.
+               See the documentation for more information.
+
+clear-vnet   - Removes the vnet configuration. The --vnetResourceGroup
+               parameter is required.
 '''))
 
     allowed_skus = [
@@ -362,6 +402,18 @@ def main():
     service_parser.add_argument(
         '--custom-tools-path', default=None, required=False)
 
+    service_parser.add_argument(
+        '--vnetName', default=None, required=False)
+
+    service_parser.add_argument(
+        '--vnetSubnetName', default=None, required=False)
+
+    service_parser.add_argument(
+        '--vnetResourceGroup', default=None, required=False)
+
+    service_parser.add_argument(
+        '--vnetRegion', default=None, required=False)
+
     # Add the positional argument.
     job_parser = sub_parser.add_parser(
         'job',

cli/raft_sdk/raft_deploy.py

@@ -1213,6 +1213,69 @@ def service_info(self):
         else:
             raise RaftApiException(info.text, info.status_code)
 
+    def config_vnet(self, vnetName, vnetSubnetName, vnetResourceGroup, vnetRegion):
+        networkProfileName = (f'{self.definitions.deployment}'
+                              f'-raft-aci-network-profile')
+
+        uri = (f"https://management.azure.com/subscriptions/"
+               f"{self.definitions.subscription}/"
+               f"resourceGroups/{vnetResourceGroup}/"
+               f"providers/Microsoft.Network/networkProfiles/"
+               f"{networkProfileName}/"
+               f"?api-version=2020-05-01")
+
+        body = (f'{{"properties" : {{'
+                f'"containerNetworkInterfaceConfigurations":[{{'
+                f'"properties":{{'
+                f'"ipConfigurations":[{{'
+                f'"properties": {{'
+                f'"subnet":{{'
+                f'"id":"/subscriptions/{self.definitions.subscription}'
+                f'/resourceGroups/{vnetResourceGroup}/providers'
+                f'/Microsoft.Network/virtualNetworks/{vnetName}'
+                f'/subnets/{vnetSubnetName}"'
+                f'}}}},"name":"ipconfig1"}}]}},"name":"eth1"}}]}},'
+                f'"location" : "{vnetRegion}"}}')
+
+        body_path = os.path.join(script_dir, 'body.json')
+        with open(body_path, 'w') as f:
+            f.write(body)
+
+        print(f'Creating network profile: {networkProfileName}')
+        result = az_json(f'rest --method put --uri {uri} '
+                         f'--body @{body_path} --output json')
+        os.remove(body_path)
+
+        print('Updating orchestrator settings')
+        az('functionapp config appsettings set'
+            f' --name {self.definitions.orchestrator}'
+            f' --resource-group {self.definitions.resource_group}'
+            f' --settings "RAFT_NETWORK_PROFILE_NAME={networkProfileName}"')
+            
+        az('functionapp config appsettings set'
+            f' --name {self.definitions.orchestrator}'
+            f' --resource-group {self.definitions.resource_group}'
+            f' --settings "RAFT_VNET_RESOURCE_GROUP={vnetResourceGroup}"')
+
+    def clear_vnet(self, vnetResourceGroup):
+        networkProfileName = (f'{self.definitions.deployment}'
+                              f'-raft-aci-network-profile')
+        
+        print('Removing network profile')
+        az(f'network profile delete --name {networkProfileName} '
+           f'--resource-group {vnetResourceGroup} --yes')
+
+        print('Updating orchestrator settings')
+        az('functionapp config appsettings set'
+            f' --name {self.definitions.orchestrator}'
+            f' --resource-group {self.definitions.resource_group}'
+            f' --settings "RAFT_NETWORK_PROFILE_NAME="')
+
+        az('functionapp config appsettings set'
+            f' --name {self.definitions.orchestrator}'
+            f' --resource-group {self.definitions.resource_group}'
+            f' --settings "RAFT_VNET_RESOURCE_GROUP="')
+
     def wait_for_service_to_start(self, old_info=None):
         new_info = old_info
         while True:

docs/how-to-vnet.md

@@ -0,0 +1,77 @@
+# How to use a VNet
+
+If you have a VNET connected to your company network and you need
+to be able to deploy your tests within that VNET, this document will tell
+you how to configure your VNET and RAFT deployment.
+
+RAFT assumes that all your test jobs will either be deployed in the VNET or not
+in the VNET. It is a system level configuration, not a job-by-job configuration.
+
+Note that deploying into a VNET does take a little longer than deploying outside
+a VNET.
+
+#### Step 1: Understanding the limitations
+
+RAFT uses Azure Container Instances and they require a specific configuration of your
+VNET with specific limitations. See those limitations [here](https://docs.microsoft.com/en-us/azure/container-instances/container-instances-virtual-network-concepts).
+
+When you deploy your RAFT job, it must be deployed into the same region as your VNET.
+This can be accomplished in two different ways.
+
+One way is on a per-job basis. Using
+this way, you will use the `--region` parameter whenever you submit your job to specify
+your VNET region.
+
+The second way is to deploy your RAFT service into the same region as your VNET. Using this
+method you will not need to use the `--region` parameter as RAFT will default the job's
+region to the system's deployment region. For example if you deploy RAFT to `westus` all
+jobs will also be deployed to `westus` by default.
+
+#### Step 2: VNET Configuration
+
+Your VNET can exist in any region and in any resource group.
+
+**Create a Subnet on your VNET.** When RAFT creates a Container Instance for your job and it is deployed into your VNET and subnet,
+it will be assigned an
+IP address from the subnet address range. Be sure to create a subnet with enough address space to hold
+as many simultaneous jobs as you think you will need. For most deployments 8 bits of
+address space is more than enough.
+
+The subnet can be created via the Azure Portal.
+
+It is important that the subnet be created only for use with Azure Container Instances. 
+You do this by specifing the Subnet Delegation.
+For more information on why, see the limitations referenced above.
+
+![](images/subnet-delegation.jpg)
+
+Additionally you must enable the storage service endpoint. RAFT mounts fileshares from
+your deployment's storage account onto the running containers. Setting the service endpoint
+in the subnet enables this behavior.
+
+![](images/vnet-services.jpg)
+
+#### Step 3: Tell RAFT about your VNET
+
+After your VNET is configured and RAFT is deployed, use the following CLI command 
+to configure RAFT to use your VNET.
+
+`python raft.py service config-vnet --vnetName myVnet --vnetSubnetName myVnetSubnet --vnetResourceGroup myVnetRG --vnetRegion myVnetRegion`
+
+| Parameter | Definition |
+|--- | :--|
+| `--vnetName` | The name of your VNET resource in Azure. |
+| `--vnetSubnetName` | The name of the subnet within the VNET. |
+| `--vnetResourceGroup` | The name of the resource group where the VNET lives. |
+| `--vnetRegion` | The region the VNET is deployed. |
+
+This command creates a Network Profile that is used when deploying the Container Instance
+and updates the orchestrator's configuration settings `RAFT_NETWORK_PROFILE_NAME` and
+`RAFT_VNET_RESOURCE_GROUP`. These orchestrator settings inform
+the orchestrator how to deploy your job into your VNET.
+
+Your VNET will need to have already been configured for this command to succeed. 
+
+If you decide to remove the VNET configuration from RAFT use the command:
+`python raft.py service clear-vnet --vnetResourceGroup myVnetRG`
+

docs/images/subnet-delegation.jpg

@@ -150,7 +150,10 @@ static Orchestrator()
                         utilsStorageAccountKey: utilsStorageAccountKey.Result,
                         utilsFileShare: GetSetting("RAFT_UTILS_FILESHARE"),
                         resultsStorageAccount: resultsStorageAccount,
-                        resultsStorageAccountKey: resultsStorageAccountKey.Result
+                        resultsStorageAccountKey: resultsStorageAccountKey.Result,
+
+                        networkProfileName: GetSetting("RAFT_NETWORK_PROFILE_NAME"),
+                        vNetResourceGroup: GetSetting("RAFT_VNET_RESOURCE_GROUP")
                     );
 
                 var allSecrets = OrchestratorLogic.ContainerInstances.initializeSecretsFromKeyvault(azure, agentConfig);