Refactoring (#194)

- Remove all azure dependencies from tooling and move it to dedicated container
- Tools only need Http Get and Post to talk to Azure infrastructure
- Move all authentication code into dedicated container
- Move all Azure communication from test tools into a dedicated container
- Remove all authentication code from test tools
- Enable network driver selection: host and bridge

Co-authored-by: stas <[email protected]>

Author: stishkin

Scripts/CustomTools/agent-utilities/config.json

@@ -0,0 +1,4 @@
+{
+	"container" : "{PrivateRegistryRestler}/restapifuzztesting/agent-utilities:v4.latest",
+	"port" : 8085
+}

Scripts/CustomTools/tools/RESTler/config.json

@@ -1,9 +1,10 @@
 {
-	"container" : "{PrivateRegistryRestler}/restapifuzztesting/restler-agent:v3.latest",
+	"container" : "{PrivateRegistryRestler}/restapifuzztesting/restler-agent:v4.latest",
 	"shell" : "/bin/sh",
 	"run" : {
 		"shellArguments" : ["-c",
-			"date; sleep ${RAFT_STARTUP_DELAY}; date; dotnet /raft/agent/RestlerAgent.dll --agent-name ${RAFT_CONTAINER_NAME} --job-id ${RAFT_JOB_ID} --task-config-path ${RAFT_WORK_DIRECTORY}/task-config.json --work-directory ${RAFT_WORK_DIRECTORY} --restler-path /RESTler --app-insights-instrumentation-key ${RAFT_APP_INSIGHTS_KEY} --output-sas \"${RAFT_SB_OUT_SAS}\""
+			"sleep ${RAFT_STARTUP_DELAY}; dotnet /raft/agent/RestlerAgent.dll --agent-name ${RAFT_CONTAINER_NAME} --job-id ${RAFT_JOB_ID} --task-config-path ${RAFT_WORK_DIRECTORY}/task-config.json --work-directory ${RAFT_WORK_DIRECTORY} --restler-path /RESTler --agent-utilities-url ${RAFT_AGENT_UTILITIES_URL}"
+
 		]
 	},
 	"idle" : {

ado/stages/build/build.yml

@@ -19,6 +19,12 @@ stages:
           vmImage: 'ubuntu-latest'
         steps:
           - template: steps/restleragent.yml
+
+      - job: AgentUtilities
+        pool:
+          vmImage: 'ubuntu-latest'
+        steps:
+          - template: steps/agent-utilities.yml
 
       - job: Orchestrator
         pool:
@@ -43,10 +49,4 @@ stages:
           vmImage: 'ubuntu-latest'
         steps:
           - template: steps/restler2postman.yml
- 
-      - job: AzureAuthUtility
-        pool:
-          vmImage: 'ubuntu-latest'
-        steps:
-          - template: steps/azure-auth-utility.yml
 

ado/stages/build/steps/agent-utilities.yml

@@ -0,0 +1,33 @@
+steps: 
+  - task: NuGetToolInstaller@1
+    displayName: 'Use NuGet 5.8'
+    inputs:
+      versionSpec: 5.8
+
+  - task: NuGetCommand@2
+    displayName: 'NuGet restore'
+    inputs:
+      restoreSolution: '**\AgentUtilities.sln'
+
+  - task: DotNetCoreCLI@2
+    displayName: 'AgentUtilities'
+    inputs:
+      command: publish
+      publishWebProjects: false
+      projects: src/AgentUtilities/AgentUtilities/AgentUtilities.fsproj
+      arguments: '-c release /p:version=$(versionNumber)'
+      zipAfterPublish: false
+
+    # Because we are using a a release pipeline we need to publish the Dockerfile
+    # in the artifacts, since the release pipeline does not have access to the code tree.
+  - task: CopyFiles@2
+    displayName: 'Copy AgentUtilities Dockerfile'
+    inputs:
+      targetFolder: src/AgentUtilities/AgentUtilities/bin/release/net5.0/publish/
+      sourceFolder: src/AgentUtilities
+      contents: Dockerfile
+
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: src/AgentUtilities/AgentUtilities/bin/release/net5.0/publish/
+      artifactName: AgentUtilities

ado/stages/build/steps/azure-auth-utility.yml

@@ -11,27 +11,6 @@ steps:
   - script: pip3 install -r cli/requirements.txt
     displayName: Install CLI Python requirements
 
-  - task: DownloadPipelineArtifact@2
-    displayName: 'Download Local pipeline artifact'
-    condition: and(succeeded(), eq('${{parameters.BuildArtifactsLocation}}', 'current')) 
-    inputs:
-      artifact: AzureAuth
-      path: $(Build.SourcesDirectory)/cli/raft-tools/auth/dotnet-5.0
-    
-  - task: DownloadPipelineArtifact@2
-    displayName: 'Download Production pipeline artifact'
-    condition: and(always(), eq('${{parameters.BuildArtifactsLocation}}', 'production')) 
-    inputs:
-      source: 'specific'
-      project: 'raft'
-      # The pipeline value is the pipeline definition id.
-      # You can find it by looking at the url for the status badge.
-      # In this case it's the pipeline named 'build-production'
-      pipeline: 2925
-      artifact: AzureAuth
-      path: $(Build.SourcesDirectory)/cli/raft-tools/auth/dotnet-5.0
-      runVersion: 'latest'
-
   - task: AzureCLI@2
     continueOnError: false
     displayName: "Upload service utilities"

ado/stages/deploy/steps/upload-tools.yml

@@ -29,6 +29,14 @@ stages:
             parameters: 
               BuildArtifactsLocation: ${{ parameters.BuildArtifactsLocation }}
 
+      - job: AgentUtilities
+        pool:
+          vmImage: 'ubuntu-latest'
+        steps:
+          - template: steps/agent-utilities.yml
+            parameters: 
+              BuildArtifactsLocation: ${{ parameters.BuildArtifactsLocation }}
+
       - job: TestInfra
         pool:
           vmImage: 'ubuntu-latest'

ado/stages/publish/publish.yml

@@ -0,0 +1,34 @@
+parameters:
+  - name: BuildArtifactsLocation
+    type: string
+
+steps: 
+- ${{ if or(eq(parameters.BuildArtifactsLocation, 'current'), eq(parameters.BuildArtifactsLocation, '')) }}:
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Local pipeline artifact AgentUtilities'
+    inputs:
+      artifact: AgentUtilities
+      path: $(Build.SourcesDirectory)/artifacts/AgentUtilities
+
+- ${{ if eq(parameters.BuildArtifactsLocation, 'production') }}:   
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Production pipeline artifact AgentUtilities'
+    inputs:
+      source: 'specific'
+      project: 'raft'
+      pipeline: $(build-production-pipeline-id)
+      artifact: AgentUtilities
+      path: $(Build.SourcesDirectory)/artifacts/AgentUtilities
+      runVersion: 'latest'
+
+- task: Docker@2
+  displayName: 'Publish Agent Utilities imageTag'
+  inputs:
+    command: buildAndPush
+    buildContext: $(Build.SourcesDirectory)/artifacts/
+    containerRegistry: $(raft-containerRegistryServiceConnection)
+    repository: $(repositoryNamePrefix)/agent-utilities
+    Dockerfile: $(Build.SourcesDirectory)/artifacts/AgentUtilities/Dockerfile
+    tags: |
+      $(imageTag)
+      $(imageTagLatest)

ado/stages/publish/steps/agent-utilities.yml

@@ -9,18 +9,6 @@ steps:
       path: $(Build.SourcesDirectory)/artifacts/cli
       runVersion: 'latest'
 
-# Azure auth library is kept in a seperate artifact. Merge into the
-# CLI tree here before the zip file is created.
-  - task: DownloadPipelineArtifact@2
-    displayName: 'Download Production Azure Auth pipeline artifact'
-    inputs:
-      source: 'specific'
-      project: 'raft'
-      pipeline: $(build-production-pipeline-id)
-      artifact: AzureAuth
-      path: $(Build.SourcesDirectory)/artifacts/cli/raft-tools/auth/dotnet-5.0
-      runVersion: 'latest'
-
   - task: ArchiveFiles@2
     displayName: Archive CLI
     inputs:

ado/stages/publish/steps/cli.yml

@@ -28,10 +28,7 @@ steps:
     inputs:
       source: 'specific'
       project: 'raft'
-      # The pipeline value is the pipeline definition id.
-      # You can find it by looking at the url for the status badge.
-      # In this case it's the pipeline named 'build-production'
-      pipeline: 2925
+      pipeline: $(build-production-pipeline-id)
       artifact: RestlerAgent
       path: $(Build.SourcesDirectory)/artifacts/RestlerAgent
       runVersion: 'latest'