Pipeline Changes (#1629)

Fix Error: 
E: Could not get lock /var/lib/dpkg/lock-frontend. It is held by process 6341 (dpkg)
E: Unable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), is another process using it?
Cleanup the pipeline

azure-pipeline-templates/build.yml

@@ -31,16 +31,54 @@ parameters:
     default: "null"    
   - name: fuselib
     type: string
-    default: "fuse3 libfuse3-dev"
+    default: "libfuse3-dev"
 
 steps:
-  # Package manager installs for libfuse
   - script: |
+      sudo rm -rf $(ROOT_DIR)
+      sudo mkdir -p $(ROOT_DIR)
+      sudo chown -R `whoami` $(ROOT_DIR)
+      chmod 777 $(ROOT_DIR)
+    displayName: 'Create Directory Structure'
+
+  # Package manager installs for libfuse, 
+  # Before installation, stop any process such as update manager from using package manager.
+  # otherwise installation may fail
+  - script: |
+      sleep 30
+      sudo systemctl stop apt-daily.timer
+      echo "Add Lock Timeout for apt package manager"
+      sudo sh -c 'echo "DPkg::Lock::Timeout \"120\";" > /etc/apt/apt.conf.d/99timeout'
+      sudo ps -aux | grep -iE "apt"
+      sudo killall apt apt-get
+      yes | sudo fuser -vik -TERM /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock
+      echo "Released any lock if some other process has acquired"
+      sudo dpkg --configure -a
+
+      echo "Starting Updates and Installation of Packages"
       sudo apt-get update --fix-missing
-      sudo apt-get install ${{ parameters.fuselib }} -y
-      fusermount -V
+      sudo apt-get install make cmake gcc g++ git parallel -y
+      if [ "${{ parameters.tags }}" == "fuse2" ]
+      then
+        sudo apt-get install libfuse2 libfuse-dev -y
+      else
+        sudo apt-get install fuse3 libfuse3-dev -y
+      fi
     displayName: 'Libfuse Setup'
     condition: eq('${{ parameters.hostedAgent }}', true)
+    retryCountOnTaskFailure: 3
+
+  # Clone the repo
+  - script: |
+      git clone https://github.com/Azure/azure-storage-fuse
+    displayName: 'Checkout Code'
+    workingDirectory: $(ROOT_DIR)
+
+  # Checkout the branch
+  - script: |
+      git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
+    displayName: 'Checkout Branch'
+    workingDirectory: $(WORK_DIR)
 
   # Custom script to install Go-lang
   - task: ShellScript@2
@@ -136,10 +174,17 @@ steps:
 
   # Install azcli
   - script: |
+      sleep 5
+      sudo ps -aux | grep -iE "apt"
+      sudo killall apt apt-get
+      yes | sudo fuser -vik -TERM /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock
+      echo "Released any lock if some other process has acquired"
+      sudo dpkg --configure -a
       curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
       az --version
     displayName: 'Install Azure CLI'
     condition: eq('${{ parameters.skip_azcli }}', 'false')
+    retryCountOnTaskFailure: 3
 
   # azcli login
   - script: |

azure-pipeline-templates/distro-tests.yml

@@ -58,20 +58,8 @@ steps:
       sudo mkdir -p $(ROOT_DIR)
       sudo chown -R `whoami` $(ROOT_DIR)
       chmod 777 $(ROOT_DIR)
-      mkdir -p $(ROOT_DIR)/go/src
+      mkdir -p $(ROOT_DIR)/azure-storage-fuse
     displayName: 'Create Directory Structure'
-
-  # Clone the repo
-  - script: |
-      git clone https://github.com/Azure/azure-storage-fuse
-    displayName: 'Checkout Code'
-    workingDirectory: $(ROOT_DIR)/go/src
-
-  # Checkout the branch
-  - script: |
-      git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
-    displayName: 'Checkout Branch'
-    workingDirectory: ${{ parameters.working_dir }}
     
   # Build the code
   - template: 'build.yml'

blobfuse2-code-coverage.yaml

@@ -62,39 +62,12 @@ stages:
               value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'
 
           steps:
-            - checkout: none
-
-            # Clone the repo
-            - script: |
-                git clone https://github.com/Azure/azure-storage-fuse
-              displayName: 'Checkout Code'
-              workingDirectory: $(ROOT_DIR)
-
-            # Checkout the branch
-            - script: |
-                git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
-              displayName: 'Checkout Branch'
-              workingDirectory: $(WORK_DIR)
-          
-            - script: |
-                sudo apt-get update --fix-missing -o Dpkg::Options::="--force-confnew"
-                sudo apt-get install make cmake gcc g++ parallel $(fuselib) $(fuselib2) -y -o Dpkg::Options::="--force-confnew"
-              displayName: 'Install libfuse'
-
-
-            # Create directory structure
-            - script: |
-                sudo mkdir -p $(ROOT_DIR)
-                sudo chown -R `whoami` $(ROOT_DIR)
-                chmod 777 $(ROOT_DIR)
-              displayName: 'Create Directory Structure'
-
             # -------------------------------------------------------
             # Pull and build the code
             - template: 'azure-pipeline-templates/build.yml'
               parameters:
                 working_directory: $(WORK_DIR)
-                root_dir: $(Pipeline.Workspace)
+                root_dir: $(ROOT_DIR)
                 mount_dir: $(MOUNT_DIR)
                 temp_dir: $(TEMP_DIR)
                 gopath: $(GOPATH)
@@ -663,31 +636,20 @@ stages:
               value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'
 
           steps:
-            - checkout: none
-
-            - script: |
-                sudo apt-get update --fix-missing -o Dpkg::Options::="--force-confnew"
-                sudo apt-get install make cmake gcc g++ parallel $(fuselib) $(fuselib2) git -y -o Dpkg::Options::="--force-confnew"
-              displayName: 'Install dependency'
-
-            # Clone the repo
-            - script: |
-                git clone https://github.com/Azure/azure-storage-fuse
-              displayName: 'Checkout Code'
-              workingDirectory: $(ROOT_DIR)
-
-            # Checkout the branch
-            - script: |
-                git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
-              displayName: 'Checkout Branch'
-              workingDirectory: $(WORK_DIR)
-
-            # Custom script to install Go-lang
-            - task: ShellScript@2
-              inputs:
-                scriptPath: "$(work_dir)/go_installer.sh"
-                args: "$(root_dir)/"
-              displayName: "GoTool Custom Setup"
+            # -------------------------------------------------------
+            # Pull and build the code
+            - template: 'azure-pipeline-templates/build.yml'
+              parameters:
+                working_directory: $(WORK_DIR)
+                root_dir: $(ROOT_DIR)
+                mount_dir: $(MOUNT_DIR)
+                temp_dir: $(TEMP_DIR)
+                gopath: $(GOPATH)
+                container: $(containerName)
+                tags: $(tags)
+                fuselib: $(fuselib)
+                skip_msi: "false"
+                skip_azcli: "true"
 
             # Run the block blob cleanup
             - task: Go@0