diff --git a/.github/workflows/deploy-linux.yml b/.github/workflows/deploy-linux.yml index 1cdaf6292..fd588dab3 100644 --- a/.github/workflows/deploy-linux.yml +++ b/.github/workflows/deploy-linux.yml @@ -1,4 +1,4 @@ -name: Deploy-Test-Cleanup Linux +name: Deploy-Test-Cleanup (v2) Linux on: pull_request: branches: diff --git a/.github/workflows/deploy-orchestrator.yml b/.github/workflows/deploy-orchestrator.yml index c6a9a0c05..d8743ed6c 100644 --- a/.github/workflows/deploy-orchestrator.yml +++ b/.github/workflows/deploy-orchestrator.yml @@ -100,7 +100,7 @@ jobs: secrets: inherit e2e-test: - if: false # E2E testing disabled + if: "!cancelled() && ((needs.deploy.result == 'success' && needs.deploy.outputs.WEB_APP_URL != '') || (inputs.existing_webapp_url != '' && inputs.existing_webapp_url != null)) && (inputs.trigger_type != 'workflow_dispatch' || (inputs.run_e2e_tests && inputs.run_e2e_tests != 'None'))" needs: [docker-build, deploy] uses: ./.github/workflows/job-test-automation.yml with: diff --git a/.github/workflows/deploy-windows.yml b/.github/workflows/deploy-windows.yml index 7f8b709dc..ded590190 100644 --- a/.github/workflows/deploy-windows.yml +++ b/.github/workflows/deploy-windows.yml @@ -1,4 +1,4 @@ -name: Deploy-Test-Cleanup Windows +name: Deploy-Test-Cleanup (v2) Windows on: # push: # branches: diff --git a/.github/workflows/job-azure-deploy.yml b/.github/workflows/job-azure-deploy.yml index 899ab3ecf..f2fa1c1d3 100644 --- a/.github/workflows/job-azure-deploy.yml +++ b/.github/workflows/job-azure-deploy.yml @@ -457,8 +457,8 @@ jobs: echo "Current branch: $BRANCH_NAME" if [[ "$BRANCH_NAME" == "main" ]]; then - IMAGE_TAG="latest" - echo "Using main branch - image tag: latest" + IMAGE_TAG="latest_waf" + echo "Using main branch - image tag: latest_waf" elif [[ "$BRANCH_NAME" == "dev" ]]; then IMAGE_TAG="dev" echo "Using dev branch - image tag: dev" @@ -466,12 +466,14 @@ jobs: IMAGE_TAG="demo" echo "Using demo branch - image tag: demo" elif [[ "$BRANCH_NAME" == "hotfix" ]]; then - BASE_TAG="hotfix" + IMAGE_TAG="hotfix" + echo "Using hotfix branch - image tag: hotfix" elif [[ "$BRANCH_NAME" == "dependabotchanges" ]]; then - BASE_TAG="dependabotchanges" + IMAGE_TAG="dependabotchanges" + echo "Using dependabotchanges branch - image tag: dependabotchanges" else - IMAGE_TAG="latest" - echo "Using default for branch '$BRANCH_NAME' - image tag: latest" + IMAGE_TAG="latest_waf" + echo "Using default for branch '$BRANCH_NAME' - image tag: latest_waf" fi echo "Using existing Docker image tag: $IMAGE_TAG" @@ -509,9 +511,7 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY echo "| Configuration | Value |" >> $GITHUB_STEP_SUMMARY echo "|---------------|-------|" >> $GITHUB_STEP_SUMMARY - echo "| **Trigger Type** | \`${{ github.event_name }}\` |" >> $GITHUB_STEP_SUMMARY echo "| **Branch** | \`${{ env.BRANCH_NAME }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| **Runner OS** | \`$INPUT_RUNNER_OS\` |" >> $GITHUB_STEP_SUMMARY echo "| **WAF Enabled** | $WAF_ENABLED_DISPLAY |" >> $GITHUB_STEP_SUMMARY echo "| **EXP Enabled** | $EXP_DISPLAY |" >> $GITHUB_STEP_SUMMARY echo "| **Run E2E Tests** | \`${{ env.RUN_E2E_TESTS }}\` |" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/job-deploy-linux.yml b/.github/workflows/job-deploy-linux.yml index 6e9e8b569..62410d5a2 100644 --- a/.github/workflows/job-deploy-linux.yml +++ b/.github/workflows/job-deploy-linux.yml @@ -283,7 +283,7 @@ jobs: if [[ -n "$AZURE_EXISTING_AI_PROJECT_RESOURCE_ID" ]]; then EXP_AI_PROJECT_ID="$AZURE_EXISTING_AI_PROJECT_RESOURCE_ID" else - EXP_AI_PROJECT_ID="${{ secrets.AZURE_ENV_FOUNDRY_PROJECT_ID }}" + EXP_AI_PROJECT_ID="${{ secrets.AZURE_EXISTING_AI_PROJECT_RESOURCE_ID }}" fi echo "AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID: $EXP_LOG_ANALYTICS_ID" @@ -297,24 +297,54 @@ jobs: azd up --no-prompt echo "✅ Deployment succeeded." - echo "$DEPLOY_OUTPUT" echo "Extracting deployment outputs..." - DEPLOY_OUTPUT=$(azd env get-values --output json) - echo "Deployment output: $DEPLOY_OUTPUT" - - if [[ -z "$DEPLOY_OUTPUT" ]]; then - echo "Error: Deployment output is empty. Please check the deployment logs." - exit 1 + API_APP_URL=$(azd env get-value API_APP_URL) + echo "API_APP_URL=${API_APP_URL}" >> $GITHUB_ENV + echo "API_APP_URL=${API_APP_URL}" >> $GITHUB_OUTPUT + + WEB_APP_URL=$(azd env get-value WEB_APP_URL) + echo "WEB_APP_URL=${WEB_APP_URL}" >> $GITHUB_ENV + echo "WEB_APP_URL=${WEB_APP_URL}" >> $GITHUB_OUTPUT + + - name: Install ODBC Driver 18 for SQL Server + shell: bash + run: | + if ! [[ "18.04 20.04 22.04 24.04 24.10" == *"$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)"* ]]; + then + echo "Ubuntu $(grep VERSION_ID /etc/os-release | cut -d '"' -f 2) is not currently supported."; + exit 1; fi + + # Download the package to configure the Microsoft repo + curl -fsSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb || { echo "Failed to download Microsoft packages config"; exit 1; } + # Install the package + sudo dpkg -i packages-microsoft-prod.deb + # Delete the file + rm packages-microsoft-prod.deb + + # Install the driver + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 + echo "✅ ODBC Driver 18 for SQL Server installed successfully." - API_APP_URL="$(echo "$DEPLOY_OUTPUT" | jq -r '.API_APP_URL // empty')" - echo "API_APP_URL=$API_APP_URL" >> $GITHUB_ENV - echo "API_APP_URL=$API_APP_URL" >> $GITHUB_OUTPUT + - name: Process Sample Data + id: process_sample_data + continue-on-error: true + shell: bash + run: | + echo "Running process_sample_data.sh..." + bash ./infra/scripts/process_sample_data.sh + echo "✅ Sample data processing completed successfully." - WEB_APP_URL="$(echo "$DEPLOY_OUTPUT" | jq -r '.WEB_APP_URL // empty')" - echo "WEB_APP_URL=$WEB_APP_URL" >> $GITHUB_ENV - echo "WEB_APP_URL=$WEB_APP_URL" >> $GITHUB_OUTPUT + - name: Retry Process Sample Data + if: steps.process_sample_data.outcome == 'failure' + shell: bash + run: | + echo "⚠️ First attempt failed. Retrying process_sample_data.sh..." + sleep 20 + bash ./infra/scripts/process_sample_data.sh + echo "✅ Sample data processing completed successfully on retry." - name: Generate Deployment Summary if: always() diff --git a/.github/workflows/job-deploy-windows.yml b/.github/workflows/job-deploy-windows.yml index 4787ddbd3..06ead37b3 100644 --- a/.github/workflows/job-deploy-windows.yml +++ b/.github/workflows/job-deploy-windows.yml @@ -287,7 +287,7 @@ jobs: if ($env:INPUT_AZURE_EXISTING_AI_PROJECT_RESOURCE_ID -ne "") { $EXP_AI_PROJECT_ID = $env:INPUT_AZURE_EXISTING_AI_PROJECT_RESOURCE_ID } else { - $EXP_AI_PROJECT_ID = "${{ secrets.AZURE_ENV_FOUNDRY_PROJECT_ID }}" + $EXP_AI_PROJECT_ID = "${{ secrets.AZURE_EXISTING_AI_PROJECT_RESOURCE_ID }}" } Write-Host "AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID: $EXP_LOG_ANALYTICS_ID" @@ -305,22 +305,77 @@ jobs: # Get deployment outputs using azd Write-Host "Extracting deployment outputs..." - $DEPLOY_OUTPUT = azd env get-values --output json | ConvertFrom-Json - Write-Host "Deployment output: $($DEPLOY_OUTPUT | ConvertTo-Json -Depth 10)" - if (-not $DEPLOY_OUTPUT) { - Write-Host "Error: Deployment output is empty. Please check the deployment logs." - exit 1 - } - - $API_APP_URL = $DEPLOY_OUTPUT.API_APP_URL + $API_APP_URL = azd env get-value API_APP_URL "API_APP_URL=$API_APP_URL" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append "API_APP_URL=$API_APP_URL" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - $WEB_APP_URL = $DEPLOY_OUTPUT.WEB_APP_URL + $WEB_APP_URL = azd env get-value WEB_APP_URL "WEB_APP_URL=$WEB_APP_URL" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append "WEB_APP_URL=$WEB_APP_URL" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + - name: Install ODBC Driver 18 for SQL Server + shell: pwsh + run: | + Write-Host "Installing ODBC Driver 18 for SQL Server on Windows..." + + # Download and install the ODBC Driver installer with error handling + $installerUrl = "https://go.microsoft.com/fwlink/?linkid=2249004" + $installerPath = "$env:TEMP\msodbcsql.msi" + + try { + Write-Host "Downloading ODBC Driver installer from $installerUrl ..." + Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing -ErrorAction Stop + + if (-not (Test-Path $installerPath)) { + throw "ODBC Driver installer was not downloaded to expected path: $installerPath" + } + + Write-Host "Running ODBC Driver installer..." + $msiArgs = "/i `"$installerPath`" /quiet IACCEPTMSODBCSQLLICENSETERMS=YES" + $process = Start-Process msiexec.exe -ArgumentList $msiArgs -Wait -NoNewWindow -PassThru + + if ($process.ExitCode -ne 0) { + throw "ODBC Driver installer exited with code $($process.ExitCode)." + } + + Write-Host "✅ ODBC Driver 18 installation complete" + } + catch { + Write-Error "Failed to install ODBC Driver 18 for SQL Server: $($_.Exception.Message)" + if (Test-Path $installerPath) { + Remove-Item $installerPath -Force -ErrorAction SilentlyContinue + } + exit 1 + } + finally { + if (Test-Path $installerPath) { + Remove-Item $installerPath -Force -ErrorAction SilentlyContinue + } + } + + - name: Process Sample Data + id: process_sample_data + continue-on-error: true + shell: bash + env: + PYTHONIOENCODING: utf-8 + run: | + echo "Running process_sample_data.sh..." + bash ./infra/scripts/process_sample_data.sh + echo "✅ Sample data processing completed successfully." + + - name: Retry Process Sample Data + if: steps.process_sample_data.outcome == 'failure' + shell: bash + env: + PYTHONIOENCODING: utf-8 + run: | + echo "⚠️ First attempt failed. Retrying process_sample_data.sh..." + sleep 20 + bash ./infra/scripts/process_sample_data.sh + echo "✅ Sample data processing completed successfully on retry." + - name: Generate Deployment Summary if: always() shell: bash diff --git a/.github/workflows/job-docker-build.yml b/.github/workflows/job-docker-build.yml index 7f7d99902..dd3461a36 100644 --- a/.github/workflows/job-docker-build.yml +++ b/.github/workflows/job-docker-build.yml @@ -56,6 +56,8 @@ jobs: - name: Build and Push Docker Image for WebApp uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_SUMMARY: false with: context: ./src/App file: ./src/App/WebApp.Dockerfile @@ -66,6 +68,8 @@ jobs: - name: Build and Push Docker Image for API uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_SUMMARY: false with: context: ./src/api file: ./src/api/ApiApp.Dockerfile diff --git a/.github/workflows/job-test-automation.yml b/.github/workflows/job-test-automation.yml index 4faa36323..2b188bce0 100644 --- a/.github/workflows/job-test-automation.yml +++ b/.github/workflows/job-test-automation.yml @@ -98,10 +98,47 @@ jobs: attempt=$((attempt + 1)) done + - name: Validate Use Case and Test Suite + run: | + echo "Validating use case: '${{ env.azure_env_use_case }}'" + echo "Validating test suite: '${{ env.test_suite }}'" + + # Validate use case + if [ -z "${{ env.azure_env_use_case }}" ]; then + echo "ERROR: AZURE_ENV_USE_CASE is empty or not provided" + exit 1 + elif [ "${{ env.azure_env_use_case }}" != "telecom" ] && [ "${{ env.azure_env_use_case }}" != "IT_helpdesk" ]; then + echo "ERROR: Invalid AZURE_ENV_USE_CASE '${{ env.azure_env_use_case }}'. Must be 'telecom' or 'IT_helpdesk'" + exit 1 + fi + + # Validate test suite + if [ -z "${{ env.test_suite }}" ]; then + echo "ERROR: TEST_SUITE is empty or not provided" + exit 1 + elif [ "${{ env.test_suite }}" != "GoldenPath-Testing" ] && [ "${{ env.test_suite }}" != "Smoke-Testing" ]; then + echo "ERROR: Invalid TEST_SUITE '${{ env.test_suite }}'. Must be 'GoldenPath-Testing' or 'Smoke-Testing'" + exit 1 + fi + + echo "✅ Use case '${{ env.azure_env_use_case }}' and test suite '${{ env.test_suite }}' are valid" + - name: Run tests(1) id: test1 run: | - xvfb-run pytest --headed --html=report/report.html --self-contained-html + if [ "${{ env.azure_env_use_case }}" == "telecom" ]; then + if [ "${{ env.test_suite }}" == "GoldenPath-Testing" ]; then + xvfb-run pytest tests/test_telecom_gp_tc.py --headed --html=report/report.html --self-contained-html + elif [ "${{ env.test_suite }}" == "Smoke-Testing" ]; then + xvfb-run pytest tests/test_telecom_gp_tc.py tests/test_telecom_smoke_tc.py --headed --html=report/report.html --self-contained-html + fi + elif [ "${{ env.azure_env_use_case }}" == "IT_helpdesk" ]; then + if [ "${{ env.test_suite }}" == "GoldenPath-Testing" ]; then + xvfb-run pytest tests/test_ithelpdesk_gp_tc.py --headed --html=report/report.html --self-contained-html + elif [ "${{ env.test_suite }}" == "Smoke-Testing" ]; then + xvfb-run pytest tests/test_ithelpdesk_gp_tc.py tests/test_ithelpdesk_smoke_tc.py --headed --html=report/report.html --self-contained-html + fi + fi working-directory: tests/e2e-test continue-on-error: true @@ -114,7 +151,19 @@ jobs: if: ${{ steps.test1.outcome == 'failure' }} id: test2 run: | - xvfb-run pytest --headed --html=report/report.html --self-contained-html + if [ "${{ env.azure_env_use_case }}" == "telecom" ]; then + if [ "${{ env.test_suite }}" == "GoldenPath-Testing" ]; then + xvfb-run pytest tests/test_telecom_gp_tc.py --headed --html=report/report.html --self-contained-html + elif [ "${{ env.test_suite }}" == "Smoke-Testing" ]; then + xvfb-run pytest tests/test_telecom_gp_tc.py tests/test_telecom_smoke_tc.py --headed --html=report/report.html --self-contained-html + fi + elif [ "${{ env.azure_env_use_case }}" == "IT_helpdesk" ]; then + if [ "${{ env.test_suite }}" == "GoldenPath-Testing" ]; then + xvfb-run pytest tests/test_ithelpdesk_gp_tc.py --headed --html=report/report.html --self-contained-html + elif [ "${{ env.test_suite }}" == "Smoke-Testing" ]; then + xvfb-run pytest tests/test_ithelpdesk_gp_tc.py tests/test_ithelpdesk_smoke_tc.py --headed --html=report/report.html --self-contained-html + fi + fi working-directory: tests/e2e-test continue-on-error: true @@ -127,7 +176,19 @@ jobs: if: ${{ steps.test2.outcome == 'failure' }} id: test3 run: | - xvfb-run pytest --headed --html=report/report.html --self-contained-html + if [ "${{ env.azure_env_use_case }}" == "telecom" ]; then + if [ "${{ env.test_suite }}" == "GoldenPath-Testing" ]; then + xvfb-run pytest tests/test_telecom_gp_tc.py --headed --html=report/report.html --self-contained-html + elif [ "${{ env.test_suite }}" == "Smoke-Testing" ]; then + xvfb-run pytest tests/test_telecom_gp_tc.py tests/test_telecom_smoke_tc.py --headed --html=report/report.html --self-contained-html + fi + elif [ "${{ env.azure_env_use_case }}" == "IT_helpdesk" ]; then + if [ "${{ env.test_suite }}" == "GoldenPath-Testing" ]; then + xvfb-run pytest tests/test_ithelpdesk_gp_tc.py --headed --html=report/report.html --self-contained-html + elif [ "${{ env.test_suite }}" == "Smoke-Testing" ]; then + xvfb-run pytest tests/test_ithelpdesk_gp_tc.py tests/test_ithelpdesk_smoke_tc.py --headed --html=report/report.html --self-contained-html + fi + fi working-directory: tests/e2e-test - name: Upload test report @@ -137,7 +198,9 @@ jobs: with: name: test-report-${{ github.run_id }} - path: tests/e2e-test/report/* + path: | + tests/e2e-test/report/* + tests/e2e-test/screenshots/* - name: Generate E2E Test Summary if: always() diff --git a/infra/main.parameters.json b/infra/main.parameters.json index c6a19270d..fbef1e178 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -58,6 +58,9 @@ }, "frontendContainerRegistryHostname": { "value": "${AZURE_ENV_CONTAINER_REGISTRY_ENDPOINT}" + }, + "usecase":{ + "value": "${AZURE_ENV_USE_CASE}" } } } diff --git a/infra/main.waf.parameters.json b/infra/main.waf.parameters.json index b64026821..a78210cf2 100644 --- a/infra/main.waf.parameters.json +++ b/infra/main.waf.parameters.json @@ -73,6 +73,9 @@ }, "virtualMachineAdminPassword": { "value": "${AZURE_ENV_VM_ADMIN_PASSWORD}" + }, + "usecase":{ + "value": "${AZURE_ENV_USE_CASE}" } } } diff --git a/infra/scripts/add_user_scripts/assign_sql_roles.py b/infra/scripts/add_user_scripts/assign_sql_roles.py index 2c42e7a15..dad4dac05 100644 --- a/infra/scripts/add_user_scripts/assign_sql_roles.py +++ b/infra/scripts/add_user_scripts/assign_sql_roles.py @@ -7,11 +7,29 @@ import json import sys import struct +import uuid import pyodbc from azure.identity import AzureCliCredential SQL_COPT_SS_ACCESS_TOKEN = 1256 + +def client_id_to_sid(client_id: str) -> str: + """ + Convert a client ID (GUID) to a SQL Server SID format. + This allows creating users without requiring MS Graph permissions. + + Args: + client_id: The client ID (GUID) of the managed identity + + Returns: + str: Hexadecimal SID string for use in CREATE USER statement + """ + # Convert the client ID to bytes using UUID + guid_bytes = uuid.UUID(client_id).bytes_le + # Convert to hexadecimal string + return "0x" + guid_bytes.hex().upper() + def connect_with_token(server: str, database: str, credential: AzureCliCredential): """ Connect to SQL Server using Azure CLI credential token. @@ -63,6 +81,7 @@ def assign_sql_roles(server, database, roles_json): client_id = role_assignment.get("clientId") display_name = role_assignment.get("displayName") role = role_assignment.get("role") + is_service_principal = role_assignment.get("isServicePrincipal", False) if not client_id or not display_name or not role: continue @@ -73,9 +92,16 @@ def assign_sql_roles(server, database, roles_json): user_exists = cursor.fetchone()[0] > 0 if not user_exists: - # Create user from external provider - create_user_sql = f"CREATE USER [{display_name}] FROM EXTERNAL PROVIDER" try: + if is_service_principal: + # For service principals/managed identities, use SID-based approach + # This doesn't require MS Graph permissions on SQL Server + sid = client_id_to_sid(client_id) + create_user_sql = f"CREATE USER [{display_name}] WITH SID = {sid}, TYPE = E" + else: + # For regular users, use standard external provider approach + create_user_sql = f"CREATE USER [{display_name}] FROM EXTERNAL PROVIDER" + cursor.execute(create_user_sql) conn.commit() print(f"✓ Created user: {display_name}") diff --git a/infra/scripts/copy_kb_files.sh b/infra/scripts/copy_kb_files.sh index b47d8fedc..d57f541e7 100644 --- a/infra/scripts/copy_kb_files.sh +++ b/infra/scripts/copy_kb_files.sh @@ -45,16 +45,41 @@ if ! az account show &> /dev/null; then az login --use-device-code fi -# Check and assign Storage Blob Data Contributor role to current user -signed_user_id=$(az ad signed-in-user show --query id --output tsv 2>&1) -if [ -z "$signed_user_id" ] || [[ "$signed_user_id" == *"ERROR"* ]] || [[ "$signed_user_id" == *"InteractionRequired"* ]]; then - echo "✗ Failed to get signed-in user ID. Token may have expired. Re-authenticating..." - az login --use-device-code - signed_user_id=$(az ad signed-in-user show --query id --output tsv) - if [ -z "$signed_user_id" ]; then - echo "✗ Failed to get signed-in user ID after re-authentication" +# Check and assign Storage Blob Data Contributor role to current identity (user or service principal) +# First, determine if we're running as a user or service principal +account_type=$(az account show --query user.type --output tsv 2>/dev/null) + +if [ "$account_type" == "user" ]; then + # Running as a user - get signed-in user ID + signed_user_id=$(az ad signed-in-user show --query id --output tsv 2>&1) + if [ -z "$signed_user_id" ] || [[ "$signed_user_id" == *"ERROR"* ]] || [[ "$signed_user_id" == *"InteractionRequired"* ]]; then + echo "✗ Failed to get signed-in user ID. Token may have expired. Re-authenticating..." + az login --use-device-code + signed_user_id=$(az ad signed-in-user show --query id --output tsv) + if [ -z "$signed_user_id" ]; then + echo "✗ Failed to get signed-in user ID after re-authentication" + exit 1 + fi + fi + echo "✓ Running as user: $signed_user_id" +elif [ "$account_type" == "servicePrincipal" ]; then + # Running as a service principal - get SP object ID + client_id=$(az account show --query user.name --output tsv 2>/dev/null) + if [ -z "$client_id" ]; then + echo "✗ Failed to get service principal client ID" + exit 1 + fi + sp_show_output=$(az ad sp show --id "$client_id" --query id --output tsv 2>&1) + if [ $? -ne 0 ] || [ -z "$sp_show_output" ]; then + echo "✗ Failed to get service principal object ID using client ID '$client_id'. Azure CLI output:" + echo "$sp_show_output" exit 1 fi + signed_user_id="$sp_show_output" + echo "✓ Running as service principal: $signed_user_id" +else + echo "✗ Unknown account type: $account_type" + exit 1 fi storage_resource_id=$(az storage account show --name "$storageAccountName" --resource-group "$resourceGroupName" --query id --output tsv) @@ -71,39 +96,66 @@ if [ -z "$role_assignment" ]; then echo "✗ Failed to assign Storage Blob Data Contributor role" exit 1 fi +fi + +# Wait for role assignment to propagate by testing storage access +echo "⏳ Waiting for role assignment to propagate..." +max_retries=30 +retry_count=0 +while [ $retry_count -lt $max_retries ]; do + if az storage container list --account-name "$storageAccountName" --auth-mode login --output none 2>/dev/null; then + echo "✓ Role assignment propagated successfully" + break + fi + retry_count=$((retry_count + 1)) + echo " Attempt $retry_count/$max_retries - waiting 10 seconds..." sleep 10 +done + +if [ $retry_count -eq $max_retries ]; then + echo "✗ Role assignment did not propagate within expected time" + exit 1 fi -# Upload files to storage account +# Upload files to storage account with retry logic +upload_with_retry() { + local source_folder="$1" + local dest_path="$2" + local description="$3" + local upload_retries=5 + local upload_attempt=0 + + while [ $upload_attempt -lt $upload_retries ]; do + if az storage blob upload-batch \ + --account-name "$storageAccountName" \ + --destination "$dest_path" \ + --source "$source_folder" \ + --auth-mode login \ + --pattern '*' \ + --overwrite \ + --output none 2>/dev/null; then + echo "✓ Uploaded $description successfully" + return 0 + fi + upload_attempt=$((upload_attempt + 1)) + echo " Upload attempt $upload_attempt/$upload_retries failed - waiting 15 seconds..." + sleep 15 + done + echo "✗ Failed to upload $description after $upload_retries attempts" + return 1 +} + if [ -d "$extractedFolder1" ]; then - echo "✓ Uploading call transcripts" - az storage blob upload-batch \ - --account-name "$storageAccountName" \ - --destination "$containerName/$extractedFolder1" \ - --source "$extractedFolder1" \ - --auth-mode login \ - --pattern '*' \ - --overwrite \ - --output none - if [ $? -ne 0 ]; then - echo "✗ Failed to upload call transcripts" + echo "⏳ Uploading call transcripts..." + if ! upload_with_retry "$extractedFolder1" "$containerName/$extractedFolder1" "call transcripts"; then exit 1 fi fi if [ "$usecase" == "telecom" ]; then if [ -d "$extractedFolder2" ]; then - echo "✓ Uploading audio data" - az storage blob upload-batch \ - --account-name "$storageAccountName" \ - --destination "$containerName/$extractedFolder2" \ - --source "$extractedFolder2" \ - --auth-mode login \ - --pattern '*' \ - --overwrite \ - --output none - if [ $? -ne 0 ]; then - echo "✗ Failed to upload audio data" + echo "⏳ Uploading audio data..." + if ! upload_with_retry "$extractedFolder2" "$containerName/$extractedFolder2" "audio data"; then exit 1 fi fi diff --git a/infra/scripts/fabric_scripts/run_fabric_items_scripts.sh b/infra/scripts/fabric_scripts/run_fabric_items_scripts.sh index ab5390909..13e03defe 100644 --- a/infra/scripts/fabric_scripts/run_fabric_items_scripts.sh +++ b/infra/scripts/fabric_scripts/run_fabric_items_scripts.sh @@ -6,13 +6,42 @@ keyvaultName="$1" fabricWorkspaceId="$2" solutionName="$3" -# get signed user -echo "Getting signed in user id" -signed_user_id=$(az ad signed-in-user show --query id -o tsv) +# Determine if we're running as a user or service principal +echo "Getting signed in user/service principal id" +account_type=$(az account show --query user.type --output tsv 2>/dev/null) -# Check if the user_id is empty -if [ -z "$signed_user_id" ]; then - echo "Error: User ID not found. Please check the user principal name or email address." +if [ "$account_type" == "user" ]; then + # Running as a user - get signed-in user ID + signed_user_id=$(az ad signed-in-user show --query id --output tsv 2>&1) + if [ -z "$signed_user_id" ] || [[ "$signed_user_id" == *"ERROR"* ]] || [[ "$signed_user_id" == *"InteractionRequired"* ]]; then + echo "✗ Failed to get signed-in user ID. Token may have expired. Re-authenticating..." + az login --use-device-code + signed_user_id=$(az ad signed-in-user show --query id --output tsv) + if [ -z "$signed_user_id" ]; then + echo "✗ Failed to get signed-in user ID after re-authentication" + exit 1 + fi + fi + echo "✓ Running as user: $signed_user_id" +elif [ "$account_type" == "servicePrincipal" ]; then + # Running as a service principal - get SP object ID + client_id=$(az account show --query user.name --output tsv 2>/dev/null) + if [ -n "$client_id" ]; then + signed_user_id=$(az ad sp show --id "$client_id" --query id --output tsv 2>&1) + # Check if the command failed or returned an empty/erroneous ID + if [ $? -ne 0 ] || [ -z "$signed_user_id" ] || [[ "$signed_user_id" == *"ERROR"* ]]; then + echo "✗ Failed to get service principal object ID using client ID: $client_id" + echo "Azure CLI output:" + echo "$signed_user_id" + exit 1 + fi + else + echo "✗ Failed to get service principal client ID" + exit 1 + fi + echo "✓ Running as service principal: $signed_user_id" +else + echo "✗ Unknown account type: $account_type" exit 1 fi diff --git a/infra/scripts/run_create_index_scripts.sh b/infra/scripts/run_create_index_scripts.sh index 882c48d66..5145691c6 100644 --- a/infra/scripts/run_create_index_scripts.sh +++ b/infra/scripts/run_create_index_scripts.sh @@ -30,19 +30,45 @@ if ! az account show &> /dev/null; then az login --use-device-code fi -# Get signed in user and store the output -signed_user=$(az ad signed-in-user show --query "{id:id, displayName:displayName}" -o json 2>&1) -if [[ "$signed_user" == *"ERROR"* ]] || [[ "$signed_user" == *"InteractionRequired"* ]] || [[ "$signed_user" == *"AADSTS"* ]]; then - echo "✗ Failed to get signed-in user. Token may have expired. Re-authenticating..." - az login --use-device-code - signed_user=$(az ad signed-in-user show --query "{id:id, displayName:displayName}" -o json) -fi - -signed_user_id=$(echo "$signed_user" | grep -o '"id": *"[^"]*"' | head -1 | sed 's/"id": *"\([^"]*\)"/\1/') -signed_user_display_name=$(echo "$signed_user" | grep -o '"displayName": *"[^"]*"' | sed 's/"displayName": *"\([^"]*\)"/\1/') - -if [ -z "$signed_user_id" ] || [ -z "$signed_user_display_name" ]; then - echo "✗ Failed to extract user information after authentication" +# Determine if we're running as a user or service principal +account_type=$(az account show --query user.type --output tsv 2>/dev/null) + +if [ "$account_type" == "user" ]; then + # Running as a user - get signed-in user info + signed_user=$(az ad signed-in-user show --query "{id:id, displayName:displayName}" -o json 2>&1) + if [[ "$signed_user" == *"ERROR"* ]] || [[ "$signed_user" == *"InteractionRequired"* ]] || [[ "$signed_user" == *"AADSTS"* ]]; then + echo "✗ Failed to get signed-in user. Token may have expired. Re-authenticating..." + az login --use-device-code + signed_user=$(az ad signed-in-user show --query "{id:id, displayName:displayName}" -o json) + fi + signed_user_id=$(echo "$signed_user" | grep -o '"id": *"[^"]*"' | head -1 | sed 's/"id": *"\([^"]*\)"/\1/') + signed_user_display_name=$(echo "$signed_user" | grep -o '"displayName": *"[^"]*"' | sed 's/"displayName": *"\([^"]*\)"/\1/') + + if [ -z "$signed_user_id" ] || [ -z "$signed_user_display_name" ]; then + echo "✗ Failed to extract user information after authentication" + exit 1 + fi + echo "✓ Running as user: $signed_user_display_name ($signed_user_id)" +elif [ "$account_type" == "servicePrincipal" ]; then + # Running as a service principal - get SP object ID and display name + client_id=$(az account show --query user.name --output tsv 2>/dev/null) + if [ -n "$client_id" ]; then + sp_info=$(az ad sp show --id "$client_id" --query "{id:id, displayName:displayName}" -o json 2>&1) + if [ $? -ne 0 ]; then + echo "✗ Failed to retrieve service principal information for client ID: $client_id" + echo "$sp_info" + exit 1 + fi + signed_user_id=$(echo "$sp_info" | grep -o '"id": *"[^"]*"' | head -1 | sed 's/"id": *"\([^"]*\)"/\1/') + signed_user_display_name=$(echo "$sp_info" | grep -o '"displayName": *"[^"]*"' | sed 's/"displayName": *"\([^"]*\)"/\1/') + fi + if [ -z "$signed_user_id" ] || [ -z "$signed_user_display_name" ]; then + echo "✗ Failed to get service principal information" + exit 1 + fi + echo "✓ Running as service principal: $signed_user_display_name ($signed_user_id)" +else + echo "✗ Unknown account type: $account_type" exit 1 fi @@ -144,7 +170,18 @@ fi if [ -n "$backendManagedIdentityClientId" ] && [ -n "$backendManagedIdentityDisplayName" ] && [ -n "$sqlDatabaseName" ]; then mi_display_name="$backendManagedIdentityDisplayName" server_fqdn="$sqlServerName.database.windows.net" - roles_json="[{\"clientId\":\"$backendManagedIdentityClientId\",\"displayName\":\"$mi_display_name\",\"role\":\"db_datareader\"},{\"clientId\":\"$backendManagedIdentityClientId\",\"displayName\":\"$mi_display_name\",\"role\":\"db_datawriter\"}]" + + # Determine isServicePrincipal based on account type + # When running as servicePrincipal, use SID-based approach + # When running as user, use FROM EXTERNAL PROVIDER + if [ "$account_type" == "servicePrincipal" ]; then + is_sp="true" + else + is_sp="false" + fi + + # Managed identity role assignments + roles_json="[{\"clientId\":\"$backendManagedIdentityClientId\",\"displayName\":\"$mi_display_name\",\"role\":\"db_datareader\",\"isServicePrincipal\":$is_sp},{\"clientId\":\"$backendManagedIdentityClientId\",\"displayName\":\"$mi_display_name\",\"role\":\"db_datawriter\",\"isServicePrincipal\":$is_sp}]" if [ -f "$SCRIPT_DIR/add_user_scripts/assign_sql_roles.py" ]; then echo "✓ Assigning SQL roles to managed identity"