Add retry logic to improve connection reliability

.github/workflows/integration-manual.yml

@@ -104,19 +104,24 @@ jobs:
             FAIL_FLAG="-fail"
           fi
 
-          go test -v ./tests -tags=integration -storage $FAIL_FLAG \
+          go test -v ./tests -tags=integration -storage ${FAIL_FLAG} \
             -timeout=10m
         env:
           STORAGE_TYPE: mock-s3
 
+      - name: Create test results directory
+        if: always()
+        run: |
+          mkdir -p test-results
+          echo "Storage test completed at $(date)" > test-results/storage-test.log
+          echo "Test status: ${{ job.status }}" >> test-results/storage-test.log
+
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: storage-test-results
-          path: |
-            test-results/
-            *.log
+          path: test-results/
 
   database-integration:
     name: Database Integration Tests (Simulated)
@@ -142,19 +147,24 @@ jobs:
             FAIL_FLAG="-fail"
           fi
 
-          go test -v ./tests -tags=integration -database $FAIL_FLAG \
+          go test -v ./tests -tags=integration -database ${FAIL_FLAG} \
             -timeout=10m
         env:
           DB_TYPE: mock-postgres
 
+      - name: Create test results directory
+        if: always()
+        run: |
+          mkdir -p test-results
+          echo "Database test completed at $(date)" > test-results/database-test.log
+          echo "Test status: ${{ job.status }}" >> test-results/database-test.log
+
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: database-test-results
-          path: |
-            test-results/
-            *.log
+          path: test-results/
 
   api-integration:
     name: External API Integration Tests (Simulated)
@@ -180,19 +190,24 @@ jobs:
             FAIL_FLAG="-fail"
           fi
 
-          go test -v ./tests -tags=integration -api $FAIL_FLAG \
+          go test -v ./tests -tags=integration -api ${FAIL_FLAG} \
             -timeout=10m
         env:
           API_TYPE: mock-rest
 
+      - name: Create test results directory
+        if: always()
+        run: |
+          mkdir -p test-results
+          echo "API test completed at $(date)" > test-results/api-test.log
+          echo "Test status: ${{ job.status }}" >> test-results/api-test.log
+
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: api-test-results
-          path: |
-            test-results/
-            *.log
+          path: test-results/
 
   summary:
     name: Test Summary
@@ -230,14 +245,21 @@ jobs:
       - name: Comment on PR (if applicable)
         if: github.event.inputs.pr_number != ''
         uses: actions/github-script@v7
+        continue-on-error: true
         with:
           script: |
             const pr_number = ${{ github.event.inputs.pr_number }};
             const run_url = `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`;
 
-            await github.rest.issues.createComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: pr_number,
-              body: `🧪 Integration tests have been run manually by @${{ github.actor }}\n\n[View test results](${run_url})`
-            });
+            try {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: pr_number,
+                body: `🧪 Integration tests have been run manually by @${{ github.actor }}\n\n[View test results](${run_url})`
+              });
+              console.log(`Successfully commented on PR #${pr_number}`);
+            } catch (error) {
+              console.log(`Could not comment on PR #${pr_number}: ${error.message}`);
+              console.log('This is expected if the PR does not exist yet.');
+            }

docs/RETRY_LOGIC.md

@@ -0,0 +1,55 @@
+# Retry Logic Implementation
+
+## Overview
+
+This PR adds automatic retry logic to the connection handling in our simulated integration tests. This improvement makes the tests more resilient to transient failures, which better simulates real-world scenarios.
+
+## Changes
+
+### Connection Retry Logic
+
+The `Connect` method now implements a retry mechanism:
+
+- **Maximum Retries**: 3 attempts
+- **Retry Behavior**: Automatically retries on connection failure
+- **Logging**: Shows retry attempts for visibility
+- **Failure Reporting**: Clear error message after all retries exhausted
+
+### Benefits
+
+1. **Improved Reliability**: Reduces false negatives from transient failures
+2. **Better Simulation**: More accurately represents real-world connection patterns
+3. **Enhanced Debugging**: Clear logging of retry attempts helps identify issues
+4. **Production-Ready Pattern**: Demonstrates best practices for connection handling
+
+## Testing
+
+To test the retry logic:
+
+```bash
+# Run with normal failure rates
+go run src/main.go
+
+# Run integration tests with failure simulation
+go test -tags=integration ./tests -storage -fail -v
+```
+
+## Example Output
+
+When a retry occurs, you'll see output like:
+
+```
+Initializing S3-like Storage service (mock)...
+  Retry 1/2 connecting to S3-like Storage...
+  Retry 2/2 connecting to S3-like Storage...
+✓ Connected to S3-like Storage
+```
+
+## Future Improvements
+
+Potential enhancements for future iterations:
+
+- Configurable retry count via environment variables
+- Exponential backoff between retries
+- Circuit breaker pattern for persistent failures
+- Metrics collection for retry attempts

src/main.go

@@ -36,14 +36,20 @@ func NewMockService(name string, responseTime time.Duration, failureRate float32
 	}
 }
 
-// Connect simulates connecting to the service
+// Connect simulates connecting to the service with retry logic
 func (m *MockService) Connect(ctx context.Context) error {
-	time.Sleep(m.responseTime)
-	if m.shouldFail() {
-		return fmt.Errorf("failed to connect to %s", m.name)
+	retries := 3
+	for i := 0; i < retries; i++ {
+		time.Sleep(m.responseTime)
+		if !m.shouldFail() {
+			fmt.Printf("✓ Connected to %s\n", m.name)
+			return nil
+		}
+		if i < retries-1 {
+			fmt.Printf("  Retry %d/%d connecting to %s...\n", i+1, retries-1, m.name)
+		}
 	}
-	fmt.Printf("✓ Connected to %s\n", m.name)
-	return nil
+	return fmt.Errorf("failed to connect to %s after %d retries", m.name, retries)
 }
 
 // Ping simulates a health check
@@ -200,4 +206,5 @@ func main() {
 	}
 
 	fmt.Println("\n=== Integration Tests Complete ===")
+	fmt.Println("\nNote: Connection logic now includes automatic retries for improved reliability!")
 }