Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions internal/files/files.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ func downloadAndProcessURL(ctx *log.Context, url, downloadDir string, fileName s

//If there was an error downloading using SAS URI or SAS was not provided, download using managedIdentity or publicly.
if scriptSASDownloadErr != nil || scriptSAS == "" {
ctx.Log("info",fmt.Sprintf("Downloading script using SAS token failed: %v. Attempting download using managed identity or public access.", scriptSASDownloadErr))
downloaders, getDownloadersError := getDownloaders(url, sourceManagedIdentity, download.ProdMsiDownloader{})
if getDownloadersError == nil {
const mode = 0500 // we assume users download scripts to execute
Expand Down
13 changes: 7 additions & 6 deletions pkg/download/blob.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,12 +96,13 @@ func GetSASBlob(blobURI, blobSas, targetDir string) (string, error) {
// Extract container name from Path of the url https://<hostName>/<Path> For ex. Path = "containerName/dir1/dir2/file.sh"
trimmedPath := strings.Trim(blobParsedurl.Path, "/")
splitStrings := strings.Split(trimmedPath, "/")
containerName := splitStrings[0]

// Extract the blob path after container name
fileName, blobPathError := getBlobPathAfterContainerName(blobURI, containerName)
if fileName == "" || blobPathError != nil {
return "", errors.Wrapf(blobPathError, "Failed to extract blob path name from URL: %q", loggableBlobUri)
if len(splitStrings) == 0 {
return "", fmt.Errorf("cannot extract file name from URL: %q. Trimmed path was empty", loggableBlobUri);
}

fileName := splitStrings[len(splitStrings)-1]
Comment thread
alsanmsft marked this conversation as resolved.
if fileName == "" {
return "", fmt.Errorf("cannot extract file name from URL: %q", loggableBlobUri)
}

// Create the local file
Expand Down
73 changes: 73 additions & 0 deletions pkg/download/blob_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,79 @@ func Test_blobDownload_actualBlob(t *testing.T) {
require.EqualValues(t, chunk, b, "retrieved body is different body=%d chunk=%d", len(b), len(chunk))
}

func Test_GetSASBlob_nestedDirectories_actualBlob(t *testing.T) {
// Running this test requires real storage account values and keys.
// Create a storage account in Azure, and ensure that Shared Access Key is enabled under Configuration settings.
// Go to your Storage Account page > Security + networking > Access keys and copy either key1 or key2.
// Then, run the following commands prior to this test
// export AZURE_STORAGE_ACCOUNT="<your_storage_account_name>"
// export AZURE_STORAGE_ACCESS_KEY="<your_storage_account_access_key>"
acct := os.Getenv("AZURE_STORAGE_ACCOUNT")
Comment thread
alsanmsft marked this conversation as resolved.
key := os.Getenv("AZURE_STORAGE_ACCESS_KEY")
if acct == "" || key == "" {
t.Skipf("Skipping: AZURE_STORAGE_ACCOUNT or AZURE_STORAGE_ACCESS_KEY not specified to run this test")
}
base := storage.DefaultBaseURL

// Create a blob in a nested directory structure
client, err := storage.NewClient(acct, key, base, storage.DefaultAPIVersion, true)
require.Nil(t, err)
blobStorageClient := client.GetBlobService()

var (
content = []byte("test script content\n")
name = "dir1/dir2/script.sh" // nested path
container = fmt.Sprintf("run-command-test-%d", rand.New(rand.NewSource(time.Now().UnixNano())).Int63())
)

containerReference := blobStorageClient.GetContainerReference(container)
_, err = containerReference.DeleteIfExists(nil)
require.Nil(t, err)
_, err = containerReference.CreateIfNotExists(&storage.CreateContainerOptions{Access: storage.ContainerAccessTypePrivate})
require.Nil(t, err)
defer containerReference.Delete(nil)

blobReference := containerReference.GetBlobReference(name)
require.Nil(t, blobReference.PutAppendBlob(nil))
require.Nil(t, blobReference.AppendBlock(content, nil))

// Generate SAS URL for the blob
d := NewBlobDownload(acct, key, blobutil.AzureBlobRef{
Container: container,
Blob: name,
StorageBase: base,
})
v, ok := d.(blobDownload)
require.True(t, ok)
sasURL, err := v.getURL()
require.Nil(t, err)

// Parse the SAS URL to separate URI from SAS token
blobURI := fmt.Sprintf("https://%s.blob.%s/%s/%s", acct, base, container, name)
sasToken := sasURL[len(blobURI):] // Everything after the base URI is the SAS token

// Download the blob using GetSASBlob
tmpDir, err := ioutil.TempDir("", "nested-test-")
require.Nil(t, err)
defer os.RemoveAll(tmpDir)

scriptFilePath, err := GetSASBlob(blobURI, sasToken, tmpDir)
require.Nil(t, err)

// Verify that the file was downloaded with correct content
result, err := ioutil.ReadFile(scriptFilePath)
require.Nil(t, err)
require.EqualValues(t, content, result, "downloaded content should match")

// Verify that the file was created
require.Contains(t, scriptFilePath, tmpDir, "file path should be in temp directory")

// Verify the file exists and is not a directory
fileInfo, err := os.Stat(scriptFilePath)
require.Nil(t, err)
require.False(t, fileInfo.IsDir(), "should be a file, not a directory")
}

func Test_blobAppend_actualBlob(t *testing.T) {
// Before running the test locally prepare storage account and set the following env variables:
// export AZURE_STORAGE_BLOB="https://atanas.blob.core.windows.net/con1/output5.txt"
Expand Down
Loading