From 3093732654591e47b08f53add17ecf4ce0b0ea54 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Sun, 2 Feb 2025 15:58:11 -0800 Subject: [PATCH 01/19] feat: s3 transfer manager v2 This is an initial phase for the s3 transfer manager v2, which includes: - Progress Tracker with a default Console Progres Bar. - Dedicated Multipart Download Listener for listen to events specificly to multipart download. - Generic Transfer Listener that will be used in either a multipart upload or a multipart download. The progress tracker is dependant on the Generic Transfer Listener, and when enabled it uses the same parameter to be provided as the progress tracker. This is important because if there is a need for listening to transfer specific events and also track the progress then, a custom implementation must be done that incorporate those two needs together, otherwise one of each other must be used. - Single Object Download - Multipart Objet Download This initial implementation misses the test cases. --- .../S3Transfer/ConsoleProgressBar.php | 130 ++++++ .../S3Transfer/DefaultProgressTracker.php | 193 +++++++++ .../S3Transfer/GetMultipartDownloader.php | 50 +++ .../Features/S3Transfer/ListenerNotifier.php | 8 + .../S3Transfer/MultipartDownloadListener.php | 190 ++++++++ .../S3Transfer/MultipartDownloadType.php | 51 +++ .../S3Transfer/MultipartDownloader.php | 408 ++++++++++++++++++ .../S3Transfer/ObjectProgressTracker.php | 179 ++++++++ src/S3/Features/S3Transfer/ProgressBar.php | 14 + .../S3Transfer/ProgressBarFactory.php | 8 + .../S3Transfer/ProgressListenerHelper.php | 45 ++ .../S3Transfer/RangeMultipartDownloader.php | 64 +++ .../Features/S3Transfer/S3TransferManager.php | 143 ++++++ .../S3Transfer/S3TransferManagerTrait.php | 88 ++++ .../Features/S3Transfer/TransferListener.php | 252 +++++++++++ .../S3Transfer/TransferListenerFactory.php | 8 + 16 files changed, 1831 insertions(+) create mode 100644 src/S3/Features/S3Transfer/ConsoleProgressBar.php create mode 100644 src/S3/Features/S3Transfer/DefaultProgressTracker.php create mode 100644 src/S3/Features/S3Transfer/GetMultipartDownloader.php create mode 100644 src/S3/Features/S3Transfer/ListenerNotifier.php create mode 100644 src/S3/Features/S3Transfer/MultipartDownloadListener.php create mode 100644 src/S3/Features/S3Transfer/MultipartDownloadType.php create mode 100644 src/S3/Features/S3Transfer/MultipartDownloader.php create mode 100644 src/S3/Features/S3Transfer/ObjectProgressTracker.php create mode 100644 src/S3/Features/S3Transfer/ProgressBar.php create mode 100644 src/S3/Features/S3Transfer/ProgressBarFactory.php create mode 100644 src/S3/Features/S3Transfer/ProgressListenerHelper.php create mode 100644 src/S3/Features/S3Transfer/RangeMultipartDownloader.php create mode 100644 src/S3/Features/S3Transfer/S3TransferManager.php create mode 100644 src/S3/Features/S3Transfer/S3TransferManagerTrait.php create mode 100644 src/S3/Features/S3Transfer/TransferListener.php create mode 100644 src/S3/Features/S3Transfer/TransferListenerFactory.php diff --git a/src/S3/Features/S3Transfer/ConsoleProgressBar.php b/src/S3/Features/S3Transfer/ConsoleProgressBar.php new file mode 100644 index 0000000000..3f4312bfa7 --- /dev/null +++ b/src/S3/Features/S3Transfer/ConsoleProgressBar.php @@ -0,0 +1,130 @@ + [ + 'format' => "[|progress_bar|] |percent|%", + 'parameters' => [] + ], + 'transfer_format' => [ + 'format' => "[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit|", + 'parameters' => [ + 'transferred', + 'tobe_transferred', + 'unit' + ] + ], + 'colored_transfer_format' => [ + 'format' => "\033|color_code|[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit|\033[0m", + 'parameters' => [ + 'transferred', + 'tobe_transferred', + 'unit', + 'color_code' + ] + ], + ]; + + /** @var string */ + private string $progressBarChar; + + /** @var int */ + private int $progressBarWidth; + + /** @var int */ + private int $percentCompleted; + + /** @var ?array */ + private ?array $format; + + private ?array $args; + + /** + * @param ?string $progressBarChar + * @param ?int $progressBarWidth + * @param ?int $percentCompleted + * @param array|null $format + */ + public function __construct( + ?string $progressBarChar = null, + ?int $progressBarWidth = null, + ?int $percentCompleted = null, + ?array $format = null, + ?array $args = null + ) { + $this->progressBarChar = $progressBarChar ?? '#'; + $this->progressBarWidth = $progressBarWidth ?? 25; + $this->percentCompleted = $percentCompleted ?? 0; + $this->format = $format ?: self::$formats['transfer_format']; + $this->args = $args ?: []; + } + + /** + * Set current progress percent. + * + * @param int $percent + * + * @return void + */ + public function setPercentCompleted(int $percent): void { + $this->percentCompleted = max(0, min(100, $percent)); + } + + /** + * @param array $args + * + * @return void + */ + public function setArgs(array $args): void + { + $this->args = $args; + } + + public function setArg(string $key, mixed $value): void + { + if (array_key_exists($key, $this->args)) { + $this->args[$key] = $value; + } + } + + private function renderProgressBar(): string { + $filledWidth = (int) round(($this->progressBarWidth * $this->percentCompleted) / 100); + return str_repeat($this->progressBarChar, $filledWidth) + . str_repeat(' ', $this->progressBarWidth - $filledWidth); + } + + /** + * + * @return string + */ + public function getPaintedProgress(): string { + foreach ($this->format['parameters'] as $param) { + if (!array_key_exists($param, $this->args)) { + throw new \InvalidArgumentException("Missing '{$param}' parameter for progress bar."); + } + } + + $replacements = [ + '|progress_bar|' => $this->renderProgressBar(), + '|percent|' => $this->percentCompleted + ]; + + foreach ($this->format['parameters'] as $param) { + $replacements["|$param|"] = $this->args[$param] ?? ''; + } + + return strtr($this->format['format'], $replacements); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/DefaultProgressTracker.php b/src/S3/Features/S3Transfer/DefaultProgressTracker.php new file mode 100644 index 0000000000..ae47baab92 --- /dev/null +++ b/src/S3/Features/S3Transfer/DefaultProgressTracker.php @@ -0,0 +1,193 @@ +clear(); + $this->initializeListener(); + $this->progressBarFactory = $progressBarFactory ?? $this->defaultProgressBarFactory(); + } + + private function initializeListener(): void { + $this->transferListener = new TransferListener(); + // Object transfer initialized + $this->transferListener->onObjectTransferInitiated = $this->objectTransferInitiated(); + // Object transfer made progress + $this->transferListener->onObjectTransferProgress = $this->objectTransferProgress(); + $this->transferListener->onObjectTransferFailed = $this->objectTransferFailed(); + $this->transferListener->onObjectTransferCompleted = $this->objectTransferCompleted(); + } + + /** + * @return TransferListener + */ + public function getTransferListener(): TransferListener { + return $this->transferListener; + } + + /** + * + * @return Closure + */ + private function objectTransferInitiated(): Closure + { + return function (string $objectKey, array &$requestArgs) { + $progressBarFactoryFn = $this->progressBarFactory; + $this->objects[$objectKey] = new ObjectProgressTracker( + objectKey: $objectKey, + objectBytesTransferred: 0, + objectSizeInBytes: 0, + status: 'initiated', + progressBar: $progressBarFactoryFn() + ); + $this->objectsInProgress++; + $this->objectsCount++; + + $this->showProgress(); + }; + } + + /** + * @return Closure + */ + private function objectTransferProgress(): Closure + { + return function ( + string $objectKey, + int $objectBytesTransferred, + int $objectSizeInBytes + ): void { + $objectProgressTracker = $this->objects[$objectKey]; + if ($objectProgressTracker->getObjectSizeInBytes() === 0) { + $objectProgressTracker->setObjectSizeInBytes($objectSizeInBytes); + // Increment objectsTotalSizeInBytes just the first time we set + // the object total size. + $this->objectsTotalSizeInBytes = + $this->objectsTotalSizeInBytes + $objectSizeInBytes; + } + $objectProgressTracker->incrementTotalBytesTransferred( + $objectBytesTransferred + ); + $objectProgressTracker->setStatus('progress'); + + $this->increaseBytesTransferred($objectBytesTransferred); + + $this->showProgress(); + }; + } + + public function objectTransferFailed(): Closure + { + return function ( + string $objectKey, + int $totalObjectBytesTransferred, + \Throwable | string $reason + ): void { + $objectProgressTracker = $this->objects[$objectKey]; + $objectProgressTracker->setStatus('failed'); + + $this->objectsInProgress--; + + $this->showProgress(); + }; + } + + public function objectTransferCompleted(): Closure + { + return function ( + string $objectKey, + int $objectBytesTransferred, + ): void { + $objectProgressTracker = $this->objects[$objectKey]; + $objectProgressTracker->setStatus('completed'); + $this->showProgress(); + }; + } + + /** + * Clear the internal state holders. + * + * @return void + */ + public function clear(): void + { + $this->objects = []; + $this->totalBytesTransferred = 0; + $this->objectsTotalSizeInBytes = 0; + $this->objectsInProgress = 0; + $this->objectsCount = 0; + $this->transferPercentCompleted = 0; + } + + private function increaseBytesTransferred(int $bytesTransferred): void { + $this->totalBytesTransferred += $bytesTransferred; + if ($this->objectsTotalSizeInBytes !== 0) { + $this->transferPercentCompleted = floor(($this->totalBytesTransferred / $this->objectsTotalSizeInBytes) * 100); + } + } + + private function showProgress(): void { + // Clear screen + fwrite(STDOUT, "\033[2J\033[H"); + + // Display progress header + echo sprintf( + "\r%d%% [%s/%s]\n", + $this->transferPercentCompleted, + $this->objectsInProgress, + $this->objectsCount + ); + + foreach ($this->objects as $name => $object) { + echo sprintf("\r%s:\n%s\n", $name, $object->getProgressBar()->getPaintedProgress()); + } + } + + private function defaultProgressBarFactory(): Closure| ProgressBarFactory { + return function () { + return new ConsoleProgressBar( + format: ConsoleProgressBar::$formats[ + ConsoleProgressBar::COLORED_TRANSFER_FORMAT + ], + args: [ + 'transferred' => 0, + 'tobe_transferred' => 0, + 'unit' => 'MB', + 'color_code' => ConsoleProgressBar::BLACK_COLOR_CODE, + ] + ); + }; + } + +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/GetMultipartDownloader.php b/src/S3/Features/S3Transfer/GetMultipartDownloader.php new file mode 100644 index 0000000000..9548d95ded --- /dev/null +++ b/src/S3/Features/S3Transfer/GetMultipartDownloader.php @@ -0,0 +1,50 @@ +currentPartNo === 0) { + $this->currentPartNo = 1; + } else { + $this->currentPartNo++; + } + + $nextRequestArgs = array_slice($this->requestArgs, 0); + $nextRequestArgs['PartNumber'] = $this->currentPartNo; + + return $this->s3Client->getCommand( + self::GET_OBJECT_COMMAND, + $nextRequestArgs + ); + } + + /** + * @inheritDoc + * + * @param Result $result + * + * @return void + */ + protected function computeObjectDimensions(ResultInterface $result): void + { + if (!empty($result['PartsCount'])) { + $this->objectPartsCount = $result['PartsCount']; + } + + $this->objectSizeInBytes = $this->computeObjectSize($result['ContentRange'] ?? ""); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/ListenerNotifier.php b/src/S3/Features/S3Transfer/ListenerNotifier.php new file mode 100644 index 0000000000..836dfcd66e --- /dev/null +++ b/src/S3/Features/S3Transfer/ListenerNotifier.php @@ -0,0 +1,8 @@ +notify('onDownloadInitiated', [&$commandArgs, $initialPart]); + } + + /** + * Event for when a download fails. + * Warning: If this method is overridden, it is recommended + * to call parent::downloadFailed() in order to + * keep the states maintained in this implementation. + * + * @param \Throwable $reason + * @param int $totalPartsTransferred + * @param int $totalBytesTransferred + * @param int $lastPartTransferred + * + * @return void + */ + public function downloadFailed(\Throwable $reason, int $totalPartsTransferred, int $totalBytesTransferred, int $lastPartTransferred): void { + $this->notify('onDownloadFailed', [$reason, $totalPartsTransferred, $totalBytesTransferred, $lastPartTransferred]); + } + + /** + * Event for when a download completes. + * Warning: If this method is overridden, it is recommended + * to call parent::onDownloadCompleted() in order to + * keep the states maintained in this implementation. + * + * @param resource $stream + * @param int $totalPartsDownloaded + * @param int $totalBytesDownloaded + * + * @return void + */ + public function downloadCompleted($stream, int $totalPartsDownloaded, int $totalBytesDownloaded): void { + $this->notify('onDownloadCompleted', [$stream, $totalPartsDownloaded, $totalBytesDownloaded]); + } + + /** + * Event for when a part download is initiated. + * Warning: If this method is overridden, it is recommended + * to call parent::partDownloadInitiated() in order to + * keep the states maintained in this implementation. + * + * @param mixed $partDownloadCommand + * @param int $partNo + * + * @return void + */ + public function partDownloadInitiated(CommandInterface $partDownloadCommand, int $partNo): void { + $this->notify('onPartDownloadInitiated', [$partDownloadCommand, $partNo]); + } + + /** + * Event for when a part download completes. + * Warning: If this method is overridden, it is recommended + * to call parent::onPartDownloadCompleted() in order to + * keep the states maintained in this implementation. + * + * @param ResultInterface $result + * @param int $partNo + * @param int $partTotalBytes + * @param int $totalParts + * @param int $objectBytesTransferred + * @param int $objectSizeInBytes + * @return void + */ + public function partDownloadCompleted( + ResultInterface $result, + int $partNo, + int $partTotalBytes, + int $totalParts, + int $objectBytesTransferred, + int $objectSizeInBytes + ): void + { + $this->notify('onPartDownloadCompleted', [ + $result, + $partNo, + $partTotalBytes, + $totalParts, + $objectBytesTransferred, + $objectSizeInBytes + ]); + } + + /** + * Event for when a part download fails. + * Warning: If this method is overridden, it is recommended + * to call parent::onPartDownloadFailed() in order to + * keep the states maintained in this implementation. + * + * @param CommandInterface $partDownloadCommand + * @param \Throwable $reason + * @param int $partNo + * + * @return void + */ + public function partDownloadFailed(CommandInterface $partDownloadCommand, \Throwable $reason, int $partNo): void { + $this->notify('onPartDownloadFailed', [$partDownloadCommand, $reason, $partNo]); + } + + protected function notify(string $event, array $params = []): void + { + $listener = match ($event) { + 'onDownloadInitiated' => $this->onDownloadInitiated, + 'onDownloadFailed' => $this->onDownloadFailed, + 'onDownloadCompleted' => $this->onDownloadCompleted, + 'onPartDownloadInitiated' => $this->onPartDownloadInitiated, + 'onPartDownloadCompleted' => $this->onPartDownloadCompleted, + 'onPartDownloadFailed' => $this->onPartDownloadFailed, + default => null, + }; + + if ($listener instanceof Closure) { + $listener(...$params); + } + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/MultipartDownloadType.php b/src/S3/Features/S3Transfer/MultipartDownloadType.php new file mode 100644 index 0000000000..6bbbfd6144 --- /dev/null +++ b/src/S3/Features/S3Transfer/MultipartDownloadType.php @@ -0,0 +1,51 @@ +value = $value; + } + + /** + * @return string + */ + public function __toString(): string { + return $this->value; + } + + /** + * @param MultipartDownloadType $type + * + * @return bool + */ + public function equals(MultipartDownloadType $type): bool + { + return $this->value === $type->value; + } + + /** + * @return MultipartDownloadType + */ + public static function rangedGet(): MultipartDownloadType { + return new static(self::$rangedGetType); + } + + /** + * @return MultipartDownloadType + */ + public static function partGet(): MultipartDownloadType { + return new static(self::$partGetType); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/MultipartDownloader.php b/src/S3/Features/S3Transfer/MultipartDownloader.php new file mode 100644 index 0000000000..03b0e6430a --- /dev/null +++ b/src/S3/Features/S3Transfer/MultipartDownloader.php @@ -0,0 +1,408 @@ +clear(); + $this->s3Client = $s3Client; + $this->requestArgs = $requestArgs; + $this->config = $config; + $this->currentPartNo = $currentPartNo; + $this->listener = $listener; + $this->progressListener = $progressListener; + $this->stream = Utils::streamFor( + fopen('php://temp', 'w+') + ); + } + + + /** + * Returns that resolves a multipart download operation, + * or to a rejection in case of any failures. + * + * @return PromiseInterface + */ + public function promise(): PromiseInterface + { + return Coroutine::of(function () { + $this->downloadInitiated($this->requestArgs, $this->currentPartNo); + $initialCommand = $this->nextCommand(); + $this->partDownloadInitiated($initialCommand, $this->currentPartNo); + try { + yield $this->s3Client->executeAsync($initialCommand) + ->then(function (ResultInterface $result) { + // Calculate object size and parts count. + $this->computeObjectDimensions($result); + // Trigger first part completed + $this->partDownloadCompleted($result, $this->currentPartNo); + })->otherwise(function ($reason) use ($initialCommand) { + $this->partDownloadFailed($initialCommand, $reason, $this->currentPartNo); + + throw $reason; + }); + } catch (\Throwable $e) { + $this->downloadFailed($e, $this->objectCompletedPartsCount, $this->objectBytesTransferred, $this->currentPartNo); + // TODO: yield transfer exception modeled with a transfer failed response. + yield Create::rejectionFor($e); + } + + while ($this->currentPartNo < $this->objectPartsCount) { + $nextCommand = $this->nextCommand(); + $this->partDownloadInitiated($nextCommand, $this->currentPartNo); + try { + yield $this->s3Client->executeAsync($nextCommand) + ->then(function ($result) { + $this->partDownloadCompleted($result, $this->currentPartNo); + + return $result; + })->otherwise(function ($reason) use ($nextCommand) { + $this->partDownloadFailed($nextCommand, $reason, $this->currentPartNo); + + return $reason; + }); + } catch (\Throwable $e) { + $this->downloadFailed($e, $this->objectCompletedPartsCount, $this->objectBytesTransferred, $this->currentPartNo); + // TODO: yield transfer exception modeled with a transfer failed response. + yield Create::rejectionFor($e); + } + } + + // Transfer completed + $this->objectDownloadCompleted(); + + // TODO: yield the stream wrapped in a modeled transfer success response. + yield Create::promiseFor($this->stream); + }); + } + + /** + * Main purpose of this method is to propagate + * the download-initiated event to listeners, but + * also it does some computation regarding internal states + * that need to be maintained. + * + * @param array $commandArgs + * @param int|null $currentPartNo + * + * @return void + */ + private function downloadInitiated(array &$commandArgs, ?int $currentPartNo): void + { + $this->objectKey = $commandArgs['Key']; + $this->progressListener?->objectTransferInitiated( + $this->objectKey, + $commandArgs + ); + $this->_notifyMultipartDownloadListeners('downloadInitiated', [ + &$commandArgs, + $currentPartNo + ]); + } + + /** + * Propagates download-failed event to listeners. + * It may also do some computation in order to maintain internal states. + * + * @param \Throwable $reason + * @param int $totalPartsTransferred + * @param int $totalBytesTransferred + * @param int $lastPartTransferred + * + * @return void + */ + private function downloadFailed( + \Throwable $reason, + int $totalPartsTransferred, + int $totalBytesTransferred, + int $lastPartTransferred + ): void { + $this->progressListener?->objectTransferFailed( + $this->objectKey, + $totalBytesTransferred, + $reason + ); + $this->_notifyMultipartDownloadListeners('downloadFailed', [ + $reason, + $totalPartsTransferred, + $totalBytesTransferred, + $lastPartTransferred + ]); + } + + /** + * Propagates part-download-initiated event to listeners. + * + * @param CommandInterface $partDownloadCommand + * @param int $partNo + * + * @return void + */ + private function partDownloadInitiated(CommandInterface $partDownloadCommand, int $partNo): void { + $this->_notifyMultipartDownloadListeners('partDownloadInitiated', [ + $partDownloadCommand, + $partNo + ]); + } + + /** + * Propagates part-download-completed to listeners. + * It also does some computation in order to maintain internal states. + * In this specific method we move each part content into an accumulative + * stream, which is meant to hold the full object content once the download + * is completed. + * + * @param ResultInterface $result + * @param int $partNo + * + * @return void + */ + private function partDownloadCompleted(ResultInterface $result, int $partNo): void { + $this->objectCompletedPartsCount++; + $partDownloadBytes = $result['ContentLength']; + $this->objectBytesTransferred = $this->objectBytesTransferred + $partDownloadBytes; + if (isset($result['ETag'])) { + $this->eTag = $result['ETag']; + } + Utils::copyToStream($result['Body'], $this->stream); + + $this->progressListener?->objectTransferProgress( + $this->objectKey, + $partDownloadBytes, + $this->objectSizeInBytes + ); + + $this->_notifyMultipartDownloadListeners('partDownloadCompleted', [ + $result, + $partNo, + $partDownloadBytes, + $this->objectCompletedPartsCount, + $this->objectBytesTransferred, + $this->objectSizeInBytes + ]); + } + + /** + * Propagates part-download-failed event to listeners. + * + * @param CommandInterface $partDownloadCommand + * @param \Throwable $reason + * @param int $partNo + * + * @return void + */ + private function partDownloadFailed( + CommandInterface $partDownloadCommand, + \Throwable $reason, + int $partNo + ): void { + $this->progressListener?->objectTransferFailed( + $this->objectKey, + $this->objectBytesTransferred, + $reason + ); + $this->_notifyMultipartDownloadListeners( + 'partDownloadFailed', + [$partDownloadCommand, $reason, $partNo]); + } + + /** + * Propagates object-download-completed event to listeners. + * It also resets the pointer of the stream to the first position, + * so that the stream is ready to be consumed once returned. + * + * @return void + */ + private function objectDownloadCompleted(): void + { + $this->stream->rewind(); + $this->progressListener?->objectTransferCompleted( + $this->objectKey, + $this->objectBytesTransferred + ); + $this->_notifyMultipartDownloadListeners('downloadCompleted', [ + $this->stream, + $this->objectCompletedPartsCount, + $this->objectBytesTransferred + ]); + } + + /** + * Internal helper method for notifying listeners of specific events. + * + * @param string $listenerMethod + * @param array $args + * + * @return void + */ + private function _notifyMultipartDownloadListeners(string $listenerMethod, array $args): void + { + $this->listener?->{$listenerMethod}(...$args); + } + + /** + * Returns the next command for fetching the next object part. + * + * @return CommandInterface + */ + abstract protected function nextCommand() : CommandInterface; + + /** + * Compute the object dimensions, such as size and parts count. + * + * @param ResultInterface $result + * + * @return void + */ + abstract protected function computeObjectDimensions(ResultInterface $result): void; + + /** + * Calculates the object size dynamically. + * + * @param $sizeSource + * + * @return int + */ + protected function computeObjectSize($sizeSource): int { + if (gettype($sizeSource) === "integer") { + return (int) $sizeSource; + } + + if (empty($sizeSource)) { + throw new \RuntimeException('Range must not be empty'); + } + + if (preg_match("/\/(\d+)$/", $sizeSource, $matches)) { + return $matches[1]; + } + + throw new \RuntimeException('Invalid range format'); + } + + private function clear(): void { + $this->currentPartNo = 0; + $this->objectPartsCount = 0; + $this->objectCompletedPartsCount = 0; + $this->objectSizeInBytes = 0; + $this->objectBytesTransferred = 0; + $this->eTag = ""; + $this->objectKey = ""; + } + + /** + * MultipartDownloader factory method to return an instance + * of MultipartDownloader based on the multipart download type. + * + * @param S3ClientInterface $s3Client + * @param string $multipartDownloadType + * @param array $requestArgs + * @param array $config + * @param MultipartDownloadListener|null $listener + * @param TransferListener|null $progressTracker + * + * @return MultipartDownloader + */ + public static function chooseDownloader( + S3ClientInterface $s3Client, + string $multipartDownloadType, + array $requestArgs, + array $config, + ?MultipartDownloadListener $listener = null, + ?TransferListener $progressTracker = null + ) : MultipartDownloader + { + if ($multipartDownloadType === self::PART_GET_MULTIPART_DOWNLOADER) { + return new GetMultipartDownloader( + $s3Client, + $requestArgs, + $config, + 0, + $listener, + $progressTracker + ); + } elseif ($multipartDownloadType === self::RANGE_GET_MULTIPART_DOWNLOADER) { + return new RangeMultipartDownloader( + $s3Client, + $requestArgs, + $config, + 0, + $listener, + $progressTracker + ); + } + + throw new \RuntimeException("Unsupported download type $multipartDownloadType"); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/ObjectProgressTracker.php b/src/S3/Features/S3Transfer/ObjectProgressTracker.php new file mode 100644 index 0000000000..9c876e363a --- /dev/null +++ b/src/S3/Features/S3Transfer/ObjectProgressTracker.php @@ -0,0 +1,179 @@ +objectKey = $objectKey; + $this->objectBytesTransferred = $objectBytesTransferred; + $this->objectSizeInBytes = $objectSizeInBytes; + $this->status = $status; + $this->progressBar = $progressBar ?? $this->defaultProgressBar(); + } + + /** + * @return string + */ + public function getObjectKey(): string + { + return $this->objectKey; + } + + /** + * @param string $objectKey + * + * @return void + */ + public function setObjectKey(string $objectKey): void + { + $this->objectKey = $objectKey; + } + + /** + * @return int + */ + public function getObjectBytesTransferred(): int + { + return $this->objectBytesTransferred; + } + + /** + * @param int $objectBytesTransferred + * + * @return void + */ + public function setObjectBytesTransferred(int $objectBytesTransferred): void + { + $this->objectBytesTransferred = $objectBytesTransferred; + } + + /** + * @return int + */ + public function getObjectSizeInBytes(): int + { + return $this->objectSizeInBytes; + } + + /** + * @param int $objectSizeInBytes + * + * @return void + */ + public function setObjectSizeInBytes(int $objectSizeInBytes): void + { + $this->objectSizeInBytes = $objectSizeInBytes; + // Update progress bar + $this->progressBar->setArg('tobe_transferred', $objectSizeInBytes); + } + + /** + * @return string + */ + public function getStatus(): string + { + return $this->status; + } + + /** + * @param string $status + * + * @return void + */ + public function setStatus(string $status): void + { + $this->status = $status; + $this->setProgressColor(); + } + + private function setProgressColor(): void { + if ($this->status === 'progress') { + $this->progressBar->setArg('color_code', ConsoleProgressBar::BLUE_COLOR_CODE); + } elseif ($this->status === 'completed') { + $this->progressBar->setArg('color_code', ConsoleProgressBar::GREEN_COLOR_CODE); + } elseif ($this->status === 'failed') { + $this->progressBar->setArg('color_code', ConsoleProgressBar::RED_COLOR_CODE); + } + } + + /** + * Increments the object bytes transferred. + * + * @param int $objectBytesTransferred + * + * @return void + */ + public function incrementTotalBytesTransferred( + int $objectBytesTransferred + ): void + { + $this->objectBytesTransferred += $objectBytesTransferred; + $progressPercent = (int) floor(($this->objectBytesTransferred / $this->objectSizeInBytes) * 100); + // Update progress bar + $this->progressBar->setPercentCompleted($progressPercent); + $this->progressBar->setArg('transferred', $this->objectBytesTransferred); + } + + /** + * @return ProgressBar|null + */ + public function getProgressBar(): ?ProgressBar + { + return $this->progressBar; + } + + /** + * @return ProgressBar + */ + private function defaultProgressBar(): ProgressBar { + return new ConsoleProgressBar( + format: ConsoleProgressBar::$formats[ + ConsoleProgressBar::COLORED_TRANSFER_FORMAT + ], + args: [ + 'transferred' => 0, + 'tobe_transferred' => 0, + 'unit' => 'B', + 'color_code' => ConsoleProgressBar::BLACK_COLOR_CODE, + ] + ); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/ProgressBar.php b/src/S3/Features/S3Transfer/ProgressBar.php new file mode 100644 index 0000000000..af38858d11 --- /dev/null +++ b/src/S3/Features/S3Transfer/ProgressBar.php @@ -0,0 +1,14 @@ + 'bytesToBytes', + 'KB' => 'bytesToKB', + 'MB' => 'bytesToMB', + ]; + + public static function getUnitValue(string $displayUnit, float $bytes): float { + $displayUnit = self::validateDisplayUnit($displayUnit); + if (isset(self::$displayUnitMapping[$displayUnit])) { + return number_format(call_user_func([__CLASS__, self::$displayUnitMapping[$displayUnit]], $bytes)); + } + + throw new \RuntimeException("Unknown display unit {$displayUnit}"); + } + + private static function validateDisplayUnit(string $displayUnit): string { + if (!isset(self::$displayUnitMapping[$displayUnit])) { + throw new \InvalidArgumentException("Invalid display unit specified: $displayUnit"); + } + + return $displayUnit; + } + + private static function bytesToBytes(float $bytes): float { + return $bytes; + } + + private static function bytesToKB(float $bytes): float { + return $bytes / 1024; + } + + private static function bytesToMB(float $bytes): float { + return $bytes / 1024 / 1024; + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/RangeMultipartDownloader.php b/src/S3/Features/S3Transfer/RangeMultipartDownloader.php new file mode 100644 index 0000000000..5a76a39408 --- /dev/null +++ b/src/S3/Features/S3Transfer/RangeMultipartDownloader.php @@ -0,0 +1,64 @@ +objectSizeInBytes !== 0) { + $this->computeObjectDimensions(new Result(['ContentRange' => $this->totalBytes])); + } + + if ($this->currentPartNo === 0) { + $this->currentPartNo = 1; + $this->partSize = $this->config['targetPartSizeBytes']; + } else { + $this->currentPartNo++; + } + + $nextRequestArgs = array_slice($this->requestArgs, 0); + $from = ($this->currentPartNo - 1) * ($this->partSize + 1); + $to = $this->currentPartNo * $this->partSize; + $nextRequestArgs['Range'] = "bytes=$from-$to"; + if (!empty($this->eTag)) { + $nextRequestArgs['IfMatch'] = $this->eTag; + } + + return $this->s3Client->getCommand( + self::GET_OBJECT_COMMAND, + $nextRequestArgs + ); + } + + /** + * @inheritDoc + * + * @param Result $result + * + * @return void + */ + protected function computeObjectDimensions(ResultInterface $result): void + { + $this->objectSizeInBytes = $this->computeObjectSize($result['ContentRange'] ?? ""); + if ($this->objectSizeInBytes > $this->partSize) { + $this->objectPartsCount = intval(ceil($this->objectSizeInBytes / $this->partSize)); + } else { + $this->partSize = $this->objectSizeInBytes; + $this->currentPartNo = 1; + } + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/S3TransferManager.php b/src/S3/Features/S3Transfer/S3TransferManager.php new file mode 100644 index 0000000000..f9c56e6b7c --- /dev/null +++ b/src/S3/Features/S3Transfer/S3TransferManager.php @@ -0,0 +1,143 @@ +s3Client = $this->defaultS3Client(); + } else { + $this->s3Client = $s3Client; + } + + $this->config = $config + self::$defaultConfig; + } + + /** + * @param string|array $source The object to be downloaded from S3. + * It can be either a string with a S3 URI or an array with a Bucket and Key + * properties set. + * @param array $downloadArgs The request arguments to be provided as part + * of the service client operation. + * @param array $config The configuration to be used for this operation. + * - listener: (null|MultipartDownloadListener) \ + * A listener to be notified in every stage of a multipart download operation. + * - trackProgress: (bool) \ + * Overrides the config option set in the transfer manager instantiation + * to decide whether transfer progress should be tracked. If not + * transfer tracker factory is provided and trackProgress is true then, + * the default progress listener implementation will be used. + * + * @return PromiseInterface + */ + public function download( + string | array $source, + array $downloadArgs, + array $config = [] + ): PromiseInterface { + if (is_string($source)) { + $sourceArgs = $this->s3UriAsBucketAndKey($source); + } elseif (is_array($source)) { + $sourceArgs = [ + 'Bucket' => $this->requireNonEmpty($source['Bucket'], "A valid bucket must be provided."), + 'Key' => $this->requireNonEmpty($source['Key'], "A valid key must be provided."), + ]; + } else { + throw new \InvalidArgumentException('Source must be a string or an array of strings'); + } + + $requestArgs = $sourceArgs + $downloadArgs; + if (empty($downloadArgs['PartNumber']) && empty($downloadArgs['Range'])) { + return $this->tryMultipartDownload($requestArgs, $config); + } + + return $this->trySingleDownload($requestArgs); + } + + /** + * Tries an object multipart download. + * + * @param array $requestArgs + * @param array $config + * - listener: (?MultipartDownloadListener) \ + * A multipart download listener for watching every multipart download + * stage. + * + * @return PromiseInterface + */ + private function tryMultipartDownload( + array $requestArgs, + array $config + ): PromiseInterface { + $trackProgress = $config['trackProgress'] + ?? $this->config['trackProgress'] + ?? false; + $progressListenerFactory = $this->config['progressListenerFactory'] ?? null; + $progressListener = null; + if ($trackProgress) { + if ($progressListenerFactory !== null) { + $progressListener = $progressListenerFactory(); + } else { + $progressListener = new DefaultProgressTracker(); + } + } + $multipartDownloader = MultipartDownloader::chooseDownloader( + $this->s3Client, + $this->config['multipartDownloadType'], + $requestArgs, + $this->config, + $config['listener'] ?? null, + $progressListener?->getTransferListener() + ); + + return $multipartDownloader->promise(); + } + + /** + * Does a single object download. + * + * @param $requestArgs + * + * @return PromiseInterface + */ + private function trySingleDownload($requestArgs): PromiseInterface { + $command = $this->s3Client->getCommand(MultipartDownloader::GET_OBJECT_COMMAND, $requestArgs); + + return $this->s3Client->executeAsync($requestArgs); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/S3TransferManagerTrait.php b/src/S3/Features/S3Transfer/S3TransferManagerTrait.php new file mode 100644 index 0000000000..d924280b46 --- /dev/null +++ b/src/S3/Features/S3Transfer/S3TransferManagerTrait.php @@ -0,0 +1,88 @@ + 8 * 1024 * 1024, + 'multipartUploadThresholdBytes' => 16 * 1024 * 1024, + 'multipartDownloadThresholdBytes' => 16 * 1024 * 1024, + 'checksumValidationEnabled' => true, + 'checksumAlgorithm' => 'crc32', + 'multipartDownloadType' => 'partGet', + 'concurrency' => 5, + ]; + + /** + * Returns a default instance of S3Client. + * + * @return S3Client + */ + private function defaultS3Client(): S3ClientInterface + { + return new S3Client([]); + } + + /** + * Validates a provided value is not empty, and if so then + * it throws an exception with the provided message. + * @param mixed $value + * + * @return mixed + */ + private function requireNonEmpty(mixed $value, string $message): mixed { + if (empty($value)) { + throw new \InvalidArgumentException($message); + } + + return $value; + } + + /** + * Validates a string value is a valid S3 URI. + * Valid S3 URI Example: S3://mybucket.dev/myobject.txt + * + * @param string $uri + * + * @return bool + */ + private function isValidS3URI(string $uri): bool + { + // in the expression `substr($uri, 5)))` the 5 belongs to the size of `s3://`. + return str_starts_with(strtolower($uri), 's3://') + && count(explode('/', substr($uri, 5))) > 1; + } + + /** + * Converts a S3 URI into an array with a Bucket and Key + * properties set. + * + * @param string $uri: The S3 URI. + * + * @return array + */ + private function s3UriAsBucketAndKey(string $uri): array { + $errorMessage = "Invalid URI: $uri. A valid S3 URI must be s3://bucket/key"; + if (!$this->isValidS3URI($uri)) { + throw new \InvalidArgumentException($errorMessage); + } + + $path = substr($uri, 5); // without s3:// + $parts = explode('/', $path, 2); + + if (count($parts) < 2) { + throw new \InvalidArgumentException($errorMessage); + } + + return [ + 'Bucket' => $parts[0], + 'Key' => $parts[1], + ]; + } + +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/TransferListener.php b/src/S3/Features/S3Transfer/TransferListener.php new file mode 100644 index 0000000000..9f3f308978 --- /dev/null +++ b/src/S3/Features/S3Transfer/TransferListener.php @@ -0,0 +1,252 @@ +notify('onTransferInitiated', []); + } + + /** + * Event for when an object transfer initiated. + * + * @param string $objectKey + * @param array $requestArgs + * + * @return void + */ + public function objectTransferInitiated(string $objectKey, array &$requestArgs): void { + $this->objectsToBeTransferred++; + if ($this->objectsToBeTransferred === 1) { + $this->transferInitiated(); + } + + $this->notify('onObjectTransferInitiated', [$objectKey, &$requestArgs]); + } + + /** + * Event for when an object transfer made some progress. + * + * @param string $objectKey + * @param int $objectBytesTransferred + * @param int $objectSizeInBytes + * + * @return void + */ + public function objectTransferProgress( + string $objectKey, + int $objectBytesTransferred, + int $objectSizeInBytes + ): void { + $this->objectsBytesTransferred += $objectBytesTransferred; + $this->notify('onObjectTransferProgress', [ + $objectKey, + $objectBytesTransferred, + $objectSizeInBytes + ]); + // Needs state management + $this->notify('onTransferProgress', [ + $this->objectsTransferCompleted, + $this->objectsBytesTransferred, + $this->objectsToBeTransferred + ]); + } + + /** + * Event for when an object transfer failed. + * + * @param string $objectKey + * @param int $objectBytesTransferred + * @param \Throwable|string $reason + * + * @return void + */ + public function objectTransferFailed( + string $objectKey, + int $objectBytesTransferred, + \Throwable | string $reason + ): void { + $this->objectsTransferFailed++; + $this->notify('onObjectTransferFailed', [ + $objectKey, + $objectBytesTransferred, + $reason + ]); + } + + /** + * Event for when an object transfer is completed. + * + * @param string $objectKey + * @param int $objectBytesCompleted + * + * @return void + */ + public function objectTransferCompleted ( + string $objectKey, + int $objectBytesCompleted + ): void { + $this->objectsTransferCompleted++; + $this->validateTransferComplete(); + $this->notify('onObjectTransferCompleted', [ + $objectKey, + $objectBytesCompleted + ]); + } + + /** + * Event for when a transfer is completed. + * + * @param int $objectsTransferCompleted + * @param int $objectsBytesTransferred + * + * @return void + */ + public function transferCompleted ( + int $objectsTransferCompleted, + int $objectsBytesTransferred, + ): void { + $this->notify('onTransferCompleted', [ + $objectsTransferCompleted, + $objectsBytesTransferred + ]); + } + + /** + * Event for when a transfer is completed. + * + * @param int $objectsTransferCompleted + * @param int $objectsBytesTransferred + * @param int $objectsTransferFailed + * + * @return void + */ + public function transferFailed ( + int $objectsTransferCompleted, + int $objectsBytesTransferred, + int $objectsTransferFailed, + Throwable | string $reason + ): void { + $this->notify('onTransferFailed', [ + $objectsTransferCompleted, + $objectsBytesTransferred, + $objectsTransferFailed, + $reason + ]); + } + + /** + * Validates if a transfer is completed, and if so then the event is propagated + * to the subscribed listeners. + * + * @return void + */ + private function validateTransferComplete(): void { + if ($this->objectsToBeTransferred === ($this->objectsTransferCompleted + $this->objectsTransferFailed)) { + if ($this->objectsTransferFailed > 0) { + $this->transferFailed( + $this->objectsTransferCompleted, + $this->objectsBytesTransferred, + $this->objectsTransferFailed, + "Transfer could not have been completed successfully." + ); + } else { + $this->transferCompleted( + $this->objectsTransferCompleted, + $this->objectsBytesTransferred + ); + } + } + } + + protected function notify(string $event, array $params = []): void + { + $listener = match ($event) { + 'onTransferInitiated' => $this->onTransferInitiated, + 'onObjectTransferInitiated' => $this->onObjectTransferInitiated, + 'onObjectTransferProgress' => $this->onObjectTransferProgress, + 'onObjectTransferFailed' => $this->onObjectTransferFailed, + 'onObjectTransferCompleted' => $this->onObjectTransferCompleted, + 'onTransferProgress' => $this->onTransferProgress, + 'onTransferCompleted' => $this->onTransferCompleted, + 'onTransferFailed' => $this->onTransferFailed, + default => null, + }; + + if ($listener instanceof Closure) { + $listener(...$params); + } + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/TransferListenerFactory.php b/src/S3/Features/S3Transfer/TransferListenerFactory.php new file mode 100644 index 0000000000..dd8c6422de --- /dev/null +++ b/src/S3/Features/S3Transfer/TransferListenerFactory.php @@ -0,0 +1,8 @@ + Date: Wed, 5 Feb 2025 17:27:10 -0800 Subject: [PATCH 02/19] chore: add tests cases and refactor - Refactor set a single argument, even when not exists, in the console progress bar. - Add a specific parameter for showing the progress rendering defaulted to STDOUT. - Add test cases for ConsoleProgressBar. - Add test cases for DefaultProgressTracker. - Add test cases for ObjectProgressTracker. - Add test cases for TransferListener. --- .../S3Transfer/ConsoleProgressBar.php | 14 +- .../S3Transfer/DefaultProgressTracker.php | 88 ++++++- .../S3Transfer/ObjectProgressTracker.php | 40 +-- .../Features/S3Transfer/TransferListener.php | 31 +++ .../S3Transfer/ConsoleProgressBarTest.php | 228 ++++++++++++++++++ .../S3Transfer/DefaultProgressTrackerTest.php | 189 +++++++++++++++ .../S3Transfer/MultipartDownloaderTest.php | 12 + .../S3Transfer/ObjectProgressTrackerTest.php | 127 ++++++++++ .../S3Transfer/TransferListenerTest.php | 216 +++++++++++++++++ 9 files changed, 903 insertions(+), 42 deletions(-) create mode 100644 tests/S3/Features/S3Transfer/ConsoleProgressBarTest.php create mode 100644 tests/S3/Features/S3Transfer/DefaultProgressTrackerTest.php create mode 100644 tests/S3/Features/S3Transfer/MultipartDownloaderTest.php create mode 100644 tests/S3/Features/S3Transfer/ObjectProgressTrackerTest.php create mode 100644 tests/S3/Features/S3Transfer/TransferListenerTest.php diff --git a/src/S3/Features/S3Transfer/ConsoleProgressBar.php b/src/S3/Features/S3Transfer/ConsoleProgressBar.php index 3f4312bfa7..23f41f0e57 100644 --- a/src/S3/Features/S3Transfer/ConsoleProgressBar.php +++ b/src/S3/Features/S3Transfer/ConsoleProgressBar.php @@ -92,11 +92,17 @@ public function setArgs(array $args): void $this->args = $args; } + /** + * Sets an argument. + * + * @param string $key + * @param mixed $value + * + * @return void + */ public function setArg(string $key, mixed $value): void { - if (array_key_exists($key, $this->args)) { - $this->args[$key] = $value; - } + $this->args[$key] = $value; } private function renderProgressBar(): string { @@ -112,7 +118,7 @@ private function renderProgressBar(): string { public function getPaintedProgress(): string { foreach ($this->format['parameters'] as $param) { if (!array_key_exists($param, $this->args)) { - throw new \InvalidArgumentException("Missing '{$param}' parameter for progress bar."); + throw new \InvalidArgumentException("Missing `$param` parameter for progress bar."); } } diff --git a/src/S3/Features/S3Transfer/DefaultProgressTracker.php b/src/S3/Features/S3Transfer/DefaultProgressTracker.php index ae47baab92..3a8cd92c68 100644 --- a/src/S3/Features/S3Transfer/DefaultProgressTracker.php +++ b/src/S3/Features/S3Transfer/DefaultProgressTracker.php @@ -27,16 +27,27 @@ class DefaultProgressTracker /** @var TransferListener */ private TransferListener $transferListener; - private Closure| ProgressBarFactory | null $progressBarFactory; + /** @var Closure|ProgressBarFactory|null */ + private Closure|ProgressBarFactory|null $progressBarFactory; + + /** @var resource */ + private $output; /** * @param Closure|ProgressBarFactory|null $progressBarFactory */ - public function __construct(Closure | ProgressBarFactory | null $progressBarFactory = null) - { + public function __construct( + Closure | ProgressBarFactory | null $progressBarFactory = null, + $output = STDOUT + ) { $this->clear(); $this->initializeListener(); $this->progressBarFactory = $progressBarFactory ?? $this->defaultProgressBarFactory(); + if (get_resource_type($output) !== 'stream') { + throw new \InvalidArgumentException("The type for $output must be a stream"); + } + + $this->output = $output; } private function initializeListener(): void { @@ -56,6 +67,46 @@ public function getTransferListener(): TransferListener { return $this->transferListener; } + /** + * @return int + */ + public function getTotalBytesTransferred(): int + { + return $this->totalBytesTransferred; + } + + /** + * @return int + */ + public function getObjectsTotalSizeInBytes(): int + { + return $this->objectsTotalSizeInBytes; + } + + /** + * @return int + */ + public function getObjectsInProgress(): int + { + return $this->objectsInProgress; + } + + /** + * @return int + */ + public function getObjectsCount(): int + { + return $this->objectsCount; + } + + /** + * @return int + */ + public function getTransferPercentCompleted(): int + { + return $this->transferPercentCompleted; + } + /** * * @return Closure @@ -107,6 +158,9 @@ private function objectTransferProgress(): Closure }; } + /** + * @return Closure + */ public function objectTransferFailed(): Closure { return function ( @@ -123,6 +177,9 @@ public function objectTransferFailed(): Closure }; } + /** + * @return Closure + */ public function objectTransferCompleted(): Closure { return function ( @@ -150,6 +207,11 @@ public function clear(): void $this->transferPercentCompleted = 0; } + /** + * @param int $bytesTransferred + * + * @return void + */ private function increaseBytesTransferred(int $bytesTransferred): void { $this->totalBytesTransferred += $bytesTransferred; if ($this->objectsTotalSizeInBytes !== 0) { @@ -157,23 +219,33 @@ private function increaseBytesTransferred(int $bytesTransferred): void { } } + /** + * @return void + */ private function showProgress(): void { // Clear screen - fwrite(STDOUT, "\033[2J\033[H"); + fwrite($this->output, "\033[2J\033[H"); // Display progress header - echo sprintf( + fwrite($this->output, sprintf( "\r%d%% [%s/%s]\n", $this->transferPercentCompleted, $this->objectsInProgress, $this->objectsCount - ); + )); foreach ($this->objects as $name => $object) { - echo sprintf("\r%s:\n%s\n", $name, $object->getProgressBar()->getPaintedProgress()); + fwrite($this->output, sprintf( + "\r%s:\n%s\n", + $name, + $object->getProgressBar()->getPaintedProgress() + )); } } + /** + * @return Closure|ProgressBarFactory + */ private function defaultProgressBarFactory(): Closure| ProgressBarFactory { return function () { return new ConsoleProgressBar( @@ -183,7 +255,7 @@ private function defaultProgressBarFactory(): Closure| ProgressBarFactory { args: [ 'transferred' => 0, 'tobe_transferred' => 0, - 'unit' => 'MB', + 'unit' => 'B', 'color_code' => ConsoleProgressBar::BLACK_COLOR_CODE, ] ); diff --git a/src/S3/Features/S3Transfer/ObjectProgressTracker.php b/src/S3/Features/S3Transfer/ObjectProgressTracker.php index 9c876e363a..5eadecfdc4 100644 --- a/src/S3/Features/S3Transfer/ObjectProgressTracker.php +++ b/src/S3/Features/S3Transfer/ObjectProgressTracker.php @@ -7,45 +7,25 @@ */ class ObjectProgressTracker { - /** @var string */ - private string $objectKey; - - /** @var int */ - private int $objectBytesTransferred; - - /** @var int */ - private int $objectSizeInBytes; - - /** @var ?ProgressBar */ - private ?ProgressBar $progressBar; - - /** - * @var string - * - initiated - * - progress - * - failed - * - completed - */ - private string $status; - /** * @param string $objectKey * @param int $objectBytesTransferred * @param int $objectSizeInBytes * @param string $status + * Possible values are: + * - initiated + * - progress + * - failed + * - completed * @param ?ProgressBar $progressBar */ public function __construct( - string $objectKey, - int $objectBytesTransferred, - int $objectSizeInBytes, - string $status, - ?ProgressBar $progressBar = null + private string $objectKey, + private int $objectBytesTransferred, + private int $objectSizeInBytes, + private string $status, + private ?ProgressBar $progressBar = null ) { - $this->objectKey = $objectKey; - $this->objectBytesTransferred = $objectBytesTransferred; - $this->objectSizeInBytes = $objectSizeInBytes; - $this->status = $status; $this->progressBar = $progressBar ?? $this->defaultProgressBar(); } diff --git a/src/S3/Features/S3Transfer/TransferListener.php b/src/S3/Features/S3Transfer/TransferListener.php index 9f3f308978..1fb5a3f74a 100644 --- a/src/S3/Features/S3Transfer/TransferListener.php +++ b/src/S3/Features/S3Transfer/TransferListener.php @@ -71,6 +71,37 @@ public function __construct( private int $objectsToBeTransferred = 0 ) {} + /** + * @return int + */ + public function getObjectsTransferCompleted(): int + { + return $this->objectsTransferCompleted; + } + + /** + * @return int + */ + public function getObjectsBytesTransferred(): int + { + return $this->objectsBytesTransferred; + } + + /** + * @return int + */ + public function getObjectsTransferFailed(): int + { + return $this->objectsTransferFailed; + } + + /** + * @return int + */ + public function getObjectsToBeTransferred(): int + { + return $this->objectsToBeTransferred; + } /** * Transfer initiated event. diff --git a/tests/S3/Features/S3Transfer/ConsoleProgressBarTest.php b/tests/S3/Features/S3Transfer/ConsoleProgressBarTest.php new file mode 100644 index 0000000000..8f59c5ac4c --- /dev/null +++ b/tests/S3/Features/S3Transfer/ConsoleProgressBarTest.php @@ -0,0 +1,228 @@ +setPercentCompleted($percent); + $progressBar->setArgs([ + 'transferred' => $transferred, + 'tobe_transferred' => $toBeTransferred, + 'unit' => $unit + ]); + + $output = $progressBar->getPaintedProgress(); + $this->assertEquals($expectedProgress, $output); + } + + /** + * Data provider for testing progress bar rendering. + * + * @return array + */ + public function progressBarPercentProvider(): array { + return [ + [ + 'percent' => 25, + 'transferred' => 25, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'expected' => '[###### ] 25% 25/100 B' + ], + [ + 'percent' => 50, + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'expected' => '[############# ] 50% 50/100 B' + ], + [ + 'percent' => 75, + 'transferred' => 75, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'expected' => '[################### ] 75% 75/100 B' + ], + [ + 'percent' => 100, + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'expected' => '[#########################] 100% 100/100 B' + ], + ]; + } + + /** + * Tests progress with custom char. + * + * @return void + */ + public function testProgressBarWithCustomChar() + { + $progressBar = new ConsoleProgressBar( + progressBarChar: '*', + progressBarWidth: 30 + ); + $progressBar->setPercentCompleted(30); + $progressBar->setArgs([ + 'transferred' => '10', + 'tobe_transferred' => '100', + 'unit' => 'B' + ]); + + $output = $progressBar->getPaintedProgress(); + $this->assertStringContainsString('10/100 B', $output); + $this->assertStringContainsString(str_repeat('*', 9), $output); + } + + /** + * Tests progress with custom char. + * + * @return void + */ + public function testProgressBarWithCustomWidth() + { + $progressBar = new ConsoleProgressBar( + progressBarChar: '*', + progressBarWidth: 100 + ); + $progressBar->setPercentCompleted(10); + $progressBar->setArgs([ + 'transferred' => '10', + 'tobe_transferred' => '100', + 'unit' => 'B' + ]); + + $output = $progressBar->getPaintedProgress(); + $this->assertStringContainsString('10/100 B', $output); + $this->assertStringContainsString(str_repeat('*', 10), $output); + } + + /** + * Tests missing parameters. + * + * @dataProvider progressBarMissingArgsProvider + * + * @return void + */ + public function testProgressBarMissingArgsThrowsException( + string $formatName, + string $parameter + ) + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("Missing `$parameter` parameter for progress bar."); + + $format = ConsoleProgressBar::$formats[$formatName]; + $progressBar = new ConsoleProgressBar( + format: $format, + ); + foreach ($format['parameters'] as $param) { + if ($param === $parameter) { + continue; + } + + $progressBar->setArg($param, 'foo'); + } + + $progressBar->setPercentCompleted(20); + $progressBar->getPaintedProgress(); + } + + + /** + * Data provider for testing exception when arguments are missing. + * + * @return array + */ + public function progressBarMissingArgsProvider(): array + { + return [ + [ + 'formatName' => ConsoleProgressBar::TRANSFER_FORMAT, + 'parameter' => 'transferred', + ], + [ + 'formatName' => ConsoleProgressBar::TRANSFER_FORMAT, + 'parameter' => 'tobe_transferred', + ], + [ + 'formatName' => ConsoleProgressBar::TRANSFER_FORMAT, + 'parameter' => 'unit', + ] + ]; + } + + /** + * Tests the progress bar does not overflow when the percent is over 100. + * + * @return void + */ + public function testProgressBarDoesNotOverflowAfter100Percent() + { + $progressBar = new ConsoleProgressBar( + progressBarChar: '*', + progressBarWidth: 10, + ); + $progressBar->setPercentCompleted(110); + $progressBar->setArgs([ + 'transferred' => 'foo', + 'tobe_transferred' => 'foo', + 'unit' => 'MB' + ]); + $output = $progressBar->getPaintedProgress(); + $this->assertStringContainsString('100%', $output); + $this->assertStringContainsString('[**********]', $output); + } + + /** + * Tests the progress bar sets the arguments. + * + * @return void + */ + public function testProgressBarSetsArguments() { + $progressBar = new ConsoleProgressBar( + progressBarChar: '*', + progressBarWidth: 25, + format: ConsoleProgressBar::$formats[ConsoleProgressBar::TRANSFER_FORMAT] + ); + $progressBar->setArgs([ + 'transferred' => 'fooTransferred', + 'tobe_transferred' => 'fooToBeTransferred', + 'unit' => 'fooUnit', + ]); + $output = $progressBar->getPaintedProgress(); + $progressBar->setPercentCompleted(100); + $this->assertStringContainsString('fooTransferred', $output); + $this->assertStringContainsString('fooToBeTransferred', $output); + $this->assertStringContainsString('fooUnit', $output); + } +} diff --git a/tests/S3/Features/S3Transfer/DefaultProgressTrackerTest.php b/tests/S3/Features/S3Transfer/DefaultProgressTrackerTest.php new file mode 100644 index 0000000000..fd82cd78f5 --- /dev/null +++ b/tests/S3/Features/S3Transfer/DefaultProgressTrackerTest.php @@ -0,0 +1,189 @@ +progressTracker = new DefaultProgressTracker( + output: $this->output = fopen('php://temp', 'r+') + ); + } + + protected function tearDown(): void { + fclose($this->output); + } + + /** + * Tests initialization is clean. + * + * @return void + */ + public function testInitialization(): void + { + $this->assertInstanceOf(TransferListener::class, $this->progressTracker->getTransferListener()); + $this->assertEquals(0, $this->progressTracker->getTotalBytesTransferred()); + $this->assertEquals(0, $this->progressTracker->getObjectsTotalSizeInBytes()); + $this->assertEquals(0, $this->progressTracker->getObjectsInProgress()); + $this->assertEquals(0, $this->progressTracker->getObjectsCount()); + $this->assertEquals(0, $this->progressTracker->getTransferPercentCompleted()); + } + + /** + * Tests object transfer is initiated when the event is triggered. + * + * @return void + */ + public function testObjectTransferInitiated(): void + { + $listener = $this->progressTracker->getTransferListener(); + $fakeRequestArgs = []; + ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); + + $this->assertEquals(1, $this->progressTracker->getObjectsInProgress()); + $this->assertEquals(1, $this->progressTracker->getObjectsCount()); + } + + /** + * Tests object transfer progress is propagated correctly. + * + * @dataProvider objectTransferProgressProvider + * + * @param string $objectKey + * @param int $objectSize + * @param array $progressList + * + * @return void + */ + public function testObjectTransferProgress( + string $objectKey, + int $objectSize, + array $progressList, + ): void + { + $listener = $this->progressTracker->getTransferListener(); + $fakeRequestArgs = []; + ($listener->onObjectTransferInitiated)($objectKey, $fakeRequestArgs); + $totalProgress = 0; + foreach ($progressList as $progress) { + ($listener->onObjectTransferProgress)($objectKey, $progress, $objectSize); + $totalProgress += $progress; + } + + $this->assertEquals($totalProgress, $this->progressTracker->getTotalBytesTransferred()); + $this->assertEquals($objectSize, $this->progressTracker->getObjectsTotalSizeInBytes()); + $percentCompleted = (int) floor($totalProgress / $objectSize) * 100; + $this->assertEquals($percentCompleted, $this->progressTracker->getTransferPercentCompleted()); + + rewind($this->output); + $this->assertStringContainsString("$percentCompleted% $totalProgress/$objectSize B", stream_get_contents($this->output)); + } + + /** + * Data provider for testing object progress tracker. + * + * @return array[] + */ + public function objectTransferProgressProvider(): array + { + return [ + [ + 'objectKey' => 'FooObjectKey', + 'objectSize' => 250, + 'progressList' => [ + 50, 100, 72, 28 + ] + ], + [ + 'objectKey' => 'FooObjectKey', + 'objectSize' => 10_000, + 'progressList' => [ + 100, 500, 1_000, 2_000, 5_000, 400, 700, 300 + ] + ], + [ + 'objectKey' => 'FooObjectKey', + 'objectSize' => 10_000, + 'progressList' => [ + 5_000, 5_000 + ] + ] + ]; + } + + /** + * Tests object transfer is completed. + * + * @return void + */ + public function testObjectTransferCompleted(): void + { + $listener = $this->progressTracker->getTransferListener(); + $fakeRequestArgs = []; + ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); + ($listener->onObjectTransferProgress)('FooObjectKey', 50, 100); + ($listener->onObjectTransferProgress)('FooObjectKey', 50, 100); + ($listener->onObjectTransferCompleted)('FooObjectKey', 100); + + $this->assertEquals(100, $this->progressTracker->getTotalBytesTransferred()); + $this->assertEquals(100, $this->progressTracker->getTransferPercentCompleted()); + + // Validate it completed 100% at the progress bar side. + rewind($this->output); + $this->assertStringContainsString("[#########################] 100% 100/100 B", stream_get_contents($this->output)); + } + + /** + * Tests object transfer failed. + * + * @return void + */ + public function testObjectTransferFailed(): void + { + $listener = $this->progressTracker->getTransferListener(); + $fakeRequestArgs = []; + ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); + ($listener->onObjectTransferProgress)('FooObjectKey', 27, 100); + ($listener->onObjectTransferFailed)('FooObjectKey', 27, 'Transfer error'); + + $this->assertEquals(27, $this->progressTracker->getTotalBytesTransferred()); + $this->assertEquals(27, $this->progressTracker->getTransferPercentCompleted()); + $this->assertEquals(0, $this->progressTracker->getObjectsInProgress()); + + rewind($this->output); + $this->assertStringContainsString("27% 27/100 B", stream_get_contents($this->output)); + } + + /** + * Tests state are cleared. + * + * @return void + */ + public function testClearState(): void + { + $listener = $this->progressTracker->getTransferListener(); + $fakeRequestArgs = []; + ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); + ($listener->onObjectTransferProgress)('FooObjectKey', 10, 100); + + $this->progressTracker->clear(); + + $this->assertEquals(0, $this->progressTracker->getTotalBytesTransferred()); + $this->assertEquals(0, $this->progressTracker->getObjectsTotalSizeInBytes()); + $this->assertEquals(0, $this->progressTracker->getObjectsInProgress()); + $this->assertEquals(0, $this->progressTracker->getObjectsCount()); + $this->assertEquals(0, $this->progressTracker->getTransferPercentCompleted()); + } +} + diff --git a/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php b/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php new file mode 100644 index 0000000000..6be2e5cf4c --- /dev/null +++ b/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php @@ -0,0 +1,12 @@ +mockProgressBar = $this->createMock(ProgressBar::class); + } + + /** + * Tests getter and setters. + * + * @return void + */ + public function testGettersAndSetters(): void + { + $tracker = new ObjectProgressTracker( + '', + 0, + 0, + '' + ); + $tracker->setObjectKey('FooKey'); + $this->assertEquals('FooKey', $tracker->getObjectKey()); + + $tracker->setObjectBytesTransferred(100); + $this->assertEquals(100, $tracker->getObjectBytesTransferred()); + + $tracker->setObjectSizeInBytes(100); + $this->assertEquals(100, $tracker->getObjectSizeInBytes()); + + $tracker->setStatus('initiated'); + $this->assertEquals('initiated', $tracker->getStatus()); + } + + /** + * Tests bytes transferred increments. + * + * @return void + */ + public function testIncrementTotalBytesTransferred(): void + { + $percentProgress = 0; + $this->mockProgressBar->expects($this->atLeast(4)) + ->method('setPercentCompleted') + ->willReturnCallback(function ($percent) use (&$percentProgress) { + $this->assertEquals($percentProgress +=25, $percent); + }); + + $tracker = new ObjectProgressTracker( + objectKey: 'FooKey', + objectBytesTransferred: 0, + objectSizeInBytes: 100, + status: 'initiated', + progressBar: $this->mockProgressBar + ); + + $tracker->incrementTotalBytesTransferred(25); + $tracker->incrementTotalBytesTransferred(25); + $tracker->incrementTotalBytesTransferred(25); + $tracker->incrementTotalBytesTransferred(25); + + $this->assertEquals(100, $tracker->getObjectBytesTransferred()); + } + + + /** + * Tests progress status color based on states. + * + * @return void + */ + public function testSetStatusUpdatesProgressBarColor() + { + $statusColorMapping = [ + 'progress' => ConsoleProgressBar::BLUE_COLOR_CODE, + 'completed' => ConsoleProgressBar::GREEN_COLOR_CODE, + 'failed' => ConsoleProgressBar::RED_COLOR_CODE, + ]; + $values = array_values($statusColorMapping); + $valueIndex = 0; + $this->mockProgressBar->expects($this->exactly(3)) + ->method('setArg') + ->willReturnCallback(function ($_, $argValue) use ($values, &$valueIndex) { + $this->assertEquals($argValue, $values[$valueIndex++]); + }); + + $tracker = new ObjectProgressTracker( + objectKey: 'FooKey', + objectBytesTransferred: 0, + objectSizeInBytes: 100, + status: 'initiated', + progressBar: $this->mockProgressBar + ); + + foreach ($statusColorMapping as $status => $value) { + $tracker->setStatus($status); + } + } + + /** + * Tests the default progress bar is initialized when not provided. + * + * @return void + */ + public function testDefaultProgressBarIsInitialized() + { + $tracker = new ObjectProgressTracker( + objectKey: 'FooKey', + objectBytesTransferred: 0, + objectSizeInBytes: 100, + status: 'initiated' + ); + $this->assertInstanceOf(ProgressBar::class, $tracker->getProgressBar()); + } +} diff --git a/tests/S3/Features/S3Transfer/TransferListenerTest.php b/tests/S3/Features/S3Transfer/TransferListenerTest.php new file mode 100644 index 0000000000..ab079ee97c --- /dev/null +++ b/tests/S3/Features/S3Transfer/TransferListenerTest.php @@ -0,0 +1,216 @@ +objectTransferInitiated('FooObjectKey', $requestArgs); + $this->assertEquals(1, $listener->getObjectsToBeTransferred()); + + $this->assertTrue($called); + } + + /** + * Tests object transfer is initiated. + * + * @return void + */ + public function testObjectTransferIsInitiated(): void + { + $called = false; + $listener = new TransferListener( + onObjectTransferInitiated: function () use (&$called) { + $called = true; + } + ); + $requestArgs = []; + $listener->objectTransferInitiated('FooObjectKey', $requestArgs); + $this->assertEquals(1, $listener->getObjectsToBeTransferred()); + + $this->assertTrue($called); + } + + /** + * Tests object transfer progress. + * + * @dataProvider objectTransferProgressProvider + * + * @param array $objects + * + * @return void + */ + public function testObjectTransferProgress( + array $objects + ): void { + $called = 0; + $listener = new TransferListener( + onObjectTransferProgress: function () use (&$called) { + $called++; + } + ); + $totalTransferred = 0; + foreach ($objects as $objectKey => $transferDetails) { + $requestArgs = []; + $listener->objectTransferInitiated( + $objectKey, + $requestArgs, + ); + $listener->objectTransferProgress( + $objectKey, + $transferDetails['transferredInBytes'], + $transferDetails['sizeInBytes'] + ); + $totalTransferred += $transferDetails['transferredInBytes']; + } + + $this->assertEquals(count($objects), $called); + $this->assertEquals(count($objects), $listener->getObjectsToBeTransferred()); + $this->assertEquals($totalTransferred, $listener->getObjectsBytesTransferred()); + } + + /** + * @return array + */ + public function objectTransferProgressProvider(): array + { + return [ + [ + [ + 'FooObjectKey1' => [ + 'sizeInBytes' => 100, + 'transferredInBytes' => 95, + ], + 'FooObjectKey2' => [ + 'sizeInBytes' => 500, + 'transferredInBytes' => 345, + ], + 'FooObjectKey3' => [ + 'sizeInBytes' => 1024, + 'transferredInBytes' => 256, + ], + ] + ] + ]; + } + + /** + * Tests object transfer failed. + * + * @return void + */ + public function testObjectTransferFailed(): void + { + $expectedBytesTransferred = 45; + $expectedReason = "Transfer failed!"; + $listener = new TransferListener( + onObjectTransferFailed: function ( + string $objectKey, + int $objectBytesTransferred, + string $reason + ) use ($expectedBytesTransferred, $expectedReason) { + $this->assertEquals($expectedBytesTransferred, $objectBytesTransferred); + $this->assertEquals($expectedReason, $reason); + } + ); + $requestArgs = []; + $listener->objectTransferInitiated('FooObjectKey', $requestArgs); + $listener->objectTransferFailed( + 'FooObjectKey', + $expectedBytesTransferred, + $expectedReason + ); + + $this->assertEquals(1, $listener->getObjectsTransferFailed()); + $this->assertEquals(0, $listener->getObjectsTransferCompleted()); + } + + /** + * Tests object transfer completed. + * + * @return void + */ + public function testObjectTransferCompleted(): void + { + $expectedBytesTransferred = 100; + $listener = new TransferListener( + onObjectTransferCompleted: function ($objectKey, $objectBytesTransferred) + use ($expectedBytesTransferred) { + $this->assertEquals($expectedBytesTransferred, $objectBytesTransferred); + } + ); + $requestArgs = []; + $listener->objectTransferInitiated('FooObjectKey', $requestArgs); + $listener->objectTransferProgress( + 'FooObjectKey', + $expectedBytesTransferred, + $expectedBytesTransferred + ); + $listener->objectTransferCompleted('FooObjectKey', $expectedBytesTransferred); + + $this->assertEquals(1, $listener->getObjectsTransferCompleted()); + $this->assertEquals($expectedBytesTransferred, $listener->getObjectsBytesTransferred()); + } + + /** + * Tests transfer is completed once all the objects in progress are completed. + * + * @return void + */ + public function testTransferCompleted(): void + { + $expectedObjectsTransferred = 2; + $expectedObjectBytesTransferred = 200; + $listener = new TransferListener( + onTransferCompleted: function(int $objectsTransferredCompleted, int $objectsBytesTransferred) + use ($expectedObjectsTransferred, $expectedObjectBytesTransferred) { + $this->assertEquals($expectedObjectsTransferred, $objectsTransferredCompleted); + $this->assertEquals($expectedObjectBytesTransferred, $objectsBytesTransferred); + } + ); + $requestArgs = []; + $listener->objectTransferInitiated('FooObjectKey_1', $requestArgs); + $listener->objectTransferInitiated('FooObjectKey_2', $requestArgs); + $listener->objectTransferProgress( + 'FooObjectKey_1', + 100, + 100 + ); + $listener->objectTransferProgress( + 'FooObjectKey_2', + 100, + 100 + ); + $listener->objectTransferCompleted( + 'FooObjectKey_1', + 100, + ); + $listener->objectTransferCompleted( + 'FooObjectKey_2', + 100, + ); + + $this->assertEquals($expectedObjectsTransferred, $listener->getObjectsTransferCompleted()); + $this->assertEquals($expectedObjectBytesTransferred, $listener->getObjectsBytesTransferred()); + } +} From 1c82ab5aab901af01ef48b6e1c979d2f4154a911 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Wed, 5 Feb 2025 18:35:10 -0800 Subject: [PATCH 03/19] chore: add multipart download listener tests - Add test cases for multipart download listener. --- .../S3Transfer/MultipartDownloadListener.php | 34 +-- .../MultipartDownloadListenerTest.php | 207 ++++++++++++++++++ 2 files changed, 224 insertions(+), 17 deletions(-) create mode 100644 tests/S3/Features/S3Transfer/MultipartDownloadListenerTest.php diff --git a/src/S3/Features/S3Transfer/MultipartDownloadListener.php b/src/S3/Features/S3Transfer/MultipartDownloadListener.php index 4ffdff1559..69936cb2a5 100644 --- a/src/S3/Features/S3Transfer/MultipartDownloadListener.php +++ b/src/S3/Features/S3Transfer/MultipartDownloadListener.php @@ -16,17 +16,17 @@ class MultipartDownloadListener extends ListenerNotifier * * @param Closure|null $onDownloadFailed * Parameters that will be passed when invoked: - * - $reason: The throwable with the reason why the transfer failed. - * - $totalPartsTransferred: The total of parts transferred before failure. - * - $totalBytesTransferred: The total of bytes transferred before failure. - * - $lastPartTransferred: The number of the last part that was transferred + * - $reason: The throwable with the reason why the download failed. + * - $totalPartsDownloaded: The total of parts downloaded before failure. + * - $totalBytesDownloaded: The total of bytes downloaded before failure. + * - $lastPartDownloaded: The number of the last part that was downloaded * before failure. * * @param Closure|null $onDownloadCompleted * Parameters that will be passed when invoked: * - $stream: The stream which holds the bytes for the file downloaded. - * - $totalPartsDownloaded: The number of objects that were transferred. - * - $totalBytesDownloaded: The total of bytes that were transferred. + * - $totalPartsDownloaded: The number of parts that were downloaded. + * - $totalBytesDownloaded: The total of bytes that were downloaded. * * @param Closure|null $onPartDownloadInitiated * Parameters that will be passed when invoked: @@ -39,7 +39,7 @@ class MultipartDownloadListener extends ListenerNotifier * - $partNo: The part number just downloaded. * - $partTotalBytes: The size of the part just downloaded. * - $totalParts: The total parts for the full object to be downloaded. - * - $objectBytesTransferred: The total in bytes already downloaded. + * - $objectBytesDownloaded: The total in bytes already downloaded. * - $objectSizeInBytes: The total in bytes for the full object to be downloaded. * * @param Closure|null $onPartDownloadFailed @@ -66,11 +66,11 @@ public function __construct( * keep the states maintained in this implementation. * * @param array &$commandArgs - * @param ?int $initialPart + * @param int $initialPart * * @return void */ - public function downloadInitiated(array &$commandArgs, ?int $initialPart): void { + public function downloadInitiated(array &$commandArgs, int $initialPart): void { $this->notify('onDownloadInitiated', [&$commandArgs, $initialPart]); } @@ -81,14 +81,14 @@ public function downloadInitiated(array &$commandArgs, ?int $initialPart): void * keep the states maintained in this implementation. * * @param \Throwable $reason - * @param int $totalPartsTransferred - * @param int $totalBytesTransferred - * @param int $lastPartTransferred + * @param int $totalPartsDownloaded + * @param int $totalBytesDownloaded + * @param int $lastPartDownloaded * * @return void */ - public function downloadFailed(\Throwable $reason, int $totalPartsTransferred, int $totalBytesTransferred, int $lastPartTransferred): void { - $this->notify('onDownloadFailed', [$reason, $totalPartsTransferred, $totalBytesTransferred, $lastPartTransferred]); + public function downloadFailed(\Throwable $reason, int $totalPartsDownloaded, int $totalBytesDownloaded, int $lastPartDownloaded): void { + $this->notify('onDownloadFailed', [$reason, $totalPartsDownloaded, $totalBytesDownloaded, $lastPartDownloaded]); } /** @@ -132,7 +132,7 @@ public function partDownloadInitiated(CommandInterface $partDownloadCommand, int * @param int $partNo * @param int $partTotalBytes * @param int $totalParts - * @param int $objectBytesTransferred + * @param int $objectBytesDownloaded * @param int $objectSizeInBytes * @return void */ @@ -141,7 +141,7 @@ public function partDownloadCompleted( int $partNo, int $partTotalBytes, int $totalParts, - int $objectBytesTransferred, + int $objectBytesDownloaded, int $objectSizeInBytes ): void { @@ -150,7 +150,7 @@ public function partDownloadCompleted( $partNo, $partTotalBytes, $totalParts, - $objectBytesTransferred, + $objectBytesDownloaded, $objectSizeInBytes ]); } diff --git a/tests/S3/Features/S3Transfer/MultipartDownloadListenerTest.php b/tests/S3/Features/S3Transfer/MultipartDownloadListenerTest.php new file mode 100644 index 0000000000..b615d3ad04 --- /dev/null +++ b/tests/S3/Features/S3Transfer/MultipartDownloadListenerTest.php @@ -0,0 +1,207 @@ +assertIsArray($commandArgs); + $this->assertIsInt($initialPart); + }; + + $listener = new MultipartDownloadListener(onDownloadInitiated: $callback); + + $commandArgs = ['Foo' => 'Buzz']; + $listener->downloadInitiated($commandArgs, 1); + + $this->assertTrue($called, "Expected onDownloadInitiated to be called."); + } + + /** + * Tests download failed event is propagated. + * + * @return void + */ + public function testDownloadFailed(): void + { + $called = false; + $expectedError = new Exception('Download failed'); + $expectedTotalPartsTransferred = 5; + $expectedTotalBytesTransferred = 1024; + $expectedLastPartTransferred = 4; + $callback = function ( + $reason, + $totalPartsTransferred, + $totalBytesTransferred, + $lastPartTransferred + ) use ( + &$called, + $expectedError, + $expectedTotalPartsTransferred, + $expectedTotalBytesTransferred, + $expectedLastPartTransferred + ) { + $called = true; + $this->assertEquals($reason, $expectedError); + $this->assertEquals($expectedTotalPartsTransferred, $totalPartsTransferred); + $this->assertEquals($expectedTotalBytesTransferred, $totalBytesTransferred); + $this->assertEquals($expectedLastPartTransferred, $lastPartTransferred); + + }; + $listener = new MultipartDownloadListener(onDownloadFailed: $callback); + $listener->downloadFailed( + $expectedError, + $expectedTotalPartsTransferred, + $expectedTotalBytesTransferred, + $expectedLastPartTransferred + ); + $this->assertTrue($called, "Expected onDownloadFailed to be called."); + } + + /** + * Tests download completed event is propagated. + * + * @return void + */ + public function testDownloadCompleted(): void + { + $called = false; + $expectedStream = fopen('php://temp', 'r+'); + $expectedTotalPartsDownloaded = 10; + $expectedTotalBytesDownloaded = 2048; + $callback = function ( + $stream, + $totalPartsDownloaded, + $totalBytesDownloaded + ) use ( + &$called, + $expectedStream, + $expectedTotalPartsDownloaded, + $expectedTotalBytesDownloaded + ) { + $called = true; + $this->assertIsResource($stream); + $this->assertEquals($expectedStream, $stream); + $this->assertEquals($expectedTotalPartsDownloaded, $totalPartsDownloaded); + $this->assertEquals($expectedTotalBytesDownloaded, $totalBytesDownloaded); + }; + + $listener = new MultipartDownloadListener(onDownloadCompleted: $callback); + $listener->downloadCompleted( + $expectedStream, + $expectedTotalPartsDownloaded, + $expectedTotalBytesDownloaded + ); + $this->assertTrue($called, "Expected onDownloadCompleted to be called."); + } + + /** + * Tests part downloaded initiated event is propagated. + * + * @return void + */ + public function testPartDownloadInitiated(): void + { + $called = false; + $mockCommand = $this->createMock(CommandInterface::class); + $expectedPartNo = 3; + $callable = function ($command, $partNo) + use (&$called, $mockCommand, $expectedPartNo) { + $called = true; + $this->assertEquals($expectedPartNo, $partNo); + $this->assertEquals($mockCommand, $command); + }; + $listener = new MultipartDownloadListener(onPartDownloadInitiated: $callable); + $listener->partDownloadInitiated($mockCommand, $expectedPartNo); + $this->assertTrue($called, "Expected onPartDownloadInitiated to be called."); + } + + /** + * Tests part download completed event is propagated. + * + * @return void + */ + public function testPartDownloadCompleted(): void + { + $called = false; + $mockResult = $this->createMock(ResultInterface::class); + $expectedPartNo = 3; + $expectedPartTotalBytes = 512; + $expectedTotalParts = 5; + $expectedObjectBytesTransferred = 1024; + $expectedObjectSizeInBytes = 2048; + $callback = function ( + $result, + $partNo, + $partTotalBytes, + $totalParts, + $objectBytesDownloaded, + $objectSizeInBytes + ) use ( + &$called, + $mockResult, + $expectedPartNo, + $expectedPartTotalBytes, + $expectedTotalParts, + $expectedObjectBytesTransferred, + $expectedObjectSizeInBytes + ) { + $called = true; + $this->assertEquals($mockResult, $result); + $this->assertEquals($expectedPartNo, $partNo); + $this->assertEquals($expectedPartTotalBytes, $partTotalBytes); + $this->assertEquals($expectedTotalParts, $totalParts); + $this->assertEquals($expectedObjectBytesTransferred, $objectBytesDownloaded); + $this->assertEquals($expectedObjectSizeInBytes, $objectSizeInBytes); + }; + $listener = new MultipartDownloadListener(onPartDownloadCompleted: $callback); + $listener->partDownloadCompleted( + $mockResult, + $expectedPartNo, + $expectedPartTotalBytes, + $expectedTotalParts, + $expectedObjectBytesTransferred, + $expectedObjectSizeInBytes + ); + $this->assertTrue($called, "Expected onPartDownloadCompleted to be called."); + } + + /** + * Tests part download failed event is propagated. + * + * @return void + */ + public function testPartDownloadFailed() + { + $called = false; + $mockCommand = $this->createMock(CommandInterface::class); + $expectedReason = new Exception('Part download failed'); + $expectedPartNo = 2; + $callable = function ($command, $reason, $partNo) + use (&$called, $mockCommand, $expectedReason, $expectedPartNo) { + $called = true; + $this->assertEquals($expectedReason, $reason); + $this->assertEquals($expectedPartNo, $partNo); + $this->assertEquals($mockCommand, $command); + }; + + $listener = new MultipartDownloadListener(onPartDownloadFailed: $callable); + $listener->partDownloadFailed($mockCommand, $expectedReason, $expectedPartNo); + $this->assertTrue($called, "Expected onPartDownloadFailed to be called."); + } +} \ No newline at end of file From 237fc7b1f34c5386b839682fc63029598ba8566a Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 6 Feb 2025 09:03:16 -0800 Subject: [PATCH 04/19] chore: refactor multipart downloaders and add tests - Add a trait to the MultipartDownloader implementation to keep the main implementatio cleaner. - Add test cases for multipart downloader, in specific testing part and range get multipart downloader. --- .../S3Transfer/MultipartDownloader.php | 387 ++++++------------ .../S3Transfer/MultipartDownloaderTrait.php | 174 ++++++++ .../S3Transfer/RangeMultipartDownloader.php | 82 +++- .../Features/S3Transfer/S3TransferManager.php | 30 +- .../S3Transfer/S3TransferManagerTrait.php | 1 - .../S3Transfer/MultipartDownloaderTest.php | 150 ++++++- 6 files changed, 548 insertions(+), 276 deletions(-) create mode 100644 src/S3/Features/S3Transfer/MultipartDownloaderTrait.php diff --git a/src/S3/Features/S3Transfer/MultipartDownloader.php b/src/S3/Features/S3Transfer/MultipartDownloader.php index 03b0e6430a..50b575c455 100644 --- a/src/S3/Features/S3Transfer/MultipartDownloader.php +++ b/src/S3/Features/S3Transfer/MultipartDownloader.php @@ -3,7 +3,6 @@ namespace Aws\S3\Features\S3Transfer; use Aws\CommandInterface; -use Aws\Result; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; use GuzzleHttp\Promise\Coroutine; @@ -15,79 +14,97 @@ abstract class MultipartDownloader implements PromisorInterface { + use MultipartDownloaderTrait; public const GET_OBJECT_COMMAND = "GetObject"; public const PART_GET_MULTIPART_DOWNLOADER = "partGet"; public const RANGE_GET_MULTIPART_DOWNLOADER = "rangeGet"; - /** @var S3ClientInterface */ - protected S3ClientInterface $s3Client; - - /** @var array */ - protected array $requestArgs; - - /** @var array */ - protected array $config; - - /** @var int */ - protected int $currentPartNo; - - /** @var int */ - protected int $objectPartsCount; - - /** @var int */ - protected int $objectCompletedPartsCount; - - /** @var int */ - protected int $objectSizeInBytes; - - /** @var int */ - protected int $objectBytesTransferred; - - /** @var ?MultipartDownloadListener */ - protected ?MultipartDownloadListener $listener; - - /** @var ?TransferListener */ - protected ?TransferListener $progressListener; - - /** @var StreamInterface */ - private StreamInterface $stream; - - /** @var string */ - protected string $eTag; - - /** @var string */ - protected string $objectKey; - /** * @param S3ClientInterface $s3Client * @param array $requestArgs * @param array $config - * - targetPartSizeBytes: The minimum part size for a multipart download - * using range get. + * - minimumPartSize: The minimum part size for a multipart download + * using range get. This option MUST be set when using range get. * @param int $currentPartNo - * @param ?MultipartDownloadListener $listener - * @param ?TransferListener $transferListener + * @param int $objectPartsCount + * @param int $objectCompletedPartsCount + * @param int $objectSizeInBytes + * @param int $objectBytesTransferred + * @param string $eTag + * @param string $objectKey + * @param MultipartDownloadListener|null $listener + * @param TransferListener|null $progressListener + * @param StreamInterface|null $stream */ public function __construct( - S3ClientInterface $s3Client, - array $requestArgs, - array $config, - int $currentPartNo = 0, - ?MultipartDownloadListener $listener = null, - ?TransferListener $progressListener = null + protected readonly S3ClientInterface $s3Client, + protected array $requestArgs, + protected readonly array $config = [], + protected int $currentPartNo = 0, + protected int $objectPartsCount = 0, + protected int $objectCompletedPartsCount = 0, + protected int $objectSizeInBytes = 0, + protected int $objectBytesTransferred = 0, + protected string $eTag = "", + protected string $objectKey = "", + private readonly ?MultipartDownloadListener $listener = null, + private readonly ?TransferListener $progressListener = null, + private ?StreamInterface $stream = null ) { - $this->clear(); - $this->s3Client = $s3Client; - $this->requestArgs = $requestArgs; - $this->config = $config; - $this->currentPartNo = $currentPartNo; - $this->listener = $listener; - $this->progressListener = $progressListener; - $this->stream = Utils::streamFor( - fopen('php://temp', 'w+') - ); + if ($stream === null) { + $this->stream = Utils::streamFor( + fopen('php://temp', 'w+') + ); + } } + /** + * @return int + */ + public function getCurrentPartNo(): int + { + return $this->currentPartNo; + } + + /** + * @return int + */ + public function getObjectPartsCount(): int + { + return $this->objectPartsCount; + } + + /** + * @return int + */ + public function getObjectCompletedPartsCount(): int + { + return $this->objectCompletedPartsCount; + } + + /** + * @return int + */ + public function getObjectSizeInBytes(): int + { + return $this->objectSizeInBytes; + } + + /** + * @return int + */ + public function getObjectBytesTransferred(): int + { + return $this->objectBytesTransferred; + } + + /** + * @return string + */ + public function getObjectKey(): string + { + return $this->objectKey; + } /** * Returns that resolves a multipart download operation, @@ -138,6 +155,8 @@ public function promise(): PromiseInterface // TODO: yield transfer exception modeled with a transfer failed response. yield Create::rejectionFor($e); } + + $lastPartIncrement = $this->currentPartNo; } // Transfer completed @@ -148,170 +167,6 @@ public function promise(): PromiseInterface }); } - /** - * Main purpose of this method is to propagate - * the download-initiated event to listeners, but - * also it does some computation regarding internal states - * that need to be maintained. - * - * @param array $commandArgs - * @param int|null $currentPartNo - * - * @return void - */ - private function downloadInitiated(array &$commandArgs, ?int $currentPartNo): void - { - $this->objectKey = $commandArgs['Key']; - $this->progressListener?->objectTransferInitiated( - $this->objectKey, - $commandArgs - ); - $this->_notifyMultipartDownloadListeners('downloadInitiated', [ - &$commandArgs, - $currentPartNo - ]); - } - - /** - * Propagates download-failed event to listeners. - * It may also do some computation in order to maintain internal states. - * - * @param \Throwable $reason - * @param int $totalPartsTransferred - * @param int $totalBytesTransferred - * @param int $lastPartTransferred - * - * @return void - */ - private function downloadFailed( - \Throwable $reason, - int $totalPartsTransferred, - int $totalBytesTransferred, - int $lastPartTransferred - ): void { - $this->progressListener?->objectTransferFailed( - $this->objectKey, - $totalBytesTransferred, - $reason - ); - $this->_notifyMultipartDownloadListeners('downloadFailed', [ - $reason, - $totalPartsTransferred, - $totalBytesTransferred, - $lastPartTransferred - ]); - } - - /** - * Propagates part-download-initiated event to listeners. - * - * @param CommandInterface $partDownloadCommand - * @param int $partNo - * - * @return void - */ - private function partDownloadInitiated(CommandInterface $partDownloadCommand, int $partNo): void { - $this->_notifyMultipartDownloadListeners('partDownloadInitiated', [ - $partDownloadCommand, - $partNo - ]); - } - - /** - * Propagates part-download-completed to listeners. - * It also does some computation in order to maintain internal states. - * In this specific method we move each part content into an accumulative - * stream, which is meant to hold the full object content once the download - * is completed. - * - * @param ResultInterface $result - * @param int $partNo - * - * @return void - */ - private function partDownloadCompleted(ResultInterface $result, int $partNo): void { - $this->objectCompletedPartsCount++; - $partDownloadBytes = $result['ContentLength']; - $this->objectBytesTransferred = $this->objectBytesTransferred + $partDownloadBytes; - if (isset($result['ETag'])) { - $this->eTag = $result['ETag']; - } - Utils::copyToStream($result['Body'], $this->stream); - - $this->progressListener?->objectTransferProgress( - $this->objectKey, - $partDownloadBytes, - $this->objectSizeInBytes - ); - - $this->_notifyMultipartDownloadListeners('partDownloadCompleted', [ - $result, - $partNo, - $partDownloadBytes, - $this->objectCompletedPartsCount, - $this->objectBytesTransferred, - $this->objectSizeInBytes - ]); - } - - /** - * Propagates part-download-failed event to listeners. - * - * @param CommandInterface $partDownloadCommand - * @param \Throwable $reason - * @param int $partNo - * - * @return void - */ - private function partDownloadFailed( - CommandInterface $partDownloadCommand, - \Throwable $reason, - int $partNo - ): void { - $this->progressListener?->objectTransferFailed( - $this->objectKey, - $this->objectBytesTransferred, - $reason - ); - $this->_notifyMultipartDownloadListeners( - 'partDownloadFailed', - [$partDownloadCommand, $reason, $partNo]); - } - - /** - * Propagates object-download-completed event to listeners. - * It also resets the pointer of the stream to the first position, - * so that the stream is ready to be consumed once returned. - * - * @return void - */ - private function objectDownloadCompleted(): void - { - $this->stream->rewind(); - $this->progressListener?->objectTransferCompleted( - $this->objectKey, - $this->objectBytesTransferred - ); - $this->_notifyMultipartDownloadListeners('downloadCompleted', [ - $this->stream, - $this->objectCompletedPartsCount, - $this->objectBytesTransferred - ]); - } - - /** - * Internal helper method for notifying listeners of specific events. - * - * @param string $listenerMethod - * @param array $args - * - * @return void - */ - private function _notifyMultipartDownloadListeners(string $listenerMethod, array $args): void - { - $this->listener?->{$listenerMethod}(...$args); - } - /** * Returns the next command for fetching the next object part. * @@ -336,7 +191,7 @@ abstract protected function computeObjectDimensions(ResultInterface $result): vo * @return int */ protected function computeObjectSize($sizeSource): int { - if (gettype($sizeSource) === "integer") { + if (is_int($sizeSource)) { return (int) $sizeSource; } @@ -344,21 +199,12 @@ protected function computeObjectSize($sizeSource): int { throw new \RuntimeException('Range must not be empty'); } + // For extracting the object size from the ContentRange header value. if (preg_match("/\/(\d+)$/", $sizeSource, $matches)) { return $matches[1]; } - throw new \RuntimeException('Invalid range format'); - } - - private function clear(): void { - $this->currentPartNo = 0; - $this->objectPartsCount = 0; - $this->objectCompletedPartsCount = 0; - $this->objectSizeInBytes = 0; - $this->objectBytesTransferred = 0; - $this->eTag = ""; - $this->objectKey = ""; + throw new \RuntimeException('Invalid source size format'); } /** @@ -369,6 +215,13 @@ private function clear(): void { * @param string $multipartDownloadType * @param array $requestArgs * @param array $config + * @param int $currentPartNo + * @param int $objectPartsCount + * @param int $objectCompletedPartsCount + * @param int $objectSizeInBytes + * @param int $objectBytesTransferred + * @param string $eTag + * @param string $objectKey * @param MultipartDownloadListener|null $listener * @param TransferListener|null $progressTracker * @@ -379,30 +232,50 @@ public static function chooseDownloader( string $multipartDownloadType, array $requestArgs, array $config, + int $currentPartNo = 0, + int $objectPartsCount = 0, + int $objectCompletedPartsCount = 0, + int $objectSizeInBytes = 0, + int $objectBytesTransferred = 0, + string $eTag = "", + string $objectKey = "", ?MultipartDownloadListener $listener = null, ?TransferListener $progressTracker = null ) : MultipartDownloader { - if ($multipartDownloadType === self::PART_GET_MULTIPART_DOWNLOADER) { - return new GetMultipartDownloader( - $s3Client, - $requestArgs, - $config, - 0, - $listener, - $progressTracker - ); - } elseif ($multipartDownloadType === self::RANGE_GET_MULTIPART_DOWNLOADER) { - return new RangeMultipartDownloader( - $s3Client, - $requestArgs, - $config, - 0, - $listener, - $progressTracker - ); - } - - throw new \RuntimeException("Unsupported download type $multipartDownloadType"); + return match ($multipartDownloadType) { + self::PART_GET_MULTIPART_DOWNLOADER => new GetMultipartDownloader( + s3Client: $s3Client, + requestArgs: $requestArgs, + config: $config, + currentPartNo: $currentPartNo, + objectPartsCount: $objectPartsCount, + objectCompletedPartsCount: $objectCompletedPartsCount, + objectSizeInBytes: $objectSizeInBytes, + objectBytesTransferred: $objectBytesTransferred, + eTag: $eTag, + objectKey: $objectKey, + listener: $listener, + progressListener: $progressTracker + ), + self::RANGE_GET_MULTIPART_DOWNLOADER => new RangeMultipartDownloader( + s3Client: $s3Client, + requestArgs: $requestArgs, + config: $config, + currentPartNo: 0, + objectPartsCount: 0, + objectCompletedPartsCount: 0, + objectSizeInBytes: 0, + objectBytesTransferred: 0, + eTag: "", + objectKey: "", + listener: $listener, + progressListener: $progressTracker + ), + default => throw new \RuntimeException( + "Unsupported download type $multipartDownloadType." + ."It should be either " . self::PART_GET_MULTIPART_DOWNLOADER . + " or " . self::RANGE_GET_MULTIPART_DOWNLOADER . ".") + }; } } \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/MultipartDownloaderTrait.php b/src/S3/Features/S3Transfer/MultipartDownloaderTrait.php new file mode 100644 index 0000000000..c722da4314 --- /dev/null +++ b/src/S3/Features/S3Transfer/MultipartDownloaderTrait.php @@ -0,0 +1,174 @@ +objectKey = $commandArgs['Key']; + $this->progressListener?->objectTransferInitiated( + $this->objectKey, + $commandArgs + ); + $this->_notifyMultipartDownloadListeners('downloadInitiated', [ + &$commandArgs, + $currentPartNo + ]); + } + + /** + * Propagates download-failed event to listeners. + * It may also do some computation in order to maintain internal states. + * + * @param \Throwable $reason + * @param int $totalPartsTransferred + * @param int $totalBytesTransferred + * @param int $lastPartTransferred + * + * @return void + */ + private function downloadFailed( + \Throwable $reason, + int $totalPartsTransferred, + int $totalBytesTransferred, + int $lastPartTransferred + ): void { + $this->progressListener?->objectTransferFailed( + $this->objectKey, + $totalBytesTransferred, + $reason + ); + $this->_notifyMultipartDownloadListeners('downloadFailed', [ + $reason, + $totalPartsTransferred, + $totalBytesTransferred, + $lastPartTransferred + ]); + } + + /** + * Propagates part-download-initiated event to listeners. + * + * @param CommandInterface $partDownloadCommand + * @param int $partNo + * + * @return void + */ + private function partDownloadInitiated(CommandInterface $partDownloadCommand, int $partNo): void { + $this->_notifyMultipartDownloadListeners('partDownloadInitiated', [ + $partDownloadCommand, + $partNo + ]); + } + + /** + * Propagates part-download-completed to listeners. + * It also does some computation in order to maintain internal states. + * In this specific method we move each part content into an accumulative + * stream, which is meant to hold the full object content once the download + * is completed. + * + * @param ResultInterface $result + * @param int $partNo + * + * @return void + */ + private function partDownloadCompleted(ResultInterface $result, int $partNo): void { + $this->objectCompletedPartsCount++; + $partDownloadBytes = $result['ContentLength']; + $this->objectBytesTransferred = $this->objectBytesTransferred + $partDownloadBytes; + if (isset($result['ETag'])) { + $this->eTag = $result['ETag']; + } + Utils::copyToStream($result['Body'], $this->stream); + + $this->progressListener?->objectTransferProgress( + $this->objectKey, + $partDownloadBytes, + $this->objectSizeInBytes + ); + + $this->_notifyMultipartDownloadListeners('partDownloadCompleted', [ + $result, + $partNo, + $partDownloadBytes, + $this->objectCompletedPartsCount, + $this->objectBytesTransferred, + $this->objectSizeInBytes + ]); + } + + /** + * Propagates part-download-failed event to listeners. + * + * @param CommandInterface $partDownloadCommand + * @param \Throwable $reason + * @param int $partNo + * + * @return void + */ + private function partDownloadFailed( + CommandInterface $partDownloadCommand, + \Throwable $reason, + int $partNo + ): void { + $this->progressListener?->objectTransferFailed( + $this->objectKey, + $this->objectBytesTransferred, + $reason + ); + $this->_notifyMultipartDownloadListeners( + 'partDownloadFailed', + [$partDownloadCommand, $reason, $partNo]); + } + + /** + * Propagates object-download-completed event to listeners. + * It also resets the pointer of the stream to the first position, + * so that the stream is ready to be consumed once returned. + * + * @return void + */ + private function objectDownloadCompleted(): void + { + $this->stream->rewind(); + $this->progressListener?->objectTransferCompleted( + $this->objectKey, + $this->objectBytesTransferred + ); + $this->_notifyMultipartDownloadListeners('downloadCompleted', [ + $this->stream, + $this->objectCompletedPartsCount, + $this->objectBytesTransferred + ]); + } + + /** + * Internal helper method for notifying listeners of specific events. + * + * @param string $listenerMethod + * @param array $args + * + * @return void + */ + private function _notifyMultipartDownloadListeners(string $listenerMethod, array $args): void + { + $this->listener?->{$listenerMethod}(...$args); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/RangeMultipartDownloader.php b/src/S3/Features/S3Transfer/RangeMultipartDownloader.php index 5a76a39408..ca26120642 100644 --- a/src/S3/Features/S3Transfer/RangeMultipartDownloader.php +++ b/src/S3/Features/S3Transfer/RangeMultipartDownloader.php @@ -5,6 +5,8 @@ use Aws\CommandInterface; use Aws\Result; use Aws\ResultInterface; +use Aws\S3\S3ClientInterface; +use Psr\Http\Message\StreamInterface; class RangeMultipartDownloader extends MultipartDownloader { @@ -12,6 +14,63 @@ class RangeMultipartDownloader extends MultipartDownloader /** @var int */ private int $partSize; + /** + * @param S3ClientInterface $s3Client + * @param array $requestArgs + * @param array $config + * @param int $currentPartNo + * @param int $objectPartsCount + * @param int $objectCompletedPartsCount + * @param int $objectSizeInBytes + * @param int $objectBytesTransferred + * @param string $eTag + * @param string $objectKey + * @param MultipartDownloadListener|null $listener + * @param TransferListener|null $progressListener + * @param StreamInterface|null $stream + */ + public function __construct( + S3ClientInterface $s3Client, + array $requestArgs = [], + array $config = [], + int $currentPartNo = 0, + int $objectPartsCount = 0, + int $objectCompletedPartsCount = 0, + int $objectSizeInBytes = 0, + int $objectBytesTransferred = 0, + string $eTag = "", + string $objectKey = "", + ?MultipartDownloadListener $listener = null, + ?TransferListener $progressListener = null, + ?StreamInterface $stream = null + ) { + parent::__construct( + $s3Client, + $requestArgs, + $config, + $currentPartNo, + $objectPartsCount, + $objectCompletedPartsCount, + $objectSizeInBytes, + $objectBytesTransferred, + $eTag, + $objectKey, + $listener, + $progressListener, + $stream + ); + if (empty($config['minimumPartSize'])) { + throw new \RuntimeException('You must provide a valid minimum part size in bytes'); + } + $this->partSize = $config['minimumPartSize']; + // If object size is known at instantiation time then, we can compute + // the object dimensions. + if ($this->objectSizeInBytes !== 0) { + $this->computeObjectDimensions(new Result(['ContentRange' => $this->objectSizeInBytes])); + } + } + + /** * @inheritDoc * @@ -19,20 +78,21 @@ class RangeMultipartDownloader extends MultipartDownloader */ protected function nextCommand(): CommandInterface { - if ($this->objectSizeInBytes !== 0) { - $this->computeObjectDimensions(new Result(['ContentRange' => $this->totalBytes])); - } - + // If currentPartNo is not know then lets initialize it to 1 + // otherwise just increment it. if ($this->currentPartNo === 0) { $this->currentPartNo = 1; - $this->partSize = $this->config['targetPartSizeBytes']; } else { $this->currentPartNo++; } $nextRequestArgs = array_slice($this->requestArgs, 0); - $from = ($this->currentPartNo - 1) * ($this->partSize + 1); - $to = $this->currentPartNo * $this->partSize; + $from = ($this->currentPartNo - 1) * $this->partSize; + $to = ($this->currentPartNo * $this->partSize) - 1; + if ($this->objectSizeInBytes !== 0) { + $to = min($this->objectSizeInBytes, $to); + } + $nextRequestArgs['Range'] = "bytes=$from-$to"; if (!empty($this->eTag)) { $nextRequestArgs['IfMatch'] = $this->eTag; @@ -53,11 +113,17 @@ protected function nextCommand(): CommandInterface */ protected function computeObjectDimensions(ResultInterface $result): void { - $this->objectSizeInBytes = $this->computeObjectSize($result['ContentRange'] ?? ""); + // Assign object size just if needed. + if ($this->objectSizeInBytes === 0) { + $this->objectSizeInBytes = $this->computeObjectSize($result['ContentRange'] ?? ""); + } + if ($this->objectSizeInBytes > $this->partSize) { $this->objectPartsCount = intval(ceil($this->objectSizeInBytes / $this->partSize)); } else { + // Single download since partSize will be set to full object size. $this->partSize = $this->objectSizeInBytes; + $this->objectPartsCount = 1; $this->currentPartNo = 1; } } diff --git a/src/S3/Features/S3Transfer/S3TransferManager.php b/src/S3/Features/S3Transfer/S3TransferManager.php index f9c56e6b7c..aef8921881 100644 --- a/src/S3/Features/S3Transfer/S3TransferManager.php +++ b/src/S3/Features/S3Transfer/S3TransferManager.php @@ -21,8 +21,6 @@ class S3TransferManager * The minimum part size to be used in a multipart upload/download. * - multipartUploadThresholdBytes: (int, default=(16777216 `16 MB`)) \ * The threshold to decided whether a multipart upload is needed. - * - multipartDownloadThresholdBytes: (int, default=(16777216 `16 MB`)) \ - * The threshold to decided whether a multipart download is needed. * - checksumValidationEnabled: (bool, default=true) \ * To decide whether a checksum validation will be applied to the response. * - checksumAlgorithm: (string, default='crc32') \ @@ -62,6 +60,8 @@ public function __construct(?S3ClientInterface $s3Client, array $config = []) { * to decide whether transfer progress should be tracked. If not * transfer tracker factory is provided and trackProgress is true then, * the default progress listener implementation will be used. + * - minimumPartSize: (int) \ + * The minimum part size in bytes to be used in a range multipart download. * * @return PromiseInterface */ @@ -83,7 +83,10 @@ public function download( $requestArgs = $sourceArgs + $downloadArgs; if (empty($downloadArgs['PartNumber']) && empty($downloadArgs['Range'])) { - return $this->tryMultipartDownload($requestArgs, $config); + return $this->tryMultipartDownload( + $requestArgs, + $config + ); } return $this->trySingleDownload($requestArgs); @@ -97,6 +100,10 @@ public function download( * - listener: (?MultipartDownloadListener) \ * A multipart download listener for watching every multipart download * stage. + * - minimumPartSize: (int) \ + * The minimum part size in bytes for a range multipart download. If + * this parameter is not provided then it fallbacks to the transfer + * manager `targetPartSizeBytes` config value. * * @return PromiseInterface */ @@ -117,12 +124,17 @@ private function tryMultipartDownload( } } $multipartDownloader = MultipartDownloader::chooseDownloader( - $this->s3Client, - $this->config['multipartDownloadType'], - $requestArgs, - $this->config, - $config['listener'] ?? null, - $progressListener?->getTransferListener() + s3Client: $this->s3Client, + multipartDownloadType: $this->config['multipartDownloadType'], + requestArgs: $requestArgs, + config: [ + 'minimumPartSize' => max( + $config['minimumPartSize'] ?? 0, + $this->config['targetPartSizeBytes'] + ) + ], + listener: $config['listener'] ?? null, + progressTracker: $progressListener?->getTransferListener() ); return $multipartDownloader->promise(); diff --git a/src/S3/Features/S3Transfer/S3TransferManagerTrait.php b/src/S3/Features/S3Transfer/S3TransferManagerTrait.php index d924280b46..6d2542cb04 100644 --- a/src/S3/Features/S3Transfer/S3TransferManagerTrait.php +++ b/src/S3/Features/S3Transfer/S3TransferManagerTrait.php @@ -11,7 +11,6 @@ trait S3TransferManagerTrait private static array $defaultConfig = [ 'targetPartSizeBytes' => 8 * 1024 * 1024, 'multipartUploadThresholdBytes' => 16 * 1024 * 1024, - 'multipartDownloadThresholdBytes' => 16 * 1024 * 1024, 'checksumValidationEnabled' => true, 'checksumAlgorithm' => 'crc32', 'multipartDownloadType' => 'partGet', diff --git a/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php b/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php index 6be2e5cf4c..dbe3625479 100644 --- a/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php +++ b/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php @@ -2,11 +2,159 @@ namespace Aws\Test\S3\Features\S3Transfer; +use Aws\Command; +use Aws\Result; +use Aws\S3\Features\S3Transfer\MultipartDownloader; +use Aws\S3\S3Client; +use GuzzleHttp\Promise\Create; +use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\TestCase; +use Psr\Http\Message\StreamInterface; +/** + * Tests multipart download implementation. + */ class MultipartDownloaderTest extends TestCase { - public function testMultipartDownloader(): void { + /** + * Tests part and range get multipart downloader. + * + * @param string $multipartDownloadType + * @param string $objectKey + * @param int $objectSizeInBytes + * @param int $targetPartSize + * + * @return void + * @dataProvider partGetMultipartDownloaderProvider + * + */ + public function testMultipartDownloader( + string $multipartDownloadType, + string $objectKey, + int $objectSizeInBytes, + int $targetPartSize + ): void { + $partsCount = (int) ceil($objectSizeInBytes / $targetPartSize); + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $remainingToTransfer = $objectSizeInBytes; + $mockClient->method('executeAsync') + -> willReturnCallback(function ($command) + use ( + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer + ) { + $currentPartLength = min( + $targetPartSize, + $remainingToTransfer + ); + $from = $objectSizeInBytes - $remainingToTransfer; + $to = $from + $currentPartLength; + $remainingToTransfer -= $currentPartLength; + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('Foo'), + 'PartsCount' => $partsCount, + 'PartNumber' => $command['PartNumber'], + 'ContentRange' => "bytes $from-$to/$objectSizeInBytes", + 'ContentLength' => $currentPartLength + ])); + }); + $mockClient->method('getCommand') + -> willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $downloader = MultipartDownloader::chooseDownloader( + $mockClient, + $multipartDownloadType, + [ + 'Bucket' => 'FooBucket', + 'Key' => $objectKey, + ], + [ + 'minimumPartSize' => $targetPartSize, + ] + ); + $stream = $downloader->promise()->wait(); + + $this->assertInstanceOf(StreamInterface::class, $stream); + $this->assertEquals($objectKey, $downloader->getObjectKey()); + $this->assertEquals($objectSizeInBytes, $downloader->getObjectSizeInBytes()); + $this->assertEquals($objectSizeInBytes, $downloader->getObjectBytesTransferred()); + $this->assertEquals($partsCount, $downloader->getObjectPartsCount()); + $this->assertEquals($partsCount, $downloader->getObjectCompletedPartsCount()); + } + + /** + * Part get multipart downloader data provider. + * + * @return array[] + */ + public function partGetMultipartDownloaderProvider(): array { + return [ + [ + 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_1', + 'objectSizeInBytes' => 1024 * 10, + 'targetPartSize' => 1024 * 2, + ], + [ + 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_2', + 'objectSizeInBytes' => 1024 * 100, + 'targetPartSize' => 1024 * 5, + ], + [ + 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_3', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 512, + ], + [ + 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_4', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 256, + ], + [ + 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_5', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 458, + ], + [ + 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_1', + 'objectSizeInBytes' => 1024 * 10, + 'targetPartSize' => 1024 * 2, + ], + [ + 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_2', + 'objectSizeInBytes' => 1024 * 100, + 'targetPartSize' => 1024 * 5, + ], + [ + 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_3', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 512, + ], + [ + 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_4', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 256, + ], + [ + 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_5', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 458, + ] + ]; } } \ No newline at end of file From c28c1657473b403756a75f9d6f91a2298ed9185b Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 13 Feb 2025 08:57:10 -0800 Subject: [PATCH 05/19] feat: add download directory and refactor code Refactor: - Moves opening braces into a new line. - Make requestArgs an optional argument. - Remove unnecessary traits. - Use traditional declarations. Adds: - Download directory feature. --- .../S3Transfer/ConsoleProgressBar.php | 14 +- .../S3Transfer/DefaultProgressTracker.php | 42 +- src/S3/Features/S3Transfer/DownloadResult.php | 23 + .../Exceptions/S3TransferException.php | 7 + .../S3Transfer/MultipartDownloadListener.php | 62 ++- .../S3Transfer/MultipartDownloadType.php | 51 --- .../S3Transfer/MultipartDownloader.php | 246 ++++++++++- .../S3Transfer/MultipartDownloaderTrait.php | 174 -------- .../S3Transfer/ObjectProgressTracker.php | 6 +- ...der.php => PartGetMultipartDownloader.php} | 9 +- ...er.php => RangeGetMultipartDownloader.php} | 19 +- .../Features/S3Transfer/S3TransferManager.php | 396 ++++++++++++++++-- .../S3Transfer/S3TransferManagerTrait.php | 87 ---- .../Features/S3Transfer/TransferListener.php | 24 +- 14 files changed, 749 insertions(+), 411 deletions(-) create mode 100644 src/S3/Features/S3Transfer/DownloadResult.php create mode 100644 src/S3/Features/S3Transfer/Exceptions/S3TransferException.php delete mode 100644 src/S3/Features/S3Transfer/MultipartDownloadType.php delete mode 100644 src/S3/Features/S3Transfer/MultipartDownloaderTrait.php rename src/S3/Features/S3Transfer/{GetMultipartDownloader.php => PartGetMultipartDownloader.php} (80%) rename src/S3/Features/S3Transfer/{RangeMultipartDownloader.php => RangeGetMultipartDownloader.php} (86%) delete mode 100644 src/S3/Features/S3Transfer/S3TransferManagerTrait.php diff --git a/src/S3/Features/S3Transfer/ConsoleProgressBar.php b/src/S3/Features/S3Transfer/ConsoleProgressBar.php index 23f41f0e57..2478812b63 100644 --- a/src/S3/Features/S3Transfer/ConsoleProgressBar.php +++ b/src/S3/Features/S3Transfer/ConsoleProgressBar.php @@ -49,7 +49,8 @@ class ConsoleProgressBar implements ProgressBar /** @var ?array */ private ?array $format; - private ?array $args; + /** @var array */ + private array $args; /** * @param ?string $progressBarChar @@ -62,7 +63,7 @@ public function __construct( ?int $progressBarWidth = null, ?int $percentCompleted = null, ?array $format = null, - ?array $args = null + array $args = [], ) { $this->progressBarChar = $progressBarChar ?? '#'; $this->progressBarWidth = $progressBarWidth ?? 25; @@ -78,7 +79,8 @@ public function __construct( * * @return void */ - public function setPercentCompleted(int $percent): void { + public function setPercentCompleted(int $percent): void + { $this->percentCompleted = max(0, min(100, $percent)); } @@ -105,7 +107,8 @@ public function setArg(string $key, mixed $value): void $this->args[$key] = $value; } - private function renderProgressBar(): string { + private function renderProgressBar(): string + { $filledWidth = (int) round(($this->progressBarWidth * $this->percentCompleted) / 100); return str_repeat($this->progressBarChar, $filledWidth) . str_repeat(' ', $this->progressBarWidth - $filledWidth); @@ -115,7 +118,8 @@ private function renderProgressBar(): string { * * @return string */ - public function getPaintedProgress(): string { + public function getPaintedProgress(): string + { foreach ($this->format['parameters'] as $param) { if (!array_key_exists($param, $this->args)) { throw new \InvalidArgumentException("Missing `$param` parameter for progress bar."); diff --git a/src/S3/Features/S3Transfer/DefaultProgressTracker.php b/src/S3/Features/S3Transfer/DefaultProgressTracker.php index 3a8cd92c68..e91d7c4d77 100644 --- a/src/S3/Features/S3Transfer/DefaultProgressTracker.php +++ b/src/S3/Features/S3Transfer/DefaultProgressTracker.php @@ -6,6 +6,9 @@ class DefaultProgressTracker { + public const TRACKING_OPERATION_DOWNLOADING = 'Downloading'; + public const TRACKING_OPERATION_UPLOADING = 'Uploading'; + /** @var ObjectProgressTracker[] */ private array $objects; @@ -33,12 +36,21 @@ class DefaultProgressTracker /** @var resource */ private $output; + /** @var string */ + private string $trackingOperation; + /** * @param Closure|ProgressBarFactory|null $progressBarFactory + * @param false|resource $output + * @param string $trackingOperation + * Valid values should be: + * - Downloading + * - Uploading */ public function __construct( Closure | ProgressBarFactory | null $progressBarFactory = null, - $output = STDOUT + mixed $output = STDOUT, + string $trackingOperation = '', ) { $this->clear(); $this->initializeListener(); @@ -48,9 +60,15 @@ public function __construct( } $this->output = $output; + if (!in_array(strtolower($trackingOperation), ['downloading', 'Uploading'], true)) { + throw new \InvalidArgumentException("Tracking operation '$trackingOperation' should be one of 'Downloading', 'Uploading'"); + } + + $this->trackingOperation = $trackingOperation; } - private function initializeListener(): void { + private function initializeListener(): void + { $this->transferListener = new TransferListener(); // Object transfer initialized $this->transferListener->onObjectTransferInitiated = $this->objectTransferInitiated(); @@ -63,7 +81,8 @@ private function initializeListener(): void { /** * @return TransferListener */ - public function getTransferListener(): TransferListener { + public function getTransferListener(): TransferListener + { return $this->transferListener; } @@ -212,7 +231,8 @@ public function clear(): void * * @return void */ - private function increaseBytesTransferred(int $bytesTransferred): void { + private function increaseBytesTransferred(int $bytesTransferred): void + { $this->totalBytesTransferred += $bytesTransferred; if ($this->objectsTotalSizeInBytes !== 0) { $this->transferPercentCompleted = floor(($this->totalBytesTransferred / $this->objectsTotalSizeInBytes) * 100); @@ -222,16 +242,18 @@ private function increaseBytesTransferred(int $bytesTransferred): void { /** * @return void */ - private function showProgress(): void { + private function showProgress(): void + { // Clear screen fwrite($this->output, "\033[2J\033[H"); // Display progress header fwrite($this->output, sprintf( - "\r%d%% [%s/%s]\n", - $this->transferPercentCompleted, + "\r%s [%d/%d] %d%%\n", + $this->trackingOperation, $this->objectsInProgress, - $this->objectsCount + $this->objectsCount, + $this->transferPercentCompleted )); foreach ($this->objects as $name => $object) { @@ -246,7 +268,8 @@ private function showProgress(): void { /** * @return Closure|ProgressBarFactory */ - private function defaultProgressBarFactory(): Closure| ProgressBarFactory { + private function defaultProgressBarFactory(): Closure| ProgressBarFactory + { return function () { return new ConsoleProgressBar( format: ConsoleProgressBar::$formats[ @@ -261,5 +284,4 @@ private function defaultProgressBarFactory(): Closure| ProgressBarFactory { ); }; } - } \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/DownloadResult.php b/src/S3/Features/S3Transfer/DownloadResult.php new file mode 100644 index 0000000000..4f40ca1623 --- /dev/null +++ b/src/S3/Features/S3Transfer/DownloadResult.php @@ -0,0 +1,23 @@ +content; + } + + public function getMetadata(): array + { + return $this->metadata; + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/Exceptions/S3TransferException.php b/src/S3/Features/S3Transfer/Exceptions/S3TransferException.php new file mode 100644 index 0000000000..d1de07840b --- /dev/null +++ b/src/S3/Features/S3Transfer/Exceptions/S3TransferException.php @@ -0,0 +1,7 @@ +notify('onDownloadInitiated', [&$commandArgs, $initialPart]); } @@ -80,15 +83,25 @@ public function downloadInitiated(array &$commandArgs, int $initialPart): void { * to call parent::downloadFailed() in order to * keep the states maintained in this implementation. * - * @param \Throwable $reason + * @param Throwable $reason * @param int $totalPartsDownloaded * @param int $totalBytesDownloaded * @param int $lastPartDownloaded * * @return void */ - public function downloadFailed(\Throwable $reason, int $totalPartsDownloaded, int $totalBytesDownloaded, int $lastPartDownloaded): void { - $this->notify('onDownloadFailed', [$reason, $totalPartsDownloaded, $totalBytesDownloaded, $lastPartDownloaded]); + public function downloadFailed( + Throwable $reason, + int $totalPartsDownloaded, + int $totalBytesDownloaded, + int $lastPartDownloaded): void + { + $this->notify('onDownloadFailed', [ + $reason, + $totalPartsDownloaded, + $totalBytesDownloaded, + $lastPartDownloaded + ]); } /** @@ -97,14 +110,23 @@ public function downloadFailed(\Throwable $reason, int $totalPartsDownloaded, in * to call parent::onDownloadCompleted() in order to * keep the states maintained in this implementation. * - * @param resource $stream + * @param StreamInterface $stream * @param int $totalPartsDownloaded * @param int $totalBytesDownloaded * * @return void */ - public function downloadCompleted($stream, int $totalPartsDownloaded, int $totalBytesDownloaded): void { - $this->notify('onDownloadCompleted', [$stream, $totalPartsDownloaded, $totalBytesDownloaded]); + public function downloadCompleted( + StreamInterface $stream, + int $totalPartsDownloaded, + int $totalBytesDownloaded + ): void + { + $this->notify('onDownloadCompleted', [ + $stream, + $totalPartsDownloaded, + $totalBytesDownloaded + ]); } /** @@ -118,8 +140,15 @@ public function downloadCompleted($stream, int $totalPartsDownloaded, int $total * * @return void */ - public function partDownloadInitiated(CommandInterface $partDownloadCommand, int $partNo): void { - $this->notify('onPartDownloadInitiated', [$partDownloadCommand, $partNo]); + public function partDownloadInitiated( + CommandInterface $partDownloadCommand, + int $partNo + ): void + { + $this->notify('onPartDownloadInitiated', [ + $partDownloadCommand, + $partNo + ]); } /** @@ -162,13 +191,22 @@ public function partDownloadCompleted( * keep the states maintained in this implementation. * * @param CommandInterface $partDownloadCommand - * @param \Throwable $reason + * @param Throwable $reason * @param int $partNo * * @return void */ - public function partDownloadFailed(CommandInterface $partDownloadCommand, \Throwable $reason, int $partNo): void { - $this->notify('onPartDownloadFailed', [$partDownloadCommand, $reason, $partNo]); + public function partDownloadFailed( + CommandInterface $partDownloadCommand, + Throwable $reason, + int $partNo + ): void + { + $this->notify('onPartDownloadFailed', [ + $partDownloadCommand, + $reason, + $partNo + ]); } protected function notify(string $event, array $params = []): void diff --git a/src/S3/Features/S3Transfer/MultipartDownloadType.php b/src/S3/Features/S3Transfer/MultipartDownloadType.php deleted file mode 100644 index 6bbbfd6144..0000000000 --- a/src/S3/Features/S3Transfer/MultipartDownloadType.php +++ /dev/null @@ -1,51 +0,0 @@ -value = $value; - } - - /** - * @return string - */ - public function __toString(): string { - return $this->value; - } - - /** - * @param MultipartDownloadType $type - * - * @return bool - */ - public function equals(MultipartDownloadType $type): bool - { - return $this->value === $type->value; - } - - /** - * @return MultipartDownloadType - */ - public static function rangedGet(): MultipartDownloadType { - return new static(self::$rangedGetType); - } - - /** - * @return MultipartDownloadType - */ - public static function partGet(): MultipartDownloadType { - return new static(self::$partGetType); - } -} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/MultipartDownloader.php b/src/S3/Features/S3Transfer/MultipartDownloader.php index 50b575c455..e3c1143da1 100644 --- a/src/S3/Features/S3Transfer/MultipartDownloader.php +++ b/src/S3/Features/S3Transfer/MultipartDownloader.php @@ -14,11 +14,37 @@ abstract class MultipartDownloader implements PromisorInterface { - use MultipartDownloaderTrait; public const GET_OBJECT_COMMAND = "GetObject"; public const PART_GET_MULTIPART_DOWNLOADER = "partGet"; public const RANGE_GET_MULTIPART_DOWNLOADER = "rangeGet"; + /** @var array */ + protected array $requestArgs; + + /** @var int */ + protected int $currentPartNo; + + /** @var int */ + protected int $objectPartsCount; + + /** @var int */ + protected int $objectCompletedPartsCount; + + /** @var int */ + protected int $objectSizeInBytes; + + /** @var int */ + protected int $objectBytesTransferred; + + /** @var string */ + protected string $eTag; + + /** @var string */ + protected string $objectKey; + + /** @var StreamInterface */ + private StreamInterface $stream; + /** * @param S3ClientInterface $s3Client * @param array $requestArgs @@ -38,23 +64,33 @@ abstract class MultipartDownloader implements PromisorInterface */ public function __construct( protected readonly S3ClientInterface $s3Client, - protected array $requestArgs, + array $requestArgs, protected readonly array $config = [], - protected int $currentPartNo = 0, - protected int $objectPartsCount = 0, - protected int $objectCompletedPartsCount = 0, - protected int $objectSizeInBytes = 0, - protected int $objectBytesTransferred = 0, - protected string $eTag = "", - protected string $objectKey = "", + int $currentPartNo = 0, + int $objectPartsCount = 0, + int $objectCompletedPartsCount = 0, + int $objectSizeInBytes = 0, + int $objectBytesTransferred = 0, + string $eTag = "", + string $objectKey = "", private readonly ?MultipartDownloadListener $listener = null, private readonly ?TransferListener $progressListener = null, - private ?StreamInterface $stream = null + ?StreamInterface $stream = null ) { + $this->requestArgs = $requestArgs; + $this->currentPartNo = $currentPartNo; + $this->objectPartsCount = $objectPartsCount; + $this->objectCompletedPartsCount = $objectCompletedPartsCount; + $this->objectSizeInBytes = $objectSizeInBytes; + $this->objectBytesTransferred = $objectBytesTransferred; + $this->eTag = $eTag; + $this->objectKey = $objectKey; if ($stream === null) { $this->stream = Utils::streamFor( fopen('php://temp', 'w+') ); + } else { + $this->stream = $stream; } } @@ -156,14 +192,16 @@ public function promise(): PromiseInterface yield Create::rejectionFor($e); } - $lastPartIncrement = $this->currentPartNo; } // Transfer completed $this->objectDownloadCompleted(); // TODO: yield the stream wrapped in a modeled transfer success response. - yield Create::promiseFor($this->stream); + yield Create::promiseFor(new DownloadResult( + $this->stream, + [] + )); }); } @@ -190,7 +228,8 @@ abstract protected function computeObjectDimensions(ResultInterface $result): vo * * @return int */ - protected function computeObjectSize($sizeSource): int { + protected function computeObjectSize($sizeSource): int + { if (is_int($sizeSource)) { return (int) $sizeSource; } @@ -244,7 +283,7 @@ public static function chooseDownloader( ) : MultipartDownloader { return match ($multipartDownloadType) { - self::PART_GET_MULTIPART_DOWNLOADER => new GetMultipartDownloader( + self::PART_GET_MULTIPART_DOWNLOADER => new PartGetMultipartDownloader( s3Client: $s3Client, requestArgs: $requestArgs, config: $config, @@ -258,7 +297,7 @@ public static function chooseDownloader( listener: $listener, progressListener: $progressTracker ), - self::RANGE_GET_MULTIPART_DOWNLOADER => new RangeMultipartDownloader( + self::RANGE_GET_MULTIPART_DOWNLOADER => new RangeGetMultipartDownloader( s3Client: $s3Client, requestArgs: $requestArgs, config: $config, @@ -278,4 +317,181 @@ public static function chooseDownloader( " or " . self::RANGE_GET_MULTIPART_DOWNLOADER . ".") }; } + + /** + * Main purpose of this method is to propagate + * the download-initiated event to listeners, but + * also it does some computation regarding internal states + * that need to be maintained. + * + * @param array $commandArgs + * @param int|null $currentPartNo + * + * @return void + */ + private function downloadInitiated(array &$commandArgs, ?int $currentPartNo): void + { + $this->objectKey = $commandArgs['Key']; + $this->progressListener?->objectTransferInitiated( + $this->objectKey, + $commandArgs + ); + $this->_notifyMultipartDownloadListeners('downloadInitiated', [ + &$commandArgs, + $currentPartNo + ]); + } + + /** + * Propagates download-failed event to listeners. + * It may also do some computation in order to maintain internal states. + * + * @param \Throwable $reason + * @param int $totalPartsTransferred + * @param int $totalBytesTransferred + * @param int $lastPartTransferred + * + * @return void + */ + private function downloadFailed( + \Throwable $reason, + int $totalPartsTransferred, + int $totalBytesTransferred, + int $lastPartTransferred + ): void + { + $this->progressListener?->objectTransferFailed( + $this->objectKey, + $totalBytesTransferred, + $reason + ); + $this->_notifyMultipartDownloadListeners('downloadFailed', [ + $reason, + $totalPartsTransferred, + $totalBytesTransferred, + $lastPartTransferred + ]); + } + + /** + * Propagates part-download-initiated event to listeners. + * + * @param CommandInterface $partDownloadCommand + * @param int $partNo + * + * @return void + */ + private function partDownloadInitiated( + CommandInterface $partDownloadCommand, + int $partNo + ): void + { + $this->_notifyMultipartDownloadListeners('partDownloadInitiated', [ + $partDownloadCommand, + $partNo + ]); + } + + /** + * Propagates part-download-completed to listeners. + * It also does some computation in order to maintain internal states. + * In this specific method we move each part content into an accumulative + * stream, which is meant to hold the full object content once the download + * is completed. + * + * @param ResultInterface $result + * @param int $partNo + * + * @return void + */ + private function partDownloadCompleted( + ResultInterface $result, + int $partNo + ): void + { + $this->objectCompletedPartsCount++; + $partDownloadBytes = $result['ContentLength']; + $this->objectBytesTransferred = $this->objectBytesTransferred + $partDownloadBytes; + if (isset($result['ETag'])) { + $this->eTag = $result['ETag']; + } + Utils::copyToStream($result['Body'], $this->stream); + + $this->progressListener?->objectTransferProgress( + $this->objectKey, + $partDownloadBytes, + $this->objectSizeInBytes + ); + + $this->_notifyMultipartDownloadListeners('partDownloadCompleted', [ + $result, + $partNo, + $partDownloadBytes, + $this->objectCompletedPartsCount, + $this->objectBytesTransferred, + $this->objectSizeInBytes + ]); + } + + /** + * Propagates part-download-failed event to listeners. + * + * @param CommandInterface $partDownloadCommand + * @param \Throwable $reason + * @param int $partNo + * + * @return void + */ + private function partDownloadFailed( + CommandInterface $partDownloadCommand, + \Throwable $reason, + int $partNo + ): void + { + $this->progressListener?->objectTransferFailed( + $this->objectKey, + $this->objectBytesTransferred, + $reason + ); + $this->_notifyMultipartDownloadListeners( + 'partDownloadFailed', + [$partDownloadCommand, $reason, $partNo]); + } + + /** + * Propagates object-download-completed event to listeners. + * It also resets the pointer of the stream to the first position, + * so that the stream is ready to be consumed once returned. + * + * @return void + */ + private function objectDownloadCompleted(): void + { + $this->stream->rewind(); + $this->progressListener?->objectTransferCompleted( + $this->objectKey, + $this->objectBytesTransferred + ); + $this->_notifyMultipartDownloadListeners('downloadCompleted', [ + $this->stream, + $this->objectCompletedPartsCount, + $this->objectBytesTransferred + ]); + } + + /** + * Internal helper method for notifying listeners of specific events. + * + * @param string $listenerMethod + * @param array $args + * + * @return void + */ + private function _notifyMultipartDownloadListeners( + string $listenerMethod, + array $args + ): void + { + $this->listener?->{$listenerMethod}(...$args); + } } \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/MultipartDownloaderTrait.php b/src/S3/Features/S3Transfer/MultipartDownloaderTrait.php deleted file mode 100644 index c722da4314..0000000000 --- a/src/S3/Features/S3Transfer/MultipartDownloaderTrait.php +++ /dev/null @@ -1,174 +0,0 @@ -objectKey = $commandArgs['Key']; - $this->progressListener?->objectTransferInitiated( - $this->objectKey, - $commandArgs - ); - $this->_notifyMultipartDownloadListeners('downloadInitiated', [ - &$commandArgs, - $currentPartNo - ]); - } - - /** - * Propagates download-failed event to listeners. - * It may also do some computation in order to maintain internal states. - * - * @param \Throwable $reason - * @param int $totalPartsTransferred - * @param int $totalBytesTransferred - * @param int $lastPartTransferred - * - * @return void - */ - private function downloadFailed( - \Throwable $reason, - int $totalPartsTransferred, - int $totalBytesTransferred, - int $lastPartTransferred - ): void { - $this->progressListener?->objectTransferFailed( - $this->objectKey, - $totalBytesTransferred, - $reason - ); - $this->_notifyMultipartDownloadListeners('downloadFailed', [ - $reason, - $totalPartsTransferred, - $totalBytesTransferred, - $lastPartTransferred - ]); - } - - /** - * Propagates part-download-initiated event to listeners. - * - * @param CommandInterface $partDownloadCommand - * @param int $partNo - * - * @return void - */ - private function partDownloadInitiated(CommandInterface $partDownloadCommand, int $partNo): void { - $this->_notifyMultipartDownloadListeners('partDownloadInitiated', [ - $partDownloadCommand, - $partNo - ]); - } - - /** - * Propagates part-download-completed to listeners. - * It also does some computation in order to maintain internal states. - * In this specific method we move each part content into an accumulative - * stream, which is meant to hold the full object content once the download - * is completed. - * - * @param ResultInterface $result - * @param int $partNo - * - * @return void - */ - private function partDownloadCompleted(ResultInterface $result, int $partNo): void { - $this->objectCompletedPartsCount++; - $partDownloadBytes = $result['ContentLength']; - $this->objectBytesTransferred = $this->objectBytesTransferred + $partDownloadBytes; - if (isset($result['ETag'])) { - $this->eTag = $result['ETag']; - } - Utils::copyToStream($result['Body'], $this->stream); - - $this->progressListener?->objectTransferProgress( - $this->objectKey, - $partDownloadBytes, - $this->objectSizeInBytes - ); - - $this->_notifyMultipartDownloadListeners('partDownloadCompleted', [ - $result, - $partNo, - $partDownloadBytes, - $this->objectCompletedPartsCount, - $this->objectBytesTransferred, - $this->objectSizeInBytes - ]); - } - - /** - * Propagates part-download-failed event to listeners. - * - * @param CommandInterface $partDownloadCommand - * @param \Throwable $reason - * @param int $partNo - * - * @return void - */ - private function partDownloadFailed( - CommandInterface $partDownloadCommand, - \Throwable $reason, - int $partNo - ): void { - $this->progressListener?->objectTransferFailed( - $this->objectKey, - $this->objectBytesTransferred, - $reason - ); - $this->_notifyMultipartDownloadListeners( - 'partDownloadFailed', - [$partDownloadCommand, $reason, $partNo]); - } - - /** - * Propagates object-download-completed event to listeners. - * It also resets the pointer of the stream to the first position, - * so that the stream is ready to be consumed once returned. - * - * @return void - */ - private function objectDownloadCompleted(): void - { - $this->stream->rewind(); - $this->progressListener?->objectTransferCompleted( - $this->objectKey, - $this->objectBytesTransferred - ); - $this->_notifyMultipartDownloadListeners('downloadCompleted', [ - $this->stream, - $this->objectCompletedPartsCount, - $this->objectBytesTransferred - ]); - } - - /** - * Internal helper method for notifying listeners of specific events. - * - * @param string $listenerMethod - * @param array $args - * - * @return void - */ - private function _notifyMultipartDownloadListeners(string $listenerMethod, array $args): void - { - $this->listener?->{$listenerMethod}(...$args); - } -} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/ObjectProgressTracker.php b/src/S3/Features/S3Transfer/ObjectProgressTracker.php index 5eadecfdc4..ea6ac4f0d3 100644 --- a/src/S3/Features/S3Transfer/ObjectProgressTracker.php +++ b/src/S3/Features/S3Transfer/ObjectProgressTracker.php @@ -104,7 +104,8 @@ public function setStatus(string $status): void $this->setProgressColor(); } - private function setProgressColor(): void { + private function setProgressColor(): void + { if ($this->status === 'progress') { $this->progressBar->setArg('color_code', ConsoleProgressBar::BLUE_COLOR_CODE); } elseif ($this->status === 'completed') { @@ -143,7 +144,8 @@ public function getProgressBar(): ?ProgressBar /** * @return ProgressBar */ - private function defaultProgressBar(): ProgressBar { + private function defaultProgressBar(): ProgressBar + { return new ConsoleProgressBar( format: ConsoleProgressBar::$formats[ ConsoleProgressBar::COLORED_TRANSFER_FORMAT diff --git a/src/S3/Features/S3Transfer/GetMultipartDownloader.php b/src/S3/Features/S3Transfer/PartGetMultipartDownloader.php similarity index 80% rename from src/S3/Features/S3Transfer/GetMultipartDownloader.php rename to src/S3/Features/S3Transfer/PartGetMultipartDownloader.php index 9548d95ded..4a9ce76bad 100644 --- a/src/S3/Features/S3Transfer/GetMultipartDownloader.php +++ b/src/S3/Features/S3Transfer/PartGetMultipartDownloader.php @@ -9,14 +9,15 @@ /** * Multipart downloader using the part get approach. */ -class GetMultipartDownloader extends MultipartDownloader +class PartGetMultipartDownloader extends MultipartDownloader { /** * @inheritDoc * * @return CommandInterface */ - protected function nextCommand() : CommandInterface { + protected function nextCommand() : CommandInterface + { if ($this->currentPartNo === 0) { $this->currentPartNo = 1; } else { @@ -45,6 +46,8 @@ protected function computeObjectDimensions(ResultInterface $result): void $this->objectPartsCount = $result['PartsCount']; } - $this->objectSizeInBytes = $this->computeObjectSize($result['ContentRange'] ?? ""); + $this->objectSizeInBytes = $this->computeObjectSize( + $result['ContentRange'] ?? "" + ); } } \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/RangeMultipartDownloader.php b/src/S3/Features/S3Transfer/RangeGetMultipartDownloader.php similarity index 86% rename from src/S3/Features/S3Transfer/RangeMultipartDownloader.php rename to src/S3/Features/S3Transfer/RangeGetMultipartDownloader.php index ca26120642..615a947b46 100644 --- a/src/S3/Features/S3Transfer/RangeMultipartDownloader.php +++ b/src/S3/Features/S3Transfer/RangeGetMultipartDownloader.php @@ -5,10 +5,11 @@ use Aws\CommandInterface; use Aws\Result; use Aws\ResultInterface; +use Aws\S3\Features\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3ClientInterface; use Psr\Http\Message\StreamInterface; -class RangeMultipartDownloader extends MultipartDownloader +class RangeGetMultipartDownloader extends MultipartDownloader { /** @var int */ @@ -60,13 +61,17 @@ public function __construct( $stream ); if (empty($config['minimumPartSize'])) { - throw new \RuntimeException('You must provide a valid minimum part size in bytes'); + throw new S3TransferException( + 'You must provide a valid minimum part size in bytes' + ); } $this->partSize = $config['minimumPartSize']; // If object size is known at instantiation time then, we can compute // the object dimensions. if ($this->objectSizeInBytes !== 0) { - $this->computeObjectDimensions(new Result(['ContentRange' => $this->objectSizeInBytes])); + $this->computeObjectDimensions( + new Result(['ContentRange' => $this->objectSizeInBytes]) + ); } } @@ -115,11 +120,15 @@ protected function computeObjectDimensions(ResultInterface $result): void { // Assign object size just if needed. if ($this->objectSizeInBytes === 0) { - $this->objectSizeInBytes = $this->computeObjectSize($result['ContentRange'] ?? ""); + $this->objectSizeInBytes = $this->computeObjectSize( + $result['ContentRange'] ?? "" + ); } if ($this->objectSizeInBytes > $this->partSize) { - $this->objectPartsCount = intval(ceil($this->objectSizeInBytes / $this->partSize)); + $this->objectPartsCount = intval( + ceil($this->objectSizeInBytes / $this->partSize) + ); } else { // Single download since partSize will be set to full object size. $this->partSize = $this->objectSizeInBytes; diff --git a/src/S3/Features/S3Transfer/S3TransferManager.php b/src/S3/Features/S3Transfer/S3TransferManager.php index aef8921881..3a199839ca 100644 --- a/src/S3/Features/S3Transfer/S3TransferManager.php +++ b/src/S3/Features/S3Transfer/S3TransferManager.php @@ -2,15 +2,31 @@ namespace Aws\S3\Features\S3Transfer; +use Aws\Command; +use Aws\S3\Features\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; +use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; +use function Aws\filter; +use function Aws\map; class S3TransferManager { - use S3TransferManagerTrait; + private static array $defaultConfig = [ + 'targetPartSizeBytes' => 8 * 1024 * 1024, + 'multipartUploadThresholdBytes' => 16 * 1024 * 1024, + 'checksumValidationEnabled' => true, + 'checksumAlgorithm' => 'crc32', + 'multipartDownloadType' => 'partGet', + 'concurrency' => 5, + 'trackProgress' => false, + 'region' => 'us-east-1', + ]; + /** @var S3Client */ private S3ClientInterface $s3Client; + /** @var array */ private array $config; @@ -30,13 +46,14 @@ class S3TransferManager * - concurrency: (int, default=5) \ * Maximum number of concurrent operations allowed during a multipart * upload/download. - * - trackProgress: (bool, default=true) \ + * - trackProgress: (bool, default=false) \ * To enable progress tracker in a multipart upload/download. - * - progressListenerFactory: (callable|TransferListenerFactory) + * - progressTrackerFactory: (callable|TransferListenerFactory) \ * A factory to create the listener which will receive notifications - * based in the different stages in an upload/download operation. + * based in the different stages an upload/download is. */ - public function __construct(?S3ClientInterface $s3Client, array $config = []) { + public function __construct(?S3ClientInterface $s3Client, array $config = []) + { if ($s3Client === null) { $this->s3Client = $this->defaultS3Client(); } else { @@ -50,16 +67,21 @@ public function __construct(?S3ClientInterface $s3Client, array $config = []) { * @param string|array $source The object to be downloaded from S3. * It can be either a string with a S3 URI or an array with a Bucket and Key * properties set. - * @param array $downloadArgs The request arguments to be provided as part - * of the service client operation. + * @param array $downloadArgs The getObject request arguments to be provided as part + * of each get object operation, except for the bucket and key, which + * are already provided as the source. + * @param MultipartDownloadListener|null $downloadListener A multipart download + * specific listener of the different states a multipart download can be. + * @param TransferListener|null $progressTracker A transfer listener implementation + * aimed to track the progress of a transfer. If not provided and trackProgress + * is resolved as true then, the default progressTrackerFactory will be used. * @param array $config The configuration to be used for this operation. - * - listener: (null|MultipartDownloadListener) \ - * A listener to be notified in every stage of a multipart download operation. * - trackProgress: (bool) \ * Overrides the config option set in the transfer manager instantiation - * to decide whether transfer progress should be tracked. If not - * transfer tracker factory is provided and trackProgress is true then, - * the default progress listener implementation will be used. + * to decide whether transfer progress should be tracked. If a `progressListenerFactory` + * was not provided when the transfer manager instance was created + * and trackProgress resolved as true then, a default progress listener + * implementation will be used. * - minimumPartSize: (int) \ * The minimum part size in bytes to be used in a range multipart download. * @@ -67,9 +89,12 @@ public function __construct(?S3ClientInterface $s3Client, array $config = []) { */ public function download( string | array $source, - array $downloadArgs, + array $downloadArgs = [], + ?MultipartDownloadListener $downloadListener = null, + ?TransferListener $progressTracker = null, array $config = [] - ): PromiseInterface { + ): PromiseInterface + { if (is_string($source)) { $sourceArgs = $this->s3UriAsBucketAndKey($source); } elseif (is_array($source)) { @@ -81,25 +106,149 @@ public function download( throw new \InvalidArgumentException('Source must be a string or an array of strings'); } + if ($progressTracker === null + && ($config['trackProgress'] ?? $this->config['trackProgress'])) { + $progressTracker = $this->resolveDefaultProgressTracker( + DefaultProgressTracker::TRACKING_OPERATION_DOWNLOADING + ); + } + $requestArgs = $sourceArgs + $downloadArgs; if (empty($downloadArgs['PartNumber']) && empty($downloadArgs['Range'])) { return $this->tryMultipartDownload( $requestArgs, - $config + $downloadListener, + $progressTracker, + [ + 'minimumPartSize' => $config['minimumPartSize'] + ?? 0 + ] ); } - return $this->trySingleDownload($requestArgs); + return $this->trySingleDownload($requestArgs, $progressTracker); + } + + /** + * @param string $bucket The bucket from where the files are going to be + * downloaded from. + * @param string $destinationDirectory The destination path where the downloaded + * files will be placed in. + * @param array $downloadArgs The getObject request arguments to be provided + * as part of each get object request sent to the service, except for the + * bucket and key which will be resolved internally. + * @param MultipartDownloadListenerFactory|null $downloadListenerFactory + * A factory of multipart download listeners `MultipartDownloadListenerFactory` + * for listening to multipart download events. + * @param TransferListener|null $progressTracker + * @param array $config The config options for this download directory operation. \ + * - trackProgress: (bool) \ + * Overrides the config option set in the transfer manager instantiation + * to decide whether transfer progress should be tracked. If a `progressListenerFactory` + * was not provided when the transfer manager instance was created + * and trackProgress resolved as true then, a default progress listener + * implementation will be used. + * - minimumPartSize: (int) \ + * The minimum part size in bytes to be used in a range multipart download. + * - listObjectV2Args: (array) \ + * The arguments to be included as part of the listObjectV2 request in + * order to fetch the objects to be downloaded. The most common arguments + * would be: + * - MaxKeys: (int) Sets the maximum number of keys returned in the response. + * - Prefix: (string) To limit the response to keys that begin with the + * specified prefix. + * - filter: (Closure) \ + * A callable which will receive an object key as parameter and should return + * true or false in order to determine whether the object should be downloaded. + * @return PromiseInterface + */ + public function downloadDirectory( + string $bucket, + string $destinationDirectory, + array $downloadArgs, + ?MultipartDownloadListenerFactory $downloadListenerFactory = null, + ?TransferListener $progressTracker = null, + array $config = [] + ): PromiseInterface + { + if (!file_exists($destinationDirectory)) { + throw new \InvalidArgumentException( + "Destination directory '$destinationDirectory' MUST exists." + ); + } + + if ($progressTracker === null + && ($config['trackProgress'] ?? $this->config['trackProgress'])) { + $progressTracker = $this->resolveDefaultProgressTracker( + DefaultProgressTracker::TRACKING_OPERATION_DOWNLOADING + ); + } + + $listArgs = [ + 'Bucket' => $bucket + ] + ($config['listObjectV2Args'] ?? []); + $objects = $this->s3Client + ->getPaginator('ListObjectsV2', $listArgs) + ->search('Contents[].Key'); + $objects = map($objects, function (string $key) use ($bucket) { + return "s3://$bucket/$key"; + }); + if (isset($config['filter'])) { + if (!is_callable($config['filter'])) { + throw new \InvalidArgumentException("The parameter \$config['filter'] must be callable."); + } + + $filter = $config['filter']; + $objects = filter($objects, function (string $key) use ($filter) { + return call_user_func($filter, $key); + }); + } + + $promises = []; + foreach ($objects as $object) { + $objectKey = $this->s3UriAsBucketAndKey($object)['Key']; + $destinationFile = $destinationDirectory . '/' . $objectKey; + if ($this->resolvesOutsideTargetDirectory($destinationFile, $objectKey)) { + throw new S3TransferException( + "Cannot download key ' . $objectKey + . ', its relative path resolves outside the' + . ' parent directory" + ); + } + + $downloadListener = null; + if ($downloadListenerFactory !== null) { + $downloadListener = $downloadListenerFactory(); + } + + $promises[] = $this->download( + $object, + $downloadArgs, + $downloadListener, + $progressTracker, + [ + 'minimumPartSize' => $config['minimumPartSize'] ?? 0, + ] + )->then(function (DownloadResult $result) use ($destinationFile) { + $directory = dirname($destinationFile); + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + + file_put_contents($destinationFile, $result->getContent()); + }); + } + + return Each::ofLimitAll($promises, $this->config['concurrency']); } /** * Tries an object multipart download. * * @param array $requestArgs + * @param MultipartDownloadListener|null $downloadListener + * @param TransferListener|null $progressTracker * @param array $config - * - listener: (?MultipartDownloadListener) \ - * A multipart download listener for watching every multipart download - * stage. * - minimumPartSize: (int) \ * The minimum part size in bytes for a range multipart download. If * this parameter is not provided then it fallbacks to the transfer @@ -109,20 +258,11 @@ public function download( */ private function tryMultipartDownload( array $requestArgs, - array $config - ): PromiseInterface { - $trackProgress = $config['trackProgress'] - ?? $this->config['trackProgress'] - ?? false; - $progressListenerFactory = $this->config['progressListenerFactory'] ?? null; - $progressListener = null; - if ($trackProgress) { - if ($progressListenerFactory !== null) { - $progressListener = $progressListenerFactory(); - } else { - $progressListener = new DefaultProgressTracker(); - } - } + ?MultipartDownloadListener $downloadListener = null, + ?TransferListener $progressTracker = null, + array $config = [] + ): PromiseInterface + { $multipartDownloader = MultipartDownloader::chooseDownloader( s3Client: $this->s3Client, multipartDownloadType: $this->config['multipartDownloadType'], @@ -133,8 +273,8 @@ private function tryMultipartDownload( $this->config['targetPartSizeBytes'] ) ], - listener: $config['listener'] ?? null, - progressTracker: $progressListener?->getTransferListener() + listener: $downloadListener, + progressTracker: $progressTracker, ); return $multipartDownloader->promise(); @@ -143,13 +283,191 @@ private function tryMultipartDownload( /** * Does a single object download. * - * @param $requestArgs + * @param array $requestArgs + * @param TransferListener|null $progressTracker * * @return PromiseInterface */ - private function trySingleDownload($requestArgs): PromiseInterface { - $command = $this->s3Client->getCommand(MultipartDownloader::GET_OBJECT_COMMAND, $requestArgs); + private function trySingleDownload( + array $requestArgs, + ?TransferListener $progressTracker + ): PromiseInterface + { + if ($progressTracker !== null) { + $progressTracker->objectTransferInitiated($requestArgs['Key'], $requestArgs); + $command = $this->s3Client->getCommand( + MultipartDownloader::GET_OBJECT_COMMAND, + $requestArgs + ); + + return $this->s3Client->executeAsync($command)->then( + function ($result) use ($progressTracker, $requestArgs) { + // Notify progress + $progressTracker->objectTransferProgress( + $requestArgs['Key'], + $result['Content-Length'] ?? 0, + $result['Content-Length'] ?? 0, + ); + + // Notify Completion + $progressTracker->objectTransferCompleted( + $requestArgs['Key'], + $result['Content-Length'] ?? 0, + ); + + return new DownloadResult( + content: $result['Body'], + metadata: $result['@metadata'], + ); + } + )->otherwise(function ($reason) use ($requestArgs, $progressTracker) { + $progressTracker->objectTransferFailed( + $requestArgs['Key'], + 0, + $reason->getMessage(), + ); + + return $reason; + }); + } + + $command = $this->s3Client->getCommand( + MultipartDownloader::GET_OBJECT_COMMAND, + $requestArgs + ); + + return $this->s3Client->executeAsync($command) + ->then(function ($result) { + return new DownloadResult( + content: $result['Body'], + metadata: $result['@metadata'], + ); + }); + } + + /** + * Returns a default instance of S3Client. + * + * @return S3Client + */ + private function defaultS3Client(): S3ClientInterface + { + return new S3Client([ + 'region' => $this->config['region'], + ]); + } + + /** + * Validates a provided value is not empty, and if so then + * it throws an exception with the provided message. + * @param mixed $value + * @param string $message + * + * @return mixed + */ + private function requireNonEmpty(mixed $value, string $message): mixed + { + if (empty($value)) { + throw new \InvalidArgumentException($message); + } + + return $value; + } + + /** + * Validates a string value is a valid S3 URI. + * Valid S3 URI Example: S3://mybucket.dev/myobject.txt + * + * @param string $uri + * + * @return bool + */ + private function isValidS3URI(string $uri): bool + { + // in the expression `substr($uri, 5)))` the 5 belongs to the size of `s3://`. + return str_starts_with(strtolower($uri), 's3://') + && count(explode('/', substr($uri, 5))) > 1; + } + + /** + * Converts a S3 URI into an array with a Bucket and Key + * properties set. + * + * @param string $uri: The S3 URI. + * + * @return array + */ + private function s3UriAsBucketAndKey(string $uri): array + { + $errorMessage = "Invalid URI: $uri. A valid S3 URI must be s3://bucket/key"; + if (!$this->isValidS3URI($uri)) { + throw new \InvalidArgumentException($errorMessage); + } + + $path = substr($uri, 5); // without s3:// + $parts = explode('/', $path, 2); + + if (count($parts) < 2) { + throw new \InvalidArgumentException($errorMessage); + } + + return [ + 'Bucket' => $parts[0], + 'Key' => $parts[1], + ]; + } + + /** + * Resolves the progress tracker to be used in the + * transfer operation if `$trackProgress` is true. + * + * @param string $trackingOperation + * + * @return TransferListener|null + */ + private function resolveDefaultProgressTracker( + string $trackingOperation + ): ?TransferListener + { + $progressTrackerFactory = $this->config['progressTrackerFactory'] ?? null; + if ($progressTrackerFactory === null) { + return (new DefaultProgressTracker(trackingOperation: $trackingOperation))->getTransferListener(); + } + + return $progressTrackerFactory([]); + } + + /** + * @param string $sink + * @param string $objectKey + * + * @return bool + */ + private function resolvesOutsideTargetDirectory( + string $sink, + string $objectKey + ): bool + { + $resolved = []; + $sections = explode('/', $sink); + $targetSectionsLength = count(explode('/', $objectKey)); + $targetSections = array_slice($sections, -($targetSectionsLength + 1)); + $targetDirectory = $targetSections[0]; + + foreach ($targetSections as $section) { + if ($section === '.' || $section === '') { + continue; + } + if ($section === '..') { + array_pop($resolved); + if (empty($resolved) || $resolved[0] !== $targetDirectory) { + return true; + } + } else { + $resolved []= $section; + } + } - return $this->s3Client->executeAsync($requestArgs); + return false; } } \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/S3TransferManagerTrait.php b/src/S3/Features/S3Transfer/S3TransferManagerTrait.php deleted file mode 100644 index 6d2542cb04..0000000000 --- a/src/S3/Features/S3Transfer/S3TransferManagerTrait.php +++ /dev/null @@ -1,87 +0,0 @@ - 8 * 1024 * 1024, - 'multipartUploadThresholdBytes' => 16 * 1024 * 1024, - 'checksumValidationEnabled' => true, - 'checksumAlgorithm' => 'crc32', - 'multipartDownloadType' => 'partGet', - 'concurrency' => 5, - ]; - - /** - * Returns a default instance of S3Client. - * - * @return S3Client - */ - private function defaultS3Client(): S3ClientInterface - { - return new S3Client([]); - } - - /** - * Validates a provided value is not empty, and if so then - * it throws an exception with the provided message. - * @param mixed $value - * - * @return mixed - */ - private function requireNonEmpty(mixed $value, string $message): mixed { - if (empty($value)) { - throw new \InvalidArgumentException($message); - } - - return $value; - } - - /** - * Validates a string value is a valid S3 URI. - * Valid S3 URI Example: S3://mybucket.dev/myobject.txt - * - * @param string $uri - * - * @return bool - */ - private function isValidS3URI(string $uri): bool - { - // in the expression `substr($uri, 5)))` the 5 belongs to the size of `s3://`. - return str_starts_with(strtolower($uri), 's3://') - && count(explode('/', substr($uri, 5))) > 1; - } - - /** - * Converts a S3 URI into an array with a Bucket and Key - * properties set. - * - * @param string $uri: The S3 URI. - * - * @return array - */ - private function s3UriAsBucketAndKey(string $uri): array { - $errorMessage = "Invalid URI: $uri. A valid S3 URI must be s3://bucket/key"; - if (!$this->isValidS3URI($uri)) { - throw new \InvalidArgumentException($errorMessage); - } - - $path = substr($uri, 5); // without s3:// - $parts = explode('/', $path, 2); - - if (count($parts) < 2) { - throw new \InvalidArgumentException($errorMessage); - } - - return [ - 'Bucket' => $parts[0], - 'Key' => $parts[1], - ]; - } - -} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/TransferListener.php b/src/S3/Features/S3Transfer/TransferListener.php index 1fb5a3f74a..892f159085 100644 --- a/src/S3/Features/S3Transfer/TransferListener.php +++ b/src/S3/Features/S3Transfer/TransferListener.php @@ -106,7 +106,8 @@ public function getObjectsToBeTransferred(): int /** * Transfer initiated event. */ - public function transferInitiated(): void { + public function transferInitiated(): void + { $this->notify('onTransferInitiated', []); } @@ -118,7 +119,8 @@ public function transferInitiated(): void { * * @return void */ - public function objectTransferInitiated(string $objectKey, array &$requestArgs): void { + public function objectTransferInitiated(string $objectKey, array &$requestArgs): void + { $this->objectsToBeTransferred++; if ($this->objectsToBeTransferred === 1) { $this->transferInitiated(); @@ -140,7 +142,8 @@ public function objectTransferProgress( string $objectKey, int $objectBytesTransferred, int $objectSizeInBytes - ): void { + ): void + { $this->objectsBytesTransferred += $objectBytesTransferred; $this->notify('onObjectTransferProgress', [ $objectKey, @@ -168,7 +171,8 @@ public function objectTransferFailed( string $objectKey, int $objectBytesTransferred, \Throwable | string $reason - ): void { + ): void + { $this->objectsTransferFailed++; $this->notify('onObjectTransferFailed', [ $objectKey, @@ -188,7 +192,8 @@ public function objectTransferFailed( public function objectTransferCompleted ( string $objectKey, int $objectBytesCompleted - ): void { + ): void + { $this->objectsTransferCompleted++; $this->validateTransferComplete(); $this->notify('onObjectTransferCompleted', [ @@ -208,7 +213,8 @@ public function objectTransferCompleted ( public function transferCompleted ( int $objectsTransferCompleted, int $objectsBytesTransferred, - ): void { + ): void + { $this->notify('onTransferCompleted', [ $objectsTransferCompleted, $objectsBytesTransferred @@ -229,7 +235,8 @@ public function transferFailed ( int $objectsBytesTransferred, int $objectsTransferFailed, Throwable | string $reason - ): void { + ): void + { $this->notify('onTransferFailed', [ $objectsTransferCompleted, $objectsBytesTransferred, @@ -244,7 +251,8 @@ public function transferFailed ( * * @return void */ - private function validateTransferComplete(): void { + private function validateTransferComplete(): void + { if ($this->objectsToBeTransferred === ($this->objectsTransferCompleted + $this->objectsTransferFailed)) { if ($this->objectsTransferFailed > 0) { $this->transferFailed( From 34321b29e521327bf778a50a70374c3b9b93ccb9 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 13 Feb 2025 17:39:07 -0800 Subject: [PATCH 06/19] chore: add upload and refactor code Refactor: - Add a message placeholder for progress status. For example in case of errors. Adds: - Upload feature, missing multipart functionality. --- .../S3Transfer/ConsoleProgressBar.php | 14 +- .../S3Transfer/DefaultProgressTracker.php | 11 +- .../MultipartDownloadListenerFactory.php | 11 + .../S3Transfer/MultipartUploadListener.php | 58 +++++ .../S3Transfer/ObjectProgressTracker.php | 40 +++- .../Features/S3Transfer/S3TransferManager.php | 208 ++++++++++++++---- 6 files changed, 285 insertions(+), 57 deletions(-) create mode 100644 src/S3/Features/S3Transfer/MultipartDownloadListenerFactory.php create mode 100644 src/S3/Features/S3Transfer/MultipartUploadListener.php diff --git a/src/S3/Features/S3Transfer/ConsoleProgressBar.php b/src/S3/Features/S3Transfer/ConsoleProgressBar.php index 2478812b63..e370362651 100644 --- a/src/S3/Features/S3Transfer/ConsoleProgressBar.php +++ b/src/S3/Features/S3Transfer/ConsoleProgressBar.php @@ -27,12 +27,13 @@ class ConsoleProgressBar implements ProgressBar ] ], 'colored_transfer_format' => [ - 'format' => "\033|color_code|[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit|\033[0m", + 'format' => "\033|color_code|[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit| |message|\033[0m", 'parameters' => [ 'transferred', 'tobe_transferred', 'unit', - 'color_code' + 'color_code', + 'message' ] ], ]; @@ -57,6 +58,7 @@ class ConsoleProgressBar implements ProgressBar * @param ?int $progressBarWidth * @param ?int $percentCompleted * @param array|null $format + * @param array $args */ public function __construct( ?string $progressBarChar = null, @@ -68,8 +70,8 @@ public function __construct( $this->progressBarChar = $progressBarChar ?? '#'; $this->progressBarWidth = $progressBarWidth ?? 25; $this->percentCompleted = $percentCompleted ?? 0; - $this->format = $format ?: self::$formats['transfer_format']; - $this->args = $args ?: []; + $this->format = $format ?? self::$formats['transfer_format']; + $this->args = $args ?? []; } /** @@ -122,13 +124,13 @@ public function getPaintedProgress(): string { foreach ($this->format['parameters'] as $param) { if (!array_key_exists($param, $this->args)) { - throw new \InvalidArgumentException("Missing `$param` parameter for progress bar."); + $this->args[$param] = ''; } } $replacements = [ '|progress_bar|' => $this->renderProgressBar(), - '|percent|' => $this->percentCompleted + '|percent|' => $this->percentCompleted, ]; foreach ($this->format['parameters'] as $param) { diff --git a/src/S3/Features/S3Transfer/DefaultProgressTracker.php b/src/S3/Features/S3Transfer/DefaultProgressTracker.php index e91d7c4d77..c09e302a6d 100644 --- a/src/S3/Features/S3Transfer/DefaultProgressTracker.php +++ b/src/S3/Features/S3Transfer/DefaultProgressTracker.php @@ -60,7 +60,10 @@ public function __construct( } $this->output = $output; - if (!in_array(strtolower($trackingOperation), ['downloading', 'Uploading'], true)) { + if (!in_array(strtolower($trackingOperation), [ + strtolower(self::TRACKING_OPERATION_DOWNLOADING), + strtolower(self::TRACKING_OPERATION_UPLOADING), + ], true)) { throw new \InvalidArgumentException("Tracking operation '$trackingOperation' should be one of 'Downloading', 'Uploading'"); } @@ -188,7 +191,7 @@ public function objectTransferFailed(): Closure \Throwable | string $reason ): void { $objectProgressTracker = $this->objects[$objectKey]; - $objectProgressTracker->setStatus('failed'); + $objectProgressTracker->setStatus('failed', $reason); $this->objectsInProgress--; @@ -235,7 +238,9 @@ private function increaseBytesTransferred(int $bytesTransferred): void { $this->totalBytesTransferred += $bytesTransferred; if ($this->objectsTotalSizeInBytes !== 0) { - $this->transferPercentCompleted = floor(($this->totalBytesTransferred / $this->objectsTotalSizeInBytes) * 100); + $this->transferPercentCompleted = floor( + ($this->totalBytesTransferred / $this->objectsTotalSizeInBytes) * 100 + ); } } diff --git a/src/S3/Features/S3Transfer/MultipartDownloadListenerFactory.php b/src/S3/Features/S3Transfer/MultipartDownloadListenerFactory.php new file mode 100644 index 0000000000..5516ae8a46 --- /dev/null +++ b/src/S3/Features/S3Transfer/MultipartDownloadListenerFactory.php @@ -0,0 +1,11 @@ +objectKey = $objectKey; + $this->objectBytesTransferred = $objectBytesTransferred; + $this->objectSizeInBytes = $objectSizeInBytes; + $this->status = $status; $this->progressBar = $progressBar ?? $this->defaultProgressBar(); } @@ -95,13 +117,18 @@ public function getStatus(): string /** * @param string $status + * @param string|null $message * * @return void */ - public function setStatus(string $status): void + public function setStatus(string $status, ?string $message = null): void { $this->status = $status; $this->setProgressColor(); + // To show specific messages for specific status. + if (!empty($message)) { + $this->progressBar->setArg('message', "$status: $message"); + } } private function setProgressColor(): void @@ -155,6 +182,7 @@ private function defaultProgressBar(): ProgressBar 'tobe_transferred' => 0, 'unit' => 'B', 'color_code' => ConsoleProgressBar::BLACK_COLOR_CODE, + 'message' => '' ] ); } diff --git a/src/S3/Features/S3Transfer/S3TransferManager.php b/src/S3/Features/S3Transfer/S3TransferManager.php index 3a199839ca..ec5052b525 100644 --- a/src/S3/Features/S3Transfer/S3TransferManager.php +++ b/src/S3/Features/S3Transfer/S3TransferManager.php @@ -8,6 +8,8 @@ use Aws\S3\S3ClientInterface; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; +use GuzzleHttp\Psr7\Utils; +use Psr\Http\Message\StreamInterface; use function Aws\filter; use function Aws\map; @@ -24,6 +26,8 @@ class S3TransferManager 'region' => 'us-east-1', ]; + private const MIN_PART_SIZE = 5 * 1024 * 1024; + /** @var S3Client */ private S3ClientInterface $s3Client; @@ -70,29 +74,29 @@ public function __construct(?S3ClientInterface $s3Client, array $config = []) * @param array $downloadArgs The getObject request arguments to be provided as part * of each get object operation, except for the bucket and key, which * are already provided as the source. + * @param array $config The configuration to be used for this operation. + * - trackProgress: (bool) \ + * Overrides the config option set in the transfer manager instantiation + * to decide whether transfer progress should be tracked. If a `progressListenerFactory` + * was not provided when the transfer manager instance was created + * and trackProgress resolved as true then, a default progress listener + * implementation will be used. + * - minimumPartSize: (int) \ + * The minimum part size in bytes to be used in a range multipart download. * @param MultipartDownloadListener|null $downloadListener A multipart download * specific listener of the different states a multipart download can be. * @param TransferListener|null $progressTracker A transfer listener implementation * aimed to track the progress of a transfer. If not provided and trackProgress * is resolved as true then, the default progressTrackerFactory will be used. - * @param array $config The configuration to be used for this operation. - * - trackProgress: (bool) \ - * Overrides the config option set in the transfer manager instantiation - * to decide whether transfer progress should be tracked. If a `progressListenerFactory` - * was not provided when the transfer manager instance was created - * and trackProgress resolved as true then, a default progress listener - * implementation will be used. - * - minimumPartSize: (int) \ - * The minimum part size in bytes to be used in a range multipart download. * * @return PromiseInterface */ public function download( string | array $source, array $downloadArgs = [], + array $config = [], ?MultipartDownloadListener $downloadListener = null, ?TransferListener $progressTracker = null, - array $config = [] ): PromiseInterface { if (is_string($source)) { @@ -117,12 +121,12 @@ public function download( if (empty($downloadArgs['PartNumber']) && empty($downloadArgs['Range'])) { return $this->tryMultipartDownload( $requestArgs, - $downloadListener, - $progressTracker, [ 'minimumPartSize' => $config['minimumPartSize'] ?? 0 - ] + ], + $downloadListener, + $progressTracker, ); } @@ -137,38 +141,39 @@ public function download( * @param array $downloadArgs The getObject request arguments to be provided * as part of each get object request sent to the service, except for the * bucket and key which will be resolved internally. + * @param array $config The config options for this download directory operation. \ + * - trackProgress: (bool) \ + * Overrides the config option set in the transfer manager instantiation + * to decide whether transfer progress should be tracked. If a `progressListenerFactory` + * was not provided when the transfer manager instance was created + * and trackProgress resolved as true then, a default progress listener + * implementation will be used. + * - minimumPartSize: (int) \ + * The minimum part size in bytes to be used in a range multipart download. + * - listObjectV2Args: (array) \ + * The arguments to be included as part of the listObjectV2 request in + * order to fetch the objects to be downloaded. The most common arguments + * would be: + * - MaxKeys: (int) Sets the maximum number of keys returned in the response. + * - Prefix: (string) To limit the response to keys that begin with the + * specified prefix. + * - filter: (Closure) \ + * A callable which will receive an object key as parameter and should return + * true or false in order to determine whether the object should be downloaded. * @param MultipartDownloadListenerFactory|null $downloadListenerFactory * A factory of multipart download listeners `MultipartDownloadListenerFactory` * for listening to multipart download events. * @param TransferListener|null $progressTracker - * @param array $config The config options for this download directory operation. \ - * - trackProgress: (bool) \ - * Overrides the config option set in the transfer manager instantiation - * to decide whether transfer progress should be tracked. If a `progressListenerFactory` - * was not provided when the transfer manager instance was created - * and trackProgress resolved as true then, a default progress listener - * implementation will be used. - * - minimumPartSize: (int) \ - * The minimum part size in bytes to be used in a range multipart download. - * - listObjectV2Args: (array) \ - * The arguments to be included as part of the listObjectV2 request in - * order to fetch the objects to be downloaded. The most common arguments - * would be: - * - MaxKeys: (int) Sets the maximum number of keys returned in the response. - * - Prefix: (string) To limit the response to keys that begin with the - * specified prefix. - * - filter: (Closure) \ - * A callable which will receive an object key as parameter and should return - * true or false in order to determine whether the object should be downloaded. + * * @return PromiseInterface */ public function downloadDirectory( string $bucket, string $destinationDirectory, array $downloadArgs, + array $config = [], ?MultipartDownloadListenerFactory $downloadListenerFactory = null, ?TransferListener $progressTracker = null, - array $config = [] ): PromiseInterface { if (!file_exists($destinationDirectory)) { @@ -224,11 +229,11 @@ public function downloadDirectory( $promises[] = $this->download( $object, $downloadArgs, - $downloadListener, - $progressTracker, [ 'minimumPartSize' => $config['minimumPartSize'] ?? 0, - ] + ], + $downloadListener, + $progressTracker, )->then(function (DownloadResult $result) use ($destinationFile) { $directory = dirname($destinationFile); if (!is_dir($directory)) { @@ -242,25 +247,97 @@ public function downloadDirectory( return Each::ofLimitAll($promises, $this->config['concurrency']); } + /** + * @param string|StreamInterface $source + * @param string $bucketTo + * @param string $key + * @param array $requestArgs + * @param array $config The config options for this upload operation. + * - mup_threshold: (int, optional) To override the default threshold + * for when to use multipart upload. + * - trackProgress: (bool, optional) To override the + * + * @param MultipartUploadListener|null $uploadListener + * @param TransferListener|null $progressTracker + * + * @return PromiseInterface + */ + public function upload( + string | StreamInterface $source, + string $bucketTo, + string $key, + array $requestArgs = [], + array $config = [], + ?MultipartUploadListener $uploadListener = null, + ?TransferListener $progressTracker = null, + ): PromiseInterface + { + if (is_string($source) && !is_readable($source)) { + throw new \InvalidArgumentException("Please provide a valid readable file path or a valid stream."); + } + + $mupThreshold = $config['mup_threshold'] ?? $this->config['multipartUploadThresholdBytes']; + if ($mupThreshold < self::MIN_PART_SIZE) { + throw new \InvalidArgumentException("\$config['mup_threshold'] must be greater than or equal to " . self::MIN_PART_SIZE); + } + + if ($source instanceof StreamInterface) { + $sourceSize = $source->getSize(); + $requestArgs['Body'] = $source; + } else { + $sourceSize = filesize($source); + $requestArgs['SourceFile'] = $source; + } + + $requestArgs['Bucket'] = $bucketTo; + $requestArgs['Key'] = $key; + $requestArgs['Size'] = $sourceSize; + if ($progressTracker === null + && ($config['trackProgress'] ?? $this->config['trackProgress'])) { + $progressTracker = $this->resolveDefaultProgressTracker( + DefaultProgressTracker::TRACKING_OPERATION_UPLOADING + ); + } + + if ($sourceSize < $mupThreshold) { + return $this->trySingleUpload( + $requestArgs, + $progressTracker + )->then(function ($result) { + $streams = get_resources("stream"); + + echo "Open file handles:\n"; + foreach ($streams as $stream) { + $metadata = stream_get_meta_data($stream); + echo "\nFile: " . ($metadata['uri'] ?? "") . "\n"; + } + + return $result; + }); + } + + throw new S3TransferException("Not implemented yet."); + } + /** * Tries an object multipart download. * * @param array $requestArgs + * @param array $config + * - minimumPartSize: (int) \ + * The minimum part size in bytes for a range multipart download. If + * this parameter is not provided then it fallbacks to the transfer + * manager `targetPartSizeBytes` config value. * @param MultipartDownloadListener|null $downloadListener * @param TransferListener|null $progressTracker - * @param array $config - * - minimumPartSize: (int) \ - * The minimum part size in bytes for a range multipart download. If - * this parameter is not provided then it fallbacks to the transfer - * manager `targetPartSizeBytes` config value. * * @return PromiseInterface */ private function tryMultipartDownload( array $requestArgs, + array $config = [], ?MultipartDownloadListener $downloadListener = null, ?TransferListener $progressTracker = null, - array $config = [] ): PromiseInterface { $multipartDownloader = MultipartDownloader::chooseDownloader( @@ -345,6 +422,53 @@ function ($result) use ($progressTracker, $requestArgs) { }); } + /** + * @param array $requestArgs + * @param TransferListener|null $progressTracker + * + * @return PromiseInterface + */ + private function trySingleUpload( + array $requestArgs, + ?TransferListener $progressTracker = null + ): PromiseInterface { + if ($progressTracker !== null) { + $progressTracker->objectTransferInitiated( + $requestArgs['Key'], + $requestArgs + ); + $command = $this->s3Client->getCommand('PutObject', $requestArgs); + return $this->s3Client->executeAsync($command)->then( + function ($result) use ($progressTracker, $requestArgs) { + $progressTracker->objectTransferProgress( + $requestArgs['Key'], + $requestArgs['Size'], + $requestArgs['Size'], + ); + + $progressTracker->objectTransferCompleted( + $requestArgs['Key'], + $requestArgs['Size'], + ); + + return $result; + } + )->otherwise(function ($reason) use ($requestArgs, $progressTracker) { + $progressTracker->objectTransferFailed( + $requestArgs['Key'], + 0, + $reason->getMessage() + ); + + return $reason; + }); + } + + $command = $this->s3Client->getCommand('PutObject', $requestArgs); + + return $this->s3Client->executeAsync($command); + } + /** * Returns a default instance of S3Client. * From 5289f7ce4db523d59f01e17448e65a4682b1564c Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Fri, 14 Feb 2025 10:00:46 -0800 Subject: [PATCH 07/19] feat: add upload directory feature - Add upload directory feature --- .../Features/S3Transfer/S3TransferManager.php | 109 ++++++++++++++++-- 1 file changed, 99 insertions(+), 10 deletions(-) diff --git a/src/S3/Features/S3Transfer/S3TransferManager.php b/src/S3/Features/S3Transfer/S3TransferManager.php index ec5052b525..51ed767101 100644 --- a/src/S3/Features/S3Transfer/S3TransferManager.php +++ b/src/S3/Features/S3Transfer/S3TransferManager.php @@ -8,10 +8,10 @@ use Aws\S3\S3ClientInterface; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; -use GuzzleHttp\Psr7\Utils; use Psr\Http\Message\StreamInterface; use function Aws\filter; use function Aws\map; +use function Aws\recursive_dir_iterator; class S3TransferManager { @@ -303,20 +303,85 @@ public function upload( return $this->trySingleUpload( $requestArgs, $progressTracker - )->then(function ($result) { - $streams = get_resources("stream"); + ); + } + + return $this->tryMultipartUpload( + $source, + $requestArgs, + $uploadListener, + $progressTracker, + ); + } + + /** + * @param string $directory + * @param string $bucketTo + * @param array $requestArgs + * @param array $config + * @param MultipartUploadListener|null $uploadListener + * @param TransferListener|null $progressTracker + * + * @return PromiseInterface + */ + public function uploadDirectory( + string $directory, + string $bucketTo, + array $requestArgs = [], + array $config = [], + ?MultipartUploadListener $uploadListener = null, + ?TransferListener $progressTracker = null, + ): PromiseInterface + { + if (!file_exists($directory)) { + throw new \InvalidArgumentException( + "Source directory '$directory' MUST exists." + ); + } + + if ($progressTracker === null + && ($config['trackProgress'] ?? $this->config['trackProgress'])) { + $progressTracker = $this->resolveDefaultProgressTracker( + DefaultProgressTracker::TRACKING_OPERATION_UPLOADING + ); + } + + $filter = null; + if (isset($config['filter'])) { + if (!is_callable($config['filter'])) { + throw new \InvalidArgumentException("The parameter \$config['filter'] must be callable."); + } + + $filter = $config['filter']; + } - echo "Open file handles:\n"; - foreach ($streams as $stream) { - $metadata = stream_get_meta_data($stream); - echo "\nFile: " . ($metadata['uri'] ?? "") . "\n"; + $files = filter( + recursive_dir_iterator($directory), + function ($file) use ($filter) { + if ($filter !== null) { + return !is_dir($file) && $filter($file); } - return $result; - }); + return !is_dir($file); + } + ); + + $promises = []; + foreach ($files as $file) { + $fileParts = explode("/", $file); + $key = end($fileParts); + $promises[] = $this->upload( + $file, + $bucketTo, + $key, + $requestArgs, + $config, + $uploadListener, + $progressTracker, + ); } - throw new S3TransferException("Not implemented yet."); + return Each::ofLimitAll($promises, $this->config['concurrency']); } /** @@ -469,6 +534,30 @@ function ($result) use ($progressTracker, $requestArgs) { return $this->s3Client->executeAsync($command); } + /** + * @param array $requestArgs + * + * @return PromiseInterface + */ + private function tryMultipartUpload( + string | StreamInterface $source, + array $requestArgs, + ?MultipartUploadListener $uploadListener = null, + ?TransferListener $progressTracker = null, + ): PromiseInterface { + return (new MultipartUploaderV2( + $this->s3Client, + $source, + $requestArgs, + [ + 'target_part_size_bytes' => $this->config['targetPartSizeBytes'], + 'concurrency' => $this->config['concurrency'], + ], + $uploadListener, + $progressTracker, + ))->promise(); + } + /** * Returns a default instance of S3Client. * From c6c07800620f3e1d73c38b03f588f3d6be65ebae Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 17 Feb 2025 08:02:47 -0800 Subject: [PATCH 08/19] feat: multipart upload and some refactor - Add a dedicated multipart upload implementation - Add transfer progress to multipart upload - Add upload directory with the required options. - Create specific response models for upload, and upload directory. - Add multipart upload test cases. - Fix transfer listener completation eval. --- ...ownloadResult.php => DownloadResponse.php} | 2 +- .../S3Transfer/MultipartDownloader.php | 2 +- .../Features/S3Transfer/MultipartUploader.php | 427 +++++++++++++++ .../Features/S3Transfer/S3TransferManager.php | 513 +++++++++++------- .../Features/S3Transfer/TransferListener.php | 1 + .../S3Transfer/UploadDirectoryResponse.php | 38 ++ src/S3/Features/S3Transfer/UploadResponse.php | 21 + .../S3Transfer/MultipartUploaderTest.php | 214 ++++++++ 8 files changed, 1029 insertions(+), 189 deletions(-) rename src/S3/Features/S3Transfer/{DownloadResult.php => DownloadResponse.php} (94%) create mode 100644 src/S3/Features/S3Transfer/MultipartUploader.php create mode 100644 src/S3/Features/S3Transfer/UploadDirectoryResponse.php create mode 100644 src/S3/Features/S3Transfer/UploadResponse.php create mode 100644 tests/S3/Features/S3Transfer/MultipartUploaderTest.php diff --git a/src/S3/Features/S3Transfer/DownloadResult.php b/src/S3/Features/S3Transfer/DownloadResponse.php similarity index 94% rename from src/S3/Features/S3Transfer/DownloadResult.php rename to src/S3/Features/S3Transfer/DownloadResponse.php index 4f40ca1623..acd0365559 100644 --- a/src/S3/Features/S3Transfer/DownloadResult.php +++ b/src/S3/Features/S3Transfer/DownloadResponse.php @@ -4,7 +4,7 @@ use Psr\Http\Message\StreamInterface; -class DownloadResult +class DownloadResponse { public function __construct( private readonly StreamInterface $content, diff --git a/src/S3/Features/S3Transfer/MultipartDownloader.php b/src/S3/Features/S3Transfer/MultipartDownloader.php index e3c1143da1..98de586c02 100644 --- a/src/S3/Features/S3Transfer/MultipartDownloader.php +++ b/src/S3/Features/S3Transfer/MultipartDownloader.php @@ -198,7 +198,7 @@ public function promise(): PromiseInterface $this->objectDownloadCompleted(); // TODO: yield the stream wrapped in a modeled transfer success response. - yield Create::promiseFor(new DownloadResult( + yield Create::promiseFor(new DownloadResponse( $this->stream, [] )); diff --git a/src/S3/Features/S3Transfer/MultipartUploader.php b/src/S3/Features/S3Transfer/MultipartUploader.php new file mode 100644 index 0000000000..47e8c72ad2 --- /dev/null +++ b/src/S3/Features/S3Transfer/MultipartUploader.php @@ -0,0 +1,427 @@ +s3Client = $s3Client; + $this->createMultipartArgs = $createMultipartArgs; + $this->uploadPartArgs = $uploadPartArgs; + $this->completeMultipartArgs = $completeMultipartArgs; + $this->config = $config; + $this->body = $this->parseBody($source); + $this->objectSizeInBytes = $objectSizeInBytes; + $this->uploadId = $uploadId; + $this->parts = $parts; + $this->progressTracker = $progressTracker; + } + + /** + * @return string|null + */ + public function getUploadId(): ?string + { + return $this->uploadId; + } + + /** + * @return array + */ + public function getParts(): array + { + return $this->parts; + } + + /** + * @return int + */ + public function getObjectSizeInBytes(): int + { + return $this->objectSizeInBytes; + } + + /** + * @return int + */ + public function getObjectBytesTransferred(): int + { + return $this->objectBytesTransferred; + } + + /** + * @return PromiseInterface + */ + public function promise(): PromiseInterface + { + return Coroutine::of(function () { + try { + yield $this->createMultipartUpload(); + yield $this->uploadParts(); + $result = yield $this->completeMultipartUpload(); + yield Create::promiseFor( + new UploadResponse($result->toArray()) + ); + } catch (Throwable $e) { + $this->uploadFailed($e); + throw $e; + } finally { + $this->callDeferredFns(); + } + }); + } + + /** + * @return PromiseInterface + */ + public function createMultipartUpload(): PromiseInterface { + $requestArgs = [...$this->createMultipartArgs]; + $this->uploadInitiated($requestArgs); + $command = $this->s3Client->getCommand( + 'CreateMultipartUpload', + $requestArgs + ); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) { + $this->uploadId = $result['UploadId']; + + return $result; + }); + } + + /** + * @return PromiseInterface + */ + public function uploadParts(): PromiseInterface + { + $this->objectSizeInBytes = 0; // To repopulate + $isSeekable = $this->body->isSeekable(); + $partSize = $this->config['part_size'] ?? self::PART_MIN_SIZE; + if ($partSize > self::PART_MAX_SIZE) { + return Create::rejectionFor( + "The part size should not exceed " . self::PART_MAX_SIZE . " bytes." + ); + } + + $commands = []; + for ($partNo = 1; + $isSeekable + ? $this->body->tell() < $this->body->getSize() + : !$this->body->eof(); + $partNo++ + ) { + if ($isSeekable) { + $readSize = min($partSize, $this->body->getSize() - $this->body->tell()); + } else { + $readSize = $partSize; + } + + $partBody = Utils::streamFor( + $this->body->read($readSize) + ); + // To make sure we do not create an empty part when + // we already reached end of file. + if (!$isSeekable && $this->body->eof() && $partBody->getSize() === 0) { + break; + } + + $uploadPartCommandArgs = [ + 'UploadId' => $this->uploadId, + 'PartNumber' => $partNo, + 'Body' => $partBody, + 'ContentLength' => $partBody->getSize(), + ] + $this->uploadPartArgs; + + $command = $this->s3Client->getCommand('UploadPart', $uploadPartCommandArgs); + $commands[] = $command; + $this->objectSizeInBytes += $partBody->getSize(); + + if ($partNo > self::PART_MAX_NUM) { + return Create::rejectionFor( + "The max number of parts has been exceeded. " . + "Max = " . self::PART_MAX_NUM + ); + } + } + + return (new CommandPool( + $this->s3Client, + $commands, + [ + 'concurrency' => $this->config['concurrency'], + 'fulfilled' => function (ResultInterface $result, $index) + use ($commands) { + $command = $commands[$index]; + $this->collectPart( + $result, + $command + ); + + // Part Upload Completed Event + $this->partUploadCompleted($result, $command['ContentLength']); + }, + 'rejected' => function (Throwable $e) { + $this->partUploadFailed($e); + } + ] + ))->promise(); + } + + /** + * @return PromiseInterface + */ + public function completeMultipartUpload(): PromiseInterface + { + $this->sortParts(); + $command = $this->s3Client->getCommand('CompleteMultipartUpload', [ + 'UploadId' => $this->uploadId, + 'MpuObjectSize' => $this->objectSizeInBytes, + 'MultipartUpload' => [ + 'Parts' => $this->parts, + ] + ] + $this->completeMultipartArgs + ); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) { + $this->uploadCompleted($result); + + return $result; + }); + } + + /** + * @return PromiseInterface + */ + public function abortMultipartUpload(): PromiseInterface { + $command = $this->s3Client->getCommand('AbortMultipartUpload', [ + ...$this->createMultipartArgs, + 'UploadId' => $this->uploadId, + ]); + + return $this->s3Client->executeAsync($command); + } + + /** + * @param ResultInterface $result + * @param CommandInterface $command + * + * @return void + */ + private function collectPart( + ResultInterface $result, + CommandInterface $command, + ): void + { + $checksumResult = $command->getName() === 'UploadPart' + ? $result + : $result[$command->getName() . 'Result']; + $partData = [ + 'PartNumber' => $command['PartNumber'], + 'ETag' => $result['ETag'], + ]; + if (isset($command['ChecksumAlgorithm'])) { + $checksumMemberName = 'Checksum' . strtoupper($command['ChecksumAlgorithm']); + $partData[$checksumMemberName] = $checksumResult[$checksumMemberName] ?? null; + } + + $this->parts[] = $partData; + } + + /** + * @return void + */ + private function sortParts(): void + { + usort($this->parts, function($partOne, $partTwo) { + return $partOne['PartNumber'] <=> $partTwo['PartNumber']; // Ascending order by age + }); + } + + /** + * @param string|StreamInterface $source + * @return StreamInterface + */ + private function parseBody(string | StreamInterface $source): StreamInterface + { + if (is_string($source)) { + // Make sure the files exists + if (!is_readable($source)) { + throw new \InvalidArgumentException( + "The source for this upload must be either a readable file or a valid stream." + ); + } + $file = Utils::tryFopen($source, 'r'); + // To make sure the resource is closed. + $this->deferFns[] = function () use ($file) { + fclose($file); + }; + $body = Utils::streamFor($file); + } elseif ($source instanceof StreamInterface) { + $body = $source; + } else { + throw new \InvalidArgumentException( + "The source must be a string or a StreamInterface." + ); + } + + return $body; + } + + /** + * @return void + */ + private function uploadInitiated(array &$requestArgs): void { + $this->objectKey = $this->createMultipartArgs['Key']; + $this->progressTracker?->objectTransferInitiated( + $this->objectKey, + $requestArgs + ); + } + + /** + * @param Throwable $reason + * + * @return void + */ + private function uploadFailed(Throwable $reason): void { + if (!empty($this->uploadId)) { + $this->abortMultipartUpload()->wait(); + } + $this->progressTracker?->objectTransferFailed( + $this->objectKey, + $this->objectBytesTransferred, + $reason + ); + } + + /** + * @param ResultInterface $result + * + * @return void + */ + private function uploadCompleted(ResultInterface $result): void { + $this->progressTracker?->objectTransferCompleted( + $this->objectKey, + $this->objectBytesTransferred, + ); + } + + /** + * @param ResultInterface $result + * @param int $partSize + * + * @return void + */ + private function partUploadCompleted(ResultInterface $result, int $partSize): void { + $this->objectBytesTransferred = $this->objectBytesTransferred + $partSize; + $this->progressTracker?->objectTransferProgress( + $this->objectKey, + $partSize, + $this->objectSizeInBytes + ); + } + + /** + * @param Throwable $reason + * + * @return void + */ + private function partUploadFailed(Throwable $reason): void + { + } + + /** + * @return void + */ + private function callDeferredFns(): void + { + foreach ($this->deferFns as $fn) { + $fn(); + } + + $this->deferFns = []; + } +} diff --git a/src/S3/Features/S3Transfer/S3TransferManager.php b/src/S3/Features/S3Transfer/S3TransferManager.php index 51ed767101..75a5b5124e 100644 --- a/src/S3/Features/S3Transfer/S3TransferManager.php +++ b/src/S3/Features/S3Transfer/S3TransferManager.php @@ -2,7 +2,6 @@ namespace Aws\S3\Features\S3Transfer; -use Aws\Command; use Aws\S3\Features\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; @@ -16,13 +15,13 @@ class S3TransferManager { private static array $defaultConfig = [ - 'targetPartSizeBytes' => 8 * 1024 * 1024, - 'multipartUploadThresholdBytes' => 16 * 1024 * 1024, - 'checksumValidationEnabled' => true, - 'checksumAlgorithm' => 'crc32', - 'multipartDownloadType' => 'partGet', + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'multipart_upload_threshold_bytes' => 16 * 1024 * 1024, + 'checksum_validation_enabled' => true, + 'checksum_algorithm' => 'crc32', + 'multipart_download_type' => 'partGet', 'concurrency' => 5, - 'trackProgress' => false, + 'track_progress' => false, 'region' => 'us-east-1', ]; @@ -35,29 +34,34 @@ class S3TransferManager private array $config; /** - * @param ?S3ClientInterface $s3Client + * @param S3ClientInterface | null $s3Client If provided as null then, + * a default client will be created where its region will be the one + * resolved from either the default from the config or the provided. * @param array $config - * - targetPartSizeBytes: (int, default=(8388608 `8MB`)) \ + * - target_part_size_bytes: (int, default=(8388608 `8MB`)) \ * The minimum part size to be used in a multipart upload/download. - * - multipartUploadThresholdBytes: (int, default=(16777216 `16 MB`)) \ + * - multipart_upload_threshold_bytes: (int, default=(16777216 `16 MB`)) \ * The threshold to decided whether a multipart upload is needed. - * - checksumValidationEnabled: (bool, default=true) \ + * - checksum_validation_enabled: (bool, default=true) \ * To decide whether a checksum validation will be applied to the response. - * - checksumAlgorithm: (string, default='crc32') \ + * - checksum_algorithm: (string, default='crc32') \ * The checksum algorithm to be used in an upload request. - * - multipartDownloadType: (string, default='partGet') + * - multipart_download_type: (string, default='partGet') * The download type to be used in a multipart download. * - concurrency: (int, default=5) \ * Maximum number of concurrent operations allowed during a multipart * upload/download. - * - trackProgress: (bool, default=false) \ + * - track_progress: (bool, default=false) \ * To enable progress tracker in a multipart upload/download. - * - progressTrackerFactory: (callable|TransferListenerFactory) \ + * - region: (string, default="us-east-2") + * - progress_tracker_factory: (callable|TransferListenerFactory) \ * A factory to create the listener which will receive notifications * based in the different stages an upload/download is. */ - public function __construct(?S3ClientInterface $s3Client, array $config = []) - { + public function __construct( + ?S3ClientInterface $s3Client = null, + array $config = [] + ) { if ($s3Client === null) { $this->s3Client = $this->defaultS3Client(); } else { @@ -67,6 +71,196 @@ public function __construct(?S3ClientInterface $s3Client, array $config = []) $this->config = $config + self::$defaultConfig; } + /** + * @param string|StreamInterface $source + * @param array $requestArgs The putObject request arguments. + * Required parameters would be: + * - Bucket: (string, required) + * - Key: (string, required) + * @param array $config The config options for this upload operation. + * - multipart_upload_threshold_bytes: (int, optional) + * To override the default threshold for when to use multipart upload. + * - target_part_size_bytes: (int, optional) To override the default + * target part size in bytes. + * - track_progress: (bool, optional) To override the default option for + * enabling progress tracking. + * @param MultipartUploadListener|null $uploadListener + * @param TransferListener|null $progressTracker + * + * @return PromiseInterface + */ + public function upload( + string | StreamInterface $source, + array $requestArgs = [], + array $config = [], + ?MultipartUploadListener $uploadListener = null, + ?TransferListener $progressTracker = null, + ): PromiseInterface + { + // Make sure the source is what is expected + if (!is_string($source) && !$source instanceof StreamInterface) { + throw new \InvalidArgumentException( + '`source` must be a string or a StreamInterface' + ); + } + // Make sure it is a valid in path in case of a string + if (is_string($source) && !is_readable($source)) { + throw new \InvalidArgumentException( + "Please provide a valid readable file path or a valid stream as source." + ); + } + // Valid required parameters + foreach (['Bucket', 'Key'] as $reqParam) { + $this->requireNonEmpty( + $requestArgs[$reqParam] ?? null, + "The `$reqParam` parameter must be provided as part of the request arguments." + ); + } + + $mupThreshold = $config['multipart_upload_threshold_bytes'] + ?? $this->config['multipart_upload_threshold_bytes']; + if ($mupThreshold < self::MIN_PART_SIZE) { + throw new \InvalidArgumentException( + "The provided config `multipart_upload_threshold_bytes`" + ."must be greater than or equal to " . self::MIN_PART_SIZE + ); + } + + if ($progressTracker === null + && ($config['track_progress'] ?? $this->config['track_progress'])) { + $progressTracker = $this->resolveDefaultProgressTracker( + DefaultProgressTracker::TRACKING_OPERATION_UPLOADING + ); + } + + if ($this->requiresMultipartUpload($source, $mupThreshold)) { + return $this->tryMultipartUpload( + $source, + $requestArgs, + $config['target_part_size_bytes'] + ?? $this->config['target_part_size_bytes'], + $uploadListener, + $progressTracker, + ); + } + + return $this->trySingleUpload( + $source, + $requestArgs, + $progressTracker + ); + } + + + /** + * @param string $directory + * @param string $bucketTo + * @param array $requestArgs + * @param array $config + * - follow_symbolic_links: (bool, optional, defaulted to false) + * - recursive: (bool, optional, defaulted to false) + * - s3_prefix: (string, optional, defaulted to null) + * - filter: (Closure(string), optional) + * - s3_delimiter: (string, optional, defaulted to `/`) + * - put_object_request_callback: (Closure, optional) + * - failure_policy: (Closure, optional) + * @param MultipartUploadListener|null $uploadListener + * @param TransferListener|null $progressTracker + * + * @return PromiseInterface + */ + public function uploadDirectory( + string $directory, + string $bucketTo, + array $requestArgs = [], + array $config = [], + ?MultipartUploadListener $uploadListener = null, + ?TransferListener $progressTracker = null, + ): PromiseInterface + { + if (!is_dir($directory)) { + throw new \InvalidArgumentException( + "Please provide a valid directory path. " + . "Provided = " . $directory + ); + } + + if ($progressTracker === null + && ($config['track_progress'] ?? $this->config['track_progress'])) { + $progressTracker = $this->resolveDefaultProgressTracker( + DefaultProgressTracker::TRACKING_OPERATION_UPLOADING + ); + } + + $filter = null; + if (isset($config['filter'])) { + if (!is_callable($config['filter'])) { + throw new \InvalidArgumentException("The parameter \$config['filter'] must be callable."); + } + + $filter = $config['filter']; + } + + $files = filter( + recursive_dir_iterator($directory), + function ($file) use ($filter) { + if ($filter !== null) { + return !is_dir($file) && $filter($file); + } + + return !is_dir($file); + } + ); + + $prefix = $config['s3_prefix'] ?? ''; + if ($prefix !== '' && !str_ends_with($prefix, '/')) { + $prefix .= '/'; + } + $delimiter = $config['s3_delimiter'] ?? '/'; + $promises = []; + $objectsUploaded = 0; + $objectsFailed = 0; + foreach ($files as $file) { + $baseDir = rtrim($directory, '/') . '/'; + $relativePath = substr($file, strlen($baseDir)); + if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { + throw new S3TransferException( + "The filename must not contain the provided delimiter `". $delimiter ."`" + ); + } + $objectKey = $prefix.$relativePath; + $objectKey = str_replace( + DIRECTORY_SEPARATOR, + $delimiter, + $objectKey + ); + $promises[] = $this->upload( + $file, + [ + ...$requestArgs, + 'Bucket' => $bucketTo, + 'Key' => $objectKey, + ], + $config, + $uploadListener, + $progressTracker, + )->then(function ($result) use (&$objectsUploaded) { + $objectsUploaded++; + + return $result; + })->otherwise(function ($reason) use (&$objectsFailed) { + $objectsFailed++; + + return $reason; + }); + } + + return Each::ofLimitAll($promises, $this->config['concurrency']) + ->then(function ($results) use ($objectsUploaded, $objectsFailed) { + return new UploadDirectoryResponse($objectsUploaded, $objectsFailed); + }); + } + /** * @param string|array $source The object to be downloaded from S3. * It can be either a string with a S3 URI or an array with a Bucket and Key @@ -75,19 +269,19 @@ public function __construct(?S3ClientInterface $s3Client, array $config = []) * of each get object operation, except for the bucket and key, which * are already provided as the source. * @param array $config The configuration to be used for this operation. - * - trackProgress: (bool) \ + * - track_progress: (bool) \ * Overrides the config option set in the transfer manager instantiation * to decide whether transfer progress should be tracked. If a `progressListenerFactory` * was not provided when the transfer manager instance was created - * and trackProgress resolved as true then, a default progress listener + * and track_progress resolved as true then, a default progress listener * implementation will be used. * - minimumPartSize: (int) \ * The minimum part size in bytes to be used in a range multipart download. * @param MultipartDownloadListener|null $downloadListener A multipart download * specific listener of the different states a multipart download can be. * @param TransferListener|null $progressTracker A transfer listener implementation - * aimed to track the progress of a transfer. If not provided and trackProgress - * is resolved as true then, the default progressTrackerFactory will be used. + * aimed to track the progress of a transfer. If not provided and track_progress + * is resolved as true then, the default progress_tracker_factory will be used. * * @return PromiseInterface */ @@ -111,7 +305,7 @@ public function download( } if ($progressTracker === null - && ($config['trackProgress'] ?? $this->config['trackProgress'])) { + && ($config['track_progress'] ?? $this->config['track_progress'])) { $progressTracker = $this->resolveDefaultProgressTracker( DefaultProgressTracker::TRACKING_OPERATION_DOWNLOADING ); @@ -142,11 +336,11 @@ public function download( * as part of each get object request sent to the service, except for the * bucket and key which will be resolved internally. * @param array $config The config options for this download directory operation. \ - * - trackProgress: (bool) \ + * - track_progress: (bool) \ * Overrides the config option set in the transfer manager instantiation * to decide whether transfer progress should be tracked. If a `progressListenerFactory` * was not provided when the transfer manager instance was created - * and trackProgress resolved as true then, a default progress listener + * and track_progress resolved as true then, a default progress listener * implementation will be used. * - minimumPartSize: (int) \ * The minimum part size in bytes to be used in a range multipart download. @@ -183,7 +377,7 @@ public function downloadDirectory( } if ($progressTracker === null - && ($config['trackProgress'] ?? $this->config['trackProgress'])) { + && ($config['track_progress'] ?? $this->config['track_progress'])) { $progressTracker = $this->resolveDefaultProgressTracker( DefaultProgressTracker::TRACKING_OPERATION_DOWNLOADING ); @@ -234,7 +428,7 @@ public function downloadDirectory( ], $downloadListener, $progressTracker, - )->then(function (DownloadResult $result) use ($destinationFile) { + )->then(function (DownloadResponse $result) use ($destinationFile) { $directory = dirname($destinationFile); if (!is_dir($directory)) { mkdir($directory, 0777, true); @@ -247,143 +441,6 @@ public function downloadDirectory( return Each::ofLimitAll($promises, $this->config['concurrency']); } - /** - * @param string|StreamInterface $source - * @param string $bucketTo - * @param string $key - * @param array $requestArgs - * @param array $config The config options for this upload operation. - * - mup_threshold: (int, optional) To override the default threshold - * for when to use multipart upload. - * - trackProgress: (bool, optional) To override the - * - * @param MultipartUploadListener|null $uploadListener - * @param TransferListener|null $progressTracker - * - * @return PromiseInterface - */ - public function upload( - string | StreamInterface $source, - string $bucketTo, - string $key, - array $requestArgs = [], - array $config = [], - ?MultipartUploadListener $uploadListener = null, - ?TransferListener $progressTracker = null, - ): PromiseInterface - { - if (is_string($source) && !is_readable($source)) { - throw new \InvalidArgumentException("Please provide a valid readable file path or a valid stream."); - } - - $mupThreshold = $config['mup_threshold'] ?? $this->config['multipartUploadThresholdBytes']; - if ($mupThreshold < self::MIN_PART_SIZE) { - throw new \InvalidArgumentException("\$config['mup_threshold'] must be greater than or equal to " . self::MIN_PART_SIZE); - } - - if ($source instanceof StreamInterface) { - $sourceSize = $source->getSize(); - $requestArgs['Body'] = $source; - } else { - $sourceSize = filesize($source); - $requestArgs['SourceFile'] = $source; - } - - $requestArgs['Bucket'] = $bucketTo; - $requestArgs['Key'] = $key; - $requestArgs['Size'] = $sourceSize; - if ($progressTracker === null - && ($config['trackProgress'] ?? $this->config['trackProgress'])) { - $progressTracker = $this->resolveDefaultProgressTracker( - DefaultProgressTracker::TRACKING_OPERATION_UPLOADING - ); - } - - if ($sourceSize < $mupThreshold) { - return $this->trySingleUpload( - $requestArgs, - $progressTracker - ); - } - - return $this->tryMultipartUpload( - $source, - $requestArgs, - $uploadListener, - $progressTracker, - ); - } - - /** - * @param string $directory - * @param string $bucketTo - * @param array $requestArgs - * @param array $config - * @param MultipartUploadListener|null $uploadListener - * @param TransferListener|null $progressTracker - * - * @return PromiseInterface - */ - public function uploadDirectory( - string $directory, - string $bucketTo, - array $requestArgs = [], - array $config = [], - ?MultipartUploadListener $uploadListener = null, - ?TransferListener $progressTracker = null, - ): PromiseInterface - { - if (!file_exists($directory)) { - throw new \InvalidArgumentException( - "Source directory '$directory' MUST exists." - ); - } - - if ($progressTracker === null - && ($config['trackProgress'] ?? $this->config['trackProgress'])) { - $progressTracker = $this->resolveDefaultProgressTracker( - DefaultProgressTracker::TRACKING_OPERATION_UPLOADING - ); - } - - $filter = null; - if (isset($config['filter'])) { - if (!is_callable($config['filter'])) { - throw new \InvalidArgumentException("The parameter \$config['filter'] must be callable."); - } - - $filter = $config['filter']; - } - - $files = filter( - recursive_dir_iterator($directory), - function ($file) use ($filter) { - if ($filter !== null) { - return !is_dir($file) && $filter($file); - } - - return !is_dir($file); - } - ); - - $promises = []; - foreach ($files as $file) { - $fileParts = explode("/", $file); - $key = end($fileParts); - $promises[] = $this->upload( - $file, - $bucketTo, - $key, - $requestArgs, - $config, - $uploadListener, - $progressTracker, - ); - } - - return Each::ofLimitAll($promises, $this->config['concurrency']); - } - /** * Tries an object multipart download. * @@ -392,7 +449,7 @@ function ($file) use ($filter) { * - minimumPartSize: (int) \ * The minimum part size in bytes for a range multipart download. If * this parameter is not provided then it fallbacks to the transfer - * manager `targetPartSizeBytes` config value. + * manager `target_part_size_bytes` config value. * @param MultipartDownloadListener|null $downloadListener * @param TransferListener|null $progressTracker * @@ -407,13 +464,10 @@ private function tryMultipartDownload( { $multipartDownloader = MultipartDownloader::chooseDownloader( s3Client: $this->s3Client, - multipartDownloadType: $this->config['multipartDownloadType'], + multipartDownloadType: $this->config['multipart_download_type'], requestArgs: $requestArgs, config: [ - 'minimumPartSize' => max( - $config['minimumPartSize'] ?? 0, - $this->config['targetPartSizeBytes'] - ) + 'target_part_size_bytes' => $config['target_part_size_bytes'] ?? 0, ], listener: $downloadListener, progressTracker: $progressTracker, @@ -457,7 +511,7 @@ function ($result) use ($progressTracker, $requestArgs) { $result['Content-Length'] ?? 0, ); - return new DownloadResult( + return new DownloadResponse( content: $result['Body'], metadata: $result['@metadata'], ); @@ -480,7 +534,7 @@ function ($result) use ($progressTracker, $requestArgs) { return $this->s3Client->executeAsync($command) ->then(function ($result) { - return new DownloadResult( + return new DownloadResponse( content: $result['Body'], metadata: $result['@metadata'], ); @@ -488,35 +542,50 @@ function ($result) use ($progressTracker, $requestArgs) { } /** + * @param string|StreamInterface $source * @param array $requestArgs * @param TransferListener|null $progressTracker * * @return PromiseInterface */ private function trySingleUpload( + string | StreamInterface $source, array $requestArgs, ?TransferListener $progressTracker = null ): PromiseInterface { + if (is_string($source) && is_readable($source)) { + $requestArgs['SourceFile'] = $source; + $objectSize = filesize($source); + } elseif ($source instanceof StreamInterface && $source->isSeekable()) { + $requestArgs['Body'] = $source; + $objectSize = $source->getSize(); + } else { + throw new S3TransferException( + "Unable to process upload request due to the type of the source" + ); + } + if ($progressTracker !== null) { $progressTracker->objectTransferInitiated( $requestArgs['Key'], $requestArgs ); + $command = $this->s3Client->getCommand('PutObject', $requestArgs); return $this->s3Client->executeAsync($command)->then( - function ($result) use ($progressTracker, $requestArgs) { + function ($result) use ($objectSize, $progressTracker, $requestArgs) { $progressTracker->objectTransferProgress( $requestArgs['Key'], - $requestArgs['Size'], - $requestArgs['Size'], + $objectSize, + $objectSize, ); $progressTracker->objectTransferCompleted( $requestArgs['Key'], - $requestArgs['Size'], + $objectSize, ); - return $result; + return new UploadResponse($result->toArray()); } )->otherwise(function ($reason) use ($requestArgs, $progressTracker) { $progressTracker->objectTransferFailed( @@ -531,33 +600,78 @@ function ($result) use ($progressTracker, $requestArgs) { $command = $this->s3Client->getCommand('PutObject', $requestArgs); - return $this->s3Client->executeAsync($command); + return $this->s3Client->executeAsync($command) + ->then(function ($result) { + return new UploadResponse($result->toArray()); + }); } /** + * @param string|StreamInterface $source * @param array $requestArgs + * @param array $config + * @param MultipartUploadListener|null $uploadListener + * @param TransferListener|null $progressTracker * * @return PromiseInterface */ private function tryMultipartUpload( string | StreamInterface $source, array $requestArgs, + int $partSizeBytes, ?MultipartUploadListener $uploadListener = null, ?TransferListener $progressTracker = null, ): PromiseInterface { - return (new MultipartUploaderV2( + $createMultipartArgs = [...$requestArgs]; + $uploadPartArgs = [...$requestArgs]; + $completeMultipartArgs = [...$requestArgs]; + if ($this->containsChecksum($requestArgs)) { + $completeMultipartArgs['ChecksumType'] = 'FULL_OBJECT'; + } + + return (new MultipartUploader( $this->s3Client, - $source, - $requestArgs, + $createMultipartArgs, + $uploadPartArgs, + $completeMultipartArgs, [ - 'target_part_size_bytes' => $this->config['targetPartSizeBytes'], + 'part_size_bytes' => $partSizeBytes, 'concurrency' => $this->config['concurrency'], ], - $uploadListener, - $progressTracker, + $source, + progressTracker: $progressTracker, ))->promise(); } + /** + * @param string|StreamInterface $source + * @param int $mupThreshold + * + * @return bool + */ + private function requiresMultipartUpload( + string | StreamInterface $source, + int $mupThreshold + ): bool + { + if (is_string($source)) { + $sourceSize = filesize($source); + + return $sourceSize > $mupThreshold; + } elseif ($source instanceof StreamInterface) { + // When the stream's size is unknown then we could try a multipart upload. + if (empty($source->getSize())) { + return true; + } + + if (!empty($source->getSize())) { + return $source->getSize() > $mupThreshold; + } + } + + return false; + } + /** * Returns a default instance of S3Client. * @@ -632,7 +746,7 @@ private function s3UriAsBucketAndKey(string $uri): array /** * Resolves the progress tracker to be used in the - * transfer operation if `$trackProgress` is true. + * transfer operation if `$track_progress` is true. * * @param string $trackingOperation * @@ -642,12 +756,12 @@ private function resolveDefaultProgressTracker( string $trackingOperation ): ?TransferListener { - $progressTrackerFactory = $this->config['progressTrackerFactory'] ?? null; - if ($progressTrackerFactory === null) { + $progress_tracker_factory = $this->config['progress_tracker_factory'] ?? null; + if ($progress_tracker_factory === null) { return (new DefaultProgressTracker(trackingOperation: $trackingOperation))->getTransferListener(); } - return $progressTrackerFactory([]); + return $progress_tracker_factory([]); } /** @@ -683,4 +797,29 @@ private function resolvesOutsideTargetDirectory( return false; } + + /** + * Verifies if a checksum was provided. + * + * @param array $requestArgs + * + * @return bool + */ + private function containsChecksum(array $requestArgs): bool + { + $algorithms = [ + 'ChecksumCRC32', + 'ChecksumCRC32C', + 'ChecksumCRC64NVME', + 'ChecksumSHA1', + 'ChecksumSHA256', + ]; + foreach ($algorithms as $algorithm) { + if (isset($requestArgs[$algorithm])) { + return true; + } + } + + return false; + } } \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/TransferListener.php b/src/S3/Features/S3Transfer/TransferListener.php index 892f159085..30b620c799 100644 --- a/src/S3/Features/S3Transfer/TransferListener.php +++ b/src/S3/Features/S3Transfer/TransferListener.php @@ -174,6 +174,7 @@ public function objectTransferFailed( ): void { $this->objectsTransferFailed++; + $this->validateTransferComplete(); $this->notify('onObjectTransferFailed', [ $objectKey, $objectBytesTransferred, diff --git a/src/S3/Features/S3Transfer/UploadDirectoryResponse.php b/src/S3/Features/S3Transfer/UploadDirectoryResponse.php new file mode 100644 index 0000000000..df8d804cf8 --- /dev/null +++ b/src/S3/Features/S3Transfer/UploadDirectoryResponse.php @@ -0,0 +1,38 @@ +objectsUploaded = $objectsUploaded; + $this->objectsFailed = $objectsFailed; + } + + /** + * @return int + */ + public function getObjectsUploaded(): int + { + return $this->objectsUploaded; + } + + /** + * @return int + */ + public function getObjectsFailed(): int + { + return $this->objectsFailed; + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/UploadResponse.php b/src/S3/Features/S3Transfer/UploadResponse.php new file mode 100644 index 0000000000..f39c16362d --- /dev/null +++ b/src/S3/Features/S3Transfer/UploadResponse.php @@ -0,0 +1,21 @@ +uploadResponse = $uploadResponse; + } + + public function getUploadResponse(): array + { + return $this->uploadResponse; + } +} \ No newline at end of file diff --git a/tests/S3/Features/S3Transfer/MultipartUploaderTest.php b/tests/S3/Features/S3Transfer/MultipartUploaderTest.php new file mode 100644 index 0000000000..0404393b9e --- /dev/null +++ b/tests/S3/Features/S3Transfer/MultipartUploaderTest.php @@ -0,0 +1,214 @@ +getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + -> willReturnCallback(function ($command) + { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'FooUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result([ + 'ETag' => 'FooETag' + ])); + } + + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + -> willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $requestArgs = [ + 'Key' => 'FooKey', + 'Bucket' => 'FooBucket', + ]; + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + $requestArgs, + $requestArgs, + $config + [ + 'concurrency' => 3, + ], + $stream + ); + $multipartUploader->promise()->wait(); + + $this->assertCount($expected['parts'], $multipartUploader->getParts()); + $this->assertEquals($expected['bytesUploaded'], $multipartUploader->getObjectBytesTransferred()); + $this->assertEquals($expected['bytesUploaded'], $multipartUploader->getObjectSizeInBytes()); + } + + /** + * @return array[] + */ + public function multipartUploadProvider(): array { + return [ + '5_parts_upload' => [ + 'stream' => Utils::streamFor( + str_repeat('*', 1024 * 5), + ), + 'config' => [ + 'part_size' => 1024 + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 5, + 'bytesUploaded' => 1024 * 5, + ] + ], + '100_parts_upload' => [ + 'stream' => Utils::streamFor( + str_repeat('*', 1024 * 100), + ), + 'config' => [ + 'part_size' => 1024 + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 100, + 'bytesUploaded' => 1024 * 100, + ] + ], + '5_parts_no_seekable_stream' => [ + 'stream' => new NoSeekStream( + Utils::streamFor( + str_repeat('*', 1024 * 5) + ) + ), + 'config' => [ + 'part_size' => 1024 + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 5, + 'bytesUploaded' => 1024 * 5, + ] + ], + '100_parts_no_seekable_stream' => [ + 'stream' => new NoSeekStream( + Utils::streamFor( + str_repeat('*', 1024 * 100) + ) + ), + 'config' => [ + 'part_size' => 1024 + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 100, + 'bytesUploaded' => 1024 * 100, + ] + ] + ]; + } + + /** + * @return S3ClientInterface + */ + private function multipartUploadS3Client(): S3ClientInterface + { + return new S3Client([ + 'region' => 'us-east-2', + 'http_handler' => function (RequestInterface $request) { + $uri = $request->getUri(); + // Create multipart upload + if ($uri->getQuery() === 'uploads') { + $body = << + + Foo + Test file + FooUploadId + +EOF; + return new Response(200, [], $body); + } + + // Parts upload + if (str_starts_with($request->getUri(), 'uploadId=') && str_contains($request->getUri(), 'partNumber=')) { + return new Response(200, ['ETag' => random_bytes(16)]); + } + + // Complete multipart upload + return new Response(200, [], null); + } + ]); + } + + /** + * @return void + */ + public function testInvalidSourceStringThrowsException(): void + { + $nonExistentFile = '/path/to/nonexistent/file.txt'; + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage( + "The source for this upload must be either a readable file or a valid stream." + ); + new MultipartUploader( + $this->multipartUploadS3Client(), + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + [], + [], + [], + $nonExistentFile + ); + } + + /** + * @return void + */ + public function testInvalidSourceTypeThrowsException(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage( + "The source for this upload must be either a readable file or a valid stream." + ); + new MultipartUploader( + $this->multipartUploadS3Client(), + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + [], + [], + [], + 12345 + ); + } +} \ No newline at end of file From 034b50d942daeeb8d234a59755e7a845b1228448 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 17 Feb 2025 08:07:26 -0800 Subject: [PATCH 09/19] chore: short namespace Short namespace from `Aws\S3\Features\S3Transfer` to `Aws\S3\S3Transfer`. --- src/S3/{Features => }/S3Transfer/ConsoleProgressBar.php | 2 +- .../{Features => }/S3Transfer/DefaultProgressTracker.php | 2 +- src/S3/{Features => }/S3Transfer/DownloadResponse.php | 2 +- .../S3Transfer/Exceptions/S3TransferException.php | 2 +- src/S3/{Features => }/S3Transfer/ListenerNotifier.php | 2 +- .../S3Transfer/MultipartDownloadListener.php | 2 +- .../S3Transfer/MultipartDownloadListenerFactory.php | 2 +- src/S3/{Features => }/S3Transfer/MultipartDownloader.php | 2 +- .../{Features => }/S3Transfer/MultipartUploadListener.php | 2 +- src/S3/{Features => }/S3Transfer/MultipartUploader.php | 2 +- .../{Features => }/S3Transfer/ObjectProgressTracker.php | 2 +- .../S3Transfer/PartGetMultipartDownloader.php | 2 +- src/S3/{Features => }/S3Transfer/ProgressBar.php | 2 +- src/S3/{Features => }/S3Transfer/ProgressBarFactory.php | 2 +- .../{Features => }/S3Transfer/ProgressListenerHelper.php | 2 +- .../S3Transfer/RangeGetMultipartDownloader.php | 4 ++-- src/S3/{Features => }/S3Transfer/S3TransferManager.php | 4 ++-- src/S3/{Features => }/S3Transfer/TransferListener.php | 2 +- .../{Features => }/S3Transfer/TransferListenerFactory.php | 2 +- .../{Features => }/S3Transfer/UploadDirectoryResponse.php | 2 +- src/S3/{Features => }/S3Transfer/UploadResponse.php | 2 +- .../{Features => }/S3Transfer/ConsoleProgressBarTest.php | 4 ++-- .../S3Transfer/DefaultProgressTrackerTest.php | 6 +++--- .../S3Transfer/MultipartDownloadListenerTest.php | 4 ++-- .../{Features => }/S3Transfer/MultipartDownloaderTest.php | 4 ++-- .../{Features => }/S3Transfer/MultipartUploaderTest.php | 4 ++-- .../S3Transfer/ObjectProgressTrackerTest.php | 8 ++++---- .../S3/{Features => }/S3Transfer/TransferListenerTest.php | 5 ++--- 28 files changed, 40 insertions(+), 41 deletions(-) rename src/S3/{Features => }/S3Transfer/ConsoleProgressBar.php (99%) rename src/S3/{Features => }/S3Transfer/DefaultProgressTracker.php (99%) rename src/S3/{Features => }/S3Transfer/DownloadResponse.php (91%) rename src/S3/{Features => }/S3Transfer/Exceptions/S3TransferException.php (56%) rename src/S3/{Features => }/S3Transfer/ListenerNotifier.php (76%) rename src/S3/{Features => }/S3Transfer/MultipartDownloadListener.php (99%) rename src/S3/{Features => }/S3Transfer/MultipartDownloadListenerFactory.php (82%) rename src/S3/{Features => }/S3Transfer/MultipartDownloader.php (99%) rename src/S3/{Features => }/S3Transfer/MultipartUploadListener.php (98%) rename src/S3/{Features => }/S3Transfer/MultipartUploader.php (99%) rename src/S3/{Features => }/S3Transfer/ObjectProgressTracker.php (99%) rename src/S3/{Features => }/S3Transfer/PartGetMultipartDownloader.php (96%) rename src/S3/{Features => }/S3Transfer/ProgressBar.php (86%) rename src/S3/{Features => }/S3Transfer/ProgressBarFactory.php (69%) rename src/S3/{Features => }/S3Transfer/ProgressListenerHelper.php (96%) rename src/S3/{Features => }/S3Transfer/RangeGetMultipartDownloader.php (97%) rename src/S3/{Features => }/S3Transfer/S3TransferManager.php (99%) rename src/S3/{Features => }/S3Transfer/TransferListener.php (99%) rename src/S3/{Features => }/S3Transfer/TransferListenerFactory.php (73%) rename src/S3/{Features => }/S3Transfer/UploadDirectoryResponse.php (94%) rename src/S3/{Features => }/S3Transfer/UploadResponse.php (89%) rename tests/S3/{Features => }/S3Transfer/ConsoleProgressBarTest.php (98%) rename tests/S3/{Features => }/S3Transfer/DefaultProgressTrackerTest.php (97%) rename tests/S3/{Features => }/S3Transfer/MultipartDownloadListenerTest.php (98%) rename tests/S3/{Features => }/S3Transfer/MultipartDownloaderTest.php (98%) rename tests/S3/{Features => }/S3Transfer/MultipartUploaderTest.php (98%) rename tests/S3/{Features => }/S3Transfer/ObjectProgressTrackerTest.php (94%) rename tests/S3/{Features => }/S3Transfer/TransferListenerTest.php (98%) diff --git a/src/S3/Features/S3Transfer/ConsoleProgressBar.php b/src/S3/S3Transfer/ConsoleProgressBar.php similarity index 99% rename from src/S3/Features/S3Transfer/ConsoleProgressBar.php rename to src/S3/S3Transfer/ConsoleProgressBar.php index e370362651..ebbadd771b 100644 --- a/src/S3/Features/S3Transfer/ConsoleProgressBar.php +++ b/src/S3/S3Transfer/ConsoleProgressBar.php @@ -1,6 +1,6 @@ Date: Fri, 21 Feb 2025 17:22:04 -0800 Subject: [PATCH 10/19] chore: refactor and address feedback - Implement progress tracker based on SEP spec. - Add a default progress bar implementation. - Add different progress tracker formats: -- Plain progress format: [|progress_bar|] |percent|% -- Transfer progress format: [|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit| -- Colored progress format: |object_name|:\n\033|color_code|[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit| |message|\033[0m - Add a default single progress tracker implementation. - Add a default multi progress tracker implementation for tracking directory transfers. - Include tests unit just for console progress bar. --- src/S3/S3Transfer/ConsoleProgressBar.php | 142 ------- src/S3/S3Transfer/DefaultProgressTracker.php | 292 --------------- src/S3/S3Transfer/ListenerNotifier.php | 8 - .../S3Transfer/MultipartDownloadListener.php | 228 ------------ .../MultipartDownloadListenerFactory.php | 11 - src/S3/S3Transfer/MultipartDownloader.php | 313 ++++------------ src/S3/S3Transfer/MultipartUploadListener.php | 58 --- src/S3/S3Transfer/MultipartUploader.php | 217 +++++++---- src/S3/S3Transfer/ObjectProgressTracker.php | 189 ---------- .../ColoredTransferProgressBarFormat.php | 45 +++ .../Progress/ConsoleProgressBar.php | 103 ++++++ .../Progress/MultiProgressTracker.php | 169 +++++++++ .../Progress/PlainProgressBarFormat.php | 24 ++ .../Progress/ProgressBarColorEnum.php | 16 + .../S3Transfer/Progress/ProgressBarFormat.php | 88 +++++ .../Progress/ProgressBarInterface.php | 31 ++ .../Progress/ProgressTrackerInterface.php | 13 + .../Progress/SingleProgressTracker.php | 216 +++++++++++ .../Progress/TransferProgressBarFormat.php | 33 ++ .../Progress/TransferProgressSnapshot.php | 78 ++++ src/S3/S3Transfer/ProgressBar.php | 14 - src/S3/S3Transfer/ProgressBarFactory.php | 8 - src/S3/S3Transfer/ProgressListenerHelper.php | 45 --- .../RangeGetMultipartDownloader.php | 33 +- src/S3/S3Transfer/S3TransferManager.php | 345 +++++++++--------- src/S3/S3Transfer/TransferListener.php | 289 ++------------- src/S3/S3Transfer/TransferListenerFactory.php | 8 - .../S3Transfer/TransferListenerNotifier.php | 74 ++++ .../S3/S3Transfer/ConsoleProgressBarTest.php | 228 ------------ .../Progress/ConsoleProgressBarTest.php | 261 +++++++++++++ 30 files changed, 1570 insertions(+), 2009 deletions(-) delete mode 100644 src/S3/S3Transfer/ConsoleProgressBar.php delete mode 100644 src/S3/S3Transfer/DefaultProgressTracker.php delete mode 100644 src/S3/S3Transfer/ListenerNotifier.php delete mode 100644 src/S3/S3Transfer/MultipartDownloadListener.php delete mode 100644 src/S3/S3Transfer/MultipartDownloadListenerFactory.php delete mode 100644 src/S3/S3Transfer/MultipartUploadListener.php delete mode 100644 src/S3/S3Transfer/ObjectProgressTracker.php create mode 100644 src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php create mode 100644 src/S3/S3Transfer/Progress/ConsoleProgressBar.php create mode 100644 src/S3/S3Transfer/Progress/MultiProgressTracker.php create mode 100644 src/S3/S3Transfer/Progress/PlainProgressBarFormat.php create mode 100644 src/S3/S3Transfer/Progress/ProgressBarColorEnum.php create mode 100644 src/S3/S3Transfer/Progress/ProgressBarFormat.php create mode 100644 src/S3/S3Transfer/Progress/ProgressBarInterface.php create mode 100644 src/S3/S3Transfer/Progress/ProgressTrackerInterface.php create mode 100644 src/S3/S3Transfer/Progress/SingleProgressTracker.php create mode 100644 src/S3/S3Transfer/Progress/TransferProgressBarFormat.php create mode 100644 src/S3/S3Transfer/Progress/TransferProgressSnapshot.php delete mode 100644 src/S3/S3Transfer/ProgressBar.php delete mode 100644 src/S3/S3Transfer/ProgressBarFactory.php delete mode 100644 src/S3/S3Transfer/ProgressListenerHelper.php delete mode 100644 src/S3/S3Transfer/TransferListenerFactory.php create mode 100644 src/S3/S3Transfer/TransferListenerNotifier.php delete mode 100644 tests/S3/S3Transfer/ConsoleProgressBarTest.php create mode 100644 tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php diff --git a/src/S3/S3Transfer/ConsoleProgressBar.php b/src/S3/S3Transfer/ConsoleProgressBar.php deleted file mode 100644 index ebbadd771b..0000000000 --- a/src/S3/S3Transfer/ConsoleProgressBar.php +++ /dev/null @@ -1,142 +0,0 @@ - [ - 'format' => "[|progress_bar|] |percent|%", - 'parameters' => [] - ], - 'transfer_format' => [ - 'format' => "[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit|", - 'parameters' => [ - 'transferred', - 'tobe_transferred', - 'unit' - ] - ], - 'colored_transfer_format' => [ - 'format' => "\033|color_code|[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit| |message|\033[0m", - 'parameters' => [ - 'transferred', - 'tobe_transferred', - 'unit', - 'color_code', - 'message' - ] - ], - ]; - - /** @var string */ - private string $progressBarChar; - - /** @var int */ - private int $progressBarWidth; - - /** @var int */ - private int $percentCompleted; - - /** @var ?array */ - private ?array $format; - - /** @var array */ - private array $args; - - /** - * @param ?string $progressBarChar - * @param ?int $progressBarWidth - * @param ?int $percentCompleted - * @param array|null $format - * @param array $args - */ - public function __construct( - ?string $progressBarChar = null, - ?int $progressBarWidth = null, - ?int $percentCompleted = null, - ?array $format = null, - array $args = [], - ) { - $this->progressBarChar = $progressBarChar ?? '#'; - $this->progressBarWidth = $progressBarWidth ?? 25; - $this->percentCompleted = $percentCompleted ?? 0; - $this->format = $format ?? self::$formats['transfer_format']; - $this->args = $args ?? []; - } - - /** - * Set current progress percent. - * - * @param int $percent - * - * @return void - */ - public function setPercentCompleted(int $percent): void - { - $this->percentCompleted = max(0, min(100, $percent)); - } - - /** - * @param array $args - * - * @return void - */ - public function setArgs(array $args): void - { - $this->args = $args; - } - - /** - * Sets an argument. - * - * @param string $key - * @param mixed $value - * - * @return void - */ - public function setArg(string $key, mixed $value): void - { - $this->args[$key] = $value; - } - - private function renderProgressBar(): string - { - $filledWidth = (int) round(($this->progressBarWidth * $this->percentCompleted) / 100); - return str_repeat($this->progressBarChar, $filledWidth) - . str_repeat(' ', $this->progressBarWidth - $filledWidth); - } - - /** - * - * @return string - */ - public function getPaintedProgress(): string - { - foreach ($this->format['parameters'] as $param) { - if (!array_key_exists($param, $this->args)) { - $this->args[$param] = ''; - } - } - - $replacements = [ - '|progress_bar|' => $this->renderProgressBar(), - '|percent|' => $this->percentCompleted, - ]; - - foreach ($this->format['parameters'] as $param) { - $replacements["|$param|"] = $this->args[$param] ?? ''; - } - - return strtr($this->format['format'], $replacements); - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/DefaultProgressTracker.php b/src/S3/S3Transfer/DefaultProgressTracker.php deleted file mode 100644 index ed2d28683c..0000000000 --- a/src/S3/S3Transfer/DefaultProgressTracker.php +++ /dev/null @@ -1,292 +0,0 @@ -clear(); - $this->initializeListener(); - $this->progressBarFactory = $progressBarFactory ?? $this->defaultProgressBarFactory(); - if (get_resource_type($output) !== 'stream') { - throw new \InvalidArgumentException("The type for $output must be a stream"); - } - - $this->output = $output; - if (!in_array(strtolower($trackingOperation), [ - strtolower(self::TRACKING_OPERATION_DOWNLOADING), - strtolower(self::TRACKING_OPERATION_UPLOADING), - ], true)) { - throw new \InvalidArgumentException("Tracking operation '$trackingOperation' should be one of 'Downloading', 'Uploading'"); - } - - $this->trackingOperation = $trackingOperation; - } - - private function initializeListener(): void - { - $this->transferListener = new TransferListener(); - // Object transfer initialized - $this->transferListener->onObjectTransferInitiated = $this->objectTransferInitiated(); - // Object transfer made progress - $this->transferListener->onObjectTransferProgress = $this->objectTransferProgress(); - $this->transferListener->onObjectTransferFailed = $this->objectTransferFailed(); - $this->transferListener->onObjectTransferCompleted = $this->objectTransferCompleted(); - } - - /** - * @return TransferListener - */ - public function getTransferListener(): TransferListener - { - return $this->transferListener; - } - - /** - * @return int - */ - public function getTotalBytesTransferred(): int - { - return $this->totalBytesTransferred; - } - - /** - * @return int - */ - public function getObjectsTotalSizeInBytes(): int - { - return $this->objectsTotalSizeInBytes; - } - - /** - * @return int - */ - public function getObjectsInProgress(): int - { - return $this->objectsInProgress; - } - - /** - * @return int - */ - public function getObjectsCount(): int - { - return $this->objectsCount; - } - - /** - * @return int - */ - public function getTransferPercentCompleted(): int - { - return $this->transferPercentCompleted; - } - - /** - * - * @return Closure - */ - private function objectTransferInitiated(): Closure - { - return function (string $objectKey, array &$requestArgs) { - $progressBarFactoryFn = $this->progressBarFactory; - $this->objects[$objectKey] = new ObjectProgressTracker( - objectKey: $objectKey, - objectBytesTransferred: 0, - objectSizeInBytes: 0, - status: 'initiated', - progressBar: $progressBarFactoryFn() - ); - $this->objectsInProgress++; - $this->objectsCount++; - - $this->showProgress(); - }; - } - - /** - * @return Closure - */ - private function objectTransferProgress(): Closure - { - return function ( - string $objectKey, - int $objectBytesTransferred, - int $objectSizeInBytes - ): void { - $objectProgressTracker = $this->objects[$objectKey]; - if ($objectProgressTracker->getObjectSizeInBytes() === 0) { - $objectProgressTracker->setObjectSizeInBytes($objectSizeInBytes); - // Increment objectsTotalSizeInBytes just the first time we set - // the object total size. - $this->objectsTotalSizeInBytes = - $this->objectsTotalSizeInBytes + $objectSizeInBytes; - } - $objectProgressTracker->incrementTotalBytesTransferred( - $objectBytesTransferred - ); - $objectProgressTracker->setStatus('progress'); - - $this->increaseBytesTransferred($objectBytesTransferred); - - $this->showProgress(); - }; - } - - /** - * @return Closure - */ - public function objectTransferFailed(): Closure - { - return function ( - string $objectKey, - int $totalObjectBytesTransferred, - \Throwable | string $reason - ): void { - $objectProgressTracker = $this->objects[$objectKey]; - $objectProgressTracker->setStatus('failed', $reason); - - $this->objectsInProgress--; - - $this->showProgress(); - }; - } - - /** - * @return Closure - */ - public function objectTransferCompleted(): Closure - { - return function ( - string $objectKey, - int $objectBytesTransferred, - ): void { - $objectProgressTracker = $this->objects[$objectKey]; - $objectProgressTracker->setStatus('completed'); - $this->showProgress(); - }; - } - - /** - * Clear the internal state holders. - * - * @return void - */ - public function clear(): void - { - $this->objects = []; - $this->totalBytesTransferred = 0; - $this->objectsTotalSizeInBytes = 0; - $this->objectsInProgress = 0; - $this->objectsCount = 0; - $this->transferPercentCompleted = 0; - } - - /** - * @param int $bytesTransferred - * - * @return void - */ - private function increaseBytesTransferred(int $bytesTransferred): void - { - $this->totalBytesTransferred += $bytesTransferred; - if ($this->objectsTotalSizeInBytes !== 0) { - $this->transferPercentCompleted = floor( - ($this->totalBytesTransferred / $this->objectsTotalSizeInBytes) * 100 - ); - } - } - - /** - * @return void - */ - private function showProgress(): void - { - // Clear screen - fwrite($this->output, "\033[2J\033[H"); - - // Display progress header - fwrite($this->output, sprintf( - "\r%s [%d/%d] %d%%\n", - $this->trackingOperation, - $this->objectsInProgress, - $this->objectsCount, - $this->transferPercentCompleted - )); - - foreach ($this->objects as $name => $object) { - fwrite($this->output, sprintf( - "\r%s:\n%s\n", - $name, - $object->getProgressBar()->getPaintedProgress() - )); - } - } - - /** - * @return Closure|ProgressBarFactory - */ - private function defaultProgressBarFactory(): Closure| ProgressBarFactory - { - return function () { - return new ConsoleProgressBar( - format: ConsoleProgressBar::$formats[ - ConsoleProgressBar::COLORED_TRANSFER_FORMAT - ], - args: [ - 'transferred' => 0, - 'tobe_transferred' => 0, - 'unit' => 'B', - 'color_code' => ConsoleProgressBar::BLACK_COLOR_CODE, - ] - ); - }; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/ListenerNotifier.php b/src/S3/S3Transfer/ListenerNotifier.php deleted file mode 100644 index c3ab9d8844..0000000000 --- a/src/S3/S3Transfer/ListenerNotifier.php +++ /dev/null @@ -1,8 +0,0 @@ -notify('onDownloadInitiated', [&$commandArgs, $initialPart]); - } - - /** - * Event for when a download fails. - * Warning: If this method is overridden, it is recommended - * to call parent::downloadFailed() in order to - * keep the states maintained in this implementation. - * - * @param Throwable $reason - * @param int $totalPartsDownloaded - * @param int $totalBytesDownloaded - * @param int $lastPartDownloaded - * - * @return void - */ - public function downloadFailed( - Throwable $reason, - int $totalPartsDownloaded, - int $totalBytesDownloaded, - int $lastPartDownloaded): void - { - $this->notify('onDownloadFailed', [ - $reason, - $totalPartsDownloaded, - $totalBytesDownloaded, - $lastPartDownloaded - ]); - } - - /** - * Event for when a download completes. - * Warning: If this method is overridden, it is recommended - * to call parent::onDownloadCompleted() in order to - * keep the states maintained in this implementation. - * - * @param StreamInterface $stream - * @param int $totalPartsDownloaded - * @param int $totalBytesDownloaded - * - * @return void - */ - public function downloadCompleted( - StreamInterface $stream, - int $totalPartsDownloaded, - int $totalBytesDownloaded - ): void - { - $this->notify('onDownloadCompleted', [ - $stream, - $totalPartsDownloaded, - $totalBytesDownloaded - ]); - } - - /** - * Event for when a part download is initiated. - * Warning: If this method is overridden, it is recommended - * to call parent::partDownloadInitiated() in order to - * keep the states maintained in this implementation. - * - * @param mixed $partDownloadCommand - * @param int $partNo - * - * @return void - */ - public function partDownloadInitiated( - CommandInterface $partDownloadCommand, - int $partNo - ): void - { - $this->notify('onPartDownloadInitiated', [ - $partDownloadCommand, - $partNo - ]); - } - - /** - * Event for when a part download completes. - * Warning: If this method is overridden, it is recommended - * to call parent::onPartDownloadCompleted() in order to - * keep the states maintained in this implementation. - * - * @param ResultInterface $result - * @param int $partNo - * @param int $partTotalBytes - * @param int $totalParts - * @param int $objectBytesDownloaded - * @param int $objectSizeInBytes - * @return void - */ - public function partDownloadCompleted( - ResultInterface $result, - int $partNo, - int $partTotalBytes, - int $totalParts, - int $objectBytesDownloaded, - int $objectSizeInBytes - ): void - { - $this->notify('onPartDownloadCompleted', [ - $result, - $partNo, - $partTotalBytes, - $totalParts, - $objectBytesDownloaded, - $objectSizeInBytes - ]); - } - - /** - * Event for when a part download fails. - * Warning: If this method is overridden, it is recommended - * to call parent::onPartDownloadFailed() in order to - * keep the states maintained in this implementation. - * - * @param CommandInterface $partDownloadCommand - * @param Throwable $reason - * @param int $partNo - * - * @return void - */ - public function partDownloadFailed( - CommandInterface $partDownloadCommand, - Throwable $reason, - int $partNo - ): void - { - $this->notify('onPartDownloadFailed', [ - $partDownloadCommand, - $reason, - $partNo - ]); - } - - protected function notify(string $event, array $params = []): void - { - $listener = match ($event) { - 'onDownloadInitiated' => $this->onDownloadInitiated, - 'onDownloadFailed' => $this->onDownloadFailed, - 'onDownloadCompleted' => $this->onDownloadCompleted, - 'onPartDownloadInitiated' => $this->onPartDownloadInitiated, - 'onPartDownloadCompleted' => $this->onPartDownloadCompleted, - 'onPartDownloadFailed' => $this->onPartDownloadFailed, - default => null, - }; - - if ($listener instanceof Closure) { - $listener(...$params); - } - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/MultipartDownloadListenerFactory.php b/src/S3/S3Transfer/MultipartDownloadListenerFactory.php deleted file mode 100644 index 0c5f38a2ce..0000000000 --- a/src/S3/S3Transfer/MultipartDownloadListenerFactory.php +++ /dev/null @@ -1,11 +0,0 @@ -requestArgs = $requestArgs; $this->currentPartNo = $currentPartNo; $this->objectPartsCount = $objectPartsCount; - $this->objectCompletedPartsCount = $objectCompletedPartsCount; $this->objectSizeInBytes = $objectSizeInBytes; - $this->objectBytesTransferred = $objectBytesTransferred; $this->eTag = $eTag; - $this->objectKey = $objectKey; if ($stream === null) { $this->stream = Utils::streamFor( fopen('php://temp', 'w+') @@ -92,6 +83,8 @@ public function __construct( } else { $this->stream = $stream; } + $this->currentSnapshot = $currentSnapshot; + $this->listenerNotifier = $listenerNotifier; } /** @@ -110,14 +103,6 @@ public function getObjectPartsCount(): int return $this->objectPartsCount; } - /** - * @return int - */ - public function getObjectCompletedPartsCount(): int - { - return $this->objectCompletedPartsCount; - } - /** * @return int */ @@ -127,19 +112,11 @@ public function getObjectSizeInBytes(): int } /** - * @return int + * @return TransferProgressSnapshot */ - public function getObjectBytesTransferred(): int + public function getCurrentSnapshot(): TransferProgressSnapshot { - return $this->objectBytesTransferred; - } - - /** - * @return string - */ - public function getObjectKey(): string - { - return $this->objectKey; + return $this->currentSnapshot; } /** @@ -151,51 +128,47 @@ public function getObjectKey(): string public function promise(): PromiseInterface { return Coroutine::of(function () { - $this->downloadInitiated($this->requestArgs, $this->currentPartNo); - $initialCommand = $this->nextCommand(); - $this->partDownloadInitiated($initialCommand, $this->currentPartNo); + $this->downloadInitiated($this->requestArgs); try { - yield $this->s3Client->executeAsync($initialCommand) + yield $this->s3Client->executeAsync($this->nextCommand()) ->then(function (ResultInterface $result) { // Calculate object size and parts count. $this->computeObjectDimensions($result); // Trigger first part completed - $this->partDownloadCompleted($result, $this->currentPartNo); - })->otherwise(function ($reason) use ($initialCommand) { - $this->partDownloadFailed($initialCommand, $reason, $this->currentPartNo); + $this->partDownloadCompleted($result); + })->otherwise(function ($reason) { + $this->partDownloadFailed($reason); throw $reason; }); } catch (\Throwable $e) { - $this->downloadFailed($e, $this->objectCompletedPartsCount, $this->objectBytesTransferred, $this->currentPartNo); + $this->downloadFailed($e); // TODO: yield transfer exception modeled with a transfer failed response. yield Create::rejectionFor($e); } while ($this->currentPartNo < $this->objectPartsCount) { - $nextCommand = $this->nextCommand(); - $this->partDownloadInitiated($nextCommand, $this->currentPartNo); try { - yield $this->s3Client->executeAsync($nextCommand) + yield $this->s3Client->executeAsync($this->nextCommand()) ->then(function ($result) { - $this->partDownloadCompleted($result, $this->currentPartNo); + $this->partDownloadCompleted($result); return $result; - })->otherwise(function ($reason) use ($nextCommand) { - $this->partDownloadFailed($nextCommand, $reason, $this->currentPartNo); + })->otherwise(function ($reason) { + $this->partDownloadFailed($reason); return $reason; }); - } catch (\Throwable $e) { - $this->downloadFailed($e, $this->objectCompletedPartsCount, $this->objectBytesTransferred, $this->currentPartNo); + } catch (\Throwable $reason) { + $this->downloadFailed($reason); // TODO: yield transfer exception modeled with a transfer failed response. - yield Create::rejectionFor($e); + yield Create::rejectionFor($reason); } } // Transfer completed - $this->objectDownloadCompleted(); + $this->downloadComplete(); // TODO: yield the stream wrapped in a modeled transfer success response. yield Create::promiseFor(new DownloadResponse( @@ -235,7 +208,7 @@ protected function computeObjectSize($sizeSource): int } if (empty($sizeSource)) { - throw new \RuntimeException('Range must not be empty'); + return 0; } // For extracting the object size from the ContentRange header value. @@ -246,78 +219,6 @@ protected function computeObjectSize($sizeSource): int throw new \RuntimeException('Invalid source size format'); } - /** - * MultipartDownloader factory method to return an instance - * of MultipartDownloader based on the multipart download type. - * - * @param S3ClientInterface $s3Client - * @param string $multipartDownloadType - * @param array $requestArgs - * @param array $config - * @param int $currentPartNo - * @param int $objectPartsCount - * @param int $objectCompletedPartsCount - * @param int $objectSizeInBytes - * @param int $objectBytesTransferred - * @param string $eTag - * @param string $objectKey - * @param MultipartDownloadListener|null $listener - * @param TransferListener|null $progressTracker - * - * @return MultipartDownloader - */ - public static function chooseDownloader( - S3ClientInterface $s3Client, - string $multipartDownloadType, - array $requestArgs, - array $config, - int $currentPartNo = 0, - int $objectPartsCount = 0, - int $objectCompletedPartsCount = 0, - int $objectSizeInBytes = 0, - int $objectBytesTransferred = 0, - string $eTag = "", - string $objectKey = "", - ?MultipartDownloadListener $listener = null, - ?TransferListener $progressTracker = null - ) : MultipartDownloader - { - return match ($multipartDownloadType) { - self::PART_GET_MULTIPART_DOWNLOADER => new PartGetMultipartDownloader( - s3Client: $s3Client, - requestArgs: $requestArgs, - config: $config, - currentPartNo: $currentPartNo, - objectPartsCount: $objectPartsCount, - objectCompletedPartsCount: $objectCompletedPartsCount, - objectSizeInBytes: $objectSizeInBytes, - objectBytesTransferred: $objectBytesTransferred, - eTag: $eTag, - objectKey: $objectKey, - listener: $listener, - progressListener: $progressTracker - ), - self::RANGE_GET_MULTIPART_DOWNLOADER => new RangeGetMultipartDownloader( - s3Client: $s3Client, - requestArgs: $requestArgs, - config: $config, - currentPartNo: 0, - objectPartsCount: 0, - objectCompletedPartsCount: 0, - objectSizeInBytes: 0, - objectBytesTransferred: 0, - eTag: "", - objectKey: "", - listener: $listener, - progressListener: $progressTracker - ), - default => throw new \RuntimeException( - "Unsupported download type $multipartDownloadType." - ."It should be either " . self::PART_GET_MULTIPART_DOWNLOADER . - " or " . self::RANGE_GET_MULTIPART_DOWNLOADER . ".") - }; - } - /** * Main purpose of this method is to propagate * the download-initiated event to listeners, but @@ -325,70 +226,45 @@ public static function chooseDownloader( * that need to be maintained. * * @param array $commandArgs - * @param int|null $currentPartNo * * @return void */ - private function downloadInitiated(array &$commandArgs, ?int $currentPartNo): void + private function downloadInitiated(array $commandArgs): void { - $this->objectKey = $commandArgs['Key']; - $this->progressListener?->objectTransferInitiated( - $this->objectKey, - $commandArgs - ); - $this->_notifyMultipartDownloadListeners('downloadInitiated', [ - &$commandArgs, - $currentPartNo + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + $commandArgs['Key'], + 0, + $this->objectSizeInBytes + ); + } else { + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse() + ); + } + + $this->listenerNotifier?->transferInitiated([ + 'request_args' => $commandArgs, + 'progress_snapshot' => $this->currentSnapshot, ]); } /** * Propagates download-failed event to listeners. - * It may also do some computation in order to maintain internal states. * * @param \Throwable $reason - * @param int $totalPartsTransferred - * @param int $totalBytesTransferred - * @param int $lastPartTransferred * * @return void */ - private function downloadFailed( - \Throwable $reason, - int $totalPartsTransferred, - int $totalBytesTransferred, - int $lastPartTransferred - ): void + private function downloadFailed(\Throwable $reason): void { - $this->progressListener?->objectTransferFailed( - $this->objectKey, - $totalBytesTransferred, - $reason - ); - $this->_notifyMultipartDownloadListeners('downloadFailed', [ - $reason, - $totalPartsTransferred, - $totalBytesTransferred, - $lastPartTransferred - ]); - } - - /** - * Propagates part-download-initiated event to listeners. - * - * @param CommandInterface $partDownloadCommand - * @param int $partNo - * - * @return void - */ - private function partDownloadInitiated( - CommandInterface $partDownloadCommand, - int $partNo - ): void - { - $this->_notifyMultipartDownloadListeners('partDownloadInitiated', [ - $partDownloadCommand, - $partNo + $this->listenerNotifier?->transferFail([ + 'request_args' => $this->requestArgs, + 'progress_snapshot' => $this->currentSnapshot, + 'reason' => $reason, ]); } @@ -400,62 +276,43 @@ private function partDownloadInitiated( * is completed. * * @param ResultInterface $result - * @param int $partNo * * @return void */ private function partDownloadCompleted( - ResultInterface $result, - int $partNo + ResultInterface $result ): void { - $this->objectCompletedPartsCount++; $partDownloadBytes = $result['ContentLength']; - $this->objectBytesTransferred = $this->objectBytesTransferred + $partDownloadBytes; if (isset($result['ETag'])) { $this->eTag = $result['ETag']; } Utils::copyToStream($result['Body'], $this->stream); - - $this->progressListener?->objectTransferProgress( - $this->objectKey, - $partDownloadBytes, - $this->objectSizeInBytes + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes() + $partDownloadBytes, + $this->objectSizeInBytes, + $result->toArray() ); - - $this->_notifyMultipartDownloadListeners('partDownloadCompleted', [ - $result, - $partNo, - $partDownloadBytes, - $this->objectCompletedPartsCount, - $this->objectBytesTransferred, - $this->objectSizeInBytes + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->bytesTransferred([ + 'request_args' => $this->requestArgs, + 'progress_snapshot' => $this->currentSnapshot, ]); } /** * Propagates part-download-failed event to listeners. * - * @param CommandInterface $partDownloadCommand * @param \Throwable $reason - * @param int $partNo * * @return void */ private function partDownloadFailed( - CommandInterface $partDownloadCommand, \Throwable $reason, - int $partNo ): void { - $this->progressListener?->objectTransferFailed( - $this->objectKey, - $this->objectBytesTransferred, - $reason - ); - $this->_notifyMultipartDownloadListeners( - 'partDownloadFailed', - [$partDownloadCommand, $reason, $partNo]); + $this->downloadFailed($reason); } /** @@ -465,33 +322,19 @@ private function partDownloadFailed( * * @return void */ - private function objectDownloadCompleted(): void + private function downloadComplete(): void { $this->stream->rewind(); - $this->progressListener?->objectTransferCompleted( - $this->objectKey, - $this->objectBytesTransferred + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->objectSizeInBytes, + $this->currentSnapshot->getResponse() ); - $this->_notifyMultipartDownloadListeners('downloadCompleted', [ - $this->stream, - $this->objectCompletedPartsCount, - $this->objectBytesTransferred + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->transferComplete([ + 'request_args' => $this->requestArgs, + 'progress_snapshot' => $this->currentSnapshot, ]); } - - /** - * Internal helper method for notifying listeners of specific events. - * - * @param string $listenerMethod - * @param array $args - * - * @return void - */ - private function _notifyMultipartDownloadListeners( - string $listenerMethod, - array $args - ): void - { - $this->listener?->{$listenerMethod}(...$args); - } } \ No newline at end of file diff --git a/src/S3/S3Transfer/MultipartUploadListener.php b/src/S3/S3Transfer/MultipartUploadListener.php deleted file mode 100644 index b69597a421..0000000000 --- a/src/S3/S3Transfer/MultipartUploadListener.php +++ /dev/null @@ -1,58 +0,0 @@ -s3Client = $s3Client; $this->createMultipartArgs = $createMultipartArgs; - $this->uploadPartArgs = $uploadPartArgs; - $this->completeMultipartArgs = $completeMultipartArgs; $this->config = $config; $this->body = $this->parseBody($source); - $this->objectSizeInBytes = $objectSizeInBytes; $this->uploadId = $uploadId; $this->parts = $parts; - $this->progressTracker = $progressTracker; + $this->currentSnapshot = $currentSnapshot; + $this->listenerNotifier = $listenerNotifier; } /** @@ -117,17 +104,17 @@ public function getParts(): array /** * @return int */ - public function getObjectSizeInBytes(): int + public function getCalculatedObjectSize(): int { - return $this->objectSizeInBytes; + return $this->calculatedObjectSize; } /** - * @return int + * @return TransferProgressSnapshot|null */ - public function getObjectBytesTransferred(): int + public function getCurrentSnapshot(): ?TransferProgressSnapshot { - return $this->objectBytesTransferred; + return $this->currentSnapshot; } /** @@ -155,7 +142,8 @@ public function promise(): PromiseInterface /** * @return PromiseInterface */ - public function createMultipartUpload(): PromiseInterface { + public function createMultipartUpload(): PromiseInterface + { $requestArgs = [...$this->createMultipartArgs]; $this->uploadInitiated($requestArgs); $command = $this->s3Client->getCommand( @@ -176,7 +164,7 @@ public function createMultipartUpload(): PromiseInterface { */ public function uploadParts(): PromiseInterface { - $this->objectSizeInBytes = 0; // To repopulate + $this->calculatedObjectSize = 0; $isSeekable = $this->body->isSeekable(); $partSize = $this->config['part_size'] ?? self::PART_MIN_SIZE; if ($partSize > self::PART_MAX_SIZE) { @@ -202,22 +190,23 @@ public function uploadParts(): PromiseInterface $this->body->read($readSize) ); // To make sure we do not create an empty part when - // we already reached end of file. + // we already reached the end of file. if (!$isSeekable && $this->body->eof() && $partBody->getSize() === 0) { break; } $uploadPartCommandArgs = [ - 'UploadId' => $this->uploadId, - 'PartNumber' => $partNo, - 'Body' => $partBody, - 'ContentLength' => $partBody->getSize(), - ] + $this->uploadPartArgs; - + ...$this->createMultipartArgs, + 'UploadId' => $this->uploadId, + 'PartNumber' => $partNo, + 'Body' => $partBody, + 'ContentLength' => $partBody->getSize(), + ]; + // To get `requestArgs` when notifying the bytesTransfer listeners. + $uploadPartCommandArgs['requestArgs'] = $uploadPartCommandArgs; $command = $this->s3Client->getCommand('UploadPart', $uploadPartCommandArgs); $commands[] = $command; - $this->objectSizeInBytes += $partBody->getSize(); - + $this->calculatedObjectSize += $partBody->getSize(); if ($partNo > self::PART_MAX_NUM) { return Create::rejectionFor( "The max number of parts has been exceeded. " . @@ -238,9 +227,11 @@ public function uploadParts(): PromiseInterface $result, $command ); - // Part Upload Completed Event - $this->partUploadCompleted($result, $command['ContentLength']); + $this->partUploadCompleted( + $command['ContentLength'], + $command['requestArgs'] + ); }, 'rejected' => function (Throwable $e) { $this->partUploadFailed($e); @@ -255,13 +246,21 @@ public function uploadParts(): PromiseInterface public function completeMultipartUpload(): PromiseInterface { $this->sortParts(); - $command = $this->s3Client->getCommand('CompleteMultipartUpload', [ - 'UploadId' => $this->uploadId, - 'MpuObjectSize' => $this->objectSizeInBytes, - 'MultipartUpload' => [ - 'Parts' => $this->parts, - ] - ] + $this->completeMultipartArgs + $completeMultipartUploadArgs = [ + ...$this->createMultipartArgs, + 'UploadId' => $this->uploadId, + 'MpuObjectSize' => $this->calculatedObjectSize, + 'MultipartUpload' => [ + 'Parts' => $this->parts, + ] + ]; + if ($this->containsChecksum($this->createMultipartArgs)) { + $completeMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; + } + + $command = $this->s3Client->getCommand( + 'CompleteMultipartUpload', + $completeMultipartUploadArgs ); return $this->s3Client->executeAsync($command) @@ -275,7 +274,8 @@ public function completeMultipartUpload(): PromiseInterface /** * @return PromiseInterface */ - public function abortMultipartUpload(): PromiseInterface { + public function abortMultipartUpload(): PromiseInterface + { $command = $this->s3Client->getCommand('AbortMultipartUpload', [ ...$this->createMultipartArgs, 'UploadId' => $this->uploadId, @@ -316,12 +316,13 @@ private function collectPart( private function sortParts(): void { usort($this->parts, function($partOne, $partTwo) { - return $partOne['PartNumber'] <=> $partTwo['PartNumber']; // Ascending order by age + return $partOne['PartNumber'] <=> $partTwo['PartNumber']; }); } /** * @param string|StreamInterface $source + * * @return StreamInterface */ private function parseBody(string | StreamInterface $source): StreamInterface @@ -333,12 +334,11 @@ private function parseBody(string | StreamInterface $source): StreamInterface "The source for this upload must be either a readable file or a valid stream." ); } - $file = Utils::tryFopen($source, 'r'); + $body = new LazyOpenStream($source, 'r'); // To make sure the resource is closed. - $this->deferFns[] = function () use ($file) { - fclose($file); + $this->deferFns[] = function () use ($body) { + $body->close(); }; - $body = Utils::streamFor($file); } elseif ($source instanceof StreamInterface) { $body = $source; } else { @@ -351,14 +351,31 @@ private function parseBody(string | StreamInterface $source): StreamInterface } /** + * @param array $requestArgs + * * @return void */ - private function uploadInitiated(array &$requestArgs): void { - $this->objectKey = $this->createMultipartArgs['Key']; - $this->progressTracker?->objectTransferInitiated( - $this->objectKey, - $requestArgs - ); + private function uploadInitiated(array $requestArgs): void + { + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + $this->body->getSize(), + ); + } else { + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse() + ); + } + + $this->listenerNotifier?->transferInitiated([ + 'request_args' => $requestArgs, + 'progress_snapshot' => $this->currentSnapshot + ]); } /** @@ -370,11 +387,11 @@ private function uploadFailed(Throwable $reason): void { if (!empty($this->uploadId)) { $this->abortMultipartUpload()->wait(); } - $this->progressTracker?->objectTransferFailed( - $this->objectKey, - $this->objectBytesTransferred, - $reason - ); + $this->listenerNotifier?->transferFail([ + 'request_args' => $this->createMultipartArgs, + 'progress_snapshot' => $this->currentSnapshot, + 'reason' => $reason, + ]); } /** @@ -383,25 +400,41 @@ private function uploadFailed(Throwable $reason): void { * @return void */ private function uploadCompleted(ResultInterface $result): void { - $this->progressTracker?->objectTransferCompleted( - $this->objectKey, - $this->objectBytesTransferred, + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $result->toArray() ); + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->transferComplete([ + 'request_args' => $this->createMultipartArgs, + 'progress_snapshot' => $this->currentSnapshot, + ]); } /** - * @param ResultInterface $result - * @param int $partSize + * @param int $partCompletedBytes + * @param array $requestArgs * * @return void */ - private function partUploadCompleted(ResultInterface $result, int $partSize): void { - $this->objectBytesTransferred = $this->objectBytesTransferred + $partSize; - $this->progressTracker?->objectTransferProgress( - $this->objectKey, - $partSize, - $this->objectSizeInBytes + private function partUploadCompleted( + int $partCompletedBytes, + array $requestArgs + ): void + { + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes() + $partCompletedBytes, + $this->currentSnapshot->getTotalBytes() ); + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->bytesTransferred([ + 'request_args' => $requestArgs, + 'progress_snapshot' => $this->currentSnapshot, + $this->currentSnapshot + ]); } /** @@ -411,6 +444,7 @@ private function partUploadCompleted(ResultInterface $result, int $partSize): vo */ private function partUploadFailed(Throwable $reason): void { + $this->uploadFailed($reason); } /** @@ -424,4 +458,29 @@ private function callDeferredFns(): void $this->deferFns = []; } + + /** + * Verifies if a checksum was provided. + * + * @param array $requestArgs + * + * @return bool + */ + private function containsChecksum(array $requestArgs): bool + { + $algorithms = [ + 'ChecksumCRC32', + 'ChecksumCRC32C', + 'ChecksumCRC64NVME', + 'ChecksumSHA1', + 'ChecksumSHA256', + ]; + foreach ($algorithms as $algorithm) { + if (isset($requestArgs[$algorithm])) { + return true; + } + } + + return false; + } } diff --git a/src/S3/S3Transfer/ObjectProgressTracker.php b/src/S3/S3Transfer/ObjectProgressTracker.php deleted file mode 100644 index a6a30f1648..0000000000 --- a/src/S3/S3Transfer/ObjectProgressTracker.php +++ /dev/null @@ -1,189 +0,0 @@ -objectKey = $objectKey; - $this->objectBytesTransferred = $objectBytesTransferred; - $this->objectSizeInBytes = $objectSizeInBytes; - $this->status = $status; - $this->progressBar = $progressBar ?? $this->defaultProgressBar(); - } - - /** - * @return string - */ - public function getObjectKey(): string - { - return $this->objectKey; - } - - /** - * @param string $objectKey - * - * @return void - */ - public function setObjectKey(string $objectKey): void - { - $this->objectKey = $objectKey; - } - - /** - * @return int - */ - public function getObjectBytesTransferred(): int - { - return $this->objectBytesTransferred; - } - - /** - * @param int $objectBytesTransferred - * - * @return void - */ - public function setObjectBytesTransferred(int $objectBytesTransferred): void - { - $this->objectBytesTransferred = $objectBytesTransferred; - } - - /** - * @return int - */ - public function getObjectSizeInBytes(): int - { - return $this->objectSizeInBytes; - } - - /** - * @param int $objectSizeInBytes - * - * @return void - */ - public function setObjectSizeInBytes(int $objectSizeInBytes): void - { - $this->objectSizeInBytes = $objectSizeInBytes; - // Update progress bar - $this->progressBar->setArg('tobe_transferred', $objectSizeInBytes); - } - - /** - * @return string - */ - public function getStatus(): string - { - return $this->status; - } - - /** - * @param string $status - * @param string|null $message - * - * @return void - */ - public function setStatus(string $status, ?string $message = null): void - { - $this->status = $status; - $this->setProgressColor(); - // To show specific messages for specific status. - if (!empty($message)) { - $this->progressBar->setArg('message', "$status: $message"); - } - } - - private function setProgressColor(): void - { - if ($this->status === 'progress') { - $this->progressBar->setArg('color_code', ConsoleProgressBar::BLUE_COLOR_CODE); - } elseif ($this->status === 'completed') { - $this->progressBar->setArg('color_code', ConsoleProgressBar::GREEN_COLOR_CODE); - } elseif ($this->status === 'failed') { - $this->progressBar->setArg('color_code', ConsoleProgressBar::RED_COLOR_CODE); - } - } - - /** - * Increments the object bytes transferred. - * - * @param int $objectBytesTransferred - * - * @return void - */ - public function incrementTotalBytesTransferred( - int $objectBytesTransferred - ): void - { - $this->objectBytesTransferred += $objectBytesTransferred; - $progressPercent = (int) floor(($this->objectBytesTransferred / $this->objectSizeInBytes) * 100); - // Update progress bar - $this->progressBar->setPercentCompleted($progressPercent); - $this->progressBar->setArg('transferred', $this->objectBytesTransferred); - } - - /** - * @return ProgressBar|null - */ - public function getProgressBar(): ?ProgressBar - { - return $this->progressBar; - } - - /** - * @return ProgressBar - */ - private function defaultProgressBar(): ProgressBar - { - return new ConsoleProgressBar( - format: ConsoleProgressBar::$formats[ - ConsoleProgressBar::COLORED_TRANSFER_FORMAT - ], - args: [ - 'transferred' => 0, - 'tobe_transferred' => 0, - 'unit' => 'B', - 'color_code' => ConsoleProgressBar::BLACK_COLOR_CODE, - 'message' => '' - ] - ); - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php new file mode 100644 index 0000000000..e5a3eb95f0 --- /dev/null +++ b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php @@ -0,0 +1,45 @@ + ColoredTransferProgressBarFormat::BLACK_COLOR_CODE, + ]; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Progress/ConsoleProgressBar.php b/src/S3/S3Transfer/Progress/ConsoleProgressBar.php new file mode 100644 index 0000000000..6fe118b167 --- /dev/null +++ b/src/S3/S3Transfer/Progress/ConsoleProgressBar.php @@ -0,0 +1,103 @@ +progressBarChar = $progressBarChar; + $this->progressBarWidth = min( + $progressBarWidth, + self::MAX_PROGRESS_BAR_WIDTH + ); + $this->percentCompleted = $percentCompleted; + $this->progressBarFormat = $progressBarFormat; + } + + /** + * @return string + */ + public function getProgressBarChar(): string + { + return $this->progressBarChar; + } + + /** + * @return int + */ + public function getProgressBarWidth(): int + { + return $this->progressBarWidth; + } + + /** + * @return int + */ + public function getPercentCompleted(): int + { + return $this->percentCompleted; + } + + /** + * @return ProgressBarFormat + */ + public function getProgressBarFormat(): ProgressBarFormat + { + return $this->progressBarFormat; + } + + /** + * Set current progress percent. + * + * @param int $percent + * + * @return void + */ + public function setPercentCompleted(int $percent): void + { + $this->percentCompleted = max(0, min(100, $percent)); + } + + /** + * @inheritDoc + */ + public function render(): string + { + $filledWidth = (int) round(($this->progressBarWidth * $this->percentCompleted) / 100); + $progressBar = str_repeat($this->progressBarChar, $filledWidth) + . str_repeat(' ', $this->progressBarWidth - $filledWidth); + + // Common arguments + $this->progressBarFormat->setArg('progress_bar', $progressBar); + $this->progressBarFormat->setArg('percent', $this->percentCompleted); + + return $this->progressBarFormat->format(); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Progress/MultiProgressTracker.php b/src/S3/S3Transfer/Progress/MultiProgressTracker.php new file mode 100644 index 0000000000..6dcba0053c --- /dev/null +++ b/src/S3/S3Transfer/Progress/MultiProgressTracker.php @@ -0,0 +1,169 @@ +singleProgressTrackers = $singleProgressTrackers; + $this->output = $output; + $this->transferCount = $transferCount; + $this->completed = $completed; + $this->failed = $failed; + } + + /** + * @return array + */ + public function getSingleProgressTrackers(): array + { + return $this->singleProgressTrackers; + } + + /** + * @return mixed + */ + public function getOutput(): mixed + { + return $this->output; + } + + /** + * @return int + */ + public function getTransferCount(): int + { + return $this->transferCount; + } + + /** + * @return int + */ + public function getCompleted(): int + { + return $this->completed; + } + + /** + * @return int + */ + public function getFailed(): int + { + return $this->failed; + } + + /** + * @inheritDoc + */ + public function transferInitiated(array $context): void + { + $this->transferCount++; + $snapshot = $context['progress_snapshot']; + $progressTracker = new SingleProgressTracker( + clear: false, + ); + $progressTracker->transferInitiated($context); + $this->singleProgressTrackers[$snapshot->getIdentifier()] = $progressTracker; + $this->showProgress(); + } + + /** + * @inheritDoc + */ + public function bytesTransferred(array $context): void + { + $snapshot = $context['progress_snapshot']; + $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; + $progressTracker->bytesTransferred($context); + $this->showProgress(); + } + + /** + * @inheritDoc + */ + public function transferComplete(array $context): void + { + $this->completed++; + $snapshot = $context['progress_snapshot']; + $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; + $progressTracker->transferComplete($context); + $this->showProgress(); + } + + /** + * @inheritDoc + */ + public function transferFail(array $context): void + { + $this->failed++; + $snapshot = $context['progress_snapshot']; + $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; + $progressTracker->transferFail($context); + $this->showProgress(); + } + + /** + * @inheritDoc + */ + public function showProgress(): void + { + fwrite($this->output, "\033[2J\033[H"); + $percentsSum = 0; + foreach ($this->singleProgressTrackers as $_ => $progressTracker) { + $progressTracker->showProgress(); + $percentsSum += $progressTracker->getProgressBar()->getPercentCompleted(); + } + + $percent = (int) floor($percentsSum / $this->transferCount); + $allTransferProgressBar = new ConsoleProgressBar( + percentCompleted: $percent, + progressBarFormat: new PlainProgressBarFormat() + ); + fwrite($this->output, "\n" . str_repeat( + '-', + $allTransferProgressBar->getProgressBarWidth()) + ); + fwrite( + $this->output, + sprintf( + "\n%s Completed: %d/%d, Failed: %d/%d\n", + $allTransferProgressBar->render(), + $this->completed, + $this->transferCount, + $this->failed, + $this->transferCount + ) + ); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php new file mode 100644 index 0000000000..55a2ec1cba --- /dev/null +++ b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php @@ -0,0 +1,24 @@ +args = $args; + } + + public function getArgs(): array { + return $this->args; + } + + /** + * To set multiple arguments at once. + * It does not override all the values, instead + * it adds the arguments individually and if a value + * already exists then that value will be overridden. + * + * @param array $args + * + * @return void + */ + public function setArgs(array $args): void + { + foreach ($args as $key => $value) { + $this->args[$key] = $value; + } + } + + /** + * @param string $key + * @param mixed $value + * + * @return void + */ + public function setArg(string $key, mixed $value): void + { + $this->args[$key] = $value; + } + + /** + * @return string + */ + public function format(): string { + $parameters = $this->getFormatParameters(); + $defaultParameterValues = $this->getFormatDefaultParameterValues(); + foreach ($parameters as $param) { + if (!array_key_exists($param, $this->args)) { + $this->args[$param] = $defaultParameterValues[$param] ?? ''; + } + } + + $replacements = []; + foreach ($parameters as $param) { + $replacements["|$param|"] = $this->args[$param] ?? ''; + } + + return strtr($this->getFormatTemplate(), $replacements); + } + + /** + * @return string + */ + abstract public function getFormatTemplate(): string; + + /** + * @return array + */ + abstract public function getFormatParameters(): array; + + /** + * @return array + */ + abstract protected function getFormatDefaultParameterValues(): array; +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Progress/ProgressBarInterface.php b/src/S3/S3Transfer/Progress/ProgressBarInterface.php new file mode 100644 index 0000000000..ed8de193c2 --- /dev/null +++ b/src/S3/S3Transfer/Progress/ProgressBarInterface.php @@ -0,0 +1,31 @@ +progressBar = $progressBar; + if (get_resource_type($output) !== 'stream') { + throw new \InvalidArgumentException("The type for $output must be a stream"); + } + $this->output = $output; + $this->objectName = $objectName; + $this->clear = $clear; + } + + /** + * @return ProgressBarInterface + */ + public function getProgressBar(): ProgressBarInterface + { + return $this->progressBar; + } + + /** + * @return mixed + */ + public function getOutput(): mixed + { + return $this->output; + } + + /** + * @return string + */ + public function getObjectName(): string + { + return $this->objectName; + } + + /** + * @return bool + */ + public function isClear(): bool { + return $this->clear; + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferInitiated(array $context): void + { + $snapshot = $context['progress_snapshot']; + $this->objectName = $snapshot->getIdentifier(); + $progressFormat = $this->progressBar->getProgressBarFormat(); + if ($progressFormat instanceof ColoredTransferProgressBarFormat) { + $progressFormat->setArg( + 'object_name', + $this->objectName + ); + } + + $this->updateProgressBar($snapshot); + } + + /** + * @inheritDoc + * + * @return void + */ + public function bytesTransferred(array $context): void + { + $progressFormat = $this->progressBar->getProgressBarFormat(); + if ($progressFormat instanceof ColoredTransferProgressBarFormat) { + $progressFormat->setArg( + 'color_code', + ColoredTransferProgressBarFormat::BLUE_COLOR_CODE + ); + } + + $this->updateProgressBar($context['progress_snapshot']); + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferComplete(array $context): void + { + $progressFormat = $this->progressBar->getProgressBarFormat(); + if ($progressFormat instanceof ColoredTransferProgressBarFormat) { + $progressFormat->setArg( + 'color_code', + ColoredTransferProgressBarFormat::GREEN_COLOR_CODE + ); + } + + $snapshot = $context['progress_snapshot']; + $this->updateProgressBar( + $snapshot, + $snapshot->getTotalBytes() === 0 + ); + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferFail(array $context): void + { + $progressFormat = $this->progressBar->getProgressBarFormat(); + if ($progressFormat instanceof ColoredTransferProgressBarFormat) { + $progressFormat->setArg( + 'color_code', + ColoredTransferProgressBarFormat::RED_COLOR_CODE + ); + $progressFormat->setArg( + 'message', + $context['reason'] + ); + } + + $this->updateProgressBar($context['progress_snapshot']); + } + + /** + * Updates the progress bar with the transfer snapshot + * and also call showProgress. + * + * @param TransferProgressSnapshot $snapshot + * @param bool $forceCompletion To force the progress bar to be + * completed. This is useful for files where its size is zero, + * for which a ratio will return zero, and hence the percent + * will be zero. + * + * @return void + */ + private function updateProgressBar( + TransferProgressSnapshot $snapshot, + bool $forceCompletion = false + ): void + { + if (!$forceCompletion) { + $this->progressBar->setPercentCompleted( + ((int)floor($snapshot->ratioTransferred() * 100)) + ); + } else { + $this->progressBar->setPercentCompleted(100); + } + + $this->progressBar->getProgressBarFormat()->setArgs([ + 'transferred' => $snapshot->getTransferredBytes(), + 'tobe_transferred' => $snapshot->getTotalBytes(), + 'unit' => 'B', + ]); + // Display progress + $this->showProgress(); + } + + /** + * @inheritDoc + * + * @return void + */ + public function showProgress(): void + { + if (empty($this->objectName)) { + throw new \RuntimeException( + "Progress tracker requires an object name to be set." + ); + } + + if ($this->clear) { + fwrite($this->output, "\033[2J\033[H"); + } + + fwrite($this->output, sprintf( + "\r\n%s", + $this->progressBar->render() + )); + fflush($this->output); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php new file mode 100644 index 0000000000..0ac129c43c --- /dev/null +++ b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php @@ -0,0 +1,33 @@ +identifier = $identifier; + $this->transferredBytes = $transferredBytes; + $this->totalBytes = $totalBytes; + $this->response = $response; + } + + public function getIdentifier(): string + { + return $this->identifier; + } + + /** + * @return int + */ + public function getTransferredBytes(): int + { + return $this->transferredBytes; + } + + /** + * @return int + */ + public function getTotalBytes(): int + { + return $this->totalBytes; + } + + /** + * @return array + */ + public function getResponse(): array + { + return $this->response; + } + + /** + * @return float + */ + public function ratioTransferred(): float + { + if ($this->totalBytes === 0) { + // Unable to calculate ratio + return 0; + } + + return $this->transferredBytes / $this->totalBytes; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/ProgressBar.php b/src/S3/S3Transfer/ProgressBar.php deleted file mode 100644 index 0421c5a039..0000000000 --- a/src/S3/S3Transfer/ProgressBar.php +++ /dev/null @@ -1,14 +0,0 @@ - 'bytesToBytes', - 'KB' => 'bytesToKB', - 'MB' => 'bytesToMB', - ]; - - public static function getUnitValue(string $displayUnit, float $bytes): float { - $displayUnit = self::validateDisplayUnit($displayUnit); - if (isset(self::$displayUnitMapping[$displayUnit])) { - return number_format(call_user_func([__CLASS__, self::$displayUnitMapping[$displayUnit]], $bytes)); - } - - throw new \RuntimeException("Unknown display unit {$displayUnit}"); - } - - private static function validateDisplayUnit(string $displayUnit): string { - if (!isset(self::$displayUnitMapping[$displayUnit])) { - throw new \InvalidArgumentException("Invalid display unit specified: $displayUnit"); - } - - return $displayUnit; - } - - private static function bytesToBytes(float $bytes): float { - return $bytes; - } - - private static function bytesToKB(float $bytes): float { - return $bytes / 1024; - } - - private static function bytesToMB(float $bytes): float { - return $bytes / 1024 / 1024; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php index f2da868473..d9107ac5cd 100644 --- a/src/S3/S3Transfer/RangeGetMultipartDownloader.php +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -7,6 +7,7 @@ use Aws\ResultInterface; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Psr\Http\Message\StreamInterface; class RangeGetMultipartDownloader extends MultipartDownloader @@ -19,31 +20,29 @@ class RangeGetMultipartDownloader extends MultipartDownloader * @param S3ClientInterface $s3Client * @param array $requestArgs * @param array $config + * - minimum_part_size: The minimum part size for a multipart download + * using range get. This option MUST be set when using range get. * @param int $currentPartNo * @param int $objectPartsCount * @param int $objectCompletedPartsCount * @param int $objectSizeInBytes * @param int $objectBytesTransferred * @param string $eTag - * @param string $objectKey - * @param MultipartDownloadListener|null $listener - * @param TransferListener|null $progressListener * @param StreamInterface|null $stream + * @param TransferProgressSnapshot|null $currentSnapshot + * @param TransferListenerNotifier|null $listenerNotifier */ public function __construct( S3ClientInterface $s3Client, - array $requestArgs = [], + array $requestArgs, array $config = [], int $currentPartNo = 0, int $objectPartsCount = 0, - int $objectCompletedPartsCount = 0, int $objectSizeInBytes = 0, - int $objectBytesTransferred = 0, string $eTag = "", - string $objectKey = "", - ?MultipartDownloadListener $listener = null, - ?TransferListener $progressListener = null, - ?StreamInterface $stream = null + ?StreamInterface $stream = null, + ?TransferProgressSnapshot $currentSnapshot = null, + ?TransferListenerNotifier $listenerNotifier = null, ) { parent::__construct( $s3Client, @@ -51,21 +50,18 @@ public function __construct( $config, $currentPartNo, $objectPartsCount, - $objectCompletedPartsCount, $objectSizeInBytes, - $objectBytesTransferred, $eTag, - $objectKey, - $listener, - $progressListener, - $stream + $stream, + $currentSnapshot, + $listenerNotifier, ); - if (empty($config['minimumPartSize'])) { + if (empty($config['minimum_part_size'])) { throw new S3TransferException( 'You must provide a valid minimum part size in bytes' ); } - $this->partSize = $config['minimumPartSize']; + $this->partSize = $config['minimum_part_size']; // If object size is known at instantiation time then, we can compute // the object dimensions. if ($this->objectSizeInBytes !== 0) { @@ -75,7 +71,6 @@ public function __construct( } } - /** * @inheritDoc * diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index e5e5a5eca3..c29ad896a9 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -5,6 +5,9 @@ use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Progress\MultiProgressTracker; +use Aws\S3\S3Transfer\Progress\SingleProgressTracker; +use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; use Psr\Http\Message\StreamInterface; @@ -25,8 +28,6 @@ class S3TransferManager 'region' => 'us-east-1', ]; - private const MIN_PART_SIZE = 5 * 1024 * 1024; - /** @var S3Client */ private S3ClientInterface $s3Client; @@ -52,11 +53,9 @@ class S3TransferManager * Maximum number of concurrent operations allowed during a multipart * upload/download. * - track_progress: (bool, default=false) \ - * To enable progress tracker in a multipart upload/download. + * To enable progress tracker in a multipart upload/download, and or + * a directory upload/download operation. * - region: (string, default="us-east-2") - * - progress_tracker_factory: (callable|TransferListenerFactory) \ - * A factory to create the listener which will receive notifications - * based in the different stages an upload/download is. */ public function __construct( ?S3ClientInterface $s3Client = null, @@ -80,11 +79,13 @@ public function __construct( * @param array $config The config options for this upload operation. * - multipart_upload_threshold_bytes: (int, optional) * To override the default threshold for when to use multipart upload. - * - target_part_size_bytes: (int, optional) To override the default + * - part_size: (int, optional) To override the default * target part size in bytes. * - track_progress: (bool, optional) To override the default option for - * enabling progress tracking. - * @param MultipartUploadListener|null $uploadListener + * enabling progress tracking. If this option is resolved as true and + * a progressTracker parameter is not provided then, a default implementation + * will be resolved. + * @param TransferListener[]|null $listeners * @param TransferListener|null $progressTracker * * @return PromiseInterface @@ -93,7 +94,7 @@ public function upload( string | StreamInterface $source, array $requestArgs = [], array $config = [], - ?MultipartUploadListener $uploadListener = null, + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { @@ -119,35 +120,39 @@ public function upload( $mupThreshold = $config['multipart_upload_threshold_bytes'] ?? $this->config['multipart_upload_threshold_bytes']; - if ($mupThreshold < self::MIN_PART_SIZE) { + if ($mupThreshold < MultipartUploader::PART_MIN_SIZE) { throw new \InvalidArgumentException( "The provided config `multipart_upload_threshold_bytes`" - ."must be greater than or equal to " . self::MIN_PART_SIZE + ."must be greater than or equal to " . MultipartUploader::PART_MIN_SIZE ); } if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { - $progressTracker = $this->resolveDefaultProgressTracker( - DefaultProgressTracker::TRACKING_OPERATION_UPLOADING - ); + $progressTracker = new SingleProgressTracker(); + } + + if ($progressTracker !== null) { + $listeners[] = $progressTracker; } + $listenerNotifier = new TransferListenerNotifier($listeners); if ($this->requiresMultipartUpload($source, $mupThreshold)) { return $this->tryMultipartUpload( $source, $requestArgs, - $config['target_part_size_bytes'] - ?? $this->config['target_part_size_bytes'], - $uploadListener, - $progressTracker, + [ + 'part_size' => $config['part_size'] ?? $this->config['target_part_size_bytes'], + 'concurrency' => $this->config['concurrency'], + ], + $listenerNotifier ); } return $this->trySingleUpload( $source, $requestArgs, - $progressTracker + $listenerNotifier ); } @@ -164,9 +169,16 @@ public function upload( * - s3_delimiter: (string, optional, defaulted to `/`) * - put_object_request_callback: (Closure, optional) * - failure_policy: (Closure, optional) - * @param MultipartUploadListener|null $uploadListener - * @param TransferListener|null $progressTracker - * + * - track_progress: (bool, optional) To override the default option for + * enabling progress tracking. If this option is resolved as true and + * a progressTracker parameter is not provided then, a default implementation + * will be resolved. + * @param TransferListener[]|null $listeners The listeners for watching + * transfer events. Each listener will be cloned per file upload. + * @param TransferListener|null $progressTracker Ideally the progress + * tracker implementation provided here should be able to track multiple + * transfers at once. Please see MultiProgressTracker implementation. + * * @return PromiseInterface */ public function uploadDirectory( @@ -174,7 +186,7 @@ public function uploadDirectory( string $bucketTo, array $requestArgs = [], array $config = [], - ?MultipartUploadListener $uploadListener = null, + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { @@ -187,9 +199,7 @@ public function uploadDirectory( if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { - $progressTracker = $this->resolveDefaultProgressTracker( - DefaultProgressTracker::TRACKING_OPERATION_UPLOADING - ); + $progressTracker = new MultiProgressTracker(); } $filter = null; @@ -242,7 +252,7 @@ function ($file) use ($filter) { 'Key' => $objectKey, ], $config, - $uploadListener, + array_map(function ($listener) { return clone $listener; }, $listeners), $progressTracker, )->then(function ($result) use (&$objectsUploaded) { $objectsUploaded++; @@ -251,7 +261,7 @@ function ($file) use ($filter) { })->otherwise(function ($reason) use (&$objectsFailed) { $objectsFailed++; - return $reason; + throw $reason; }); } @@ -269,19 +279,20 @@ function ($file) use ($filter) { * of each get object operation, except for the bucket and key, which * are already provided as the source. * @param array $config The configuration to be used for this operation. + * - multipart_download_type: (string, optional) \ + * Overrides the resolved value from the transfer manager config. * - track_progress: (bool) \ * Overrides the config option set in the transfer manager instantiation * to decide whether transfer progress should be tracked. If a `progressListenerFactory` * was not provided when the transfer manager instance was created * and track_progress resolved as true then, a default progress listener * implementation will be used. - * - minimumPartSize: (int) \ + * - minimum_part_size: (int) \ * The minimum part size in bytes to be used in a range multipart download. - * @param MultipartDownloadListener|null $downloadListener A multipart download - * specific listener of the different states a multipart download can be. - * @param TransferListener|null $progressTracker A transfer listener implementation - * aimed to track the progress of a transfer. If not provided and track_progress - * is resolved as true then, the default progress_tracker_factory will be used. + * If this parameter is not provided then it fallbacks to the transfer + * manager `target_part_size_bytes` config value. + * @param TransferListener[]|null $listeners + * @param TransferListener|null $progressTracker * * @return PromiseInterface */ @@ -289,7 +300,7 @@ public function download( string | array $source, array $downloadArgs = [], array $config = [], - ?MultipartDownloadListener $downloadListener = null, + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { @@ -306,21 +317,25 @@ public function download( if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { - $progressTracker = $this->resolveDefaultProgressTracker( - DefaultProgressTracker::TRACKING_OPERATION_DOWNLOADING - ); + $progressTracker = new SingleProgressTracker(); } + if ($progressTracker !== null) { + $listeners[] = $progressTracker; + } + + $listenerNotifier = new TransferListenerNotifier($listeners); $requestArgs = $sourceArgs + $downloadArgs; if (empty($downloadArgs['PartNumber']) && empty($downloadArgs['Range'])) { return $this->tryMultipartDownload( $requestArgs, [ - 'minimumPartSize' => $config['minimumPartSize'] - ?? 0 + 'minimum_part_size' => $config['minimum_part_size'] + ?? $this->config['target_part_size_bytes'], + 'multipart_download_type' => $config['multipart_download_type'] + ?? $this->config['multipart_download_type'], ], - $downloadListener, - $progressTracker, + $listenerNotifier, ); } @@ -354,11 +369,12 @@ public function download( * - filter: (Closure) \ * A callable which will receive an object key as parameter and should return * true or false in order to determine whether the object should be downloaded. - * @param MultipartDownloadListenerFactory|null $downloadListenerFactory - * A factory of multipart download listeners `MultipartDownloadListenerFactory` - * for listening to multipart download events. - * @param TransferListener|null $progressTracker - * + * @param TransferListener[] $listeners The listeners for watching + * transfer events. Each listener will be cloned per file upload. + * @param TransferListener|null $progressTracker Ideally the progress + * tracker implementation provided here should be able to track multiple + * transfers at once. Please see MultiProgressTracker implementation. + * * @return PromiseInterface */ public function downloadDirectory( @@ -366,7 +382,7 @@ public function downloadDirectory( string $destinationDirectory, array $downloadArgs, array $config = [], - ?MultipartDownloadListenerFactory $downloadListenerFactory = null, + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { @@ -378,9 +394,7 @@ public function downloadDirectory( if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { - $progressTracker = $this->resolveDefaultProgressTracker( - DefaultProgressTracker::TRACKING_OPERATION_DOWNLOADING - ); + $progressTracker = new MultiProgressTracker(); } $listArgs = [ @@ -415,18 +429,13 @@ public function downloadDirectory( ); } - $downloadListener = null; - if ($downloadListenerFactory !== null) { - $downloadListener = $downloadListenerFactory(); - } - $promises[] = $this->download( $object, $downloadArgs, [ 'minimumPartSize' => $config['minimumPartSize'] ?? 0, ], - $downloadListener, + array_map(function ($listener) { return clone $listener; }, $listeners), $progressTracker, )->then(function (DownloadResponse $result) use ($destinationFile) { $directory = dirname($destinationFile); @@ -446,31 +455,33 @@ public function downloadDirectory( * * @param array $requestArgs * @param array $config - * - minimumPartSize: (int) \ - * The minimum part size in bytes for a range multipart download. If - * this parameter is not provided then it fallbacks to the transfer - * manager `target_part_size_bytes` config value. - * @param MultipartDownloadListener|null $downloadListener - * @param TransferListener|null $progressTracker + * - minimum_part_size: (int) \ + * The minimum part size in bytes for a range multipart download. + * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function tryMultipartDownload( array $requestArgs, array $config = [], - ?MultipartDownloadListener $downloadListener = null, - ?TransferListener $progressTracker = null, + ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { - $multipartDownloader = MultipartDownloader::chooseDownloader( - s3Client: $this->s3Client, - multipartDownloadType: $this->config['multipart_download_type'], - requestArgs: $requestArgs, - config: [ - 'target_part_size_bytes' => $config['target_part_size_bytes'] ?? 0, - ], - listener: $downloadListener, - progressTracker: $progressTracker, + $downloaderClassName = match ($config['multipart_download_type']) { + MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\PartGetMultipartDownloader', + MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\RangeGetMultipartDownloader', + default => throw new \InvalidArgumentException( + "The config value for `multipart_download_type` must be one of:\n" + . "\t* " . MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER + ."\n" + . "\t* " . MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER + ) + }; + $multipartDownloader = new $downloaderClassName( + $this->s3Client, + $requestArgs, + $config, + listenerNotifier: $listenerNotifier, ); return $multipartDownloader->promise(); @@ -480,48 +491,60 @@ private function tryMultipartDownload( * Does a single object download. * * @param array $requestArgs - * @param TransferListener|null $progressTracker + * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function trySingleDownload( array $requestArgs, - ?TransferListener $progressTracker + ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { - if ($progressTracker !== null) { - $progressTracker->objectTransferInitiated($requestArgs['Key'], $requestArgs); + if ($listenerNotifier !== null) { + $listenerNotifier->transferInitiated([ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + 0 + ) + ]); $command = $this->s3Client->getCommand( MultipartDownloader::GET_OBJECT_COMMAND, $requestArgs ); return $this->s3Client->executeAsync($command)->then( - function ($result) use ($progressTracker, $requestArgs) { + function ($result) use ($requestArgs, $listenerNotifier) { // Notify progress - $progressTracker->objectTransferProgress( - $requestArgs['Key'], - $result['Content-Length'] ?? 0, - $result['Content-Length'] ?? 0, - ); - + $progressContext = [ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + $result['Content-Length'] ?? 0, + $result['Content-Length'] ?? 0, + $result->toArray() + ) + ]; + $listenerNotifier->bytesTransferred($progressContext); // Notify Completion - $progressTracker->objectTransferCompleted( - $requestArgs['Key'], - $result['Content-Length'] ?? 0, - ); + $listenerNotifier->transferComplete($progressContext); return new DownloadResponse( content: $result['Body'], metadata: $result['@metadata'], ); } - )->otherwise(function ($reason) use ($requestArgs, $progressTracker) { - $progressTracker->objectTransferFailed( - $requestArgs['Key'], - 0, - $reason->getMessage(), - ); + )->otherwise(function ($reason) use ($requestArgs, $listenerNotifier) { + $listenerNotifier->transferFail([ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + 0, + ), + 'reason' => $reason + ]); return $reason; }); @@ -544,14 +567,14 @@ function ($result) use ($progressTracker, $requestArgs) { /** * @param string|StreamInterface $source * @param array $requestArgs - * @param TransferListener|null $progressTracker + * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function trySingleUpload( string | StreamInterface $source, array $requestArgs, - ?TransferListener $progressTracker = null + ?TransferListenerNotifier $listenerNotifier = null ): PromiseInterface { if (is_string($source) && is_readable($source)) { $requestArgs['SourceFile'] = $source; @@ -565,33 +588,57 @@ private function trySingleUpload( ); } - if ($progressTracker !== null) { - $progressTracker->objectTransferInitiated( - $requestArgs['Key'], - $requestArgs + if (!empty($listenerNotifier)) { + $listenerNotifier->transferInitiated( + [ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + $objectSize, + ), + ] ); $command = $this->s3Client->getCommand('PutObject', $requestArgs); return $this->s3Client->executeAsync($command)->then( - function ($result) use ($objectSize, $progressTracker, $requestArgs) { - $progressTracker->objectTransferProgress( - $requestArgs['Key'], - $objectSize, - $objectSize, + function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { + $listenerNotifier->bytesTransferred( + [ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + $objectSize, + $objectSize, + ), + ] ); - $progressTracker->objectTransferCompleted( - $requestArgs['Key'], - $objectSize, + $listenerNotifier->transferComplete( + [ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + $objectSize, + $objectSize, + $result->toArray() + ), + ] ); return new UploadResponse($result->toArray()); } - )->otherwise(function ($reason) use ($requestArgs, $progressTracker) { - $progressTracker->objectTransferFailed( - $requestArgs['Key'], - 0, - $reason->getMessage() + )->otherwise(function ($reason) use ($objectSize, $requestArgs, $listenerNotifier) { + $listenerNotifier->transferFail( + [ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + $objectSize, + ), + 'reason' => $reason, + ] ); return $reason; @@ -610,36 +657,23 @@ function ($result) use ($objectSize, $progressTracker, $requestArgs) { * @param string|StreamInterface $source * @param array $requestArgs * @param array $config - * @param MultipartUploadListener|null $uploadListener - * @param TransferListener|null $progressTracker + * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function tryMultipartUpload( string | StreamInterface $source, array $requestArgs, - int $partSizeBytes, - ?MultipartUploadListener $uploadListener = null, - ?TransferListener $progressTracker = null, + array $config = [], + ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { $createMultipartArgs = [...$requestArgs]; - $uploadPartArgs = [...$requestArgs]; - $completeMultipartArgs = [...$requestArgs]; - if ($this->containsChecksum($requestArgs)) { - $completeMultipartArgs['ChecksumType'] = 'FULL_OBJECT'; - } - return (new MultipartUploader( $this->s3Client, $createMultipartArgs, - $uploadPartArgs, - $completeMultipartArgs, - [ - 'part_size_bytes' => $partSizeBytes, - 'concurrency' => $this->config['concurrency'], - ], + $config, $source, - progressTracker: $progressTracker, + listenerNotifier: $listenerNotifier, ))->promise(); } @@ -744,26 +778,6 @@ private function s3UriAsBucketAndKey(string $uri): array ]; } - /** - * Resolves the progress tracker to be used in the - * transfer operation if `$track_progress` is true. - * - * @param string $trackingOperation - * - * @return TransferListener|null - */ - private function resolveDefaultProgressTracker( - string $trackingOperation - ): ?TransferListener - { - $progress_tracker_factory = $this->config['progress_tracker_factory'] ?? null; - if ($progress_tracker_factory === null) { - return (new DefaultProgressTracker(trackingOperation: $trackingOperation))->getTransferListener(); - } - - return $progress_tracker_factory([]); - } - /** * @param string $sink * @param string $objectKey @@ -797,29 +811,4 @@ private function resolvesOutsideTargetDirectory( return false; } - - /** - * Verifies if a checksum was provided. - * - * @param array $requestArgs - * - * @return bool - */ - private function containsChecksum(array $requestArgs): bool - { - $algorithms = [ - 'ChecksumCRC32', - 'ChecksumCRC32C', - 'ChecksumCRC64NVME', - 'ChecksumSHA1', - 'ChecksumSHA256', - ]; - foreach ($algorithms as $algorithm) { - if (isset($requestArgs[$algorithm])) { - return true; - } - } - - return false; - } } \ No newline at end of file diff --git a/src/S3/S3Transfer/TransferListener.php b/src/S3/S3Transfer/TransferListener.php index 8b2c88635d..2a24707d17 100644 --- a/src/S3/S3Transfer/TransferListener.php +++ b/src/S3/S3Transfer/TransferListener.php @@ -2,291 +2,46 @@ namespace Aws\S3\S3Transfer; -use Closure; -use Throwable; - -class TransferListener extends ListenerNotifier +abstract class TransferListener { /** - * @param Closure|null $onTransferInitiated - * No parameters will be passed. - * - * @param Closure|null $onObjectTransferInitiated - * Parameters that will be passed when invoked: - * - $objectKey: The key that identifies the object being transferred. - * - $objectRequestArgs: The arguments that initiated the object transfer request. - * - * @param Closure|null $onObjectTransferProgress - * Parameters that will be passed when invoked: - * - $objectKey: The key that identifies the object being transferred. - * - $objectBytesTransferred: The total of bytes transferred for this object. - * - $objectSizeInBytes: The size in bytes of the object. - * - * @param Closure|null $onObjectTransferFailed - * Parameters that will be passed when invoked: - * - $objectKey: The object key for which the transfer has failed. - * - $objectBytesTransferred: The total of bytes transferred from - * this object. - * - $reason: The reason why the transfer failed for this object. - * - * @param Closure|null $onObjectTransferCompleted - * Parameters that will be passed when invoked: - * - $objectKey: The object key for which the transfer was completed. - * - $objectBytesCompleted: The total of bytes transferred for this object. - * - * @param Closure|null $onTransferProgress - * Parameters that will be passed when invoked: - * - $totalObjectsTransferred: The number of objects transferred. - * - $totalBytesTransferred: The total of bytes already transferred on this event. - * - $totalBytes: The total of bytes to be transferred. - * - * @param Closure|null $onTransferCompleted - * Parameters that will be passed when invoked: - * - $objectsTransferCompleted: The number of objects that were transferred. - * - $objectsBytesTransferred: The total of bytes that were transferred. - * - * @param Closure|null $onTransferFailed - * Parameters that will be passed when invoked: - * - $objectsTransferCompleted: The total of objects transferred before failure. - * - $objectsBytesTransferred: The total of bytes transferred before failure. - * - $objectsTransferFailed: The total of objects that failed in the transfer. - * - $reason: The throwable with the reason why the transfer failed. - * @param int $objectsTransferCompleted - * @param int $objectsBytesTransferred - * @param int $objectsTransferFailed - * @param int $objectsToBeTransferred - */ - public function __construct( - public ?Closure $onTransferInitiated = null, - public ?Closure $onObjectTransferInitiated = null, - public ?Closure $onObjectTransferProgress = null, - public ?Closure $onObjectTransferFailed = null, - public ?Closure $onObjectTransferCompleted = null, - public ?Closure $onTransferProgress = null, - public ?Closure $onTransferCompleted = null, - public ?Closure $onTransferFailed = null, - private int $objectsTransferCompleted = 0, - private int $objectsBytesTransferred = 0, - private int $objectsTransferFailed = 0, - private int $objectsToBeTransferred = 0 - ) {} - - /** - * @return int - */ - public function getObjectsTransferCompleted(): int - { - return $this->objectsTransferCompleted; - } - - /** - * @return int - */ - public function getObjectsBytesTransferred(): int - { - return $this->objectsBytesTransferred; - } - - /** - * @return int - */ - public function getObjectsTransferFailed(): int - { - return $this->objectsTransferFailed; - } - - /** - * @return int - */ - public function getObjectsToBeTransferred(): int - { - return $this->objectsToBeTransferred; - } - - /** - * Transfer initiated event. - */ - public function transferInitiated(): void - { - $this->notify('onTransferInitiated', []); - } - - /** - * Event for when an object transfer initiated. - * - * @param string $objectKey - * @param array $requestArgs + * @param array $context + * - request_args: (array) The request arguments that will be provided + * as part of the request initialization. + * - progress_snapshot: (TransferProgressSnapshot) The transfer snapshot holder. * * @return void */ - public function objectTransferInitiated(string $objectKey, array &$requestArgs): void - { - $this->objectsToBeTransferred++; - if ($this->objectsToBeTransferred === 1) { - $this->transferInitiated(); - } - - $this->notify('onObjectTransferInitiated', [$objectKey, &$requestArgs]); - } + public function transferInitiated(array $context): void {} /** - * Event for when an object transfer made some progress. - * - * @param string $objectKey - * @param int $objectBytesTransferred - * @param int $objectSizeInBytes + * @param array $context + * - request_args: (array) The request arguments that will be provided + * as part of the operation that originated the bytes transferred event. + * - progress_snapshot: (TransferProgressSnapshot) The transfer snapshot holder. * * @return void */ - public function objectTransferProgress( - string $objectKey, - int $objectBytesTransferred, - int $objectSizeInBytes - ): void - { - $this->objectsBytesTransferred += $objectBytesTransferred; - $this->notify('onObjectTransferProgress', [ - $objectKey, - $objectBytesTransferred, - $objectSizeInBytes - ]); - // Needs state management - $this->notify('onTransferProgress', [ - $this->objectsTransferCompleted, - $this->objectsBytesTransferred, - $this->objectsToBeTransferred - ]); - } + public function bytesTransferred(array $context): void {} /** - * Event for when an object transfer failed. - * - * @param string $objectKey - * @param int $objectBytesTransferred - * @param \Throwable|string $reason + * @param array $context + * - request_args: (array) The request arguments that will be provided + * as part of the operation that originated the bytes transferred event. + * - progress_snapshot: (TransferProgressSnapshot) The transfer snapshot holder. * * @return void */ - public function objectTransferFailed( - string $objectKey, - int $objectBytesTransferred, - \Throwable | string $reason - ): void - { - $this->objectsTransferFailed++; - $this->validateTransferComplete(); - $this->notify('onObjectTransferFailed', [ - $objectKey, - $objectBytesTransferred, - $reason - ]); - } + public function transferComplete(array $context): void {} /** - * Event for when an object transfer is completed. - * - * @param string $objectKey - * @param int $objectBytesCompleted + * @param array $context + * - request_args: (array) The request arguments that will be provided + * as part of the operation that originated the bytes transferred event. + * - progress_snapshot: (TransferProgressSnapshot) The transfer snapshot holder. + * - reason: (Throwable) The exception originated by the transfer failure. * * @return void */ - public function objectTransferCompleted ( - string $objectKey, - int $objectBytesCompleted - ): void - { - $this->objectsTransferCompleted++; - $this->validateTransferComplete(); - $this->notify('onObjectTransferCompleted', [ - $objectKey, - $objectBytesCompleted - ]); - } - - /** - * Event for when a transfer is completed. - * - * @param int $objectsTransferCompleted - * @param int $objectsBytesTransferred - * - * @return void - */ - public function transferCompleted ( - int $objectsTransferCompleted, - int $objectsBytesTransferred, - ): void - { - $this->notify('onTransferCompleted', [ - $objectsTransferCompleted, - $objectsBytesTransferred - ]); - } - - /** - * Event for when a transfer is completed. - * - * @param int $objectsTransferCompleted - * @param int $objectsBytesTransferred - * @param int $objectsTransferFailed - * - * @return void - */ - public function transferFailed ( - int $objectsTransferCompleted, - int $objectsBytesTransferred, - int $objectsTransferFailed, - Throwable | string $reason - ): void - { - $this->notify('onTransferFailed', [ - $objectsTransferCompleted, - $objectsBytesTransferred, - $objectsTransferFailed, - $reason - ]); - } - - /** - * Validates if a transfer is completed, and if so then the event is propagated - * to the subscribed listeners. - * - * @return void - */ - private function validateTransferComplete(): void - { - if ($this->objectsToBeTransferred === ($this->objectsTransferCompleted + $this->objectsTransferFailed)) { - if ($this->objectsTransferFailed > 0) { - $this->transferFailed( - $this->objectsTransferCompleted, - $this->objectsBytesTransferred, - $this->objectsTransferFailed, - "Transfer could not have been completed successfully." - ); - } else { - $this->transferCompleted( - $this->objectsTransferCompleted, - $this->objectsBytesTransferred - ); - } - } - } - - protected function notify(string $event, array $params = []): void - { - $listener = match ($event) { - 'onTransferInitiated' => $this->onTransferInitiated, - 'onObjectTransferInitiated' => $this->onObjectTransferInitiated, - 'onObjectTransferProgress' => $this->onObjectTransferProgress, - 'onObjectTransferFailed' => $this->onObjectTransferFailed, - 'onObjectTransferCompleted' => $this->onObjectTransferCompleted, - 'onTransferProgress' => $this->onTransferProgress, - 'onTransferCompleted' => $this->onTransferCompleted, - 'onTransferFailed' => $this->onTransferFailed, - default => null, - }; - - if ($listener instanceof Closure) { - $listener(...$params); - } - } + public function transferFail(array $context): void {} } \ No newline at end of file diff --git a/src/S3/S3Transfer/TransferListenerFactory.php b/src/S3/S3Transfer/TransferListenerFactory.php deleted file mode 100644 index 40f9747cf2..0000000000 --- a/src/S3/S3Transfer/TransferListenerFactory.php +++ /dev/null @@ -1,8 +0,0 @@ -listeners = $listeners; + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferInitiated(array $context): void + { + foreach ($this->listeners as $name => $listener) { + $listener->transferInitiated($context); + } + } + + /** + * @inheritDoc + * + * @return void + */ + public function bytesTransferred(array $context): void + { + foreach ($this->listeners as $name => $listener) { + $listener->bytesTransferred($context); + } + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferComplete(array $context): void + { + foreach ($this->listeners as $name => $listener) { + $listener->transferComplete($context); + } + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferFail(array $context): void + { + foreach ($this->listeners as $name => $listener) { + $listener->transferFail($context); + } + } +} \ No newline at end of file diff --git a/tests/S3/S3Transfer/ConsoleProgressBarTest.php b/tests/S3/S3Transfer/ConsoleProgressBarTest.php deleted file mode 100644 index 31ea5ebd5d..0000000000 --- a/tests/S3/S3Transfer/ConsoleProgressBarTest.php +++ /dev/null @@ -1,228 +0,0 @@ -setPercentCompleted($percent); - $progressBar->setArgs([ - 'transferred' => $transferred, - 'tobe_transferred' => $toBeTransferred, - 'unit' => $unit - ]); - - $output = $progressBar->getPaintedProgress(); - $this->assertEquals($expectedProgress, $output); - } - - /** - * Data provider for testing progress bar rendering. - * - * @return array - */ - public function progressBarPercentProvider(): array { - return [ - [ - 'percent' => 25, - 'transferred' => 25, - 'tobe_transferred' => 100, - 'unit' => 'B', - 'expected' => '[###### ] 25% 25/100 B' - ], - [ - 'percent' => 50, - 'transferred' => 50, - 'tobe_transferred' => 100, - 'unit' => 'B', - 'expected' => '[############# ] 50% 50/100 B' - ], - [ - 'percent' => 75, - 'transferred' => 75, - 'tobe_transferred' => 100, - 'unit' => 'B', - 'expected' => '[################### ] 75% 75/100 B' - ], - [ - 'percent' => 100, - 'transferred' => 100, - 'tobe_transferred' => 100, - 'unit' => 'B', - 'expected' => '[#########################] 100% 100/100 B' - ], - ]; - } - - /** - * Tests progress with custom char. - * - * @return void - */ - public function testProgressBarWithCustomChar() - { - $progressBar = new ConsoleProgressBar( - progressBarChar: '*', - progressBarWidth: 30 - ); - $progressBar->setPercentCompleted(30); - $progressBar->setArgs([ - 'transferred' => '10', - 'tobe_transferred' => '100', - 'unit' => 'B' - ]); - - $output = $progressBar->getPaintedProgress(); - $this->assertStringContainsString('10/100 B', $output); - $this->assertStringContainsString(str_repeat('*', 9), $output); - } - - /** - * Tests progress with custom char. - * - * @return void - */ - public function testProgressBarWithCustomWidth() - { - $progressBar = new ConsoleProgressBar( - progressBarChar: '*', - progressBarWidth: 100 - ); - $progressBar->setPercentCompleted(10); - $progressBar->setArgs([ - 'transferred' => '10', - 'tobe_transferred' => '100', - 'unit' => 'B' - ]); - - $output = $progressBar->getPaintedProgress(); - $this->assertStringContainsString('10/100 B', $output); - $this->assertStringContainsString(str_repeat('*', 10), $output); - } - - /** - * Tests missing parameters. - * - * @dataProvider progressBarMissingArgsProvider - * - * @return void - */ - public function testProgressBarMissingArgsThrowsException( - string $formatName, - string $parameter - ) - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage("Missing `$parameter` parameter for progress bar."); - - $format = ConsoleProgressBar::$formats[$formatName]; - $progressBar = new ConsoleProgressBar( - format: $format, - ); - foreach ($format['parameters'] as $param) { - if ($param === $parameter) { - continue; - } - - $progressBar->setArg($param, 'foo'); - } - - $progressBar->setPercentCompleted(20); - $progressBar->getPaintedProgress(); - } - - - /** - * Data provider for testing exception when arguments are missing. - * - * @return array - */ - public function progressBarMissingArgsProvider(): array - { - return [ - [ - 'formatName' => ConsoleProgressBar::TRANSFER_FORMAT, - 'parameter' => 'transferred', - ], - [ - 'formatName' => ConsoleProgressBar::TRANSFER_FORMAT, - 'parameter' => 'tobe_transferred', - ], - [ - 'formatName' => ConsoleProgressBar::TRANSFER_FORMAT, - 'parameter' => 'unit', - ] - ]; - } - - /** - * Tests the progress bar does not overflow when the percent is over 100. - * - * @return void - */ - public function testProgressBarDoesNotOverflowAfter100Percent() - { - $progressBar = new ConsoleProgressBar( - progressBarChar: '*', - progressBarWidth: 10, - ); - $progressBar->setPercentCompleted(110); - $progressBar->setArgs([ - 'transferred' => 'foo', - 'tobe_transferred' => 'foo', - 'unit' => 'MB' - ]); - $output = $progressBar->getPaintedProgress(); - $this->assertStringContainsString('100%', $output); - $this->assertStringContainsString('[**********]', $output); - } - - /** - * Tests the progress bar sets the arguments. - * - * @return void - */ - public function testProgressBarSetsArguments() { - $progressBar = new ConsoleProgressBar( - progressBarChar: '*', - progressBarWidth: 25, - format: ConsoleProgressBar::$formats[ConsoleProgressBar::TRANSFER_FORMAT] - ); - $progressBar->setArgs([ - 'transferred' => 'fooTransferred', - 'tobe_transferred' => 'fooToBeTransferred', - 'unit' => 'fooUnit', - ]); - $output = $progressBar->getPaintedProgress(); - $progressBar->setPercentCompleted(100); - $this->assertStringContainsString('fooTransferred', $output); - $this->assertStringContainsString('fooToBeTransferred', $output); - $this->assertStringContainsString('fooUnit', $output); - } -} diff --git a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php new file mode 100644 index 0000000000..89a76d452b --- /dev/null +++ b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php @@ -0,0 +1,261 @@ +assertEquals( + ConsoleProgressBar::DEFAULT_PROGRESS_BAR_WIDTH, + $progressBar->getProgressBarWidth() + ); + $this->assertEquals( + ConsoleProgressBar::DEFAULT_PROGRESS_BAR_CHAR, + $progressBar->getProgressBarChar() + ); + $this->assertEquals( + 0, + $progressBar->getPercentCompleted() + ); + $this->assertInstanceOf( + ColoredTransferProgressBarFormat::class, + $progressBar->getProgressBarFormat() + ); + } + + /** + * Tests the percent is updated properly. + * + * @return void + */ + public function testSetPercentCompleted(): void { + $progressBar = new ConsoleProgressBar(); + $progressBar->setPercentCompleted(10); + $this->assertEquals(10, $progressBar->getPercentCompleted()); + $progressBar->setPercentCompleted(100); + $this->assertEquals(100, $progressBar->getPercentCompleted()); + } + + /** + * @return void + */ + public function testSetCustomValues(): void { + $progressBar = new ConsoleProgressBar( + progressBarChar: '-', + progressBarWidth: 10, + percentCompleted: 25, + progressBarFormat: new PlainProgressBarFormat() + ); + $this->assertEquals('-', $progressBar->getProgressBarChar()); + $this->assertEquals(10, $progressBar->getProgressBarWidth()); + $this->assertEquals(25, $progressBar->getPercentCompleted()); + $this->assertInstanceOf( + PlainProgressBarFormat::class, + $progressBar->getProgressBarFormat() + ); + } + + /** + * To make sure the percent is not over 100. + * + * @return void + */ + public function testPercentIsNotOverOneHundred(): void { + $progressBar = new ConsoleProgressBar(); + $progressBar->setPercentCompleted(150); + $this->assertEquals(100, $progressBar->getPercentCompleted()); + } + + /** + * @param string $progressBarChar + * @param int $progressBarWidth + * @param int $percentCompleted + * @param ProgressBarFormat $progressBarFormat + * @param array $progressBarFormatArgs + * @param string $expectedOutput + * + * @return void + * @dataProvider progressBarRenderingProvider + * + */ + public function testProgressBarRendering( + string $progressBarChar, + int $progressBarWidth, + int $percentCompleted, + ProgressBarFormat $progressBarFormat, + array $progressBarFormatArgs, + string $expectedOutput + ): void { + $progressBarFormat->setArgs($progressBarFormatArgs); + $progressBar = new ConsoleProgressBar( + $progressBarChar, + $progressBarWidth, + $percentCompleted, + $progressBarFormat, + ); + + $this->assertEquals($expectedOutput, $progressBar->render()); + } + + /** + * Data provider for testing progress bar rendering. + * + * @return array + */ + public function progressBarRenderingProvider(): array { + return [ + 'plain_progress_bar_format_1' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 50, + 'percent_completed' => 15, + 'progress_bar_format' => new PlainProgressBarFormat(), + 'progress_bar_format_args' => [], + 'expected_output' => '[######## ] 15%' + ], + 'plain_progress_bar_format_2' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 50, + 'percent_completed' => 45, + 'progress_bar_format' => new PlainProgressBarFormat(), + 'progress_bar_format_args' => [], + 'expected_output' => '[####################### ] 45%' + ], + 'plain_progress_bar_format_3' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 50, + 'percent_completed' => 100, + 'progress_bar_format' => new PlainProgressBarFormat(), + 'progress_bar_format_args' => [], + 'expected_output' => '[##################################################] 100%' + ], + 'plain_progress_bar_format_4' => [ + 'progress_bar_char' => '.', + 'progress_bar_width' => 50, + 'percent_completed' => 100, + 'progress_bar_format' => new PlainProgressBarFormat(), + 'progress_bar_format_args' => [], + 'expected_output' => '[..................................................] 100%' + ], + 'transfer_progress_bar_format_1' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 50, + 'percent_completed' => 23, + 'progress_bar_format' => new TransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'transferred' => 23, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => '[############ ] 23% 23/100 B' + ], + 'transfer_progress_bar_format_2' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 25, + 'percent_completed' => 75, + 'progress_bar_format' => new TransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'transferred' => 75, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => '[################### ] 75% 75/100 B' + ], + 'transfer_progress_bar_format_3' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 30, + 'percent_completed' => 100, + 'progress_bar_format' => new TransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => '[##############################] 100% 100/100 B' + ], + 'transfer_progress_bar_format_4' => [ + 'progress_bar_char' => '*', + 'progress_bar_width' => 30, + 'percent_completed' => 100, + 'progress_bar_format' => new TransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => '[******************************] 100% 100/100 B' + ], + 'colored_progress_bar_format_1' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 20, + 'percent_completed' => 10, + 'progress_bar_format' => new ColoredTransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'ObjectName_1', + 'transferred' => 10, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => "ObjectName_1:\n\033[30m[## ] 10% 10/100 B \033[0m" + ], + 'colored_progress_bar_format_2' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 20, + 'percent_completed' => 50, + 'progress_bar_format' => new ColoredTransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'ObjectName_2', + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE + ], + 'expected_output' => "ObjectName_2:\n\033[34m[########## ] 50% 50/100 B \033[0m" + ], + 'colored_progress_bar_format_3' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 25, + 'percent_completed' => 100, + 'progress_bar_format' => new ColoredTransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'ObjectName_3', + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE + ], + 'expected_output' => "ObjectName_3:\n\033[32m[#########################] 100% 100/100 B \033[0m" + ], + 'colored_progress_bar_format_4' => [ + 'progress_bar_char' => '=', + 'progress_bar_width' => 25, + 'percent_completed' => 100, + 'progress_bar_format' => new ColoredTransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'ObjectName_3', + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE + ], + 'expected_output' => "ObjectName_3:\n\033[32m[=========================] 100% 100/100 B \033[0m" + ] + ]; + } +} From e681d10705fa91f199c5f2b786decb98f0af4325 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 24 Feb 2025 07:07:53 -0800 Subject: [PATCH 11/19] chore: fix test cases - Fixes current test cases for: - MultipartUploader - MultipartDownloader - ProgressTracker --- src/S3/S3Transfer/MultipartDownloader.php | 23 +- src/S3/S3Transfer/MultipartUploader.php | 2 +- .../Progress/MultiProgressTracker.php | 12 +- src/S3/S3Transfer/S3TransferManager.php | 13 +- .../S3Transfer/DefaultProgressTrackerTest.php | 189 ---------------- .../MultipartDownloadListenerTest.php | 207 ------------------ .../S3/S3Transfer/MultipartDownloaderTest.php | 26 ++- tests/S3/S3Transfer/MultipartUploaderTest.php | 16 +- .../S3Transfer/ObjectProgressTrackerTest.php | 127 ----------- 9 files changed, 56 insertions(+), 559 deletions(-) delete mode 100644 tests/S3/S3Transfer/DefaultProgressTrackerTest.php delete mode 100644 tests/S3/S3Transfer/MultipartDownloadListenerTest.php delete mode 100644 tests/S3/S3Transfer/ObjectProgressTrackerTest.php diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index d110d01635..f447ac5815 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -51,9 +51,7 @@ abstract class MultipartDownloader implements PromisorInterface * using range get. This option MUST be set when using range get. * @param int $currentPartNo * @param int $objectPartsCount - * @param int $objectCompletedPartsCount * @param int $objectSizeInBytes - * @param int $objectBytesTransferred * @param string $eTag * @param StreamInterface|null $stream * @param TransferProgressSnapshot|null $currentSnapshot @@ -337,4 +335,25 @@ private function downloadComplete(): void 'progress_snapshot' => $this->currentSnapshot, ]); } + + /** + * @param mixed $multipartDownloadType + * + * @return string + */ + public static function chooseDownloaderClassName( + string $multipartDownloadType + ): string + { + return match ($multipartDownloadType) { + MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\PartGetMultipartDownloader', + MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\RangeGetMultipartDownloader', + default => throw new \InvalidArgumentException( + "The config value for `multipart_download_type` must be one of:\n" + . "\t* " . MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER + ."\n" + . "\t* " . MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER + ) + }; + } } \ No newline at end of file diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 805eb5179e..c9885bb9c9 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -132,7 +132,7 @@ public function promise(): PromiseInterface ); } catch (Throwable $e) { $this->uploadFailed($e); - throw $e; + yield Create::rejectionFor($e); } finally { $this->callDeferredFns(); } diff --git a/src/S3/S3Transfer/Progress/MultiProgressTracker.php b/src/S3/S3Transfer/Progress/MultiProgressTracker.php index 6dcba0053c..6b00ea440f 100644 --- a/src/S3/S3Transfer/Progress/MultiProgressTracker.php +++ b/src/S3/S3Transfer/Progress/MultiProgressTracker.php @@ -90,11 +90,15 @@ public function transferInitiated(array $context): void { $this->transferCount++; $snapshot = $context['progress_snapshot']; - $progressTracker = new SingleProgressTracker( - clear: false, - ); + if (isset($this->singleProgressTrackers[$snapshot['key']])) { + $progressTracker = $this->singleProgressTrackers[$snapshot['key']]; + } else { + $progressTracker = new SingleProgressTracker( + clear: false, + ); + $this->singleProgressTrackers[$snapshot->getIdentifier()] = $progressTracker; + } $progressTracker->transferInitiated($context); - $this->singleProgressTrackers[$snapshot->getIdentifier()] = $progressTracker; $this->showProgress(); } diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index c29ad896a9..e14c3037b6 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -467,16 +467,9 @@ private function tryMultipartDownload( ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { - $downloaderClassName = match ($config['multipart_download_type']) { - MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\PartGetMultipartDownloader', - MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\RangeGetMultipartDownloader', - default => throw new \InvalidArgumentException( - "The config value for `multipart_download_type` must be one of:\n" - . "\t* " . MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER - ."\n" - . "\t* " . MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER - ) - }; + $downloaderClassName = MultipartDownloader::chooseDownloaderClassName( + $config['multipart_download_type'] + ); $multipartDownloader = new $downloaderClassName( $this->s3Client, $requestArgs, diff --git a/tests/S3/S3Transfer/DefaultProgressTrackerTest.php b/tests/S3/S3Transfer/DefaultProgressTrackerTest.php deleted file mode 100644 index 0c9d8f0e53..0000000000 --- a/tests/S3/S3Transfer/DefaultProgressTrackerTest.php +++ /dev/null @@ -1,189 +0,0 @@ -progressTracker = new DefaultProgressTracker( - output: $this->output = fopen('php://temp', 'r+') - ); - } - - protected function tearDown(): void { - fclose($this->output); - } - - /** - * Tests initialization is clean. - * - * @return void - */ - public function testInitialization(): void - { - $this->assertInstanceOf(TransferListener::class, $this->progressTracker->getTransferListener()); - $this->assertEquals(0, $this->progressTracker->getTotalBytesTransferred()); - $this->assertEquals(0, $this->progressTracker->getObjectsTotalSizeInBytes()); - $this->assertEquals(0, $this->progressTracker->getObjectsInProgress()); - $this->assertEquals(0, $this->progressTracker->getObjectsCount()); - $this->assertEquals(0, $this->progressTracker->getTransferPercentCompleted()); - } - - /** - * Tests object transfer is initiated when the event is triggered. - * - * @return void - */ - public function testObjectTransferInitiated(): void - { - $listener = $this->progressTracker->getTransferListener(); - $fakeRequestArgs = []; - ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); - - $this->assertEquals(1, $this->progressTracker->getObjectsInProgress()); - $this->assertEquals(1, $this->progressTracker->getObjectsCount()); - } - - /** - * Tests object transfer progress is propagated correctly. - * - * @dataProvider objectTransferProgressProvider - * - * @param string $objectKey - * @param int $objectSize - * @param array $progressList - * - * @return void - */ - public function testObjectTransferProgress( - string $objectKey, - int $objectSize, - array $progressList, - ): void - { - $listener = $this->progressTracker->getTransferListener(); - $fakeRequestArgs = []; - ($listener->onObjectTransferInitiated)($objectKey, $fakeRequestArgs); - $totalProgress = 0; - foreach ($progressList as $progress) { - ($listener->onObjectTransferProgress)($objectKey, $progress, $objectSize); - $totalProgress += $progress; - } - - $this->assertEquals($totalProgress, $this->progressTracker->getTotalBytesTransferred()); - $this->assertEquals($objectSize, $this->progressTracker->getObjectsTotalSizeInBytes()); - $percentCompleted = (int) floor($totalProgress / $objectSize) * 100; - $this->assertEquals($percentCompleted, $this->progressTracker->getTransferPercentCompleted()); - - rewind($this->output); - $this->assertStringContainsString("$percentCompleted% $totalProgress/$objectSize B", stream_get_contents($this->output)); - } - - /** - * Data provider for testing object progress tracker. - * - * @return array[] - */ - public function objectTransferProgressProvider(): array - { - return [ - [ - 'objectKey' => 'FooObjectKey', - 'objectSize' => 250, - 'progressList' => [ - 50, 100, 72, 28 - ] - ], - [ - 'objectKey' => 'FooObjectKey', - 'objectSize' => 10_000, - 'progressList' => [ - 100, 500, 1_000, 2_000, 5_000, 400, 700, 300 - ] - ], - [ - 'objectKey' => 'FooObjectKey', - 'objectSize' => 10_000, - 'progressList' => [ - 5_000, 5_000 - ] - ] - ]; - } - - /** - * Tests object transfer is completed. - * - * @return void - */ - public function testObjectTransferCompleted(): void - { - $listener = $this->progressTracker->getTransferListener(); - $fakeRequestArgs = []; - ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); - ($listener->onObjectTransferProgress)('FooObjectKey', 50, 100); - ($listener->onObjectTransferProgress)('FooObjectKey', 50, 100); - ($listener->onObjectTransferCompleted)('FooObjectKey', 100); - - $this->assertEquals(100, $this->progressTracker->getTotalBytesTransferred()); - $this->assertEquals(100, $this->progressTracker->getTransferPercentCompleted()); - - // Validate it completed 100% at the progress bar side. - rewind($this->output); - $this->assertStringContainsString("[#########################] 100% 100/100 B", stream_get_contents($this->output)); - } - - /** - * Tests object transfer failed. - * - * @return void - */ - public function testObjectTransferFailed(): void - { - $listener = $this->progressTracker->getTransferListener(); - $fakeRequestArgs = []; - ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); - ($listener->onObjectTransferProgress)('FooObjectKey', 27, 100); - ($listener->onObjectTransferFailed)('FooObjectKey', 27, 'Transfer error'); - - $this->assertEquals(27, $this->progressTracker->getTotalBytesTransferred()); - $this->assertEquals(27, $this->progressTracker->getTransferPercentCompleted()); - $this->assertEquals(0, $this->progressTracker->getObjectsInProgress()); - - rewind($this->output); - $this->assertStringContainsString("27% 27/100 B", stream_get_contents($this->output)); - } - - /** - * Tests state are cleared. - * - * @return void - */ - public function testClearState(): void - { - $listener = $this->progressTracker->getTransferListener(); - $fakeRequestArgs = []; - ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); - ($listener->onObjectTransferProgress)('FooObjectKey', 10, 100); - - $this->progressTracker->clear(); - - $this->assertEquals(0, $this->progressTracker->getTotalBytesTransferred()); - $this->assertEquals(0, $this->progressTracker->getObjectsTotalSizeInBytes()); - $this->assertEquals(0, $this->progressTracker->getObjectsInProgress()); - $this->assertEquals(0, $this->progressTracker->getObjectsCount()); - $this->assertEquals(0, $this->progressTracker->getTransferPercentCompleted()); - } -} - diff --git a/tests/S3/S3Transfer/MultipartDownloadListenerTest.php b/tests/S3/S3Transfer/MultipartDownloadListenerTest.php deleted file mode 100644 index fc76b48e97..0000000000 --- a/tests/S3/S3Transfer/MultipartDownloadListenerTest.php +++ /dev/null @@ -1,207 +0,0 @@ -assertIsArray($commandArgs); - $this->assertIsInt($initialPart); - }; - - $listener = new MultipartDownloadListener(onDownloadInitiated: $callback); - - $commandArgs = ['Foo' => 'Buzz']; - $listener->downloadInitiated($commandArgs, 1); - - $this->assertTrue($called, "Expected onDownloadInitiated to be called."); - } - - /** - * Tests download failed event is propagated. - * - * @return void - */ - public function testDownloadFailed(): void - { - $called = false; - $expectedError = new Exception('Download failed'); - $expectedTotalPartsTransferred = 5; - $expectedTotalBytesTransferred = 1024; - $expectedLastPartTransferred = 4; - $callback = function ( - $reason, - $totalPartsTransferred, - $totalBytesTransferred, - $lastPartTransferred - ) use ( - &$called, - $expectedError, - $expectedTotalPartsTransferred, - $expectedTotalBytesTransferred, - $expectedLastPartTransferred - ) { - $called = true; - $this->assertEquals($reason, $expectedError); - $this->assertEquals($expectedTotalPartsTransferred, $totalPartsTransferred); - $this->assertEquals($expectedTotalBytesTransferred, $totalBytesTransferred); - $this->assertEquals($expectedLastPartTransferred, $lastPartTransferred); - - }; - $listener = new MultipartDownloadListener(onDownloadFailed: $callback); - $listener->downloadFailed( - $expectedError, - $expectedTotalPartsTransferred, - $expectedTotalBytesTransferred, - $expectedLastPartTransferred - ); - $this->assertTrue($called, "Expected onDownloadFailed to be called."); - } - - /** - * Tests download completed event is propagated. - * - * @return void - */ - public function testDownloadCompleted(): void - { - $called = false; - $expectedStream = fopen('php://temp', 'r+'); - $expectedTotalPartsDownloaded = 10; - $expectedTotalBytesDownloaded = 2048; - $callback = function ( - $stream, - $totalPartsDownloaded, - $totalBytesDownloaded - ) use ( - &$called, - $expectedStream, - $expectedTotalPartsDownloaded, - $expectedTotalBytesDownloaded - ) { - $called = true; - $this->assertIsResource($stream); - $this->assertEquals($expectedStream, $stream); - $this->assertEquals($expectedTotalPartsDownloaded, $totalPartsDownloaded); - $this->assertEquals($expectedTotalBytesDownloaded, $totalBytesDownloaded); - }; - - $listener = new MultipartDownloadListener(onDownloadCompleted: $callback); - $listener->downloadCompleted( - $expectedStream, - $expectedTotalPartsDownloaded, - $expectedTotalBytesDownloaded - ); - $this->assertTrue($called, "Expected onDownloadCompleted to be called."); - } - - /** - * Tests part downloaded initiated event is propagated. - * - * @return void - */ - public function testPartDownloadInitiated(): void - { - $called = false; - $mockCommand = $this->createMock(CommandInterface::class); - $expectedPartNo = 3; - $callable = function ($command, $partNo) - use (&$called, $mockCommand, $expectedPartNo) { - $called = true; - $this->assertEquals($expectedPartNo, $partNo); - $this->assertEquals($mockCommand, $command); - }; - $listener = new MultipartDownloadListener(onPartDownloadInitiated: $callable); - $listener->partDownloadInitiated($mockCommand, $expectedPartNo); - $this->assertTrue($called, "Expected onPartDownloadInitiated to be called."); - } - - /** - * Tests part download completed event is propagated. - * - * @return void - */ - public function testPartDownloadCompleted(): void - { - $called = false; - $mockResult = $this->createMock(ResultInterface::class); - $expectedPartNo = 3; - $expectedPartTotalBytes = 512; - $expectedTotalParts = 5; - $expectedObjectBytesTransferred = 1024; - $expectedObjectSizeInBytes = 2048; - $callback = function ( - $result, - $partNo, - $partTotalBytes, - $totalParts, - $objectBytesDownloaded, - $objectSizeInBytes - ) use ( - &$called, - $mockResult, - $expectedPartNo, - $expectedPartTotalBytes, - $expectedTotalParts, - $expectedObjectBytesTransferred, - $expectedObjectSizeInBytes - ) { - $called = true; - $this->assertEquals($mockResult, $result); - $this->assertEquals($expectedPartNo, $partNo); - $this->assertEquals($expectedPartTotalBytes, $partTotalBytes); - $this->assertEquals($expectedTotalParts, $totalParts); - $this->assertEquals($expectedObjectBytesTransferred, $objectBytesDownloaded); - $this->assertEquals($expectedObjectSizeInBytes, $objectSizeInBytes); - }; - $listener = new MultipartDownloadListener(onPartDownloadCompleted: $callback); - $listener->partDownloadCompleted( - $mockResult, - $expectedPartNo, - $expectedPartTotalBytes, - $expectedTotalParts, - $expectedObjectBytesTransferred, - $expectedObjectSizeInBytes - ); - $this->assertTrue($called, "Expected onPartDownloadCompleted to be called."); - } - - /** - * Tests part download failed event is propagated. - * - * @return void - */ - public function testPartDownloadFailed() - { - $called = false; - $mockCommand = $this->createMock(CommandInterface::class); - $expectedReason = new Exception('Part download failed'); - $expectedPartNo = 2; - $callable = function ($command, $reason, $partNo) - use (&$called, $mockCommand, $expectedReason, $expectedPartNo) { - $called = true; - $this->assertEquals($expectedReason, $reason); - $this->assertEquals($expectedPartNo, $partNo); - $this->assertEquals($mockCommand, $command); - }; - - $listener = new MultipartDownloadListener(onPartDownloadFailed: $callable); - $listener->partDownloadFailed($mockCommand, $expectedReason, $expectedPartNo); - $this->assertTrue($called, "Expected onPartDownloadFailed to be called."); - } -} \ No newline at end of file diff --git a/tests/S3/S3Transfer/MultipartDownloaderTest.php b/tests/S3/S3Transfer/MultipartDownloaderTest.php index 0838f86ab5..e986ac4dd6 100644 --- a/tests/S3/S3Transfer/MultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/MultipartDownloaderTest.php @@ -5,6 +5,7 @@ use Aws\Command; use Aws\Result; use Aws\S3\S3Client; +use Aws\S3\S3Transfer\DownloadResponse; use Aws\S3\S3Transfer\MultipartDownloader; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; @@ -25,9 +26,9 @@ class MultipartDownloaderTest extends TestCase * @param int $objectSizeInBytes * @param int $targetPartSize * - * @return void * @dataProvider partGetMultipartDownloaderProvider * + * @return void */ public function testMultipartDownloader( string $multipartDownloadType, @@ -67,25 +68,30 @@ public function testMultipartDownloader( -> willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); }); - $downloader = MultipartDownloader::chooseDownloader( + $downloaderClassName = MultipartDownloader::chooseDownloaderClassName( + $multipartDownloadType + ); + /** @var MultipartDownloader $downloader */ + $downloader = new $downloaderClassName( $mockClient, - $multipartDownloadType, [ 'Bucket' => 'FooBucket', 'Key' => $objectKey, ], [ - 'minimumPartSize' => $targetPartSize, + 'minimum_part_size' => $targetPartSize, ] ); - $stream = $downloader->promise()->wait(); + /** @var DownloadResponse $response */ + $response = $downloader->promise()->wait(); + $snapshot = $downloader->getCurrentSnapshot(); - $this->assertInstanceOf(StreamInterface::class, $stream); - $this->assertEquals($objectKey, $downloader->getObjectKey()); - $this->assertEquals($objectSizeInBytes, $downloader->getObjectSizeInBytes()); - $this->assertEquals($objectSizeInBytes, $downloader->getObjectBytesTransferred()); + $this->assertInstanceOf(DownloadResponse::class, $response); + $this->assertEquals($objectKey, $snapshot->getIdentifier()); + $this->assertEquals($objectSizeInBytes, $snapshot->getTotalBytes()); + $this->assertEquals($objectSizeInBytes, $snapshot->getTransferredBytes()); $this->assertEquals($partsCount, $downloader->getObjectPartsCount()); - $this->assertEquals($partsCount, $downloader->getObjectCompletedPartsCount()); + $this->assertEquals($partsCount, $downloader->getCurrentPartNo()); } /** diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index b74dd3d6e3..d727595132 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -7,6 +7,7 @@ use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\MultipartUploader; +use Aws\S3\S3Transfer\UploadResponse; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\NoSeekStream; use GuzzleHttp\Psr7\Response; @@ -62,18 +63,19 @@ public function testMultipartUpload( $multipartUploader = new MultipartUploader( $s3Client, $requestArgs, - $requestArgs, - $requestArgs, $config + [ 'concurrency' => 3, ], $stream ); - $multipartUploader->promise()->wait(); + /** @var UploadResponse $response */ + $response = $multipartUploader->promise()->wait(); + $snapshot = $multipartUploader->getCurrentSnapshot(); + $this->assertInstanceOf(UploadResponse::class, $response); $this->assertCount($expected['parts'], $multipartUploader->getParts()); - $this->assertEquals($expected['bytesUploaded'], $multipartUploader->getObjectBytesTransferred()); - $this->assertEquals($expected['bytesUploaded'], $multipartUploader->getObjectSizeInBytes()); + $this->assertEquals($expected['bytesUploaded'], $snapshot->getTransferredBytes()); + $this->assertEquals($expected['bytesUploaded'], $snapshot->getTotalBytes()); } /** @@ -187,8 +189,6 @@ public function testInvalidSourceStringThrowsException(): void $this->multipartUploadS3Client(), ['Bucket' => 'test-bucket', 'Key' => 'test-key'], [], - [], - [], $nonExistentFile ); } @@ -206,8 +206,6 @@ public function testInvalidSourceTypeThrowsException(): void $this->multipartUploadS3Client(), ['Bucket' => 'test-bucket', 'Key' => 'test-key'], [], - [], - [], 12345 ); } diff --git a/tests/S3/S3Transfer/ObjectProgressTrackerTest.php b/tests/S3/S3Transfer/ObjectProgressTrackerTest.php deleted file mode 100644 index 07795ad893..0000000000 --- a/tests/S3/S3Transfer/ObjectProgressTrackerTest.php +++ /dev/null @@ -1,127 +0,0 @@ -mockProgressBar = $this->createMock(ProgressBar::class); - } - - /** - * Tests getter and setters. - * - * @return void - */ - public function testGettersAndSetters(): void - { - $tracker = new ObjectProgressTracker( - '', - 0, - 0, - '' - ); - $tracker->setObjectKey('FooKey'); - $this->assertEquals('FooKey', $tracker->getObjectKey()); - - $tracker->setObjectBytesTransferred(100); - $this->assertEquals(100, $tracker->getObjectBytesTransferred()); - - $tracker->setObjectSizeInBytes(100); - $this->assertEquals(100, $tracker->getObjectSizeInBytes()); - - $tracker->setStatus('initiated'); - $this->assertEquals('initiated', $tracker->getStatus()); - } - - /** - * Tests bytes transferred increments. - * - * @return void - */ - public function testIncrementTotalBytesTransferred(): void - { - $percentProgress = 0; - $this->mockProgressBar->expects($this->atLeast(4)) - ->method('setPercentCompleted') - ->willReturnCallback(function ($percent) use (&$percentProgress) { - $this->assertEquals($percentProgress +=25, $percent); - }); - - $tracker = new ObjectProgressTracker( - objectKey: 'FooKey', - objectBytesTransferred: 0, - objectSizeInBytes: 100, - status: 'initiated', - progressBar: $this->mockProgressBar - ); - - $tracker->incrementTotalBytesTransferred(25); - $tracker->incrementTotalBytesTransferred(25); - $tracker->incrementTotalBytesTransferred(25); - $tracker->incrementTotalBytesTransferred(25); - - $this->assertEquals(100, $tracker->getObjectBytesTransferred()); - } - - - /** - * Tests progress status color based on states. - * - * @return void - */ - public function testSetStatusUpdatesProgressBarColor() - { - $statusColorMapping = [ - 'progress' => ConsoleProgressBar::BLUE_COLOR_CODE, - 'completed' => ConsoleProgressBar::GREEN_COLOR_CODE, - 'failed' => ConsoleProgressBar::RED_COLOR_CODE, - ]; - $values = array_values($statusColorMapping); - $valueIndex = 0; - $this->mockProgressBar->expects($this->exactly(3)) - ->method('setArg') - ->willReturnCallback(function ($_, $argValue) use ($values, &$valueIndex) { - $this->assertEquals($argValue, $values[$valueIndex++]); - }); - - $tracker = new ObjectProgressTracker( - objectKey: 'FooKey', - objectBytesTransferred: 0, - objectSizeInBytes: 100, - status: 'initiated', - progressBar: $this->mockProgressBar - ); - - foreach ($statusColorMapping as $status => $value) { - $tracker->setStatus($status); - } - } - - /** - * Tests the default progress bar is initialized when not provided. - * - * @return void - */ - public function testDefaultProgressBarIsInitialized() - { - $tracker = new ObjectProgressTracker( - objectKey: 'FooKey', - objectBytesTransferred: 0, - objectSizeInBytes: 100, - status: 'initiated' - ); - $this->assertInstanceOf(ProgressBar::class, $tracker->getProgressBar()); - } -} From 1f094cd8fc74dbd429047ca0b55831b2ef4886c8 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 24 Feb 2025 07:13:57 -0800 Subject: [PATCH 12/19] chore: remove unused implementation - Remove progress bar color enum since the colors were moved into the specific format that requires them. --- .../S3Transfer/Progress/ProgressBarColorEnum.php | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 src/S3/S3Transfer/Progress/ProgressBarColorEnum.php diff --git a/src/S3/S3Transfer/Progress/ProgressBarColorEnum.php b/src/S3/S3Transfer/Progress/ProgressBarColorEnum.php deleted file mode 100644 index a932e76432..0000000000 --- a/src/S3/S3Transfer/Progress/ProgressBarColorEnum.php +++ /dev/null @@ -1,16 +0,0 @@ - Date: Mon, 24 Feb 2025 07:19:43 -0800 Subject: [PATCH 13/19] chore: remove invalid test TransferListener must be tested from the implementations that extends and use this abstract class. --- tests/S3/S3Transfer/TransferListenerTest.php | 215 ------------------- 1 file changed, 215 deletions(-) delete mode 100644 tests/S3/S3Transfer/TransferListenerTest.php diff --git a/tests/S3/S3Transfer/TransferListenerTest.php b/tests/S3/S3Transfer/TransferListenerTest.php deleted file mode 100644 index c0d0870661..0000000000 --- a/tests/S3/S3Transfer/TransferListenerTest.php +++ /dev/null @@ -1,215 +0,0 @@ -objectTransferInitiated('FooObjectKey', $requestArgs); - $this->assertEquals(1, $listener->getObjectsToBeTransferred()); - - $this->assertTrue($called); - } - - /** - * Tests object transfer is initiated. - * - * @return void - */ - public function testObjectTransferIsInitiated(): void - { - $called = false; - $listener = new TransferListener( - onObjectTransferInitiated: function () use (&$called) { - $called = true; - } - ); - $requestArgs = []; - $listener->objectTransferInitiated('FooObjectKey', $requestArgs); - $this->assertEquals(1, $listener->getObjectsToBeTransferred()); - - $this->assertTrue($called); - } - - /** - * Tests object transfer progress. - * - * @dataProvider objectTransferProgressProvider - * - * @param array $objects - * - * @return void - */ - public function testObjectTransferProgress( - array $objects - ): void { - $called = 0; - $listener = new TransferListener( - onObjectTransferProgress: function () use (&$called) { - $called++; - } - ); - $totalTransferred = 0; - foreach ($objects as $objectKey => $transferDetails) { - $requestArgs = []; - $listener->objectTransferInitiated( - $objectKey, - $requestArgs, - ); - $listener->objectTransferProgress( - $objectKey, - $transferDetails['transferredInBytes'], - $transferDetails['sizeInBytes'] - ); - $totalTransferred += $transferDetails['transferredInBytes']; - } - - $this->assertEquals(count($objects), $called); - $this->assertEquals(count($objects), $listener->getObjectsToBeTransferred()); - $this->assertEquals($totalTransferred, $listener->getObjectsBytesTransferred()); - } - - /** - * @return array - */ - public function objectTransferProgressProvider(): array - { - return [ - [ - [ - 'FooObjectKey1' => [ - 'sizeInBytes' => 100, - 'transferredInBytes' => 95, - ], - 'FooObjectKey2' => [ - 'sizeInBytes' => 500, - 'transferredInBytes' => 345, - ], - 'FooObjectKey3' => [ - 'sizeInBytes' => 1024, - 'transferredInBytes' => 256, - ], - ] - ] - ]; - } - - /** - * Tests object transfer failed. - * - * @return void - */ - public function testObjectTransferFailed(): void - { - $expectedBytesTransferred = 45; - $expectedReason = "Transfer failed!"; - $listener = new TransferListener( - onObjectTransferFailed: function ( - string $objectKey, - int $objectBytesTransferred, - string $reason - ) use ($expectedBytesTransferred, $expectedReason) { - $this->assertEquals($expectedBytesTransferred, $objectBytesTransferred); - $this->assertEquals($expectedReason, $reason); - } - ); - $requestArgs = []; - $listener->objectTransferInitiated('FooObjectKey', $requestArgs); - $listener->objectTransferFailed( - 'FooObjectKey', - $expectedBytesTransferred, - $expectedReason - ); - - $this->assertEquals(1, $listener->getObjectsTransferFailed()); - $this->assertEquals(0, $listener->getObjectsTransferCompleted()); - } - - /** - * Tests object transfer completed. - * - * @return void - */ - public function testObjectTransferCompleted(): void - { - $expectedBytesTransferred = 100; - $listener = new TransferListener( - onObjectTransferCompleted: function ($objectKey, $objectBytesTransferred) - use ($expectedBytesTransferred) { - $this->assertEquals($expectedBytesTransferred, $objectBytesTransferred); - } - ); - $requestArgs = []; - $listener->objectTransferInitiated('FooObjectKey', $requestArgs); - $listener->objectTransferProgress( - 'FooObjectKey', - $expectedBytesTransferred, - $expectedBytesTransferred - ); - $listener->objectTransferCompleted('FooObjectKey', $expectedBytesTransferred); - - $this->assertEquals(1, $listener->getObjectsTransferCompleted()); - $this->assertEquals($expectedBytesTransferred, $listener->getObjectsBytesTransferred()); - } - - /** - * Tests transfer is completed once all the objects in progress are completed. - * - * @return void - */ - public function testTransferCompleted(): void - { - $expectedObjectsTransferred = 2; - $expectedObjectBytesTransferred = 200; - $listener = new TransferListener( - onTransferCompleted: function(int $objectsTransferredCompleted, int $objectsBytesTransferred) - use ($expectedObjectsTransferred, $expectedObjectBytesTransferred) { - $this->assertEquals($expectedObjectsTransferred, $objectsTransferredCompleted); - $this->assertEquals($expectedObjectBytesTransferred, $objectsBytesTransferred); - } - ); - $requestArgs = []; - $listener->objectTransferInitiated('FooObjectKey_1', $requestArgs); - $listener->objectTransferInitiated('FooObjectKey_2', $requestArgs); - $listener->objectTransferProgress( - 'FooObjectKey_1', - 100, - 100 - ); - $listener->objectTransferProgress( - 'FooObjectKey_2', - 100, - 100 - ); - $listener->objectTransferCompleted( - 'FooObjectKey_1', - 100, - ); - $listener->objectTransferCompleted( - 'FooObjectKey_2', - 100, - ); - - $this->assertEquals($expectedObjectsTransferred, $listener->getObjectsTransferCompleted()); - $this->assertEquals($expectedObjectBytesTransferred, $listener->getObjectsBytesTransferred()); - } -} From 09e493fce9e85131fd12acacecbbadda7e7c881f Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 24 Feb 2025 13:04:21 -0800 Subject: [PATCH 14/19] fix: add nullable type Add nullable type to listenerNotifier property in the MultipartUploader implementation. --- src/S3/S3Transfer/MultipartUploader.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index c9885bb9c9..7dcfde1b4a 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -49,7 +49,7 @@ class MultipartUploader implements PromisorInterface private array $deferFns = []; /** @var TransferListenerNotifier | null */ - private TransferListenerNotifier | null $listenerNotifier; + private ?TransferListenerNotifier $listenerNotifier; /** Tracking Members */ /** @var TransferProgressSnapshot|null */ @@ -73,7 +73,7 @@ public function __construct( ?string $uploadId = null, array $parts = [], ?TransferProgressSnapshot $currentSnapshot = null, - TransferListenerNotifier $listenerNotifier = null, + ?TransferListenerNotifier $listenerNotifier = null, ) { $this->s3Client = $s3Client; $this->createMultipartArgs = $createMultipartArgs; From f10522ba5339d41f482672ccb0d709ec349d858f Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Wed, 26 Feb 2025 09:02:09 -0800 Subject: [PATCH 15/19] chore: add more tests - Tests for MultiProgressTracker - Tests for SingleProgressTracker - Tests for ProgressBarFormat - Tests for TransferProgressSnapshot - Tests for TransferListenerNotifier --- .../Exceptions/ProgressTrackerException.php | 8 + src/S3/S3Transfer/MultipartDownloader.php | 5 +- src/S3/S3Transfer/MultipartUploader.php | 3 +- .../Progress/MultiProgressBarFormat.php | 36 + .../Progress/MultiProgressTracker.php | 78 +- .../Progress/PlainProgressBarFormat.php | 3 +- .../Progress/ProgressBarFactoryInterface.php | 8 + .../Progress/SingleProgressTracker.php | 83 +- .../{ => Progress}/TransferListener.php | 2 +- .../TransferListenerNotifier.php | 4 +- .../Progress/TransferProgressBarFormat.php | 3 +- .../RangeGetMultipartDownloader.php | 1 + src/S3/S3Transfer/S3TransferManager.php | 2 + .../Progress/ConsoleProgressBarTest.php | 56 +- .../Progress/MultiProgressTrackerTest.php | 732 ++++++++++++++++++ .../Progress/ProgressBarFormatTest.php | 153 ++++ .../Progress/SingleProgressTrackerTest.php | 299 +++++++ .../Progress/TransferListenerNotifierTest.php | 39 + .../Progress/TransferProgressSnapshotTest.php | 84 ++ 19 files changed, 1521 insertions(+), 78 deletions(-) create mode 100644 src/S3/S3Transfer/Exceptions/ProgressTrackerException.php create mode 100644 src/S3/S3Transfer/Progress/MultiProgressBarFormat.php create mode 100644 src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php rename src/S3/S3Transfer/{ => Progress}/TransferListener.php (97%) rename src/S3/S3Transfer/{ => Progress}/TransferListenerNotifier.php (96%) create mode 100644 tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php create mode 100644 tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php create mode 100644 tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php create mode 100644 tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php create mode 100644 tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php diff --git a/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php b/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php new file mode 100644 index 0000000000..66d2a90cbd --- /dev/null +++ b/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php @@ -0,0 +1,8 @@ + 'Aws\S3\S3Transfer\PartGetMultipartDownloader', - MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\RangeGetMultipartDownloader', + MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => PartGetMultipartDownloader::class, + MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, default => throw new \InvalidArgumentException( "The config value for `multipart_download_type` must be one of:\n" . "\t* " . MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 7dcfde1b4a..2172e604e8 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -5,6 +5,7 @@ use Aws\CommandPool; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; +use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Coroutine; use GuzzleHttp\Promise\Create; @@ -468,7 +469,7 @@ private function callDeferredFns(): void */ private function containsChecksum(array $requestArgs): bool { - $algorithms = [ + static $algorithms = [ 'ChecksumCRC32', 'ChecksumCRC32C', 'ChecksumCRC64NVME', diff --git a/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php b/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php new file mode 100644 index 0000000000..16f65a220c --- /dev/null +++ b/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php @@ -0,0 +1,36 @@ +singleProgressTrackers = $singleProgressTrackers; @@ -41,6 +46,7 @@ public function __construct( $this->transferCount = $transferCount; $this->completed = $completed; $this->failed = $failed; + $this->progressBarFactory = $progressBarFactory; } /** @@ -83,6 +89,14 @@ public function getFailed(): int return $this->failed; } + /** + * @return ProgressBarFactoryInterface|Closure|null + */ + public function getProgressBarFactory(): ProgressBarFactoryInterface | Closure | null + { + return $this->progressBarFactory; + } + /** * @inheritDoc */ @@ -90,14 +104,28 @@ public function transferInitiated(array $context): void { $this->transferCount++; $snapshot = $context['progress_snapshot']; - if (isset($this->singleProgressTrackers[$snapshot['key']])) { - $progressTracker = $this->singleProgressTrackers[$snapshot['key']]; + if (isset($this->singleProgressTrackers[$snapshot->getIdentifier()])) { + $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; } else { - $progressTracker = new SingleProgressTracker( - clear: false, - ); + if ($this->progressBarFactory === null) { + $progressTracker = new SingleProgressTracker( + output: $this->output, + clear: false, + showProgressOnUpdate: false, + ); + } else { + $progressBarFactoryFn = $this->progressBarFactory; + $progressTracker = new SingleProgressTracker( + progressBar: $progressBarFactoryFn(), + output: $this->output, + clear: false, + showProgressOnUpdate: false, + ); + } + $this->singleProgressTrackers[$snapshot->getIdentifier()] = $progressTracker; } + $progressTracker->transferInitiated($context); $this->showProgress(); } @@ -144,29 +172,49 @@ public function showProgress(): void { fwrite($this->output, "\033[2J\033[H"); $percentsSum = 0; + /** + * @var $_ + * @var SingleProgressTracker $progressTracker + */ foreach ($this->singleProgressTrackers as $_ => $progressTracker) { $progressTracker->showProgress(); $percentsSum += $progressTracker->getProgressBar()->getPercentCompleted(); } + $allProgressBarWidth = ConsoleProgressBar::DEFAULT_PROGRESS_BAR_WIDTH; + if (count($this->singleProgressTrackers) !== 0) { + $firstKey = array_key_first($this->singleProgressTrackers); + $allProgressBarWidth = $this->singleProgressTrackers[$firstKey] + ->getProgressBar()->getProgressBarWidth(); + } + $percent = (int) floor($percentsSum / $this->transferCount); + $multiProgressBarFormat = new MultiProgressBarFormat(); + $multiProgressBarFormat->setArgs([ + 'completed' => $this->completed, + 'failed' => $this->failed, + 'total' => $this->transferCount, + ]); $allTransferProgressBar = new ConsoleProgressBar( + progressBarWidth: $allProgressBarWidth, percentCompleted: $percent, - progressBarFormat: new PlainProgressBarFormat() + progressBarFormat: $multiProgressBarFormat ); - fwrite($this->output, "\n" . str_repeat( - '-', - $allTransferProgressBar->getProgressBarWidth()) + fwrite( + $this->output, + sprintf( + "\n%s\n", + str_repeat( + '-', + $allTransferProgressBar->getProgressBarWidth() + ) + ) ); fwrite( $this->output, sprintf( - "\n%s Completed: %d/%d, Failed: %d/%d\n", + "%s\n", $allTransferProgressBar->render(), - $this->completed, - $this->transferCount, - $this->failed, - $this->transferCount ) ); } diff --git a/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php index 55a2ec1cba..0e89093fb5 100644 --- a/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php @@ -6,12 +6,13 @@ final class PlainProgressBarFormat extends ProgressBarFormat { public function getFormatTemplate(): string { - return '[|progress_bar|] |percent|%'; + return "|object_name|:\n[|progress_bar|] |percent|%"; } public function getFormatParameters(): array { return [ + 'object_name', 'progress_bar', 'percent', ]; diff --git a/src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php b/src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php new file mode 100644 index 0000000000..87f0fea51c --- /dev/null +++ b/src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php @@ -0,0 +1,8 @@ +progressBar = $progressBar; @@ -39,8 +44,9 @@ public function __construct( throw new \InvalidArgumentException("The type for $output must be a stream"); } $this->output = $output; - $this->objectName = $objectName; $this->clear = $clear; + $this->currentSnapshot = $currentSnapshot; + $this->showProgressOnUpdate = $showProgressOnUpdate; } /** @@ -60,18 +66,26 @@ public function getOutput(): mixed } /** - * @return string + * @return bool + */ + public function isClear(): bool { + return $this->clear; + } + + /** + * @return TransferProgressSnapshot|null */ - public function getObjectName(): string + public function getCurrentSnapshot(): ?TransferProgressSnapshot { - return $this->objectName; + return $this->currentSnapshot; } /** * @return bool */ - public function isClear(): bool { - return $this->clear; + public function isShowProgressOnUpdate(): bool + { + return $this->showProgressOnUpdate; } /** @@ -81,17 +95,15 @@ public function isClear(): bool { */ public function transferInitiated(array $context): void { - $snapshot = $context['progress_snapshot']; - $this->objectName = $snapshot->getIdentifier(); + $this->currentSnapshot = $context['progress_snapshot']; $progressFormat = $this->progressBar->getProgressBarFormat(); - if ($progressFormat instanceof ColoredTransferProgressBarFormat) { - $progressFormat->setArg( - 'object_name', - $this->objectName - ); - } + // Probably a common argument + $progressFormat->setArg( + 'object_name', + $this->currentSnapshot->getIdentifier() + ); - $this->updateProgressBar($snapshot); + $this->updateProgressBar(); } /** @@ -101,6 +113,7 @@ public function transferInitiated(array $context): void */ public function bytesTransferred(array $context): void { + $this->currentSnapshot = $context['progress_snapshot']; $progressFormat = $this->progressBar->getProgressBarFormat(); if ($progressFormat instanceof ColoredTransferProgressBarFormat) { $progressFormat->setArg( @@ -109,7 +122,7 @@ public function bytesTransferred(array $context): void ); } - $this->updateProgressBar($context['progress_snapshot']); + $this->updateProgressBar(); } /** @@ -119,6 +132,7 @@ public function bytesTransferred(array $context): void */ public function transferComplete(array $context): void { + $this->currentSnapshot = $context['progress_snapshot']; $progressFormat = $this->progressBar->getProgressBarFormat(); if ($progressFormat instanceof ColoredTransferProgressBarFormat) { $progressFormat->setArg( @@ -127,10 +141,8 @@ public function transferComplete(array $context): void ); } - $snapshot = $context['progress_snapshot']; $this->updateProgressBar( - $snapshot, - $snapshot->getTotalBytes() === 0 + $this->currentSnapshot->getTotalBytes() === 0 ); } @@ -141,6 +153,7 @@ public function transferComplete(array $context): void */ public function transferFail(array $context): void { + $this->currentSnapshot = $context['progress_snapshot']; $progressFormat = $this->progressBar->getProgressBarFormat(); if ($progressFormat instanceof ColoredTransferProgressBarFormat) { $progressFormat->setArg( @@ -153,14 +166,13 @@ public function transferFail(array $context): void ); } - $this->updateProgressBar($context['progress_snapshot']); + $this->updateProgressBar(); } /** * Updates the progress bar with the transfer snapshot * and also call showProgress. * - * @param TransferProgressSnapshot $snapshot * @param bool $forceCompletion To force the progress bar to be * completed. This is useful for files where its size is zero, * for which a ratio will return zero, and hence the percent @@ -169,25 +181,26 @@ public function transferFail(array $context): void * @return void */ private function updateProgressBar( - TransferProgressSnapshot $snapshot, bool $forceCompletion = false ): void { if (!$forceCompletion) { $this->progressBar->setPercentCompleted( - ((int)floor($snapshot->ratioTransferred() * 100)) + ((int)floor($this->currentSnapshot->ratioTransferred() * 100)) ); } else { $this->progressBar->setPercentCompleted(100); } $this->progressBar->getProgressBarFormat()->setArgs([ - 'transferred' => $snapshot->getTransferredBytes(), - 'tobe_transferred' => $snapshot->getTotalBytes(), + 'transferred' => $this->currentSnapshot->getTransferredBytes(), + 'tobe_transferred' => $this->currentSnapshot->getTotalBytes(), 'unit' => 'B', ]); // Display progress - $this->showProgress(); + if ($this->showProgressOnUpdate) { + $this->showProgress(); + } } /** @@ -197,9 +210,9 @@ private function updateProgressBar( */ public function showProgress(): void { - if (empty($this->objectName)) { - throw new \RuntimeException( - "Progress tracker requires an object name to be set." + if ($this->currentSnapshot === null) { + throw new ProgressTrackerException( + "There is not snapshot to show progress for." ); } diff --git a/src/S3/S3Transfer/TransferListener.php b/src/S3/S3Transfer/Progress/TransferListener.php similarity index 97% rename from src/S3/S3Transfer/TransferListener.php rename to src/S3/S3Transfer/Progress/TransferListener.php index 2a24707d17..811a3bd47d 100644 --- a/src/S3/S3Transfer/TransferListener.php +++ b/src/S3/S3Transfer/Progress/TransferListener.php @@ -1,6 +1,6 @@ assertEquals( ConsoleProgressBar::DEFAULT_PROGRESS_BAR_WIDTH, @@ -45,7 +46,8 @@ public function testDefaultValues(): void { * * @return void */ - public function testSetPercentCompleted(): void { + public function testSetPercentCompleted(): void + { $progressBar = new ConsoleProgressBar(); $progressBar->setPercentCompleted(10); $this->assertEquals(10, $progressBar->getPercentCompleted()); @@ -56,7 +58,8 @@ public function testSetPercentCompleted(): void { /** * @return void */ - public function testSetCustomValues(): void { + public function testSetCustomValues(): void + { $progressBar = new ConsoleProgressBar( progressBarChar: '-', progressBarWidth: 10, @@ -77,7 +80,8 @@ public function testSetCustomValues(): void { * * @return void */ - public function testPercentIsNotOverOneHundred(): void { + public function testPercentIsNotOverOneHundred(): void + { $progressBar = new ConsoleProgressBar(); $progressBar->setPercentCompleted(150); $this->assertEquals(100, $progressBar->getPercentCompleted()); @@ -87,7 +91,7 @@ public function testPercentIsNotOverOneHundred(): void { * @param string $progressBarChar * @param int $progressBarWidth * @param int $percentCompleted - * @param ProgressBarFormat $progressBarFormat + * @param ProgressBarFormatTest $progressBarFormat * @param array $progressBarFormatArgs * @param string $expectedOutput * @@ -102,7 +106,8 @@ public function testProgressBarRendering( ProgressBarFormat $progressBarFormat, array $progressBarFormatArgs, string $expectedOutput - ): void { + ): void + { $progressBarFormat->setArgs($progressBarFormatArgs); $progressBar = new ConsoleProgressBar( $progressBarChar, @@ -119,39 +124,48 @@ public function testProgressBarRendering( * * @return array */ - public function progressBarRenderingProvider(): array { + public function progressBarRenderingProvider(): array + { return [ 'plain_progress_bar_format_1' => [ 'progress_bar_char' => '#', 'progress_bar_width' => 50, 'percent_completed' => 15, 'progress_bar_format' => new PlainProgressBarFormat(), - 'progress_bar_format_args' => [], - 'expected_output' => '[######## ] 15%' + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + ], + 'expected_output' => "FooObject:\n[######## ] 15%" ], 'plain_progress_bar_format_2' => [ 'progress_bar_char' => '#', 'progress_bar_width' => 50, 'percent_completed' => 45, 'progress_bar_format' => new PlainProgressBarFormat(), - 'progress_bar_format_args' => [], - 'expected_output' => '[####################### ] 45%' + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + ], + 'expected_output' => "FooObject:\n[####################### ] 45%" ], 'plain_progress_bar_format_3' => [ 'progress_bar_char' => '#', 'progress_bar_width' => 50, 'percent_completed' => 100, 'progress_bar_format' => new PlainProgressBarFormat(), - 'progress_bar_format_args' => [], - 'expected_output' => '[##################################################] 100%' + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + ], + 'expected_output' => "FooObject:\n[##################################################] 100%" ], 'plain_progress_bar_format_4' => [ 'progress_bar_char' => '.', 'progress_bar_width' => 50, 'percent_completed' => 100, 'progress_bar_format' => new PlainProgressBarFormat(), - 'progress_bar_format_args' => [], - 'expected_output' => '[..................................................] 100%' + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + ], + 'expected_output' => "FooObject:\n[..................................................] 100%" ], 'transfer_progress_bar_format_1' => [ 'progress_bar_char' => '#', @@ -159,11 +173,12 @@ public function progressBarRenderingProvider(): array { 'percent_completed' => 23, 'progress_bar_format' => new TransferProgressBarFormat(), 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', 'transferred' => 23, 'tobe_transferred' => 100, 'unit' => 'B' ], - 'expected_output' => '[############ ] 23% 23/100 B' + 'expected_output' => "FooObject:\n[############ ] 23% 23/100 B" ], 'transfer_progress_bar_format_2' => [ 'progress_bar_char' => '#', @@ -171,11 +186,12 @@ public function progressBarRenderingProvider(): array { 'percent_completed' => 75, 'progress_bar_format' => new TransferProgressBarFormat(), 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', 'transferred' => 75, 'tobe_transferred' => 100, 'unit' => 'B' ], - 'expected_output' => '[################### ] 75% 75/100 B' + 'expected_output' => "FooObject:\n[################### ] 75% 75/100 B" ], 'transfer_progress_bar_format_3' => [ 'progress_bar_char' => '#', @@ -183,11 +199,12 @@ public function progressBarRenderingProvider(): array { 'percent_completed' => 100, 'progress_bar_format' => new TransferProgressBarFormat(), 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', 'transferred' => 100, 'tobe_transferred' => 100, 'unit' => 'B' ], - 'expected_output' => '[##############################] 100% 100/100 B' + 'expected_output' => "FooObject:\n[##############################] 100% 100/100 B" ], 'transfer_progress_bar_format_4' => [ 'progress_bar_char' => '*', @@ -195,11 +212,12 @@ public function progressBarRenderingProvider(): array { 'percent_completed' => 100, 'progress_bar_format' => new TransferProgressBarFormat(), 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', 'transferred' => 100, 'tobe_transferred' => 100, 'unit' => 'B' ], - 'expected_output' => '[******************************] 100% 100/100 B' + 'expected_output' => "FooObject:\n[******************************] 100% 100/100 B" ], 'colored_progress_bar_format_1' => [ 'progress_bar_char' => '#', diff --git a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php new file mode 100644 index 0000000000..6c10b44df5 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php @@ -0,0 +1,732 @@ +assertEquals([], $progressTracker->getSingleProgressTrackers()); + $this->assertEquals(STDOUT, $progressTracker->getOutput()); + $this->assertEquals(0, $progressTracker->getTransferCount()); + $this->assertEquals(0, $progressTracker->getCompleted()); + $this->assertEquals(0, $progressTracker->getFailed()); + } + + /** + * @dataProvider customInitializationProvider + * + * @param array $progressTrackers + * @param mixed $output + * @param int $transferCount + * @param int $completed + * @param int $failed + * + * @return void + */ + public function testCustomInitialization( + array $progressTrackers, + mixed $output, + int $transferCount, + int $completed, + int $failed + ): void + { + $progressTracker = new MultiProgressTracker( + $progressTrackers, + $output, + $transferCount, + $completed, + $failed + ); + $this->assertSame($output, $progressTracker->getOutput()); + $this->assertSame($transferCount, $progressTracker->getTransferCount()); + $this->assertSame($completed, $progressTracker->getCompleted()); + $this->assertSame($failed, $progressTracker->getFailed()); + } + + /** + * @param ProgressBarFactoryInterface $progressBarFactory + * @param callable $eventInvoker + * @param array $expectedOutputs + * + * @return void + * @dataProvider multiProgressTrackerProvider + * + */ + public function testMultiProgressTracker( + Closure $progressBarFactory, + callable $eventInvoker, + array $expectedOutputs, + ): void + { + $output = fopen("php://temp", "w+"); + $progressTracker = new MultiProgressTracker( + output: $output, + progressBarFactory: $progressBarFactory + ); + $eventInvoker($progressTracker); + + $this->assertEquals( + $expectedOutputs['transfer_count'], + $progressTracker->getTransferCount() + ); + $this->assertEquals( + $expectedOutputs['completed'], + $progressTracker->getCompleted() + ); + $this->assertEquals( + $expectedOutputs['failed'], + $progressTracker->getFailed() + ); + $progress = $expectedOutputs['progress']; + if (is_array($progress)) { + $progress = join('', $progress); + } + rewind($output); + $this->assertEquals( + $progress, + stream_get_contents($output), + ); + } + + /** + * @return array + */ + public function customInitializationProvider(): array + { + return [ + 'custom_initialization_1' => [ + 'progress_trackers' => [ + new SingleProgressTracker(), + new SingleProgressTracker(), + ], + 'output' => STDOUT, + 'transfer_count' => 20, + 'completed' => 20, + 'failed' => 0, + ], + 'custom_initialization_2' => [ + 'progress_trackers' => [ + new SingleProgressTracker(), + ], + 'output' => STDOUT, + 'transfer_count' => 25, + 'completed' => 20, + 'failed' => 5, + ], + 'custom_initialization_3' => [ + 'progress_trackers' => [ + new SingleProgressTracker(), + new SingleProgressTracker(), + new SingleProgressTracker(), + new SingleProgressTracker(), + ], + 'output' => fopen("php://temp", "w"), + 'transfer_count' => 50, + 'completed' => 35, + 'failed' => 15, + ] + ]; + } + + /** + * @return array + */ + public function multiProgressTrackerProvider(): array + { + return [ + 'multi_progress_tracker_1_single_tracking_object' => [ + 'progress_bar_factory' => function() { + return new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat(), + ); + }, + 'event_invoker' => function (MultiProgressTracker $tracker): void + { + $tracker->transferInitiated([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 1024 + ) + ]); + $tracker->bytesTransferred([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 512, + 1024 + ) + ]); + }, + 'expected_outputs' => [ + 'transfer_count' => 1, + 'completed' => 0, + 'failed' => 0, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/1, Failed: 0/1\n", + "\033[2J\033[H\r\n", + "Foo:\n[########## ] 50%\n", + "--------------------\n", + "[########## ] 50% Completed: 0/1, Failed: 0/1\n" + ] + ], + ], + 'multi_progress_tracker_2' => [ + 'progress_bar_factory' => function() { + return new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat(), + ); + }, + 'event_invoker' => function (MultiProgressTracker $progressTracker): void + { + $events = [ + 'transfer_initiated' => [ + 'request_args' => [], + 'total_bytes' => 1024 + ], + 'transfer_progress_1' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 342, + ], + 'transfer_progress_2' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 684, + ], + 'transfer_progress_3' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 1024, + ], + 'transfer_complete' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 1024, + ] + ]; + foreach ($events as $eventName => $event) { + if ($eventName === 'transfer_initiated') { + for ($i = 0; $i < 3; $i++) { + $progressTracker->transferInitiated([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + 0, + $event['total_bytes'], + ) + ]); + } + } elseif (str_starts_with($eventName, 'transfer_progress')) { + for ($i = 0; $i < 3; $i++) { + $progressTracker->bytesTransferred([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + $event['bytes_transferred'], + $event['total_bytes'], + ) + ]); + } + } elseif ($eventName === 'transfer_complete') { + for ($i = 0; $i < 3; $i++) { + $progressTracker->transferComplete([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + $event['bytes_transferred'], + $event['total_bytes'], + ) + ]); + } + } + } + }, + 'expected_outputs' => [ + 'transfer_count' => 3, + 'completed' => 3, + 'failed' => 0, + 'progress' => [ + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/1, Failed: 0/1\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/2, Failed: 0/2\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\n", + "--------------------\n", + "[## ] 11% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[ ] 0%\n", + "--------------------\n", + "[#### ] 22% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[####### ] 33%\n", + "--------------------\n", + "[####### ] 33% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[####### ] 33%\n", + "--------------------\n", + "[######### ] 44% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[####### ] 33%\n", + "--------------------\n", + "[########### ] 55% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[############# ] 66%\n", + "--------------------\n", + "[############# ] 66% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[############# ] 66%\n", + "--------------------\n", + "[############### ] 77% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[############# ] 66%\n", + "--------------------\n", + "[################## ] 88% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\n", + "--------------------\n", + "[####################] 100% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\n", + "--------------------\n", + "[####################] 100% Completed: 1/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\n", + "--------------------\n", + "[####################] 100% Completed: 2/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\n", + "--------------------\n", + "[####################] 100% Completed: 3/3, Failed: 0/3\n", + + ] + ], + ], + 'multi_progress_tracker_3' => [ + 'progress_bar_factory' => function() { + return new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat(), + ); + }, + 'event_invoker' => function (MultiProgressTracker $progressTracker): void + { + $events = [ + 'transfer_initiated' => [ + 'request_args' => [], + 'total_bytes' => 1024 + ], + 'transfer_progress_1' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 342, + ], + 'transfer_progress_2' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 684, + ], + 'transfer_progress_3' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 1024, + ], + 'transfer_complete' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 1024, + ], + 'transfer_fail' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 0, + 'reason' => 'Transfer failed' + ] + ]; + foreach ($events as $eventName => $event) { + if ($eventName === 'transfer_initiated') { + for ($i = 0; $i < 5; $i++) { + $progressTracker->transferInitiated([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + 0, + $event['total_bytes'], + ) + ]); + } + } elseif (str_starts_with($eventName, 'transfer_progress')) { + for ($i = 0; $i < 3; $i++) { + $progressTracker->bytesTransferred([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + $event['bytes_transferred'], + $event['total_bytes'], + ) + ]); + } + } elseif ($eventName === 'transfer_complete') { + for ($i = 0; $i < 3; $i++) { + $progressTracker->transferComplete([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + $event['bytes_transferred'], + $event['total_bytes'], + ) + ]); + } + } elseif ($eventName === 'transfer_fail') { + // Just two of them will fail + for ($i = 3; $i < 5; $i++) { + $progressTracker->transferFail([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + 0, + $event['total_bytes'], + ), + 'reason' => $event['reason'] + ]); + } + } + } + }, + 'expected_outputs' => [ + 'transfer_count' => 5, + 'completed' => 3, + 'failed' => 2, + 'progress' => [ + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/1, Failed: 0/1\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/2, Failed: 0/2\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\r\n", + "FooObject_3:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/4, Failed: 0/4\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[# ] 6% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[ ] 0%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[### ] 13% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[####### ] 33%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[#### ] 19% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[####### ] 33%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[##### ] 26% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[####### ] 33%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[####### ] 33% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[############# ] 66%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[######## ] 39% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[############# ] 66%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[######### ] 46% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[############# ] 66%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[########### ] 53% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 1/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 2/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 3/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 3/5, Failed: 1/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 3/5, Failed: 2/5\n", + ] + ], + ] + ]; + } +} \ No newline at end of file diff --git a/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php b/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php new file mode 100644 index 0000000000..5e1c03a397 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php @@ -0,0 +1,153 @@ +setArgs($args); + + $this->assertEquals($expectedFormat, $progressBarFormat->format()); + } + + /** + * @return array[] + */ + public function progressBarFormatProvider(): array + { + return [ + 'plain_progress_bar_format_1' => [ + 'implementation_class' => PlainProgressBarFormat::class, + 'args' => [ + 'object_name' => 'foo', + 'progress_bar' => '..........', + 'percent' => 100, + ], + 'expected_format' => "foo:\n[..........] 100%", + ], + 'plain_progress_bar_format_2' => [ + 'implementation_class' => PlainProgressBarFormat::class, + 'args' => [ + 'object_name' => 'foo', + 'progress_bar' => '..... ', + 'percent' => 50, + ], + 'expected_format' => "foo:\n[..... ] 50%", + ], + 'transfer_progress_bar_format_1' => [ + 'implementation_class' => TransferProgressBarFormat::class, + 'args' => [ + 'object_name' => 'foo', + 'progress_bar' => '..........', + 'percent' => 100, + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_format' => "foo:\n[..........] 100% 100/100 B", + ], + 'transfer_progress_bar_format_2' => [ + 'implementation_class' => TransferProgressBarFormat::class, + 'args' => [ + 'object_name' => 'foo', + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_format' => "foo:\n[..... ] 50% 50/100 B", + ], + 'colored_transfer_progress_bar_format_1_color_code_black_defaulted' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject' + ], + 'expected_format' => "FooObject:\n\033[30m[..... ] 50% 50/100 B \033[0m", + ], + 'colored_transfer_progress_bar_format_1' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject', + 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE + ], + 'expected_format' => "FooObject:\n\033[34m[..... ] 50% 50/100 B \033[0m", + ], + 'colored_transfer_progress_bar_format_2' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject', + 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE + ], + 'expected_format' => "FooObject:\n\033[32m[..... ] 50% 50/100 B \033[0m", + ], + 'colored_transfer_progress_bar_format_3' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject', + 'color_code' => ColoredTransferProgressBarFormat::RED_COLOR_CODE + ], + 'expected_format' => "FooObject:\n\033[31m[..... ] 50% 50/100 B \033[0m", + ], + 'colored_transfer_progress_bar_format_4' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..........', + 'percent' => 100, + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject', + 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE + ], + 'expected_format' => "FooObject:\n\033[34m[..........] 100% 100/100 B \033[0m", + ], + ]; + } +} \ No newline at end of file diff --git a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php new file mode 100644 index 0000000000..04f8be80fe --- /dev/null +++ b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php @@ -0,0 +1,299 @@ +assertInstanceOf(ConsoleProgressBar::class, $progressTracker->getProgressBar()); + $this->assertEquals(STDOUT, $progressTracker->getOutput()); + $this->assertTrue($progressTracker->isClear()); + $this->assertNull($progressTracker->getCurrentSnapshot()); + } + + /** + * @param ProgressBarInterface $progressBar + * @param mixed $output + * @param bool $clear + * @param TransferProgressSnapshot $snapshot + * + * @dataProvider customInitializationProvider + * + * @return void + */ + public function testCustomInitialization( + ProgressBarInterface $progressBar, + mixed $output, + bool $clear, + TransferProgressSnapshot $snapshot + ): void + { + $progressTracker = new SingleProgressTracker( + $progressBar, + $output, + $clear, + $snapshot, + ); + $this->assertSame($progressBar, $progressTracker->getProgressBar()); + $this->assertSame($output, $progressTracker->getOutput()); + $this->assertSame($clear, $progressTracker->isClear()); + $this->assertSame($snapshot, $progressTracker->getCurrentSnapshot()); + } + + /** + * @return array[] + */ + public function customInitializationProvider(): array + { + return [ + 'initialization_1' => [ + 'progress_bar' => new ConsoleProgressBar(), + 'output' => STDOUT, + 'clear' => true, + 'snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 10 + ), + ], + 'initialization_2' => [ + 'progress_bar' => new ConsoleProgressBar(), + 'output' => fopen('php://temp', 'w'), + 'clear' => true, + 'snapshot' => new TransferProgressSnapshot( + 'FooTest', + 50, + 500 + ), + ], + ]; + } + + /** + * @param ProgressBarInterface $progressBar + * @param callable $eventInvoker + * @param array $expectedOutputs + * + * @dataProvider singleProgressTrackingProvider + * + * @return void + */ + public function testSingleProgressTracking( + ProgressBarInterface $progressBar, + callable $eventInvoker, + array $expectedOutputs, + ): void + { + $output = fopen('php://temp', 'w'); + $progressTracker = new SingleProgressTracker( + $progressBar, + $output, + ); + $eventInvoker($progressTracker); + $this->assertEquals( + $expectedOutputs['identifier'], + $progressTracker->getCurrentSnapshot()->getIdentifier() + ); + $this->assertEquals( + $expectedOutputs['transferred_bytes'], + $progressTracker->getCurrentSnapshot()->getTransferredBytes() + ); + $this->assertEquals( + $expectedOutputs['total_bytes'], + $progressTracker->getCurrentSnapshot()->getTotalBytes() + ); + + $progress = $expectedOutputs['progress']; + if (is_array($progress)) { + $progress = join('', $expectedOutputs['progress']); + } + rewind($output); + $this->assertEquals( + $progress, + stream_get_contents($output) + ); + } + + /** + * @return array[] + */ + public function singleProgressTrackingProvider(): array + { + return [ + 'progress_rendering_1_transfer_initiated' => [ + 'progress_bar' => new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat() + ), + 'event_invoker' => function (singleProgressTracker $progressTracker): void + { + $progressTracker->transferInitiated([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 1024 + ) + ]); + }, + 'expected_outputs' => [ + 'identifier' => 'Foo', + 'transferred_bytes' => 0, + 'total_bytes' => 1024, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%" + ] + ], + ], + 'progress_rendering_2_transfer_progress' => [ + 'progress_bar' => new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat() + ), + 'event_invoker' => function (singleProgressTracker $progressTracker): void + { + $progressTracker->transferInitiated([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 1024 + ) + ]); + $progressTracker->bytesTransferred([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 256, + 1024 + ) + ]); + }, + 'expected_outputs' => [ + 'identifier' => 'Foo', + 'transferred_bytes' => 256, + 'total_bytes' => 1024, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%", + "\033[2J\033[H\r\n", + "Foo:\n[##### ] 25%" + ] + ], + ], + 'progress_rendering_3_transfer_force_completion_when_total_bytes_zero' => [ + 'progress_bar' => new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat() + ), + 'event_invoker' => function (singleProgressTracker $progressTracker): void + { + $progressTracker->transferInitiated([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 0 + ) + ]); + $progressTracker->bytesTransferred([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 1024, + 0 + ) + ]); + $progressTracker->transferComplete([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 2048, + 0 + ) + ]); + }, + 'expected_outputs' => [ + 'identifier' => 'Foo', + 'transferred_bytes' => 2048, + 'total_bytes' => 0, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%", + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%", + "\033[2J\033[H\r\n", + "Foo:\n[####################] 100%" + ] + ], + ], + 'progress_rendering_4_transfer_fail_with_colored_transfer_format' => [ + 'progress_bar' => new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new ColoredTransferProgressBarFormat() + ), + 'event_invoker' => function (singleProgressTracker $progressTracker): void + { + $progressTracker->transferInitiated([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 1024 + ) + ]); + $progressTracker->bytesTransferred([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 512, + 1024 + ) + ]); + $progressTracker->transferFail([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 512, + 1024 + ), + 'reason' => "Error transferring!" + ]); + }, + 'expected_outputs' => [ + 'identifier' => 'Foo', + 'transferred_bytes' => 512, + 'total_bytes' => 1024, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n", + "\033[30m[ ] 0% 0/1024 B ", + "\033[0m", + "\033[2J\033[H\r\n", + "Foo:\n", + "\033[34m[########## ] 50% 512/1024 B ", + "\033[0m", + "\033[2J\033[H\r\n", + "Foo:\n", + "\033[31m[########## ] 50% 512/1024 B Error transferring!", + "\033[0m" + ] + ], + ] + ]; + } +} \ No newline at end of file diff --git a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php new file mode 100644 index 0000000000..7ccd168ede --- /dev/null +++ b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php @@ -0,0 +1,39 @@ +getMockBuilder(TransferListener::class) + ->getMock(), + $this->getMockBuilder(TransferListener::class) + ->getMock(), + $this->getMockBuilder(TransferListener::class) + ->getMock(), + $this->getMockBuilder(TransferListener::class) + ->getMock(), + $this->getMockBuilder(TransferListener::class) + ->getMock(), + ]; + foreach ($listeners as $listener) { + $listener->expects($this->once())->method('transferInitiated'); + $listener->expects($this->once())->method('bytesTransferred'); + $listener->expects($this->once())->method('transferComplete'); + $listener->expects($this->once())->method('transferFail'); + } + $listenerNotifier = new TransferListenerNotifier($listeners); + $listenerNotifier->transferInitiated([]); + $listenerNotifier->bytesTransferred([]); + $listenerNotifier->transferComplete([]); + $listenerNotifier->transferFail([]); + } +} \ No newline at end of file diff --git a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php new file mode 100644 index 0000000000..c0033324bb --- /dev/null +++ b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php @@ -0,0 +1,84 @@ + 'Bar'] + ); + + $this->assertEquals($snapshot->getIdentifier(), 'FooObject'); + $this->assertEquals($snapshot->getTransferredBytes(), 0); + $this->assertEquals($snapshot->getTotalBytes(), 10); + $this->assertEquals($snapshot->getResponse(), ['Foo' => 'Bar']); + } + + /** + * @param int $transferredBytes + * @param int $totalBytes + * @param float $expectedRatio + * + * @return void + * @dataProvider ratioTransferredProvider + * + */ + public function testRatioTransferred( + int $transferredBytes, + int $totalBytes, + float $expectedRatio + ): void + { + $snapshot = new TransferProgressSnapshot( + 'FooObject', + $transferredBytes, + $totalBytes + ); + $this->assertEquals($expectedRatio, $snapshot->ratioTransferred()); + } + + /** + * @return array + */ + public function ratioTransferredProvider(): array + { + return [ + 'ratio_1' => [ + 'transferred_bytes' => 10, + 'total_bytes' => 100, + 'expected_ratio' => 10 / 100, + ], + 'ratio_2_transferred_bytes_zero' => [ + 'transferred_bytes' => 0, + 'total_bytes' => 100, + 'expected_ratio' => 0, + ], + 'ratio_3_unknown_total_bytes' => [ + 'transferred_bytes' => 100, + 'total_bytes' => 0, + 'expected_ratio' => 0, + ], + 'ratio_4' => [ + 'transferred_bytes' => 50, + 'total_bytes' => 256, + 'expected_ratio' => 50 / 256, + ], + 'ratio_5' => [ + 'transferred_bytes' => 250, + 'total_bytes' => 256, + 'expected_ratio' => 250 / 256, + ], + ]; + } +} \ No newline at end of file From a55e1b3dc32e0414b54be15becc540ef496230ba Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 13 Mar 2025 08:23:40 -0700 Subject: [PATCH 16/19] chore: add upload unit tests and refactor - Refactor code to address some styling related feedback. - Add upload and uploadDirectory unit tests. --- src/S3/S3Transfer/DownloadResponse.php | 23 - .../Models/DownloadDirectoryResponse.php | 47 + src/S3/S3Transfer/Models/DownloadResponse.php | 33 + .../{ => Models}/UploadDirectoryResponse.php | 2 +- .../{ => Models}/UploadResponse.php | 2 +- src/S3/S3Transfer/MultipartDownloader.php | 16 +- src/S3/S3Transfer/MultipartUploader.php | 72 +- .../S3Transfer/PartGetMultipartDownloader.php | 2 +- .../Progress/SingleProgressTracker.php | 3 +- .../RangeGetMultipartDownloader.php | 2 - src/S3/S3Transfer/S3TransferManager.php | 349 +++-- .../S3/S3Transfer/MultipartDownloaderTest.php | 22 +- tests/S3/S3Transfer/MultipartUploaderTest.php | 2 +- tests/S3/S3Transfer/S3TransferManagerTest.php | 1345 +++++++++++++++++ 14 files changed, 1773 insertions(+), 147 deletions(-) delete mode 100644 src/S3/S3Transfer/DownloadResponse.php create mode 100644 src/S3/S3Transfer/Models/DownloadDirectoryResponse.php create mode 100644 src/S3/S3Transfer/Models/DownloadResponse.php rename src/S3/S3Transfer/{ => Models}/UploadDirectoryResponse.php (94%) rename src/S3/S3Transfer/{ => Models}/UploadResponse.php (90%) create mode 100644 tests/S3/S3Transfer/S3TransferManagerTest.php diff --git a/src/S3/S3Transfer/DownloadResponse.php b/src/S3/S3Transfer/DownloadResponse.php deleted file mode 100644 index d501882c24..0000000000 --- a/src/S3/S3Transfer/DownloadResponse.php +++ /dev/null @@ -1,23 +0,0 @@ -content; - } - - public function getMetadata(): array - { - return $this->metadata; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php b/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php new file mode 100644 index 0000000000..d493630e16 --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php @@ -0,0 +1,47 @@ +objectsDownloaded = $objectsUploaded; + $this->objectsFailed = $objectsFailed; + } + + /** + * @return int + */ + public function getObjectsDownloaded(): int + { + return $this->objectsDownloaded; + } + + /** + * @return int + */ + public function getObjectsFailed(): int + { + return $this->objectsFailed; + } + + public function __toString(): string + { + return sprintf( + "DownloadDirectoryResponse: %d objects downloaded, %d objects failed", + $this->objectsDownloaded, + $this->objectsFailed + ); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/DownloadResponse.php b/src/S3/S3Transfer/Models/DownloadResponse.php new file mode 100644 index 0000000000..836ba428bc --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadResponse.php @@ -0,0 +1,33 @@ +data; + } + + /** + * @return array + */ + public function getMetadata(): array + { + return $this->metadata; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/UploadDirectoryResponse.php b/src/S3/S3Transfer/Models/UploadDirectoryResponse.php similarity index 94% rename from src/S3/S3Transfer/UploadDirectoryResponse.php rename to src/S3/S3Transfer/Models/UploadDirectoryResponse.php index b098be7ac0..867442986c 100644 --- a/src/S3/S3Transfer/UploadDirectoryResponse.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryResponse.php @@ -1,6 +1,6 @@ stream = $stream; + // Position at the end of the stream + $this->stream->seek($stream->getSize()); } $this->currentSnapshot = $currentSnapshot; $this->listenerNotifier = $listenerNotifier; @@ -128,13 +131,16 @@ public function promise(): PromiseInterface { return Coroutine::of(function () { $this->downloadInitiated($this->requestArgs); + $result = ['@metadata'=>[]]; try { - yield $this->s3Client->executeAsync($this->nextCommand()) + $result = yield $this->s3Client->executeAsync($this->nextCommand()) ->then(function (ResultInterface $result) { // Calculate object size and parts count. $this->computeObjectDimensions($result); // Trigger first part completed $this->partDownloadCompleted($result); + + return $result; })->otherwise(function ($reason) { $this->partDownloadFailed($reason); @@ -156,7 +162,7 @@ public function promise(): PromiseInterface })->otherwise(function ($reason) { $this->partDownloadFailed($reason); - return $reason; + throw $reason; }); } catch (\Throwable $reason) { $this->downloadFailed($reason); @@ -169,10 +175,9 @@ public function promise(): PromiseInterface // Transfer completed $this->downloadComplete(); - // TODO: yield the stream wrapped in a modeled transfer success response. yield Create::promiseFor(new DownloadResponse( $this->stream, - [] + $result['@metadata'] )); }); } @@ -260,6 +265,7 @@ private function downloadInitiated(array $commandArgs): void */ private function downloadFailed(\Throwable $reason): void { + $this->stream->close(); $this->listenerNotifier?->transferFail([ 'request_args' => $this->requestArgs, 'progress_snapshot' => $this->currentSnapshot, @@ -342,7 +348,7 @@ private function downloadComplete(): void * * @return string */ - public static function chooseDownloaderClassName( + public static function chooseDownloaderClass( string $multipartDownloadType ): string { diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 2172e604e8..a99162d21e 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -3,8 +3,11 @@ use Aws\CommandInterface; use Aws\CommandPool; +use Aws\HashingStream; +use Aws\PhpHash; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; +use Aws\S3\S3Transfer\Models\UploadResponse; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Coroutine; @@ -14,6 +17,7 @@ use GuzzleHttp\Psr7\LazyOpenStream; use GuzzleHttp\Psr7\Utils; use Psr\Http\Message\StreamInterface; +use Psr\Http\Message\StreamInterface as Stream; use Throwable; /** @@ -21,8 +25,8 @@ */ class MultipartUploader implements PromisorInterface { - const PART_MIN_SIZE = 5242880; - const PART_MAX_SIZE = 5368709120; + public const PART_MIN_SIZE = 5 * 1024 * 1024; // 5 MBs + public const PART_MAX_SIZE = 5 * 1024 * 1024 * 1024; // 5 GBs public const PART_MAX_NUM = 10000; /** @var S3ClientInterface */ @@ -60,6 +64,8 @@ class MultipartUploader implements PromisorInterface * @param S3ClientInterface $s3Client * @param array $createMultipartArgs * @param array $config + * - part_size: (int, optional) + * - concurrency: (int, required) * @param string | StreamInterface $source * @param string|null $uploadId * @param array $parts @@ -78,6 +84,7 @@ public function __construct( ) { $this->s3Client = $s3Client; $this->createMultipartArgs = $createMultipartArgs; + $this->validateConfig($config); $this->config = $config; $this->body = $this->parseBody($source); $this->uploadId = $uploadId; @@ -86,6 +93,25 @@ public function __construct( $this->listenerNotifier = $listenerNotifier; } + /** + * @param array $config + * + * @return void + */ + private function validateConfig(array &$config): void + { + if (isset($config['part_size'])) { + if ($config['part_size'] < self::PART_MIN_SIZE || $config['part_size'] > self::PART_MAX_SIZE) { + throw new \InvalidArgumentException( + "The config `part_size` value must be between " + . self::PART_MIN_SIZE . " and " . self::PART_MAX_SIZE . "." + ); + } + } else { + $config['part_size'] = self::PART_MIN_SIZE; + } + } + /** * @return string|null */ @@ -143,7 +169,7 @@ public function promise(): PromiseInterface /** * @return PromiseInterface */ - public function createMultipartUpload(): PromiseInterface + private function createMultipartUpload(): PromiseInterface { $requestArgs = [...$this->createMultipartArgs]; $this->uploadInitiated($requestArgs); @@ -163,19 +189,13 @@ public function createMultipartUpload(): PromiseInterface /** * @return PromiseInterface */ - public function uploadParts(): PromiseInterface + private function uploadParts(): PromiseInterface { $this->calculatedObjectSize = 0; $isSeekable = $this->body->isSeekable(); - $partSize = $this->config['part_size'] ?? self::PART_MIN_SIZE; - if ($partSize > self::PART_MAX_SIZE) { - return Create::rejectionFor( - "The part size should not exceed " . self::PART_MAX_SIZE . " bytes." - ); - } - + $partSize = $this->config['part_size']; $commands = []; - for ($partNo = 1; + for ($partNo = count($this->parts) + 1; $isSeekable ? $this->body->tell() < $this->body->getSize() : !$this->body->eof(); @@ -196,15 +216,20 @@ public function uploadParts(): PromiseInterface break; } + $uploadPartCommandArgs = [ ...$this->createMultipartArgs, 'UploadId' => $this->uploadId, 'PartNumber' => $partNo, - 'Body' => $partBody, 'ContentLength' => $partBody->getSize(), ]; // To get `requestArgs` when notifying the bytesTransfer listeners. - $uploadPartCommandArgs['requestArgs'] = $uploadPartCommandArgs; + $uploadPartCommandArgs['requestArgs'] = [...$uploadPartCommandArgs]; + // Attach body + $uploadPartCommandArgs['Body'] = $this->decorateWithHashes( + $partBody, + $uploadPartCommandArgs + ); $command = $this->s3Client->getCommand('UploadPart', $uploadPartCommandArgs); $commands[] = $command; $this->calculatedObjectSize += $partBody->getSize(); @@ -244,7 +269,7 @@ public function uploadParts(): PromiseInterface /** * @return PromiseInterface */ - public function completeMultipartUpload(): PromiseInterface + private function completeMultipartUpload(): PromiseInterface { $this->sortParts(); $completeMultipartUploadArgs = [ @@ -275,7 +300,7 @@ public function completeMultipartUpload(): PromiseInterface /** * @return PromiseInterface */ - public function abortMultipartUpload(): PromiseInterface + private function abortMultipartUpload(): PromiseInterface { $command = $this->s3Client->getCommand('AbortMultipartUpload', [ ...$this->createMultipartArgs, @@ -484,4 +509,19 @@ private function containsChecksum(array $requestArgs): bool return false; } + + /** + * @param Stream $stream + * @param array $data + * + * @return Stream + */ + private function decorateWithHashes(Stream $stream, array &$data): StreamInterface + { + // Decorate source with a hashing stream + $hash = new PhpHash('sha256'); + return new HashingStream($stream, $hash, function ($result) use (&$data) { + $data['ContentSHA256'] = bin2hex($result); + }); + } } diff --git a/src/S3/S3Transfer/PartGetMultipartDownloader.php b/src/S3/S3Transfer/PartGetMultipartDownloader.php index 2ceb3987ab..bb30b3e952 100644 --- a/src/S3/S3Transfer/PartGetMultipartDownloader.php +++ b/src/S3/S3Transfer/PartGetMultipartDownloader.php @@ -16,7 +16,7 @@ class PartGetMultipartDownloader extends MultipartDownloader * * @return CommandInterface */ - protected function nextCommand() : CommandInterface + protected function nextCommand(): CommandInterface { if ($this->currentPartNo === 0) { $this->currentPartNo = 1; diff --git a/src/S3/S3Transfer/Progress/SingleProgressTracker.php b/src/S3/S3Transfer/Progress/SingleProgressTracker.php index fca06b00d1..1024068d4f 100644 --- a/src/S3/S3Transfer/Progress/SingleProgressTracker.php +++ b/src/S3/S3Transfer/Progress/SingleProgressTracker.php @@ -68,7 +68,8 @@ public function getOutput(): mixed /** * @return bool */ - public function isClear(): bool { + public function isClear(): bool + { return $this->clear; } diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php index d6dfdc5549..5fa10cde99 100644 --- a/src/S3/S3Transfer/RangeGetMultipartDownloader.php +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -25,9 +25,7 @@ class RangeGetMultipartDownloader extends MultipartDownloader * using range get. This option MUST be set when using range get. * @param int $currentPartNo * @param int $objectPartsCount - * @param int $objectCompletedPartsCount * @param int $objectSizeInBytes - * @param int $objectBytesTransferred * @param string $eTag * @param StreamInterface|null $stream * @param TransferProgressSnapshot|null $currentSnapshot diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index d2f9e81610..513908c264 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -2,20 +2,27 @@ namespace Aws\S3\S3Transfer; +use Aws\Result; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Models\DownloadDirectoryResponse; +use Aws\S3\S3Transfer\Models\DownloadResponse; +use Aws\S3\S3Transfer\Models\UploadDirectoryResponse; +use Aws\S3\S3Transfer\Models\UploadResponse; use Aws\S3\S3Transfer\Progress\MultiProgressTracker; use Aws\S3\S3Transfer\Progress\SingleProgressTracker; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; +use FilesystemIterator; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; +use InvalidArgumentException; use Psr\Http\Message\StreamInterface; +use Throwable; use function Aws\filter; use function Aws\map; -use function Aws\recursive_dir_iterator; class S3TransferManager { @@ -41,20 +48,20 @@ class S3TransferManager * a default client will be created where its region will be the one * resolved from either the default from the config or the provided. * @param array $config - * - target_part_size_bytes: (int, default=(8388608 `8MB`)) \ + * - target_part_size_bytes: (int, default=(8388608 `8MB`)) * The minimum part size to be used in a multipart upload/download. - * - multipart_upload_threshold_bytes: (int, default=(16777216 `16 MB`)) \ + * - multipart_upload_threshold_bytes: (int, default=(16777216 `16 MB`)) * The threshold to decided whether a multipart upload is needed. - * - checksum_validation_enabled: (bool, default=true) \ + * - checksum_validation_enabled: (bool, default=true) * To decide whether a checksum validation will be applied to the response. - * - checksum_algorithm: (string, default='crc32') \ + * - checksum_algorithm: (string, default='crc32') * The checksum algorithm to be used in an upload request. * - multipart_download_type: (string, default='partGet') * The download type to be used in a multipart download. - * - concurrency: (int, default=5) \ + * - concurrency: (int, default=5) * Maximum number of concurrent operations allowed during a multipart * upload/download. - * - track_progress: (bool, default=false) \ + * - track_progress: (bool, default=false) * To enable progress tracker in a multipart upload/download, and or * a directory upload/download operation. * - region: (string, default="us-east-2") @@ -63,13 +70,28 @@ public function __construct( ?S3ClientInterface $s3Client = null, array $config = [] ) { + $this->config = $config + self::$defaultConfig; if ($s3Client === null) { $this->s3Client = $this->defaultS3Client(); } else { $this->s3Client = $s3Client; } + } - $this->config = $config + self::$defaultConfig; + /** + * @return S3ClientInterface + */ + public function getS3Client(): S3ClientInterface + { + return $this->s3Client; + } + + /** + * @return array + */ + public function getConfig(): array + { + return $this->config; } /** @@ -86,7 +108,10 @@ public function __construct( * - track_progress: (bool, optional) To override the default option for * enabling progress tracking. If this option is resolved as true and * a progressTracker parameter is not provided then, a default implementation - * will be resolved. + * will be resolved. This option is intended to make the operation to use + * a default progress tracker implementation when $progressTracker is null. + * - checksum_algorithm: (bool, optional) To override the default + * checksum algorithm. * @param TransferListener[]|null $listeners * @param TransferListener|null $progressTracker * @@ -100,15 +125,9 @@ public function upload( ?TransferListener $progressTracker = null, ): PromiseInterface { - // Make sure the source is what is expected - if (!is_string($source) && !$source instanceof StreamInterface) { - throw new \InvalidArgumentException( - '`source` must be a string or a StreamInterface' - ); - } // Make sure it is a valid in path in case of a string if (is_string($source) && !is_readable($source)) { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( "Please provide a valid readable file path or a valid stream as source." ); } @@ -123,12 +142,18 @@ public function upload( $mupThreshold = $config['multipart_upload_threshold_bytes'] ?? $this->config['multipart_upload_threshold_bytes']; if ($mupThreshold < MultipartUploader::PART_MIN_SIZE) { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( "The provided config `multipart_upload_threshold_bytes`" ."must be greater than or equal to " . MultipartUploader::PART_MIN_SIZE ); } + if (!isset($requestArgs['ChecksumAlgorithm'])) { + $algorithm = $config['checksum_algorithm'] + ?? $this->config['checksum_algorithm']; + $requestArgs['ChecksumAlgorithm'] = strtoupper($algorithm); + } + if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { $progressTracker = new SingleProgressTracker(); @@ -158,19 +183,31 @@ public function upload( ); } - /** * @param string $directory * @param string $bucketTo * @param array $requestArgs - * @param array $config + * @param array $config The config options for this request that are: * - follow_symbolic_links: (bool, optional, defaulted to false) * - recursive: (bool, optional, defaulted to false) * - s3_prefix: (string, optional, defaulted to null) - * - filter: (Closure(string), optional) + * - filter: (Closure(SplFileInfo|string), optional) + * By default an instance of SplFileInfo will be provided, however + * you can annotate the parameter with a string type and by doing + * so you will get the full path of the file. * - s3_delimiter: (string, optional, defaulted to `/`) - * - put_object_request_callback: (Closure, optional) - * - failure_policy: (Closure, optional) + * - put_object_request_callback: (Closure, optional) A callback function + * to be invoked right before the request initiates and that will receive + * as parameter the request arguments for each upload request. + * - failure_policy: (Closure, optional) A function that will be invoked + * on an upload failure and that will receive as parameters: + * - $requestArgs: (array) The arguments for the request that originated + * the failure. + * - $uploadDirectoryRequestArgs: (array) The arguments for the upload + * directory request. + * - $reason: (Throwable) The exception that originated the request failure. + * - $uploadDirectoryResponse: (UploadDirectoryResponse) The upload response + * to that point in the upload process. * - track_progress: (bool, optional) To override the default option for * enabling progress tracking. If this option is resolved as true and * a progressTracker parameter is not provided then, a default implementation @@ -193,7 +230,7 @@ public function uploadDirectory( ): PromiseInterface { if (!is_dir($directory)) { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( "Please provide a valid directory path. " . "Provided = " . $directory ); @@ -207,14 +244,44 @@ public function uploadDirectory( $filter = null; if (isset($config['filter'])) { if (!is_callable($config['filter'])) { - throw new \InvalidArgumentException("The parameter \$config['filter'] must be callable."); + throw new InvalidArgumentException("The parameter \$config['filter'] must be callable."); } $filter = $config['filter']; } + $putObjectRequestCallback = null; + if (isset($config['put_object_request_callback'])) { + if (!is_callable($config['put_object_request_callback'])) { + throw new InvalidArgumentException( + "The parameter \$config['put_object_request_callback'] must be callable." + ); + } + + $putObjectRequestCallback = $config['put_object_request_callback']; + } + + $failurePolicyCallback = null; + if (isset($config['failure_policy']) && !is_callable($config['failure_policy'])) { + throw new InvalidArgumentException( + "The parameter \$config['failure_policy'] must be callable." + ); + } elseif (isset($config['failure_policy'])) { + $failurePolicyCallback = $config['failure_policy']; + } + + $dirIterator = new \RecursiveDirectoryIterator($directory); + $dirIterator->setFlags(FilesystemIterator::SKIP_DOTS); + if (($config['follow_symbolic_links'] ?? false) === true) { + $dirIterator->setFlags(FilesystemIterator::FOLLOW_SYMLINKS); + } + + if (($config['recursive'] ?? false) === true) { + $dirIterator = new \RecursiveIteratorIterator($dirIterator); + } + $files = filter( - recursive_dir_iterator($directory), + $dirIterator, function ($file) use ($filter) { if ($filter !== null) { return !is_dir($file) && $filter($file); @@ -237,7 +304,7 @@ function ($file) use ($filter) { $relativePath = substr($file, strlen($baseDir)); if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { throw new S3TransferException( - "The filename must not contain the provided delimiter `". $delimiter ."`" + "The filename `$relativePath` must not contain the provided delimiter `$delimiter`" ); } $objectKey = $prefix.$relativePath; @@ -246,29 +313,54 @@ function ($file) use ($filter) { $delimiter, $objectKey ); + $uploadRequestArgs = [ + ...$requestArgs, + 'Bucket' => $bucketTo, + 'Key' => $objectKey, + ]; + if ($putObjectRequestCallback !== null) { + $putObjectRequestCallback($uploadRequestArgs); + } + $promises[] = $this->upload( $file, - [ - ...$requestArgs, - 'Bucket' => $bucketTo, - 'Key' => $objectKey, - ], + $uploadRequestArgs, $config, array_map(function ($listener) { return clone $listener; }, $listeners), $progressTracker, - )->then(function ($result) use (&$objectsUploaded) { + )->then(function (UploadResponse $response) use (&$objectsUploaded) { $objectsUploaded++; - return $result; - })->otherwise(function ($reason) use (&$objectsFailed) { + return $response; + })->otherwise(function ($reason) use ( + $failurePolicyCallback, + $uploadRequestArgs, + $requestArgs, + &$objectsUploaded, + &$objectsFailed + ) { $objectsFailed++; + if($failurePolicyCallback !== null) { + call_user_func( + $failurePolicyCallback, + $requestArgs, + $uploadRequestArgs, + $reason, + new UploadDirectoryResponse( + $objectsUploaded, + $objectsFailed + ) + ); + + return; + } throw $reason; }); } return Each::ofLimitAll($promises, $this->config['concurrency']) - ->then(function ($results) use ($objectsUploaded, $objectsFailed) { + ->then(function ($_) use ($objectsUploaded, $objectsFailed) { return new UploadDirectoryResponse($objectsUploaded, $objectsFailed); }); } @@ -280,19 +372,20 @@ function ($file) use ($filter) { * @param array $downloadArgs The getObject request arguments to be provided as part * of each get object operation, except for the bucket and key, which * are already provided as the source. - * @param array $config The configuration to be used for this operation. - * - multipart_download_type: (string, optional) \ + * @param array $config The configuration to be used for this operation: + * - multipart_download_type: (string, optional) * Overrides the resolved value from the transfer manager config. - * - track_progress: (bool) \ - * Overrides the config option set in the transfer manager instantiation - * to decide whether transfer progress should be tracked. If a `progressListenerFactory` - * was not provided when the transfer manager instance was created - * and track_progress resolved as true then, a default progress listener - * implementation will be used. - * - minimum_part_size: (int) \ - * The minimum part size in bytes to be used in a range multipart download. - * If this parameter is not provided then it fallbacks to the transfer - * manager `target_part_size_bytes` config value. + * - checksum_validation_enabled: (bool, optional) Overrides the resolved + * value from transfer manager config for whether checksum validation + * should be done. This option will be considered just if ChecksumMode + * is not present in the request args. + * - track_progress: (bool) Overrides the config option set in the transfer + * manager instantiation to decide whether transfer progress should be + * tracked. + * - minimum_part_size: (int) The minimum part size in bytes to be used + * in a range multipart download. If this parameter is not provided + * then it fallbacks to the transfer manager `target_part_size_bytes` + * config value. * @param TransferListener[]|null $listeners * @param TransferListener|null $progressTracker * @@ -314,7 +407,16 @@ public function download( 'Key' => $this->requireNonEmpty($source['Key'], "A valid key must be provided."), ]; } else { - throw new \InvalidArgumentException('Source must be a string or an array of strings'); + throw new InvalidArgumentException('Source must be a string or an array of strings'); + } + + if (!isset($requestArgs['ChecksumMode'])) { + $checksumEnabled = $config['checksum_validation_enabled'] + ?? $this->config['checksum_validation_enabled'] + ?? false; + if ($checksumEnabled) { + $requestArgs['ChecksumMode'] = 'enabled'; + } } if ($progressTracker === null @@ -352,31 +454,44 @@ public function download( * @param array $downloadArgs The getObject request arguments to be provided * as part of each get object request sent to the service, except for the * bucket and key which will be resolved internally. - * @param array $config The config options for this download directory operation. \ - * - track_progress: (bool) \ - * Overrides the config option set in the transfer manager instantiation - * to decide whether transfer progress should be tracked. If a `progressListenerFactory` - * was not provided when the transfer manager instance was created - * and track_progress resolved as true then, a default progress listener - * implementation will be used. - * - minimumPartSize: (int) \ - * The minimum part size in bytes to be used in a range multipart download. - * - listObjectV2Args: (array) \ - * The arguments to be included as part of the listObjectV2 request in - * order to fetch the objects to be downloaded. The most common arguments - * would be: + * @param array $config The config options for this download directory operation. + * - s3_prefix: (string, optional) This parameter will be considered just if + * not provided as part of the list_object_v2_args config option. + * - s3_delimiter: (string, optional, defaulted to '/') This parameter will be + * considered just if not provided as part of the list_object_v2_args config + * option. + * - filter: (Closure, optional) A callable which will receive an object key as + * parameter and should return true or false in order to determine + * whether the object should be downloaded. + * - get_object_request_callback: (Closure, optional) A function that will + * be invoked right before the download request is performed and that will + * receive as parameter the request arguments for each request. + * - failure_policy: (Closure, optional) A function that will be invoked + * on a download failure and that will receive as parameters: + * - $requestArgs: (array) The arguments for the request that originated + * the failure. + * - $downloadDirectoryRequestArgs: (array) The arguments for the download + * directory request. + * - $reason: (Throwable) The exception that originated the request failure. + * - $downloadDirectoryResponse: (DownloadDirectoryResponse) The download response + * to that point in the upload process. + * - track_progress: (bool, optional) Overrides the config option set + * in the transfer manager instantiation to decide whether transfer + * progress should be tracked. + * - minimum_part_size: (int, optional) The minimum part size in bytes + * to be used in a range multipart download. + * - list_object_v2_args: (array, optional) The arguments to be included + * as part of the listObjectV2 request in order to fetch the objects to + * be downloaded. The most common arguments would be: * - MaxKeys: (int) Sets the maximum number of keys returned in the response. * - Prefix: (string) To limit the response to keys that begin with the * specified prefix. - * - filter: (Closure) \ - * A callable which will receive an object key as parameter and should return - * true or false in order to determine whether the object should be downloaded. * @param TransferListener[] $listeners The listeners for watching * transfer events. Each listener will be cloned per file upload. * @param TransferListener|null $progressTracker Ideally the progress * tracker implementation provided here should be able to track multiple * transfers at once. Please see MultiProgressTracker implementation. - * + * * @return PromiseInterface */ public function downloadDirectory( @@ -389,7 +504,7 @@ public function downloadDirectory( ): PromiseInterface { if (!file_exists($destinationDirectory)) { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( "Destination directory '$destinationDirectory' MUST exists." ); } @@ -401,7 +516,15 @@ public function downloadDirectory( $listArgs = [ 'Bucket' => $bucket - ] + ($config['listObjectV2Args'] ?? []); + ] + ($config['list_object_v2_args'] ?? []); + if (isset($config['s3_prefix']) && !isset($listArgs['Prefix'])) { + $listArgs['Prefix'] = $config['s3_prefix']; + } + + if (isset($config['s3_delimiter']) && !isset($listArgs['Delimiter'])) { + $listArgs['Delimiter'] = $config['s3_delimiter']; + } + $objects = $this->s3Client ->getPaginator('ListObjectsV2', $listArgs) ->search('Contents[].Key'); @@ -410,7 +533,7 @@ public function downloadDirectory( }); if (isset($config['filter'])) { if (!is_callable($config['filter'])) { - throw new \InvalidArgumentException("The parameter \$config['filter'] must be callable."); + throw new InvalidArgumentException("The parameter \$config['filter'] must be callable."); } $filter = $config['filter']; @@ -420,6 +543,8 @@ public function downloadDirectory( } $promises = []; + $objectsDownloaded = 0; + $objectsFailed = 0; foreach ($objects as $object) { $objectKey = $this->s3UriAsBucketAndKey($object)['Key']; $destinationFile = $destinationDirectory . '/' . $objectKey; @@ -431,25 +556,69 @@ public function downloadDirectory( ); } + $requestArgs = [...$downloadArgs]; + if (isset($config['get_object_request_callback'])) { + if (!is_callable($config['get_object_request_callback'])) { + throw new InvalidArgumentException( + "The parameter \$config['get_object_request_callback'] must be callable." + ); + } + + call_user_func($config['get_object_request_callback'], $requestArgs); + } + $promises[] = $this->download( $object, - $downloadArgs, + $requestArgs, [ - 'minimumPartSize' => $config['minimumPartSize'] ?? 0, + 'minimum_part_size' => $config['minimum_part_size'] ?? 0, ], array_map(function ($listener) { return clone $listener; }, $listeners), $progressTracker, - )->then(function (DownloadResponse $result) use ($destinationFile) { + )->then(function (DownloadResponse $result) use ( + &$objectsDownloaded, + $destinationFile + ) { $directory = dirname($destinationFile); if (!is_dir($directory)) { mkdir($directory, 0777, true); } - file_put_contents($destinationFile, $result->getContent()); + file_put_contents($destinationFile, $result->getData()); + // Close the stream + $result->getData()->close(); + $objectsDownloaded++; + })->otherwise(function ($reason) use ( + &$objectsDownloaded, + &$objectsFailed, + $downloadArgs, + $requestArgs + ) { + $objectsFailed++; + if (isset($config['failure_policy']) && is_callable($config['failure_policy'])) { + call_user_func( + $config['failure_policy'], + $requestArgs, + $downloadArgs, + $reason, + new DownloadDirectoryResponse( + $objectsDownloaded, + $objectsFailed + ) + ); + } + + throw $reason; }); } - return Each::ofLimitAll($promises, $this->config['concurrency']); + return Each::ofLimitAll($promises, $this->config['concurrency']) + ->then(function ($_) use (&$objectsFailed, &$objectsDownloaded) { + return new DownloadDirectoryResponse( + $objectsDownloaded, + $objectsFailed + ); + }); } /** @@ -457,8 +626,8 @@ public function downloadDirectory( * * @param array $requestArgs * @param array $config - * - minimum_part_size: (int) \ - * The minimum part size in bytes for a range multipart download. + * - minimum_part_size: (int) The minimum part size in bytes for a + * range multipart download. * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface @@ -469,7 +638,7 @@ private function tryMultipartDownload( ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { - $downloaderClassName = MultipartDownloader::chooseDownloaderClassName( + $downloaderClassName = MultipartDownloader::chooseDownloaderClass( $config['multipart_download_type'] ); $multipartDownloader = new $downloaderClassName( @@ -526,7 +695,7 @@ function ($result) use ($requestArgs, $listenerNotifier) { $listenerNotifier->transferComplete($progressContext); return new DownloadResponse( - content: $result['Body'], + data: $result['Body'], metadata: $result['@metadata'], ); } @@ -541,7 +710,7 @@ function ($result) use ($requestArgs, $listenerNotifier) { 'reason' => $reason ]); - return $reason; + throw $reason; }); } @@ -553,7 +722,7 @@ function ($result) use ($requestArgs, $listenerNotifier) { return $this->s3Client->executeAsync($command) ->then(function ($result) { return new DownloadResponse( - content: $result['Body'], + data: $result['Body'], metadata: $result['@metadata'], ); }); @@ -636,7 +805,7 @@ function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { ] ); - return $reason; + throw $reason; }); } @@ -683,22 +852,18 @@ private function requiresMultipartUpload( int $mupThreshold ): bool { - if (is_string($source)) { - $sourceSize = filesize($source); - - return $sourceSize > $mupThreshold; + if (is_string($source) && is_readable($source)) { + return filesize($source) >= $mupThreshold; } elseif ($source instanceof StreamInterface) { // When the stream's size is unknown then we could try a multipart upload. if (empty($source->getSize())) { return true; } - if (!empty($source->getSize())) { - return $source->getSize() > $mupThreshold; - } + return $source->getSize() >= $mupThreshold; } - return false; + throw new S3TransferException("Unable to determine if a multipart is required"); } /** @@ -724,7 +889,7 @@ private function defaultS3Client(): S3ClientInterface private function requireNonEmpty(mixed $value, string $message): mixed { if (empty($value)) { - throw new \InvalidArgumentException($message); + throw new InvalidArgumentException($message); } return $value; @@ -757,14 +922,14 @@ private function s3UriAsBucketAndKey(string $uri): array { $errorMessage = "Invalid URI: $uri. A valid S3 URI must be s3://bucket/key"; if (!$this->isValidS3URI($uri)) { - throw new \InvalidArgumentException($errorMessage); + throw new InvalidArgumentException($errorMessage); } $path = substr($uri, 5); // without s3:// $parts = explode('/', $path, 2); if (count($parts) < 2) { - throw new \InvalidArgumentException($errorMessage); + throw new InvalidArgumentException($errorMessage); } return [ @@ -806,4 +971,4 @@ private function resolvesOutsideTargetDirectory( return false; } -} \ No newline at end of file +} diff --git a/tests/S3/S3Transfer/MultipartDownloaderTest.php b/tests/S3/S3Transfer/MultipartDownloaderTest.php index e986ac4dd6..1ab1dd8fc8 100644 --- a/tests/S3/S3Transfer/MultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/MultipartDownloaderTest.php @@ -5,19 +5,19 @@ use Aws\Command; use Aws\Result; use Aws\S3\S3Client; -use Aws\S3\S3Transfer\DownloadResponse; +use Aws\S3\S3Transfer\Models\DownloadResponse; use Aws\S3\S3Transfer\MultipartDownloader; +use Aws\S3\S3Transfer\PartGetMultipartDownloader; +use Aws\S3\S3Transfer\RangeGetMultipartDownloader; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\TestCase; -use Psr\Http\Message\StreamInterface; /** * Tests multipart download implementation. */ class MultipartDownloaderTest extends TestCase { - /** * Tests part and range get multipart downloader. * @@ -68,7 +68,7 @@ public function testMultipartDownloader( -> willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); }); - $downloaderClassName = MultipartDownloader::chooseDownloaderClassName( + $downloaderClassName = MultipartDownloader::chooseDownloaderClass( $multipartDownloadType ); /** @var MultipartDownloader $downloader */ @@ -163,4 +163,18 @@ public function partGetMultipartDownloaderProvider(): array { ] ]; } + + /** + * @return void + */ + public function testChooseDownloaderClass(): void { + $multipartDownloadTypes = [ + MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => PartGetMultipartDownloader::class, + MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, + ]; + foreach ($multipartDownloadTypes as $multipartDownloadType => $class) { + $resolvedClass = MultipartDownloader::chooseDownloaderClass($multipartDownloadType); + $this->assertEquals($class, $resolvedClass); + } + } } \ No newline at end of file diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index d727595132..3a52b33b0c 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -6,8 +6,8 @@ use Aws\Result; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; +use Aws\S3\S3Transfer\Models\UploadResponse; use Aws\S3\S3Transfer\MultipartUploader; -use Aws\S3\S3Transfer\UploadResponse; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\NoSeekStream; use GuzzleHttp\Psr7\Response; diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php new file mode 100644 index 0000000000..cfea7c6c06 --- /dev/null +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -0,0 +1,1345 @@ +assertArrayHasKey( + 'target_part_size_bytes', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'multipart_upload_threshold_bytes', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'checksum_validation_enabled', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'checksum_algorithm', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'multipart_download_type', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'concurrency', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'track_progress', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'region', + $manager->getConfig() + ); + $this->assertInstanceOf( + S3Client::class, + $manager->getS3Client() + ); + } + + /** + * @return void + */ + public function testCustomConfigIsSet(): void + { + $manager = new S3TransferManager( + null, + [ + 'target_part_size_bytes' => 1024, + 'multipart_upload_threshold_bytes' => 1024, + 'checksum_validation_enabled' => false, + 'checksum_algorithm' => 'sha256', + 'multipart_download_type' => 'partGet', + 'concurrency' => 20, + 'track_progress' => true, + 'region' => 'us-west-1', + ] + ); + $config = $manager->getConfig(); + $this->assertEquals(1024, $config['target_part_size_bytes']); + $this->assertEquals(1024, $config['multipart_upload_threshold_bytes']); + $this->assertFalse($config['checksum_validation_enabled']); + $this->assertEquals('sha256', $config['checksum_algorithm']); + $this->assertEquals('partGet', $config['multipart_download_type']); + $this->assertEquals(20, $config['concurrency']); + $this->assertTrue($config['track_progress']); + $this->assertEquals('us-west-1', $config['region']); + } + + /** + * @return void + */ + public function testUploadExpectsAReadableSource(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("Please provide a valid readable file path or a valid stream as source."); + $manager = new S3TransferManager(); + $manager->upload( + "noreadablefile", + )->wait(); + } + + /** + * @dataProvider uploadBucketAndKeyProvider + * + * @return void + */ + public function testUploadFailsWhenBucketAndKeyAreNotProvided( + array $bucketKeyArgs, + string $missingProperty + ): void + { + $manager = new S3TransferManager(); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The `$missingProperty` parameter must be provided as part of the request arguments."); + $manager->upload( + Utils::streamFor(), + $bucketKeyArgs + )->wait(); + } + + /** + * @return array[] + */ + public function uploadBucketAndKeyProvider(): array + { + return [ + 'bucket_missing' => [ + 'bucket_key_args' => [ + 'Key' => 'Key', + ], + 'missing_property' => 'Bucket', + ], + 'key_missing' => [ + 'bucket_key_args' => [ + 'Bucket' => 'Bucket', + ], + 'missing_property' => 'Key', + ], + ]; + } + + /** + * @return void + */ + public function testUploadFailsWhenMultipartThresholdIsLessThanMinSize(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The provided config `multipart_upload_threshold_bytes`" + . "must be greater than or equal to " . MultipartUploader::PART_MIN_SIZE); + $manager = new S3TransferManager(); + $manager->upload( + Utils::streamFor(), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE - 1 + ] + )->wait(); + } + + /** + * This tests takes advantage of the transfer listeners to validate + * if a multipart upload was done. How?, it will check if bytesTransfer + * event happens more than once, which only will occur in a multipart upload. + * + * @return void + */ + public function testDoesMultipartUploadWhenApplicable(): void + { + $client = $this->getS3ClientMock();; + $manager = new S3TransferManager( + $client, + ); + $transferListener = $this->createMock(TransferListener::class); + $expectedPartCount = 2; + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + $manager->upload( + Utils::streamFor( + str_repeat("#", MultipartUploader::PART_MIN_SIZE * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'part_size' => MultipartUploader::PART_MIN_SIZE, + 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE, + ], + [ + $transferListener, + ] + )->wait(); + } + + /** + * @return void + */ + public function testDoesSingleUploadWhenApplicable(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $transferListener = $this->createMock(TransferListener::class); + $transferListener->expects($this->once()) + ->method('bytesTransferred'); + $manager->upload( + Utils::streamFor( + str_repeat("#", MultipartUploader::PART_MIN_SIZE - 1) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE, + ], + [ + $transferListener, + ] + )->wait(); + } + + /** + * @return void + */ + public function testUploadUsesTransferManagerConfigDefaultMupThreshold(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $expectedPartCount = 2; + $transferListener = $this->createMock(TransferListener::class); + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + $manager->upload( + Utils::streamFor( + str_repeat("#", $manager->getConfig()['multipart_upload_threshold_bytes']) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'part_size' => intval( + $manager->getConfig()['multipart_upload_threshold_bytes'] / $expectedPartCount + ), + ], + [ + $transferListener, + ] + )->wait(); + } + + /** + * + * @param int $mupThreshold + * @param int $expectedPartCount + * @param int $expectedPartSize + * @param bool $isMultipartUpload + * + * @dataProvider uploadUsesCustomMupThresholdProvider + * + * @return void + */ + public function testUploadUsesCustomMupThreshold( + int $mupThreshold, + int $expectedPartCount, + int $expectedPartSize, + bool $isMultipartUpload + ): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $transferListener = $this->createMock(TransferListener::class); + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + $expectedIncrementalPartSize = $expectedPartSize; + $transferListener->method('bytesTransferred') + -> willReturnCallback(function ($context) use ($expectedPartSize, &$expectedIncrementalPartSize) { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context['progress_snapshot']; + $this->assertEquals($expectedIncrementalPartSize, $snapshot->getTransferredBytes()); + $expectedIncrementalPartSize += $expectedPartSize; + }); + $manager->upload( + Utils::streamFor( + str_repeat("#", $expectedPartSize * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => $mupThreshold, + 'part_size' => $expectedPartSize, + ], + [ + $transferListener, + ] + )->wait(); + if ($isMultipartUpload) { + $this->assertGreaterThan(1, $expectedPartCount); + } + } + + /** + * @return array + */ + public function uploadUsesCustomMupThresholdProvider(): array + { + return [ + 'mup_threshold_multipart_upload' => [ + 'mup_threshold' => 1024 * 1024 * 7, + 'expected_part_count' => 3, + 'expected_part_size' => 1024 * 1024 * 7, + 'is_multipart_upload' => true, + ], + 'mup_threshold_single_upload' => [ + 'mup_threshold' => 1024 * 1024 * 7, + 'expected_part_count' => 1, + 'expected_part_size' => 1024 * 1024 * 5, + 'is_multipart_upload' => false, + ] + ]; + } + + /** + * @return void + */ + public function testUploadUsesTransferManagerConfigDefaultTargetPartSize(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $expectedPartCount = 2; + $transferListener = $this->createMock(TransferListener::class); + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + $manager->upload( + Utils::streamFor( + str_repeat("#", $manager->getConfig()['target_part_size_bytes'] * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => $manager->getConfig()['target_part_size_bytes'], + ], + [ + $transferListener, + ] + )->wait(); + } + + /** + * @return void + */ + public function testUploadUsesCustomPartSize(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $expectedPartCount = 2; + $expectedPartSize = 6 * 1024 * 1024; // 6 MBs + $transferListener = $this->getMockBuilder(TransferListener::class) + ->onlyMethods(['bytesTransferred']) + ->getMock(); + $expectedIncrementalPartSize = $expectedPartSize; + $transferListener->method('bytesTransferred') + ->willReturnCallback(function ($context) use ( + $expectedPartSize, + &$expectedIncrementalPartSize + ) { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context['progress_snapshot']; + $this->assertEquals($expectedIncrementalPartSize, $snapshot->getTransferredBytes()); + $expectedIncrementalPartSize += $expectedPartSize; + }); + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + + $manager->upload( + Utils::streamFor( + str_repeat("#", $expectedPartSize * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'part_size' => $expectedPartSize, + 'multipart_upload_threshold_bytes' => $expectedPartSize, + ], + [ + $transferListener, + ] + )->wait(); + } + + /** + * @return void + */ + public function testUploadUsesDefaultChecksumAlgorithm(): void + { + $manager = new S3TransferManager(); + $this->testUploadResolvedChecksum( + [], // No checksum provided + $manager->getConfig()['checksum_algorithm'] // default checksum algo + ); + } + + /** + * @param string $checksumAlgorithm + * + * @dataProvider uploadUsesCustomChecksumAlgorithmProvider + * + * @return void + */ + public function testUploadUsesCustomChecksumAlgorithm( + string $checksumAlgorithm, + ): void + { + $this->testUploadResolvedChecksum( + ['checksum_algorithm' => $checksumAlgorithm], + $checksumAlgorithm + ); + } + + /** + * @return array[] + */ + public function uploadUsesCustomChecksumAlgorithmProvider(): array + { + return [ + 'checksum_sha256' => [ + 'checksum_algorithm' => 'sha256', + ], + 'checksum_sha1' => [ + 'checksum_algorithm' => 'sha1', + ], + 'checksum_crc32c' => [ + 'checksum_algorithm' => 'crc32c', + ], + 'checksum_crc32' => [ + 'checksum_algorithm' => 'crc32', + ] + ]; + } + + /** + * @param array $config + * @param string $expectedChecksum + * + * @return void + */ + private function testUploadResolvedChecksum( + array $config, + string $expectedChecksum + ): void { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $manager = new S3TransferManager( + $client, + ); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use ( + $expectedChecksum + ) { + $this->assertEquals( + strtoupper($expectedChecksum), + strtoupper($args['ChecksumAlgorithm']) + ); + + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function ($command) { + return Create::promiseFor(new Result([])); + }); + $manager->upload( + Utils::streamFor(), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + $config + )->wait(); + } + + /** + * @param string $directory + * @param bool $isDirectoryValid + * + * @dataProvider uploadDirectoryValidatesProvidedDirectoryProvider + * + * @return void + */ + public function testUploadDirectoryValidatesProvidedDirectory( + string $directory, + bool $isDirectoryValid + ): void + { + if (!$isDirectoryValid) { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + "Please provide a valid directory path. " + . "Provided = " . $directory); + } else { + $this->assertTrue(true); + } + + $manager = new S3TransferManager( + $this->getS3ClientMock(), + ); + $manager->uploadDirectory( + $directory, + "Bucket", + )->wait(); + // Clean up resources + if ($isDirectoryValid) { + rmdir($directory); + } + } + + /** + * @return array[] + */ + public function uploadDirectoryValidatesProvidedDirectoryProvider(): array + { + $validDirectory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($validDirectory)) { + mkdir($validDirectory, 0777, true); + } + + return [ + 'valid_directory' => [ + 'directory' => $validDirectory, + 'is_valid_directory' => true, + ], + 'invalid_directory' => [ + 'directory' => 'invalid-directory', + 'is_valid_directory' => false, + ] + ]; + } + + /** + * @return void + */ + public function testUploadDirectoryFailsOnInvalidFilter(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'The parameter $config[\'filter\'] must be callable' + ); + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'filter' => 'invalid_filter', + ] + )->wait(); + } finally { + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFileFilter(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + + $filesCreated = []; + $validFilesCount = 0; + for ($i = 0; $i < 10; $i++) { + $fileName = "file-$i"; + if ($i % 2 === 0) { + $fileName .= "-valid"; + $validFilesCount++; + } + + $filePathName = $directory . "/" . $fileName . ".txt"; + file_put_contents($filePathName, "test"); + $filesCreated[] = $filePathName; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $calledTimes = 0; + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'filter' => function (string $objectKey) { + return str_ends_with($objectKey, "-valid.txt"); + }, + 'put_object_request_callback' => function ($requestArgs) use (&$calledTimes) { + $this->assertStringContainsString( + 'valid.txt', + $requestArgs["Key"] + ); + $calledTimes++; + } + ] + )->wait(); + $this->assertEquals($validFilesCount, $calledTimes); + } finally { + foreach ($filesCreated as $filePathName) { + unlink($filePathName); + } + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryRecursive(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + $subDirectory = $directory . "/sub-directory"; + if (!is_dir($subDirectory)) { + mkdir($subDirectory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $subDirectory . "/subdir-file-1.txt", + $subDirectory . "/subdir-file-2.txt", + ]; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + // Remove the directory from the file path to leave + // just what will be the object key + $objectKey = str_replace($directory . "/", "", $file); + $objectKeys[$objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + ] + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated); + } + } finally { + foreach ($files as $file) { + unlink($file); + } + + rmdir($subDirectory); + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryNonRecursive(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + $subDirectory = $directory . "/sub-directory"; + if (!is_dir($subDirectory)) { + mkdir($subDirectory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $subDirectory . "/subdir-file-1.txt", + $subDirectory . "/subdir-file-2.txt", + ]; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + // Remove the directory from the file path to leave + // just what will be the object key + $objectKey = str_replace($directory . "/", "", $file); + $objectKeys[$objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'recursive' => false, + ] + )->wait(); + $subDirPrefix = str_replace($directory . "/", "", $subDirectory); + foreach ($objectKeys as $key => $validated) { + if (str_starts_with($key, $subDirPrefix)) { + // Files in subdirectory should have been ignored + $this->assertFalse($validated, "Key {$key} should have not been considered"); + } else { + $this->assertTrue($validated, "Key {$key} should have been considered"); + } + } + } finally { + foreach ($files as $file) { + unlink($file); + } + + rmdir($subDirectory); + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFollowsSymbolicLink(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + $linkDirectory = sys_get_temp_dir() . "/link-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + if (!is_dir($linkDirectory)) { + mkdir($linkDirectory, 0777, true); + } + $symLinkDirectory = $directory . "/upload-directory-test-link"; + if (is_link($symLinkDirectory)) { + unlink($symLinkDirectory); + } + symlink($linkDirectory, $symLinkDirectory); + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $linkDirectory . "/symlink-file-1.txt", + $linkDirectory . "/symlink-file-2.txt", + ]; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + // Remove the directory from the file path to leave + // just what will be the object key + $objectKey = str_replace($directory . "/", "", $file); + $objectKey = str_replace($linkDirectory . "/", "", $objectKey); + if (str_contains($objectKey, 'symlink-file')) { + $objectKey = "upload-directory-test-link/" . $objectKey; + } + + $objectKeys[$objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + // First lets make sure that when follows_symbolic_link is false + // the directory in the link will not be traversed. + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + 'follow_symbolic_links' => false, + ] + )->wait(); + foreach ($objectKeys as $key => $validated) { + if (str_contains($key, "symlink")) { + // Files in subdirectory should have been ignored + $this->assertFalse($validated, "Key {$key} should have not been considered"); + } else { + $this->assertTrue($validated, "Key {$key} should have been considered"); + } + } + // Now let's enable follow_symbolic_links and all files should have + // been considered, included the ones in the symlink directory. + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + 'follow_symbolic_links' => true, + ] + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been considered"); + } + } finally { + foreach ($files as $file) { + unlink($file); + } + + unlink($symLinkDirectory); + rmdir($linkDirectory); + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryUsesProvidedPrefix(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + $directory . "/dir-file-5.txt", + ]; + $s3Prefix = 'expenses-files/'; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + $objectKey = str_replace($directory . "/", "", $file); + $objectKeys[$s3Prefix . $objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 's3_prefix' => $s3Prefix + ] + )->wait(); + + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been validated"); + } + } finally { + foreach ($files as $file) { + unlink($file); + } + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryUsesProvidedDelimiter(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + $directory . "/dir-file-5.txt", + ]; + $s3Prefix = 'expenses-files/today/records/'; + $s3Delimiter = '|'; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + $objectKey = str_replace($directory . "/", "", $file); + $objectKey = $s3Prefix . $objectKey; + $objectKey = str_replace("/", $s3Delimiter, $objectKey); + $objectKeys[$objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 's3_prefix' => $s3Prefix, + 's3_delimiter' => $s3Delimiter, + ] + )->wait(); + + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been validated"); + } + } finally { + foreach ($files as $file) { + unlink($file); + } + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFailsOnInvalidPutObjectRequestCallback(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The parameter \$config['put_object_request_callback'] must be callable."); + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'put_object_request_callback' => false, + ] + )->wait(); + } finally { + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryPutObjectRequestCallbackWorks(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + ]; + foreach ($files as $file) { + file_put_contents($file, "test"); + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function ($command) { + $this->assertEquals("Test", $command['FooParameter']); + + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $called = 0; + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'put_object_request_callback' => function ( + &$requestArgs + ) use (&$called) { + $requestArgs["FooParameter"] = "Test"; + $called++; + }, + ] + )->wait(); + $this->assertEquals(count($files), $called); + } finally { + foreach ($files as $file) { + unlink($file); + } + + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryUsesFailurePolicy(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + ]; + foreach ($files as $file) { + file_put_contents($file, "test"); + } + try { + $client = new S3Client([ + 'region' => 'us-east-2', + 'handler' => function ($command) { + if (str_contains($command['Key'], "dir-file-2.txt")) { + return Create::rejectionFor( + new Exception("Failed uploading second file") + ); + } + + return Create::promiseFor(new Result([])); + } + ]); + $manager = new S3TransferManager( + $client, + [ + 'concurrency' => 1, // To make uploads to be one after the other + ] + ); + $called = false; + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'failure_policy' => function ( + array $requestArgs, + array $uploadDirectoryRequestArgs, + \Throwable $reason, + UploadDirectoryResponse $uploadDirectoryResponse + ) use (&$called) { + $called = true; + $this->assertEquals( + "Failed uploading second file", + $reason->getMessage() + ); + $this->assertEquals( + 1, + $uploadDirectoryResponse->getObjectsUploaded() + ); + $this->assertEquals( + 1, + $uploadDirectoryResponse->getObjectsFailed() + ); + }, + ] + )->wait(); + $this->assertTrue($called); + } finally { + foreach ($files as $file) { + unlink($file); + } + + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFailsOnInvalidFailurePolicy(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The parameter \$config['failure_policy'] must be callable."); + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'failure_policy' => false, + ] + )->wait(); + } finally { + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): void + { + $s3Delimiter = "*"; + $fileNameWithDelimiter = "dir-file-$s3Delimiter.txt"; + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "The filename `$fileNameWithDelimiter` must not contain the provided delimiter `$s3Delimiter`" + ); + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + $directory . "/$fileNameWithDelimiter", + ]; + foreach ($files as $file) { + file_put_contents($file, "test"); + } + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + ['s3_delimiter' => $s3Delimiter] + )->wait(); + } finally { + foreach ($files as $file) { + unlink($file); + } + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryTracksMultipleFiles(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + ]; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + $objectKey = str_replace($directory . "/", "", $file); + $objectKeys[$objectKey] = false; + } + + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $transferListener = $this->getMockBuilder(TransferListener::class) + ->disableOriginalConstructor() + ->getMock(); + $transferListener->expects($this->exactly(count($files))) + ->method('transferInitiated'); + $transferListener->expects($this->exactly(count($files))) + ->method('transferComplete'); + $transferListener->method('bytesTransferred') + ->willReturnCallback(function(array $context) use (&$objectKeys) { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context['progress_snapshot']; + $objectKeys[$snapshot->getIdentifier()] = true; + }); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [], + [ + $transferListener + ] + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue( + $validated, + "The object key `$key` should have been validated." + ); + } + } finally { + foreach ($files as $file) { + unlink($file); + } + rmdir($directory); + } + } + + /** + * @return S3Client + */ + private function getS3ClientMock(): S3Client + { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $client->method('executeAsync')->willReturnCallback( + function ($command) { + return match ($command->getName()) { + 'CreateMultipartUpload' => Create::promiseFor(new Result([ + 'UploadId' => 'FooUploadId', + ])), + 'UploadPart', + 'CompleteMultipartUpload', + 'AbortMultipartUpload', + 'PutObject' => Create::promiseFor(new Result([])), + default => null, + }; + } + ); + + return $client; + } +} From b27189790fc89640b07d4acd07bbcf2c35ebc534 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 13 Mar 2025 08:33:35 -0700 Subject: [PATCH 17/19] chore: address naming feedback and test failures - Fix MultipartUpload tests by increasing the part size from 1024 to 10240000 so it gets between the allowed part size range 5MB-5GBs. - Rename tobe to to_be in the progress formatting. --- src/S3/S3Transfer/MultipartDownloader.php | 2 +- .../ColoredTransferProgressBarFormat.php | 4 ++-- .../Progress/SingleProgressTracker.php | 2 +- .../Progress/TransferProgressBarFormat.php | 4 ++-- .../S3/S3Transfer/MultipartDownloaderTest.php | 1 + tests/S3/S3Transfer/MultipartUploaderTest.php | 24 +++++++++---------- .../Progress/ConsoleProgressBarTest.php | 16 ++++++------- .../Progress/ProgressBarFormatTest.php | 14 +++++------ 8 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index ab42b59cd7..2e4de9c568 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -177,7 +177,7 @@ public function promise(): PromiseInterface yield Create::promiseFor(new DownloadResponse( $this->stream, - $result['@metadata'] + $result['@metadata'] ?? [] )); }); } diff --git a/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php index e5a3eb95f0..cc7f80d6f3 100644 --- a/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php @@ -16,7 +16,7 @@ public function getFormatTemplate(): string { return "|object_name|:\n" - ."\033|color_code|[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit| |message|\033[0m"; + ."\033|color_code|[|progress_bar|] |percent|% |transferred|/|to_be_transferred| |unit| |message|\033[0m"; } /** @@ -28,7 +28,7 @@ public function getFormatParameters(): array 'progress_bar', 'percent', 'transferred', - 'tobe_transferred', + 'to_be_transferred', 'unit', 'color_code', 'message', diff --git a/src/S3/S3Transfer/Progress/SingleProgressTracker.php b/src/S3/S3Transfer/Progress/SingleProgressTracker.php index 1024068d4f..23867d7e22 100644 --- a/src/S3/S3Transfer/Progress/SingleProgressTracker.php +++ b/src/S3/S3Transfer/Progress/SingleProgressTracker.php @@ -195,7 +195,7 @@ private function updateProgressBar( $this->progressBar->getProgressBarFormat()->setArgs([ 'transferred' => $this->currentSnapshot->getTransferredBytes(), - 'tobe_transferred' => $this->currentSnapshot->getTotalBytes(), + 'to_be_transferred' => $this->currentSnapshot->getTotalBytes(), 'unit' => 'B', ]); // Display progress diff --git a/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php index d6568e3b96..c7a40575a5 100644 --- a/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php @@ -9,7 +9,7 @@ final class TransferProgressBarFormat extends ProgressBarFormat */ public function getFormatTemplate(): string { - return "|object_name|:\n[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit|"; + return "|object_name|:\n[|progress_bar|] |percent|% |transferred|/|to_be_transferred| |unit|"; } /** @@ -22,7 +22,7 @@ public function getFormatParameters(): array 'progress_bar', 'percent', 'transferred', - 'tobe_transferred', + 'to_be_transferred', 'unit', ]; } diff --git a/tests/S3/S3Transfer/MultipartDownloaderTest.php b/tests/S3/S3Transfer/MultipartDownloaderTest.php index 1ab1dd8fc8..6787a93231 100644 --- a/tests/S3/S3Transfer/MultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/MultipartDownloaderTest.php @@ -68,6 +68,7 @@ public function testMultipartDownloader( -> willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); }); + $downloaderClassName = MultipartDownloader::chooseDownloaderClass( $multipartDownloadType ); diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index 3a52b33b0c..27e4ab8793 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -85,58 +85,58 @@ public function multipartUploadProvider(): array { return [ '5_parts_upload' => [ 'stream' => Utils::streamFor( - str_repeat('*', 1024 * 5), + str_repeat('*', 10240000 * 5), ), 'config' => [ - 'part_size' => 1024 + 'part_size' => 10240000 ], 'expected' => [ 'succeed' => true, 'parts' => 5, - 'bytesUploaded' => 1024 * 5, + 'bytesUploaded' => 10240000 * 5, ] ], '100_parts_upload' => [ 'stream' => Utils::streamFor( - str_repeat('*', 1024 * 100), + str_repeat('*', 10240000 * 100), ), 'config' => [ - 'part_size' => 1024 + 'part_size' => 10240000 ], 'expected' => [ 'succeed' => true, 'parts' => 100, - 'bytesUploaded' => 1024 * 100, + 'bytesUploaded' => 10240000 * 100, ] ], '5_parts_no_seekable_stream' => [ 'stream' => new NoSeekStream( Utils::streamFor( - str_repeat('*', 1024 * 5) + str_repeat('*', 10240000 * 5) ) ), 'config' => [ - 'part_size' => 1024 + 'part_size' => 10240000 ], 'expected' => [ 'succeed' => true, 'parts' => 5, - 'bytesUploaded' => 1024 * 5, + 'bytesUploaded' => 10240000 * 5, ] ], '100_parts_no_seekable_stream' => [ 'stream' => new NoSeekStream( Utils::streamFor( - str_repeat('*', 1024 * 100) + str_repeat('*', 10240000 * 100) ) ), 'config' => [ - 'part_size' => 1024 + 'part_size' => 10240000 ], 'expected' => [ 'succeed' => true, 'parts' => 100, - 'bytesUploaded' => 1024 * 100, + 'bytesUploaded' => 10240000 * 100, ] ] ]; diff --git a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php index 6de1dbdaaf..393bb13400 100644 --- a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php +++ b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php @@ -175,7 +175,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'FooObject', 'transferred' => 23, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_output' => "FooObject:\n[############ ] 23% 23/100 B" @@ -188,7 +188,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'FooObject', 'transferred' => 75, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_output' => "FooObject:\n[################### ] 75% 75/100 B" @@ -201,7 +201,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'FooObject', 'transferred' => 100, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_output' => "FooObject:\n[##############################] 100% 100/100 B" @@ -214,7 +214,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'FooObject', 'transferred' => 100, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_output' => "FooObject:\n[******************************] 100% 100/100 B" @@ -227,7 +227,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'ObjectName_1', 'transferred' => 10, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_output' => "ObjectName_1:\n\033[30m[## ] 10% 10/100 B \033[0m" @@ -240,7 +240,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'ObjectName_2', 'transferred' => 50, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE ], @@ -254,7 +254,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'ObjectName_3', 'transferred' => 100, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE ], @@ -268,7 +268,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'ObjectName_3', 'transferred' => 100, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE ], diff --git a/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php b/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php index 5e1c03a397..42fa59fc35 100644 --- a/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php +++ b/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php @@ -67,7 +67,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..........', 'percent' => 100, 'transferred' => 100, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_format' => "foo:\n[..........] 100% 100/100 B", @@ -79,7 +79,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..... ', 'percent' => 50, 'transferred' => 50, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_format' => "foo:\n[..... ] 50% 50/100 B", @@ -90,7 +90,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..... ', 'percent' => 50, 'transferred' => 50, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'object_name' => 'FooObject' ], @@ -102,7 +102,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..... ', 'percent' => 50, 'transferred' => 50, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'object_name' => 'FooObject', 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE @@ -115,7 +115,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..... ', 'percent' => 50, 'transferred' => 50, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'object_name' => 'FooObject', 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE @@ -128,7 +128,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..... ', 'percent' => 50, 'transferred' => 50, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'object_name' => 'FooObject', 'color_code' => ColoredTransferProgressBarFormat::RED_COLOR_CODE @@ -141,7 +141,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..........', 'percent' => 100, 'transferred' => 100, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'object_name' => 'FooObject', 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE From f4f1c88a4146e24facf1bc8d670b261cf5bdae74 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 13 Mar 2025 08:59:25 -0700 Subject: [PATCH 18/19] chore: address minor styling issues --- src/S3/S3Transfer/MultipartUploader.php | 4 ++-- src/S3/S3Transfer/S3TransferManager.php | 15 +++++++++++---- tests/S3/S3Transfer/MultipartUploaderTest.php | 1 - tests/S3/S3Transfer/S3TransferManagerTest.php | 1 - 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index a99162d21e..b40e1891ec 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -64,8 +64,8 @@ class MultipartUploader implements PromisorInterface * @param S3ClientInterface $s3Client * @param array $createMultipartArgs * @param array $config - * - part_size: (int, optional) - * - concurrency: (int, required) + * - part_size: (int, optional) + * - concurrency: (int, required) * @param string | StreamInterface $source * @param string|null $uploadId * @param array $parts diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 513908c264..bdb4f2eb80 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -2,7 +2,6 @@ namespace Aws\S3\S3Transfer; -use Aws\Result; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; @@ -20,7 +19,6 @@ use GuzzleHttp\Promise\PromiseInterface; use InvalidArgumentException; use Psr\Http\Message\StreamInterface; -use Throwable; use function Aws\filter; use function Aws\map; @@ -541,6 +539,14 @@ public function downloadDirectory( return call_user_func($filter, $key); }); } + $failurePolicyCallback = null; + if (isset($config['failure_policy']) && !is_callable($config['failure_policy'])) { + throw new InvalidArgumentException( + "The parameter \$config['failure_policy'] must be callable." + ); + } elseif (isset($config['failure_policy'])) { + $failurePolicyCallback = $config['failure_policy']; + } $promises = []; $objectsDownloaded = 0; @@ -589,15 +595,16 @@ public function downloadDirectory( $result->getData()->close(); $objectsDownloaded++; })->otherwise(function ($reason) use ( + $failurePolicyCallback, &$objectsDownloaded, &$objectsFailed, $downloadArgs, $requestArgs ) { $objectsFailed++; - if (isset($config['failure_policy']) && is_callable($config['failure_policy'])) { + if ($failurePolicyCallback !== null) { call_user_func( - $config['failure_policy'], + $failurePolicyCallback, $requestArgs, $downloadArgs, $reason, diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index 27e4ab8793..a5ca28bf69 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -18,7 +18,6 @@ class MultipartUploaderTest extends TestCase { - /** * @param StreamInterface $stream * @param array $config diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index cfea7c6c06..4ba9b5a0a5 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -13,7 +13,6 @@ use Aws\S3\S3Transfer\S3TransferManager; use Exception; use GuzzleHttp\Promise\Create; -use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; use InvalidArgumentException; use PHPUnit\Framework\TestCase; From d987aff968802d7e329ab4481b8d7fa1be392f78 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Sun, 16 Mar 2025 20:54:06 -0700 Subject: [PATCH 19/19] chore: add download tests - Add download tests - Add download directory tests - Minor naming refactor --- .../RangeGetMultipartDownloader.php | 2 +- src/S3/S3Transfer/S3TransferManager.php | 175 +- tests/S3/S3Transfer/S3TransferManagerTest.php | 1472 ++++++++++++++++- 3 files changed, 1557 insertions(+), 92 deletions(-) diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php index 5fa10cde99..1022edc221 100644 --- a/src/S3/S3Transfer/RangeGetMultipartDownloader.php +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -85,7 +85,7 @@ protected function nextCommand(): CommandInterface $this->currentPartNo++; } - $nextRequestArgs = array_slice($this->requestArgs, 0); + $nextRequestArgs = [...$this->requestArgs]; $from = ($this->currentPartNo - 1) * $this->partSize; $to = ($this->currentPartNo * $this->partSize) - 1; if ($this->objectSizeInBytes !== 0) { diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index bdb4f2eb80..fc761c2f1a 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -2,6 +2,7 @@ namespace Aws\S3\S3Transfer; +use Aws\Arn\ArnParser; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; @@ -68,7 +69,10 @@ public function __construct( ?S3ClientInterface $s3Client = null, array $config = [] ) { - $this->config = $config + self::$defaultConfig; + $this->config = [ + ...self::$defaultConfig, + ...$config, + ]; if ($s3Client === null) { $this->s3Client = $this->defaultS3Client(); } else { @@ -132,7 +136,8 @@ public function upload( // Valid required parameters foreach (['Bucket', 'Key'] as $reqParam) { $this->requireNonEmpty( - $requestArgs[$reqParam] ?? null, + $requestArgs, + $reqParam, "The `$reqParam` parameter must be provided as part of the request arguments." ); } @@ -182,9 +187,9 @@ public function upload( } /** - * @param string $directory + * @param string $sourceDirectory * @param string $bucketTo - * @param array $requestArgs + * @param array $uploadDirectoryRequestArgs * @param array $config The config options for this request that are: * - follow_symbolic_links: (bool, optional, defaulted to false) * - recursive: (bool, optional, defaulted to false) @@ -219,21 +224,23 @@ public function upload( * @return PromiseInterface */ public function uploadDirectory( - string $directory, - string $bucketTo, - array $requestArgs = [], - array $config = [], - array $listeners = [], + string $sourceDirectory, + string $bucketTo, + array $uploadDirectoryRequestArgs = [], + array $config = [], + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { - if (!is_dir($directory)) { + if (!is_dir($sourceDirectory)) { throw new InvalidArgumentException( "Please provide a valid directory path. " - . "Provided = " . $directory + . "Provided = " . $sourceDirectory ); } + $bucketTo = $this->parseBucket($bucketTo); + if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { $progressTracker = new MultiProgressTracker(); @@ -268,7 +275,7 @@ public function uploadDirectory( $failurePolicyCallback = $config['failure_policy']; } - $dirIterator = new \RecursiveDirectoryIterator($directory); + $dirIterator = new \RecursiveDirectoryIterator($sourceDirectory); $dirIterator->setFlags(FilesystemIterator::SKIP_DOTS); if (($config['follow_symbolic_links'] ?? false) === true) { $dirIterator->setFlags(FilesystemIterator::FOLLOW_SYMLINKS); @@ -298,7 +305,7 @@ function ($file) use ($filter) { $objectsUploaded = 0; $objectsFailed = 0; foreach ($files as $file) { - $baseDir = rtrim($directory, '/') . '/'; + $baseDir = rtrim($sourceDirectory, '/') . '/'; $relativePath = substr($file, strlen($baseDir)); if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { throw new S3TransferException( @@ -312,7 +319,7 @@ function ($file) use ($filter) { $objectKey ); $uploadRequestArgs = [ - ...$requestArgs, + ...$uploadDirectoryRequestArgs, 'Bucket' => $bucketTo, 'Key' => $objectKey, ]; @@ -331,9 +338,11 @@ function ($file) use ($filter) { return $response; })->otherwise(function ($reason) use ( + $bucketTo, + $sourceDirectory, $failurePolicyCallback, $uploadRequestArgs, - $requestArgs, + $uploadDirectoryRequestArgs, &$objectsUploaded, &$objectsFailed ) { @@ -341,8 +350,12 @@ function ($file) use ($filter) { if($failurePolicyCallback !== null) { call_user_func( $failurePolicyCallback, - $requestArgs, $uploadRequestArgs, + [ + ...$uploadDirectoryRequestArgs, + "source_directory" => $sourceDirectory, + "bucket_to" => $bucketTo, + ], $reason, new UploadDirectoryResponse( $objectsUploaded, @@ -367,7 +380,7 @@ function ($file) use ($filter) { * @param string|array $source The object to be downloaded from S3. * It can be either a string with a S3 URI or an array with a Bucket and Key * properties set. - * @param array $downloadArgs The getObject request arguments to be provided as part + * @param array $downloadRequestArgs The getObject request arguments to be provided as part * of each get object operation, except for the bucket and key, which * are already provided as the source. * @param array $config The configuration to be used for this operation: @@ -390,10 +403,10 @@ function ($file) use ($filter) { * @return PromiseInterface */ public function download( - string | array $source, - array $downloadArgs = [], - array $config = [], - array $listeners = [], + string | array $source, + array $downloadRequestArgs = [], + array $config = [], + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { @@ -401,19 +414,29 @@ public function download( $sourceArgs = $this->s3UriAsBucketAndKey($source); } elseif (is_array($source)) { $sourceArgs = [ - 'Bucket' => $this->requireNonEmpty($source['Bucket'], "A valid bucket must be provided."), - 'Key' => $this->requireNonEmpty($source['Key'], "A valid key must be provided."), + 'Bucket' => $this->requireNonEmpty( + $source, + 'Bucket', + "A valid bucket must be provided." + ), + 'Key' => $this->requireNonEmpty( + $source, + 'Key', + "A valid key must be provided." + ), ]; } else { - throw new InvalidArgumentException('Source must be a string or an array of strings'); + throw new S3TransferException( + "Unsupported source type `" . gettype($source) . "`" + ); } - if (!isset($requestArgs['ChecksumMode'])) { + if (!isset($downloadRequestArgs['ChecksumMode'])) { $checksumEnabled = $config['checksum_validation_enabled'] ?? $this->config['checksum_validation_enabled'] ?? false; if ($checksumEnabled) { - $requestArgs['ChecksumMode'] = 'enabled'; + $downloadRequestArgs['ChecksumMode'] = 'enabled'; } } @@ -427,8 +450,11 @@ public function download( } $listenerNotifier = new TransferListenerNotifier($listeners); - $requestArgs = $sourceArgs + $downloadArgs; - if (empty($downloadArgs['PartNumber']) && empty($downloadArgs['Range'])) { + $requestArgs = [ + ...$sourceArgs, + ...$downloadRequestArgs, + ]; + if (empty($downloadRequestArgs['PartNumber']) && empty($downloadRequestArgs['Range'])) { return $this->tryMultipartDownload( $requestArgs, [ @@ -449,7 +475,7 @@ public function download( * downloaded from. * @param string $destinationDirectory The destination path where the downloaded * files will be placed in. - * @param array $downloadArgs The getObject request arguments to be provided + * @param array $downloadDirectoryArgs The getObject request arguments to be provided * as part of each get object request sent to the service, except for the * bucket and key which will be resolved internally. * @param array $config The config options for this download directory operation. @@ -493,20 +519,22 @@ public function download( * @return PromiseInterface */ public function downloadDirectory( - string $bucket, - string $destinationDirectory, - array $downloadArgs, - array $config = [], - array $listeners = [], + string $bucket, + string $destinationDirectory, + array $downloadDirectoryArgs = [], + array $config = [], + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { if (!file_exists($destinationDirectory)) { throw new InvalidArgumentException( - "Destination directory '$destinationDirectory' MUST exists." + "Destination directory `$destinationDirectory` MUST exists." ); } + $bucket = $this->parseBucket($bucket); + if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { $progressTracker = new MultiProgressTracker(); @@ -526,9 +554,6 @@ public function downloadDirectory( $objects = $this->s3Client ->getPaginator('ListObjectsV2', $listArgs) ->search('Contents[].Key'); - $objects = map($objects, function (string $key) use ($bucket) { - return "s3://$bucket/$key"; - }); if (isset($config['filter'])) { if (!is_callable($config['filter'])) { throw new InvalidArgumentException("The parameter \$config['filter'] must be callable."); @@ -539,6 +564,21 @@ public function downloadDirectory( return call_user_func($filter, $key); }); } + + $objects = map($objects, function (string $key) use ($bucket) { + return "s3://$bucket/$key"; + }); + $getObjectRequestCallback = null; + if (isset($config['get_object_request_callback'])) { + if (!is_callable($config['get_object_request_callback'])) { + throw new InvalidArgumentException( + "The parameter \$config['get_object_request_callback'] must be callable." + ); + } + + $getObjectRequestCallback = $config['get_object_request_callback']; + } + $failurePolicyCallback = null; if (isset($config['failure_policy']) && !is_callable($config['failure_policy'])) { throw new InvalidArgumentException( @@ -562,15 +602,9 @@ public function downloadDirectory( ); } - $requestArgs = [...$downloadArgs]; - if (isset($config['get_object_request_callback'])) { - if (!is_callable($config['get_object_request_callback'])) { - throw new InvalidArgumentException( - "The parameter \$config['get_object_request_callback'] must be callable." - ); - } - - call_user_func($config['get_object_request_callback'], $requestArgs); + $requestArgs = [...$downloadDirectoryArgs]; + if ($getObjectRequestCallback !== null) { + call_user_func($getObjectRequestCallback, $requestArgs); } $promises[] = $this->download( @@ -595,10 +629,12 @@ public function downloadDirectory( $result->getData()->close(); $objectsDownloaded++; })->otherwise(function ($reason) use ( + $bucket, + $destinationDirectory, $failurePolicyCallback, &$objectsDownloaded, &$objectsFailed, - $downloadArgs, + $downloadDirectoryArgs, $requestArgs ) { $objectsFailed++; @@ -606,13 +642,19 @@ public function downloadDirectory( call_user_func( $failurePolicyCallback, $requestArgs, - $downloadArgs, + [ + ...$downloadDirectoryArgs, + "destination_directory" => $destinationDirectory, + "bucket" => $bucket, + ], $reason, new DownloadDirectoryResponse( $objectsDownloaded, $objectsFailed ) ); + + return; } throw $reason; @@ -888,18 +930,19 @@ private function defaultS3Client(): S3ClientInterface /** * Validates a provided value is not empty, and if so then * it throws an exception with the provided message. - * @param mixed $value + * @param array $array + * @param string $key * @param string $message * * @return mixed */ - private function requireNonEmpty(mixed $value, string $message): mixed + private function requireNonEmpty(array $array, string $key, string $message): mixed { - if (empty($value)) { + if (empty($array[$key])) { throw new InvalidArgumentException($message); } - return $value; + return $array[$key]; } /** @@ -927,7 +970,7 @@ private function isValidS3URI(string $uri): bool */ private function s3UriAsBucketAndKey(string $uri): array { - $errorMessage = "Invalid URI: $uri. A valid S3 URI must be s3://bucket/key"; + $errorMessage = "Invalid URI: `$uri` provided. \nA valid S3 URI looks as `s3://bucket/key`"; if (!$this->isValidS3URI($uri)) { throw new InvalidArgumentException($errorMessage); } @@ -945,6 +988,22 @@ private function s3UriAsBucketAndKey(string $uri): array ]; } + /** + * To parse the bucket name when the bucket is provided as an ARN. + * + * @param string $bucket + * + * @return string + */ + private function parseBucket(string $bucket): string + { + if (ArnParser::isArn($bucket)) { + return ArnParser::parse($bucket)->getResource(); + } + + return $bucket; + } + /** * @param string $sink * @param string $objectKey @@ -978,4 +1037,12 @@ private function resolvesOutsideTargetDirectory( return false; } + + /** + * @return array + */ + public static function getDefaultConfig(): array + { + return self::$defaultConfig; + } } diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 4ba9b5a0a5..f66d84571b 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -2,15 +2,21 @@ namespace Aws\Test\S3\S3Transfer; +use Aws\Api\Service; use Aws\Command; +use Aws\CommandInterface; +use Aws\HandlerList; use Aws\Result; use Aws\S3\S3Client; use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Models\DownloadDirectoryResponse; use Aws\S3\S3Transfer\Models\UploadDirectoryResponse; +use Aws\S3\S3Transfer\MultipartDownloader; use Aws\S3\S3Transfer\MultipartUploader; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Aws\S3\S3Transfer\S3TransferManager; +use Closure; use Exception; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; @@ -175,7 +181,7 @@ public function testUploadFailsWhenMultipartThresholdIsLessThanMinSize(): void */ public function testDoesMultipartUploadWhenApplicable(): void { - $client = $this->getS3ClientMock();; + $client = $this->getS3ClientMock(); $manager = new S3TransferManager( $client, ); @@ -473,15 +479,11 @@ private function testUploadResolvedChecksum( array $config, string $expectedChecksum ): void { - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync']) - ->getMock(); - $manager = new S3TransferManager( - $client, - ); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use ( + $client = $this->getS3ClientMock([ + 'getCommand' => function ( + string $commandName, + array $args + ) use ( $expectedChecksum ) { $this->assertEquals( @@ -490,11 +492,14 @@ private function testUploadResolvedChecksum( ); return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function ($command) { + }, + 'executeAsync' => function () { return Create::promiseFor(new Result([])); - }); + } + ]); + $manager = new S3TransferManager( + $client, + ); $manager->upload( Utils::streamFor(), [ @@ -1148,8 +1153,16 @@ public function testUploadDirectoryUsesFailurePolicy(): void array $uploadDirectoryRequestArgs, \Throwable $reason, UploadDirectoryResponse $uploadDirectoryResponse - ) use (&$called) { + ) use ($directory, &$called) { $called = true; + $this->assertEquals( + $directory, + $uploadDirectoryRequestArgs["source_directory"] + ); + $this->assertEquals( + "Bucket", + $uploadDirectoryRequestArgs["bucket_to"] + ); $this->assertEquals( "Failed uploading second file", $reason->getMessage() @@ -1312,32 +1325,1417 @@ public function testUploadDirectoryTracksMultipleFiles(): void } /** - * @return S3Client + * @return void */ - private function getS3ClientMock(): S3Client + public function testDownloadFailsOnInvalidS3UriSource(): void { - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync']) - ->getMock(); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) { - return new Command($commandName, $args); - }); - $client->method('executeAsync')->willReturnCallback( - function ($command) { - return match ($command->getName()) { - 'CreateMultipartUpload' => Create::promiseFor(new Result([ - 'UploadId' => 'FooUploadId', - ])), - 'UploadPart', - 'CompleteMultipartUpload', - 'AbortMultipartUpload', - 'PutObject' => Create::promiseFor(new Result([])), - default => null, - }; + $invalidS3Uri = "invalid-s3-uri"; + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("Invalid URI: `$invalidS3Uri` provided. " + . "\nA valid S3 URI looks as `s3://bucket/key`"); + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->download( + $invalidS3Uri + ); + } + + /** + * @dataProvider downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider + * + * @param array $sourceAsArray + * @param string $expectedExceptionMessage + * + * @return void + */ + public function testDownloadFailsWhenSourceAsArrayMissesBucketOrKeyProperty( + array $sourceAsArray, + string $expectedExceptionMessage, + ): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage($expectedExceptionMessage); + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->download( + $sourceAsArray + ); + } + + /** + * @return array + */ + public function downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider(): array + { + return [ + 'missing_key' => [ + 'source' => [ + 'Bucket' => 'bucket', + ], + 'expected_exception' => "A valid key must be provided." + ], + 'missing_bucket' => [ + 'source' => [ + 'Key' => 'key', + ], + 'expected_exception' => "A valid bucket must be provided." + ] + ]; + } + + /** + * @return void + */ + public function testDownloadWorksWithS3UriAsSource(): void + { + $sourceAsArray = [ + 'Bucket' => 'bucket', + 'Key' => 'key', + ]; + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function(CommandInterface $command) use ( + $sourceAsArray, + &$called + ) { + $called = true; + $this->assertEquals($sourceAsArray['Bucket'], $command['Bucket']); + $this->assertEquals($sourceAsArray['Key'], $command['Key']); + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + }, + ]); + $manager = new S3TransferManager( + $client + ); + $manager->download( + $sourceAsArray, + )->wait(); + $this->assertTrue($called); + } + + /** + * @return void + */ + public function testDownloadWorksWithBucketAndKeyAsSource(): void + { + $bucket = 'bucket'; + $key = 'key'; + $sourceAsS3Uri = "s3://$bucket/$key"; + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function(CommandInterface $command) use ( + $bucket, + $key, + &$called + ) { + $called = true; + $this->assertEquals($bucket, $command['Bucket']); + $this->assertEquals($key, $command['Key']); + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + }, + ]); + $manager = new S3TransferManager( + $client + ); + $manager->download( + $sourceAsS3Uri, + )->wait(); + $this->assertTrue($called); + } + + /** + * + * @param array $transferManagerConfig + * @param array $downloadConfig + * @param array $downloadArgs + * @param bool $expectedChecksumMode + * + * @return void + * @dataProvider downloadAppliesChecksumProvider + * + */ + public function testDownloadAppliesChecksumMode( + array $transferManagerConfig, + array $downloadConfig, + array $downloadArgs, + bool $expectedChecksumMode, + ): void + { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $expectedChecksumMode, + &$called + ) { + $called = true; + if ($expectedChecksumMode) { + $this->assertEquals( + 'enabled', + $command['ChecksumMode'], + ); + } else { + if (isset($command['ChecksumMode'])) { + $this->assertEquals( + 'disabled', + $command['ChecksumMode'], + ); + } + } + + if ($command->getName() === MultipartDownloader::GET_OBJECT_COMMAND) { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + } + + return Create::promiseFor(new Result([])); + } + ]); + $manager = new S3TransferManager( + $client, + $transferManagerConfig, + ); + $manager->download( + "s3://bucket/key", + $downloadArgs, + $downloadConfig + )->wait(); + $this->assertTrue($called); + } + + /** + * @return array + */ + public function downloadAppliesChecksumProvider(): array + { + return [ + 'checksum_mode_from_default_transfer_manager_config' => [ + 'transfer_manager_config' => [], + 'download_config' => [], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => S3TransferManager::getDefaultConfig()[ + 'checksum_validation_enabled' + ], + ], + 'checksum_mode_enabled_by_transfer_manager_config' => [ + 'transfer_manager_config' => [ + 'checksum_validation_enabled' => true + ], + 'download_config' => [], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => true, + ], + 'checksum_mode_disabled_by_transfer_manager_config' => [ + 'transfer_manager_config' => [ + 'checksum_validation_enabled' => false + ], + 'download_config' => [], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => false, + ], + 'checksum_mode_enabled_by_download_config' => [ + 'transfer_manager_config' => [], + 'download_config' => [ + 'checksum_validation_enabled' => true + ], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => true, + ], + 'checksum_mode_disabled_by_download_config' => [ + 'transfer_manager_config' => [], + 'download_config' => [ + 'checksum_validation_enabled' => false + ], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => false, + ], + 'checksum_mode_download_config_overrides_transfer_manager_config' => [ + 'transfer_manager_config' => [ + 'checksum_validation_enabled' => false + ], + 'download_config' => [ + 'checksum_validation_enabled' => true + ], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => true, + ] + ]; + } + + /** + * @param array $downloadArgs + * + * @dataProvider singleDownloadWhenPartNumberOrRangeArePresentProvider + * + * @return void + */ + public function testDoesSingleDownloadWhenPartNumberOrRangeArePresent( + array $downloadArgs, + ): void + { + $calledOnce = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use (&$calledOnce) { + if ($command->getName() === MultipartDownloader::GET_OBJECT_COMMAND) { + if ($calledOnce) { + $this->fail(MultipartDownloader::GET_OBJECT_COMMAND . " should have been called once."); + } + + $calledOnce = true; + return Create::promiseFor(new Result([ + 'PartsCount' => 2, + 'ContentRange' => 10240000, + 'Body' => Utils::streamFor( + str_repeat("*", 1024 * 1024 * 20) + ), + '@metadata' => [] + ])); + } else { + $this->fail("Unexpected command execution `" . $command->getName() . "`."); + } + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->download( + "s3://bucket/key", + $downloadArgs, + )->wait(); + $this->assertTrue($calledOnce); + } + + /** + * @return array + */ + public function singleDownloadWhenPartNumberOrRangeArePresentProvider(): array + { + return [ + 'part_number_present' => [ + 'download_args' => [ + 'PartNumber' => 1 + ] + ], + 'range_present' => [ + 'download_args' => [ + 'Range' => '100-1024' + ] + ] + ]; + } + + /** + * @param string $multipartDownloadType + * @param string $expectedParameter + * + * @dataProvider downloadChoosesMultipartDownloadTypeProvider + * + * @return void + */ + public function testDownloadChoosesMultipartDownloadType( + string $multipartDownloadType, + string $expectedParameter + ): void + { + $calledOnce = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$calledOnce, + $expectedParameter + ) { + $this->assertTrue( + isset($command[$expectedParameter]), + ); + $calledOnce = true; + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->download( + "s3://bucket/key", + [], + ['multipart_download_type' => $multipartDownloadType] + )->wait(); + $this->assertTrue($calledOnce); + } + + /** + * @return array + */ + public function downloadChoosesMultipartDownloadTypeProvider(): array + { + return [ + 'part_get_multipart_download' => [ + 'multipart_download_type' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'expected_parameter' => 'PartNumber' + ], + 'range_get_multipart_download' => [ + 'multipart_download_type' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'expected_parameter' => 'Range' + ] + ]; + } + + /** + * @param int $minimumPartSize + * @param int $objectSize + * @param array $expectedPartsSize + * + * @dataProvider rangeGetMultipartDownloadMinimumPartSizeProvider + * + * @return void + */ + public function testRangeGetMultipartDownloadMinimumPartSize( + int $minimumPartSize, + int $objectSize, + array $expectedRangeSizes + ): void + { + $calledTimes = 0; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $objectSize, + $expectedRangeSizes, + &$calledTimes, + ) { + $this->assertTrue(isset($command['Range'])); + $range = str_replace("bytes=", "", $command['Range']); + $rangeParts = explode("-", $range); + $this->assertEquals( + (intval($rangeParts[1]) - intval($rangeParts[0])) + 1, + $expectedRangeSizes[$calledTimes] + ); + $calledTimes++; + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'ContentRange' => $objectSize, + '@metadata' => [] + ])); } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->download( + "s3://bucket/key", + [], + [ + 'multipart_download_type' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'minimum_part_size' => $minimumPartSize, + ] + )->wait(); + $this->assertEquals(count($expectedRangeSizes), $calledTimes); + } + + /** + * @return array + */ + public function rangeGetMultipartDownloadMinimumPartSizeProvider(): array + { + return [ + 'minimum_part_size_1' => [ + 'minimum_part_size' => 1024, + 'object_size' => 3072, + 'expected_range_sizes' => [ + 1024, + 1024, + 1024 + ] + ], + 'minimum_part_size_2' => [ + 'minimum_part_size' => 1024, + 'object_size' => 2000, + 'expected_range_sizes' => [ + 1024, + 977, + ] + ], + 'minimum_part_size_3' => [ + 'minimum_part_size' => 1024 * 1024 * 10, + 'object_size' => 1024 * 1024 * 25, + 'expected_range_sizes' => [ + 1024 * 1024 * 10, + 1024 * 1024 * 10, + (1024 * 1024 * 5) + 1 + ] + ], + 'minimum_part_size_4' => [ + 'minimum_part_size' => 1024 * 1024 * 25, + 'object_size' => 1024 * 1024 * 100, + 'expected_range_sizes' => [ + 1024 * 1024 * 25, + 1024 * 1024 * 25, + 1024 * 1024 * 25, + 1024 * 1024 * 25, + ] + ] + ]; + } + + /** + * @return void + */ + public function testDownloadDirectoryValidatesDestinationDirectory(): void + { + $destinationDirectory = "invalid-directory"; + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("Destination directory `$destinationDirectory` MUST exists."); + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory ); + } + + /** + * @param array $config + * @param string $expectedS3Prefix + * + * @dataProvider downloadDirectoryAppliesS3PrefixProvider + * + * @return void + */ + public function testDownloadDirectoryAppliesS3Prefix( + array $config, + string $expectedS3Prefix + ): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $listObjectsCalled = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $expectedS3Prefix, + &$called, + &$listObjectsCalled, + ) { + $called = true; + if ($command->getName() === "ListObjectsV2") { + $listObjectsCalled = true; + $this->assertEquals( + $expectedS3Prefix, + $command['Prefix'] + ); + } + + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + $config + )->wait(); + + $this->assertTrue($called); + $this->assertTrue($listObjectsCalled); + } finally { + rmdir($destinationDirectory); + } + } + + /** + * @return array + */ + public function downloadDirectoryAppliesS3PrefixProvider(): array + { + return [ + 's3_prefix_from_config' => [ + 'config' => [ + 's3_prefix' => 'TestPrefix', + ], + 'expected_s3_prefix' => 'TestPrefix' + ], + 's3_prefix_from_list_object_v2_args' => [ + 'config' => [ + 'list_object_v2_args' => [ + 'Prefix' => 'PrefixFromArgs' + ], + ], + 'expected_s3_prefix' => 'PrefixFromArgs' + ], + 's3_prefix_from_config_is_ignored_when_present_in_list_object_args' => [ + 'config' => [ + 's3_prefix' => 'TestPrefix', + 'list_object_v2_args' => [ + 'Prefix' => 'PrefixFromArgs' + ], + ], + 'expected_s3_prefix' => 'PrefixFromArgs' + ], + ]; + } + + /** + * @param array $config + * @param string $expectedS3Delimiter + * + * @dataProvider downloadDirectoryAppliesDelimiterProvider + * + * @return void + */ + public function testDownloadDirectoryAppliesDelimiter( + array $config, + string $expectedS3Delimiter + ): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $listObjectsCalled = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $expectedS3Delimiter, + &$called, + &$listObjectsCalled, + ) { + $called = true; + if ($command->getName() === "ListObjectsV2") { + $listObjectsCalled = true; + $this->assertEquals( + $expectedS3Delimiter, + $command['Delimiter'] + ); + } + + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + $config + )->wait(); + + $this->assertTrue($called); + $this->assertTrue($listObjectsCalled); + } finally { + rmdir($destinationDirectory); + } + } + + /** + * @return array + */ + public function downloadDirectoryAppliesDelimiterProvider(): array + { + return [ + 's3_delimiter_from_config' => [ + 'config' => [ + 's3_delimiter' => 'FooDelimiter', + ], + 'expected_s3_delimiter' => 'FooDelimiter' + ], + 's3_delimiter_from_list_object_v2_args' => [ + 'config' => [ + 'list_object_v2_args' => [ + 'Delimiter' => 'DelimiterFromArgs' + ], + ], + 'expected_s3_delimiter' => 'DelimiterFromArgs' + ], + 's3_delimiter_from_config_is_ignored_when_present_in_list_object_args' => [ + 'config' => [ + 's3_delimiter' => 'TestDelimiter', + 'list_object_v2_args' => [ + 'Delimiter' => 'DelimiterFromArgs' + ], + ], + 'expected_s3_delimiter' => 'DelimiterFromArgs' + ], + ]; + } + + /** + * @return void + */ + public function testDownloadDirectoryFailsOnInvalidFilter(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The parameter \$config['filter'] must be callable."); + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$called, + ) { + $called = true; + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + ['filter' => false] + )->wait(); + $this->assertTrue($called); + } finally { + rmdir($destinationDirectory); + } + } + + /** + * @return void + */ + public function testDownloadDirectoryFailsOnInvalidFailurePolicy(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The parameter \$config['failure_policy'] must be callable."); + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$called, + ) { + $called = true; + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + ['failure_policy' => false] + )->wait(); + $this->assertTrue($called); + } finally { + rmdir($destinationDirectory); + } + } + + /** + * @return void + */ + public function testDownloadDirectoryUsesFailurePolicy(): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + + try { + $client = new S3Client([ + 'region' => 'us-west-2', + 'handler' => function (CommandInterface $command) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => [ + [ + 'Key' => 'file1.txt', + ], + [ + 'Key' => 'file2.txt', + ] + ] + ])); + } elseif ($command->getName() === 'GetObject') { + if ($command['Key'] === 'file2.txt') { + return Create::rejectionFor( + new Exception("Failed downloading file") + ); + } + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + ['failure_policy' => function ( + array $requestArgs, + array $uploadDirectoryRequestArgs, + \Throwable $reason, + DownloadDirectoryResponse $downloadDirectoryResponse + ) use ($destinationDirectory, &$called) { + $called = true; + $this->assertEquals( + $destinationDirectory, + $uploadDirectoryRequestArgs['destination_directory'] + ); + $this->assertEquals( + "Failed downloading file", + $reason->getMessage() + ); + $this->assertEquals( + 1, + $downloadDirectoryResponse->getObjectsDownloaded() + ); + $this->assertEquals( + 1, + $downloadDirectoryResponse->getObjectsFailed() + ); + }] + )->wait(); + $this->assertTrue($called); + } finally { + unlink($destinationDirectory . '/file1.txt'); + rmdir($destinationDirectory); + } + } + + /** + * @param Closure $filter + * @param array $objectList + * @param array $expectedObjectList + * + * @dataProvider downloadDirectoryAppliesFilter + * + * @return void + */ + public function testDownloadDirectoryAppliesFilter( + Closure $filter, + array $objectList, + array $expectedObjectList, + ): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $downloadObjectKeys = []; + foreach ($expectedObjectList as $objectKey) { + $downloadObjectKeys[$objectKey] = false; + } + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $objectList, + &$called, + &$downloadObjectKeys + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $objectList, + ])); + } elseif ($command->getName() === 'GetObject') { + $downloadObjectKeys[$command['Key']] = true; + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + ['filter' => $filter] + )->wait(); + + $this->assertTrue($called); + foreach ($downloadObjectKeys as $key => $validated) { + $this->assertTrue( + $validated, + "The key `$key` should have been validated" + ); + } + } finally { + $dirs = []; + foreach ($objectList as $object) { + if (file_exists($destinationDirectory . "/" . $object['Key'])) { + unlink($destinationDirectory . "/" . $object['Key']); + } + + $dirs [dirname($destinationDirectory . "/" . $object['Key'])] = true; + } + + foreach ($dirs as $dir => $_) { + if (is_dir($dir)) { + rmdir($dir); + } + } + + rmdir($destinationDirectory); + } + } + + /** + * @return array[] + */ + public function downloadDirectoryAppliesFilter(): array + { + return [ + 'filter_1' => [ + 'filter' => function (string $objectKey) { + return str_starts_with($objectKey, "folder_2/"); + }, + 'object_list' => [ + [ + 'Key' => 'folder_1/key_1.txt', + ], + [ + 'Key' => 'folder_1/key_2.txt' + ], + [ + 'Key' => 'folder_2/key_1.txt' + ], + [ + 'Key' => 'folder_2/key_2.txt' + ] + ], + 'expected_object_list' => [ + "folder_2/key_1.txt", + "folder_2/key_2.txt", + ] + ], + 'filter_2' => [ + 'filter' => function (string $objectKey) { + return $objectKey === "folder_2/key_1.txt"; + }, + 'object_list' => [ + [ + 'Key' => 'folder_1/key_1.txt', + ], + [ + 'Key' => 'folder_1/key_2.txt' + ], + [ + 'Key' => 'folder_2/key_1.txt' + ], + [ + 'Key' => 'folder_2/key_2.txt' + ] + ], + 'expected_object_list' => [ + "folder_2/key_1.txt", + ] + ], + 'filter_3' => [ + 'filter' => function (string $objectKey) { + return $objectKey !== "folder_2/key_1.txt"; + }, + 'object_list' => [ + [ + 'Key' => 'folder_1/key_1.txt', + ], + [ + 'Key' => 'folder_1/key_2.txt' + ], + [ + 'Key' => 'folder_2/key_1.txt' + ], + [ + 'Key' => 'folder_2/key_2.txt' + ] + ], + 'expected_object_list' => [ + "folder_2/key_2.txt", + "folder_1/key_1.txt", + "folder_1/key_1.txt", + ] + ] + ]; + } + + /** + * @return void + */ + public function testDownloadDirectoryFailsOnInvalidGetObjectRequestCallback(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + "The parameter \$config['get_object_request_callback'] must be callable." + ); + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => [], + ])); + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + ['get_object_request_callback' => false] + )->wait(); + } finally { + rmdir($destinationDirectory); + } + } + + /** + * @return void + */ + public function testDownloadDirectoryGetObjectRequestCallbackWorks(): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $listObjectsContent = [ + [ + 'Key' => 'folder_1/key_1.txt', + ] + ]; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ($listObjectsContent) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $listObjectsContent, + ])); + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $getObjectRequestCallback = function($requestArgs) use (&$called) { + $called = true; + $this->assertTrue(isset($requestArgs['CustomParameter'])); + $this->assertEquals( + 'CustomParameterValue', + $requestArgs['CustomParameter'] + ); + }; + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [ + 'CustomParameter' => 'CustomParameterValue' + ], + ['get_object_request_callback' => $getObjectRequestCallback] + )->wait(); + $this->assertTrue($called); + } finally { + $dirs = []; + foreach ($listObjectsContent as $object) { + $file = $destinationDirectory . "/" . $object['Key']; + if (file_exists($file)) { + $dirs[dirname($file)] = true; + unlink($file); + } + } + + foreach (array_keys($dirs) as $dir) { + if (is_dir($dir)) { + rmdir($dir); + } + } + + rmdir($destinationDirectory); + } + } + + /** + * @param array $listObjectsContent + * @param array $expectedFileKeys + * + * @dataProvider downloadDirectoryCreateFilesProvider + * + * @return void + */ + public function testDownloadDirectoryCreateFiles( + array $listObjectsContent, + array $expectedFileKeys, + ): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $listObjectsContent, + &$called + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $listObjectsContent, + ])); + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor( + "Test file " . $command['Key'] + ), + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + )->wait(); + $this->assertTrue($called); + foreach ($expectedFileKeys as $key) { + $file = $destinationDirectory . "/" . $key; + $this->assertFileExists($file); + $this->assertEquals( + "Test file " . $key, + file_get_contents($file) + ); + } + } finally { + $dirs = []; + foreach ($expectedFileKeys as $key) { + $file = $destinationDirectory . "/" . $key; + if (file_exists($file)) { + unlink($file); + } + + $dirs [dirname($file)] = true; + } + + foreach ($dirs as $dir => $_) { + if (is_dir($dir)) { + rmdir($dir); + } + } + + if (is_dir($destinationDirectory)) { + rmdir($destinationDirectory); + } + } + } + + /** + * @return array + */ + public function downloadDirectoryCreateFilesProvider(): array + { + return [ + 'files_1' => [ + 'list_objects_content' => [ + [ + 'Key' => 'file1.txt' + ], + [ + 'Key' => 'file2.txt' + ], + [ + 'Key' => 'file3.txt' + ], + [ + 'Key' => 'file4.txt' + ], + [ + 'Key' => 'file5.txt' + ] + ], + 'expected_file_keys' => [ + 'file1.txt', + 'file2.txt', + 'file3.txt', + 'file4.txt', + 'file5.txt' + ] + ] + ]; + } + + /** + * @param array $methodsCallback If any from the callbacks below + * is not provided then a default implementation will be provided. + * - getCommand: (Closure, optional) This callable will + * receive as parameters: + * - $commandName: (string, optional) + * - $args: (array, optional) + * - executeAsync: (Closure, optional) This callable will + * receive as parameter: + * - $command: (CommandInterface, optional) + * + * @return S3Client + */ + private function getS3ClientMock( + array $methodsCallback = [] + ): S3Client + { + if (isset($methodsCallback['getCommand']) && !is_callable($methodsCallback['getCommand'])) { + throw new InvalidArgumentException("getCommand should be callable"); + } elseif (!isset($methodsCallback['getCommand'])) { + $methodsCallback['getCommand'] = function ( + string $commandName, + array $args + ) { + return new Command($commandName, $args); + }; + } + + if (isset($methodsCallback['executeAsync']) && !is_callable($methodsCallback['executeAsync'])) { + throw new InvalidArgumentException("getObject should be callable"); + } elseif (!isset($methodsCallback['executeAsync'])) { + $methodsCallback['executeAsync'] = function ($command) { + return match ($command->getName()) { + 'CreateMultipartUpload' => Create::promiseFor(new Result([ + 'UploadId' => 'FooUploadId', + ])), + 'UploadPart', + 'CompleteMultipartUpload', + 'AbortMultipartUpload', + 'PutObject' => Create::promiseFor(new Result([])), + default => null, + }; + }; + } + + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(array_keys($methodsCallback)) + ->getMock(); + foreach ($methodsCallback as $name => $callback) { + $client->method($name)->willReturnCallback($callback); + } return $client; }