Skip to content

Commit 9bba233

Browse files
const-cloudinaryconst-cloudinary
committed
Fix upload large
1 parent 85ad336 commit 9bba233

File tree

3 files changed

+111
-29
lines changed

3 files changed

+111
-29
lines changed

‎cloudinary-core/src/main/scala/com/cloudinary/Responses.scala‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,12 @@ case class UploadResponse(public_id: String, url: String, secure_url: String, si
4747
def format:String = (raw \ "format").extractOpt[String].getOrElse(null)
4848
lazy val tags: Set[String] = (raw \ "tags").extractOpt[Array[String]].map(_.toSet).getOrElse(Set.empty)
4949
}
50-
case class LargeRawUploadResponse(public_id: String, url: String, secure_url: String, signature: String, bytes: Long,
51-
resource_type: String) extends AdvancedResponse with VersionedResponse with TimestampedResponse {
52-
override implicit val formats = DefaultFormats + new EnumNameSerializer(ModerationStatus)
53-
lazy val tags: Set[String] = (raw \ "tags").extractOpt[Array[String]].map(_.toSet).getOrElse(Set.empty)
54-
}
50+
case class LargeUploadResponse(public_id: String = "", url: String = "", secure_url: String = "", signature: String = "", bytes: Long = 0,
51+
resource_type: String = "", kind: String = "") extends LargeUploadResponseBase
52+
53+
// Keep LargeRawUploadResponse for backward compatibility
54+
case class LargeRawUploadResponse(public_id: String = "", url: String = "", secure_url: String = "", signature: String = "", bytes: Long = 0,
55+
resource_type: String = "", kind: String = "") extends LargeUploadResponseBase
5556
case class DestroyResponse(result: String) extends RawResponse
5657
case class ExplicitResponse(public_id: String, version: String, url: String, secure_url: String, signature: String, bytes: Long,
5758
format: String, eager: List[EagerInfo] = List(), `type`: String) extends RawResponse
@@ -239,6 +240,12 @@ trait AdvancedResponse extends RawResponse {
239240
lazy val pages:Int = (raw \ "pages").extractOpt[Int].getOrElse(1)
240241
}
241242

243+
trait LargeUploadResponseBase extends AdvancedResponse with VersionedResponse with TimestampedResponse {
244+
override implicit val formats = DefaultFormats + new EnumNameSerializer(ModerationStatus)
245+
lazy val tags: Set[String] = (raw \ "tags").extractOpt[Array[String]].map(_.toSet).getOrElse(Set.empty)
246+
lazy val done: Boolean = (raw \ "done").extractOpt[Boolean].getOrElse(true)
247+
}
248+
242249
class ImageAnalysis(raw: JsonAST.JValue) {
243250
implicit lazy val formats = DefaultFormats
244251

‎cloudinary-core/src/main/scala/com/cloudinary/Uploader.scala‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ class Uploader(implicit val cloudinary: Cloudinary) {
126126
}
127127

128128
def uploadLargeRaw(file: AnyRef, params: LargeUploadParameters = LargeUploadParameters(), chunkSize: Int = 20000000): Future[LargeRawUploadResponse] = {
129-
// Backwards compatibility wrapper - delegate to generic uploadLarge and convert response
129+
// Backwards compatibility wrapper - delegate to generic uploadLarge
130130
uploadLarge(file, params, "raw", chunkSize).map { uploadResponse =>
131131
val response = LargeRawUploadResponse(
132132
public_id = uploadResponse.public_id,
@@ -174,21 +174,41 @@ class Uploader(implicit val cloudinary: Cloudinary) {
174174
// HTTP headers should not be included in signature parameters
175175
val uploadParams = updatedParams.toMap
176176

177-
val responseFuture = callApiWithHeaders[UploadResponse]("upload", uploadParams, part, resourceType,
177+
val responseFuture = callApiWithHeaders[LargeUploadResponse]("upload", uploadParams, part, resourceType,
178178
Map("Content-Range" -> contentRange, "X-Unique-Upload-Id" -> uploadId), updatedParams.signed)
179179

180180
responseFuture.flatMap { response =>
181-
uploadResult = responseFuture
182-
// Update params with public_id from response for subsequent chunks
183-
updatedParams = updatedParams.publicId(response.public_id)
184-
185-
// Check if there's more data to upload
186-
if (input.available() > 0) {
187-
uploadNextChunk()
181+
if (!response.done) {
182+
// Intermediate response - continue with next chunk without updating public_id
183+
if (input.available() > 0) {
184+
uploadNextChunk()
185+
} else {
186+
// No more data but upload not done - this shouldn't happen
187+
throw new RuntimeException("No more data to upload but server says not done")
188+
}
188189
} else {
189-
// Close the stream when upload is complete
190-
try { input.close() } catch { case _: Exception => }
191-
Future.successful(response)
190+
// Final response - convert to UploadResponse and update params
191+
val uploadResponse = UploadResponse(
192+
public_id = response.public_id,
193+
url = response.url,
194+
secure_url = response.secure_url,
195+
signature = response.signature,
196+
bytes = response.bytes,
197+
resource_type = response.resource_type
198+
)
199+
uploadResponse.raw = response.raw
200+
val responseFuture = Future.successful(uploadResponse)
201+
uploadResult = responseFuture
202+
updatedParams = updatedParams.publicId(response.public_id)
203+
204+
// Check if there's more data to upload
205+
if (input.available() > 0) {
206+
uploadNextChunk()
207+
} else {
208+
// Close the stream when upload is complete
209+
try { input.close() } catch { case _: Exception => }
210+
Future.successful(uploadResponse)
211+
}
192212
}
193213
}
194214
}

‎cloudinary-core/src/test/scala/com/cloudinary/UploaderSpec.scala‎

Lines changed: 67 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,42 @@ class UploaderSpec extends MockableFlatSpec with Matchers with OptionValues with
2626
private val options = UploadParameters().tags(Set(prefix, testTag, uploadTag))
2727
private val uploader : Uploader = cloudinary.uploader()
2828

29+
// Test constants for large file uploads
30+
private val LargeFileSize = 5880138L
31+
private val LargeChunkSize = 5243000
32+
33+
// Helper function to create large test files in memory
34+
def createLargeBinaryFile(size: Long, chunkSize: Int = 4096): Array[Byte] = {
35+
val output = new java.io.ByteArrayOutputStream()
36+
37+
// BMP header for a valid binary file
38+
val header = Array[Byte](
39+
0x42, 0x4D, 0x4A, 0xB9.toByte, 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A.toByte, 0x00, 0x00, 0x00, 0x7C, 0x00,
40+
0x00, 0x00, 0x78, 0x05, 0x00, 0x00, 0x78, 0x05, 0x00, 0x00, 0x01, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00,
41+
0xC0.toByte, 0xB8.toByte, 0x59, 0x00, 0x61, 0x0F, 0x00, 0x00, 0x61, 0x0F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
42+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF.toByte, 0x00, 0x00, 0xFF.toByte, 0x00, 0x00, 0xFF.toByte, 0x00, 0x00,
43+
0x00, 0x00, 0x00, 0x00, 0xFF.toByte, 0x42, 0x47, 0x52, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
44+
0x54, 0xB8.toByte, 0x1E, 0xFC.toByte, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x66, 0x66, 0xFC.toByte,
45+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC4.toByte, 0xF5.toByte, 0x28, 0xFF.toByte, 0x00, 0x00, 0x00, 0x00,
46+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
47+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00
48+
)
49+
50+
output.write(header)
51+
var remainingSize = size - header.length
52+
53+
while (remainingSize > 0) {
54+
val currentChunkSize = Math.min(remainingSize, chunkSize).toInt
55+
val chunk = Array.fill[Byte](currentChunkSize)(0xFF.toByte)
56+
output.write(chunk)
57+
remainingSize -= currentChunkSize
58+
}
59+
60+
output.toByteArray
61+
}
62+
63+
64+
2965
private val api = cloudinary.api()
3066

3167
override def afterAll(): Unit = {
@@ -321,36 +357,55 @@ class UploaderSpec extends MockableFlatSpec with Matchers with OptionValues with
321357
}
322358

323359
it should "support uploading large raw files from Array[Byte]" in {
324-
val fileBytes = java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(s"$testResourcePath/docx.docx"))
360+
val fileBytes = createLargeBinaryFile(LargeFileSize)
325361
Await.result(for {
326-
response <- uploader.uploadLargeRaw(fileBytes, LargeUploadParameters().tags(Set("large_upload_bytes_test")))
362+
response <- uploader.uploadLargeRaw(fileBytes, LargeUploadParameters().tags(Set("large_upload_bytes_test")), LargeChunkSize)
327363
} yield {
328364
response.bytes should equal(fileBytes.length)
329365
response.tags should equal(Set("large_upload_bytes_test"))
330-
}, 10.seconds)
366+
}, 30.seconds)
331367
}
332368

333369
it should "support uploading large raw files from InputStream" in {
334-
val file = new java.io.File(s"$testResourcePath/docx.docx")
335-
val inputStream = new java.io.FileInputStream(file)
370+
val fileBytes = createLargeBinaryFile(LargeFileSize)
371+
val inputStream = new java.io.ByteArrayInputStream(fileBytes)
336372
Await.result(for {
337-
response <- uploader.uploadLargeRaw(inputStream, LargeUploadParameters().tags(Set("large_upload_stream_test")))
373+
response <- uploader.uploadLargeRaw(inputStream, LargeUploadParameters().tags(Set("large_upload_stream_test")), LargeChunkSize)
338374
} yield {
339-
response.bytes should equal(file.length())
375+
response.bytes should equal(LargeFileSize)
340376
response.tags should equal(Set("large_upload_stream_test"))
341-
}, 10.seconds)
377+
}, 30.seconds)
342378
}
343379

344-
it should "support uploading large raw files from URL" in {
380+
it should "support uploading large binary files" in {
381+
val largeBinaryData = createLargeBinaryFile(LargeFileSize)
382+
345383
Await.result(for {
346-
response <- uploader.uploadLargeRaw("http://cloudinary.com/images/logo.png", LargeUploadParameters().tags(Set("large_upload_url_test")))
384+
response <- uploader.uploadLarge(largeBinaryData, LargeUploadParameters().tags(Set("large_upload_binary_test")), "raw", LargeChunkSize)
347385
} yield {
348386
response.public_id should not be empty
349-
response.tags should equal(Set("large_upload_url_test"))
387+
response.tags should equal(Set("large_upload_binary_test"))
350388
response.resource_type should equal("raw")
351-
}, 10.seconds)
389+
response.bytes should equal(LargeFileSize)
390+
}, 60.seconds)
352391
}
353392

393+
it should "support uploading large image files" in {
394+
val largeImageData = createLargeBinaryFile(LargeFileSize) // BMP is a valid image format
395+
396+
Await.result(for {
397+
response <- uploader.uploadLarge(largeImageData, LargeUploadParameters().tags(Set("large_upload_image_test")), "image", LargeChunkSize)
398+
} yield {
399+
response.public_id should not be empty
400+
response.tags should equal(Set("large_upload_image_test"))
401+
response.resource_type should equal("image")
402+
response.bytes should equal(LargeFileSize)
403+
response.width should be > 0
404+
response.height should be > 0
405+
}, 60.seconds)
406+
}
407+
408+
354409
it should "support uploading large files with different resource types using uploadLarge" in {
355410
Await.result(for {
356411
// Test image upload

0 commit comments

Comments
 (0)