Skip to content

Commit 860e6bf

Browse files
iroquetairoqueta
committed
The method createNewFile converts all the inputstream to memory. This could cause out of memory errors.
Now the process use a buffer to avoid this. Issue: 205339
1 parent 4c656db commit 860e6bf

File tree

6 files changed

+102
-31
lines changed

6 files changed

+102
-31
lines changed

‎gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
import com.amazonaws.services.s3.AmazonS3;
1919
import com.amazonaws.services.s3.AmazonS3Client;
20-
import com.amazonaws.util.IOUtils;
2120

2221
import java.io.FileNotFoundException;
2322
import java.io.FileOutputStream;
@@ -220,23 +219,31 @@ else if (acl == ResourceAccessControlList.PublicReadWrite) {
220219
}
221220

222221
public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) {
223-
byte[] bytes;
222+
ExternalProviderHelper.InputStreamWithLength streamInfo = null;
224223
try {
225-
bytes = IOUtils.toByteArray(input);
224+
streamInfo = ExternalProviderHelper.getInputStreamContentLength(input);
225+
226226
ObjectMetadata metadata = new ObjectMetadata();
227-
metadata.setContentLength(bytes.length);
227+
metadata.setContentLength(streamInfo.contentLength);
228+
228229
if (externalFileName.endsWith(".tmp")) {
229230
metadata.setContentType("image/jpeg");
230231
}
231232
String upload = "";
232-
try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes)) {
233-
client.putObject(new PutObjectRequest(bucket, externalFileName, byteArrayInputStream, metadata).withCannedAcl(internalToAWSACL(acl)));
234-
upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes);
235-
}
233+
client.putObject(new PutObjectRequest(bucket, externalFileName, streamInfo.inputStream, metadata).withCannedAcl(internalToAWSACL(acl)));
234+
upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes);
236235
return upload;
237236
} catch (IOException ex) {
238237
logger.error("Error while uploading file to the external provider.", ex);
239238
return "";
239+
} finally {
240+
if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) {
241+
try {
242+
streamInfo.tempFile.delete();
243+
} catch (Exception e) {
244+
logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e);
245+
}
246+
}
240247
}
241248
}
242249

‎gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,11 @@
2121
import com.genexus.util.StorageUtils;
2222
import com.genexus.StructSdtMessages_Message;
2323
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
24-
import software.amazon.awssdk.utils.IoUtils;
2524

2625
import java.io.*;
2726
import java.net.URI;
2827
import java.net.URL;
2928
import java.net.URLEncoder;
30-
import java.nio.ByteBuffer;
3129
import java.nio.charset.StandardCharsets;
3230
import java.nio.file.Files;
3331
import java.nio.file.Path;
@@ -603,8 +601,10 @@ else if (acl == ResourceAccessControlList.PublicReadWrite)
603601
}
604602

605603
private String uploadWithACL(String externalFileName, InputStream input, ResourceAccessControlList acl) {
604+
ExternalProviderHelper.InputStreamWithLength streamInfo = null;
606605
try {
607-
ByteBuffer byteBuffer = ByteBuffer.wrap(IoUtils.toByteArray(input));
606+
streamInfo = ExternalProviderHelper.getInputStreamContentLength(input);
607+
608608
PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder()
609609
.bucket(bucket)
610610
.key(externalFileName)
@@ -613,7 +613,7 @@ private String uploadWithACL(String externalFileName, InputStream input, Resourc
613613
putObjectRequestBuilder = putObjectRequestBuilder.acl(internalToAWSACLWithACL(acl));
614614
PutObjectRequest putObjectRequest = putObjectRequestBuilder.build();
615615

616-
PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromByteBuffer(byteBuffer));
616+
PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromInputStream(streamInfo.inputStream, streamInfo.contentLength));
617617
if (!response.sdkHttpResponse().isSuccessful()) {
618618
logger.error("Error while uploading file: " + response.sdkHttpResponse().statusText().orElse("Unknown error"));
619619
}
@@ -622,6 +622,15 @@ private String uploadWithACL(String externalFileName, InputStream input, Resourc
622622
} catch (IOException ex) {
623623
logger.error("Error while uploading file to the external provider.", ex);
624624
return "";
625+
} finally {
626+
// Clean up the temporary file if it was created
627+
if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) {
628+
try {
629+
streamInfo.tempFile.delete();
630+
} catch (Exception e) {
631+
logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e);
632+
}
633+
}
625634
}
626635
}
627636

@@ -727,15 +736,17 @@ private String uploadWithoutACL(String localFile, String externalFileName) {
727736
}
728737

729738
private String uploadWithoutACL(String externalFileName, InputStream input) {
739+
ExternalProviderHelper.InputStreamWithLength streamInfo = null;
730740
try {
731-
ByteBuffer byteBuffer = ByteBuffer.wrap(IoUtils.toByteArray(input));
741+
streamInfo = ExternalProviderHelper.getInputStreamContentLength(input);
742+
732743
PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder()
733744
.bucket(bucket)
734745
.key(externalFileName)
735746
.contentType(externalFileName.endsWith(".tmp") ? "image/jpeg" : null);
736747
PutObjectRequest putObjectRequest = putObjectRequestBuilder.build();
737748

738-
PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromByteBuffer(byteBuffer));
749+
PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromInputStream(streamInfo.inputStream, streamInfo.contentLength));
739750
if (!response.sdkHttpResponse().isSuccessful()) {
740751
logger.error("Error while uploading file: " + response.sdkHttpResponse().statusText().orElse("Unknown error"));
741752
}
@@ -744,6 +755,15 @@ private String uploadWithoutACL(String externalFileName, InputStream input) {
744755
} catch (IOException ex) {
745756
logger.error("Error while uploading file to the external provider.", ex);
746757
return "";
758+
} finally {
759+
// Clean up the temporary file if it was created
760+
if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) {
761+
try {
762+
streamInfo.tempFile.delete();
763+
} catch (Exception e) {
764+
logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e);
765+
}
766+
}
747767
}
748768
}
749769

‎gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -146,10 +146,10 @@ public String upload(String externalFileName, InputStream input, ResourceAccessC
146146
blob.getProperties().setContentType("image/jpeg");
147147
}
148148
try (BlobOutputStream blobOutputStream = blob.openOutputStream()) {
149-
int next = input.read();
150-
while (next != -1) {
151-
blobOutputStream.write(next);
152-
next = input.read();
149+
byte[] buffer = new byte[8192];
150+
int bytesRead;
151+
while ((bytesRead = input.read(buffer)) != -1) {
152+
blobOutputStream.write(buffer, 0, bytesRead);
153153
}
154154
}
155155
return getResourceUrl(externalFileName, acl, DEFAULT_EXPIRATION_MINUTES);

‎gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
import com.genexus.util.Encryption;
44
import com.genexus.util.GXService;
55

6+
import java.io.*;
7+
68
public class ExternalProviderHelper {
79

810
public static String getServicePropertyValue(GXService s, String propName, boolean isSecure){
@@ -23,4 +25,30 @@ public static String getEnvironmentVariable(String name, boolean required) throw
2325
}
2426
return value;
2527
}
28+
29+
public static InputStreamWithLength getInputStreamContentLength(InputStream input) throws IOException {
30+
File tempFile = File.createTempFile("upload-", ".tmp");
31+
try (OutputStream out = new FileOutputStream(tempFile)) {
32+
byte[] buffer = new byte[8192];
33+
int bytesRead;
34+
while ((bytesRead = input.read(buffer)) != -1) {
35+
out.write(buffer, 0, bytesRead);
36+
}
37+
}
38+
long size = tempFile.length();
39+
InputStream newInput = new FileInputStream(tempFile);
40+
return new InputStreamWithLength(newInput, size, tempFile);
41+
}
42+
43+
public static class InputStreamWithLength {
44+
public final InputStream inputStream;
45+
public final long contentLength;
46+
public final File tempFile; // nullable
47+
48+
public InputStreamWithLength(InputStream inputStream, long contentLength, File tempFile) {
49+
this.inputStream = inputStream;
50+
this.contentLength = contentLength;
51+
this.tempFile = tempFile;
52+
}
53+
}
2654
}

‎gxcloudstorage-googlecloudstorage/src/main/java/com/genexus/db/driver/ExternalProviderGoogle.java

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
import com.google.api.services.storage.model.StorageObject;
1717
import com.google.auth.oauth2.ServiceAccountCredentials;
1818
import com.google.cloud.storage.*;
19-
import org.apache.commons.io.IOUtils;
2019
import org.apache.logging.log4j.LogManager;
2120
import org.apache.logging.log4j.Logger;
2221

@@ -170,16 +169,28 @@ private void setBlobAcl(BlobId blobId, ResourceAccessControlList acl) {
170169
}
171170

172171
public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) {
172+
ExternalProviderHelper.InputStreamWithLength streamInfo = null;
173173
try {
174+
streamInfo = ExternalProviderHelper.getInputStreamContentLength(input);
175+
174176
BlobId blobId = BlobId.of(bucket, externalFileName);
175177
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
176-
byte[] targetArray = IOUtils.toByteArray(input);
177-
storageClient.create(blobInfo, targetArray);
178+
179+
storageClient.createFrom(blobInfo, streamInfo.tempFile.toPath());
180+
178181
setBlobAcl(blobId, acl);
179182
return getResourceUrl(blobInfo, acl);
180183
} catch (IOException ex) {
181184
handleIOException(ex);
182185
return "";
186+
} finally {
187+
if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) {
188+
try {
189+
streamInfo.tempFile.delete();
190+
} catch (Exception e) {
191+
logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e);
192+
}
193+
}
183194
}
184195
}
185196

‎gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import com.ibm.cloud.objectstorage.services.s3.AmazonS3Client;
1515
import com.ibm.cloud.objectstorage.services.s3.AmazonS3ClientBuilder;
1616
import com.ibm.cloud.objectstorage.services.s3.model.*;
17-
import com.ibm.cloud.objectstorage.util.IOUtils;
1817
import org.apache.logging.log4j.LogManager;
1918
import org.apache.logging.log4j.Logger;
2019

@@ -150,24 +149,30 @@ else if (acl == ResourceAccessControlList.PublicReadWrite) {
150149
}
151150

152151
public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) {
153-
byte[] bytes;
154-
try {
155-
bytes = IOUtils.toByteArray(input);
152+
ExternalProviderHelper.InputStreamWithLength streamInfo = null;
153+
try {
154+
streamInfo = ExternalProviderHelper.getInputStreamContentLength(input);
156155
ObjectMetadata metadata = new ObjectMetadata();
157-
metadata.setContentLength(bytes.length);
156+
metadata.setContentLength(streamInfo.contentLength);
158157
if (externalFileName.endsWith(".tmp")) {
159158
metadata.setContentType("image/jpeg");
160159
}
161160
String upload = "";
162-
try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes)) {
163-
client.putObject(new PutObjectRequest(bucket, externalFileName, byteArrayInputStream, metadata).withCannedAcl(internalToAWSACL(acl)));
164-
upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes);
165-
}
161+
client.putObject(new PutObjectRequest(bucket, externalFileName, streamInfo.inputStream, metadata).withCannedAcl(internalToAWSACL(acl)));
162+
upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes);
166163
return upload;
167164
} catch (IOException ex) {
168165
logger.error("Error while uploading file to the external provider.", ex);
169166
return "";
170-
}
167+
} finally {
168+
if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) {
169+
try {
170+
streamInfo.tempFile.delete();
171+
} catch (Exception e) {
172+
logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e);
173+
}
174+
}
175+
}
171176
}
172177

173178
public String get(String externalFileName, ResourceAccessControlList acl, int expirationMinutes) {

0 commit comments

Comments
 (0)