Skip to content

Commit c02b715

Browse files
Torch3333Torch3333
committed
For Oracle, change the data type for BLOB column to support storing up to 2GB
1 parent 6ad930f commit c02b715

26 files changed

+800
-138
lines changed

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminIntegrationTestWithJdbcDatabase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,6 @@ public void alterColumnType_Sqlite_AlterColumnType_ShouldThrowUnsupportedOperati
418418

419419
@Override
420420
protected boolean isIndexOnBlobColumnSupported() {
421-
return !JdbcTestUtils.isDb2(rdbEngine);
421+
return !(JdbcTestUtils.isDb2(rdbEngine) || JdbcTestUtils.isOracle(rdbEngine));
422422
}
423423
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminCaseSensitivityIntegrationTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,6 @@ public void alterColumnType_Sqlite_AlterColumnType_ShouldThrowUnsupportedOperati
498498

499499
@Override
500500
protected boolean isIndexOnBlobColumnSupported() {
501-
return !JdbcTestUtils.isDb2(rdbEngine);
501+
return !(JdbcTestUtils.isDb2(rdbEngine) || JdbcTestUtils.isOracle(rdbEngine));
502502
}
503503
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTestUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -482,7 +482,7 @@ private LinkedHashMap<String, String> prepareColumnsForDb2() {
482482
columns.put("col17", "NCLOB(512)");
483483
columns.put("col18", "BINARY(5)");
484484
columns.put("col19", "VARBINARY(512)");
485-
columns.put("col20", "BLOB(1024)");
485+
columns.put("col20", "BLOB(2G)");
486486
columns.put("col21", "CHAR(5) FOR BIT DATA");
487487
columns.put("col22", "VARCHAR(512) FOR BIT DATA");
488488
columns.put("col23", "DATE");

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminIntegrationTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,6 @@ public void alterColumnType_Sqlite_AlterColumnType_ShouldThrowUnsupportedOperati
498498

499499
@Override
500500
protected boolean isIndexOnBlobColumnSupported() {
501-
return !JdbcTestUtils.isDb2(rdbEngine);
501+
return !(JdbcTestUtils.isDb2(rdbEngine) || JdbcTestUtils.isOracle(rdbEngine));
502502
}
503503
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseColumnValueIntegrationTest.java‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,41 @@
11
package com.scalar.db.storage.jdbc;
22

3+
import static org.assertj.core.api.Assertions.assertThat;
4+
35
import com.scalar.db.api.DistributedStorageColumnValueIntegrationTestBase;
6+
import com.scalar.db.api.Get;
7+
import com.scalar.db.api.Put;
8+
import com.scalar.db.api.PutBuilder;
9+
import com.scalar.db.api.Result;
10+
import com.scalar.db.api.TableMetadata;
411
import com.scalar.db.config.DatabaseConfig;
12+
import com.scalar.db.exception.storage.ExecutionException;
13+
import com.scalar.db.io.BigIntColumn;
14+
import com.scalar.db.io.BlobColumn;
15+
import com.scalar.db.io.BooleanColumn;
516
import com.scalar.db.io.Column;
617
import com.scalar.db.io.DataType;
18+
import com.scalar.db.io.DateColumn;
19+
import com.scalar.db.io.DoubleColumn;
20+
import com.scalar.db.io.FloatColumn;
21+
import com.scalar.db.io.IntColumn;
22+
import com.scalar.db.io.Key;
23+
import com.scalar.db.io.TextColumn;
24+
import com.scalar.db.io.TimeColumn;
25+
import com.scalar.db.io.TimestampColumn;
26+
import com.scalar.db.io.TimestampTZColumn;
727
import com.scalar.db.util.TestUtils;
28+
import java.util.ArrayList;
29+
import java.util.List;
30+
import java.util.Optional;
831
import java.util.Properties;
932
import java.util.Random;
33+
import java.util.stream.Stream;
34+
import org.junit.jupiter.api.Test;
35+
import org.junit.jupiter.api.condition.EnabledIf;
36+
import org.junit.jupiter.params.ParameterizedTest;
37+
import org.junit.jupiter.params.provider.Arguments;
38+
import org.junit.jupiter.params.provider.MethodSource;
1039

1140
public class JdbcDatabaseColumnValueIntegrationTest
1241
extends DistributedStorageColumnValueIntegrationTestBase {
@@ -68,4 +97,183 @@ protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType)
6897
}
6998
return super.getColumnWithMaxValue(columnName, dataType);
7099
}
100+
101+
@EnabledIf("isDb2OrOracle")
102+
@ParameterizedTest()
103+
@MethodSource("provideBlobSizes")
104+
public void put_largeBlobData_ShouldWorkCorrectly(int blobSize, String humanReadableBlobSize)
105+
throws ExecutionException {
106+
String tableName = TABLE + "_large_single_blob_single";
107+
try {
108+
// Arrange
109+
TableMetadata.Builder metadata =
110+
TableMetadata.newBuilder()
111+
.addColumn(COL_NAME1, DataType.INT)
112+
.addColumn(COL_NAME2, DataType.BLOB)
113+
.addPartitionKey(COL_NAME1);
114+
115+
admin.createTable(namespace, tableName, metadata.build(), true, getCreationOptions());
116+
admin.truncateTable(namespace, tableName);
117+
byte[] blobData = createLargeBlob(blobSize);
118+
Put put =
119+
Put.newBuilder()
120+
.namespace(namespace)
121+
.table(tableName)
122+
.partitionKey(Key.ofInt(COL_NAME1, 1))
123+
.blobValue(COL_NAME2, blobData)
124+
.build();
125+
126+
// Act
127+
storage.put(put);
128+
129+
// Assert
130+
Optional<Result> optionalResult =
131+
storage.get(
132+
Get.newBuilder()
133+
.namespace(namespace)
134+
.table(tableName)
135+
.partitionKey(Key.ofInt(COL_NAME1, 1))
136+
.build());
137+
assertThat(optionalResult).isPresent();
138+
Result result = optionalResult.get();
139+
assertThat(result.getColumns().get(COL_NAME2).getBlobValueAsBytes()).isEqualTo(blobData);
140+
} finally {
141+
admin.dropTable(namespace, tableName, true);
142+
}
143+
}
144+
145+
Stream<Arguments> provideBlobSizes() {
146+
List<Arguments> args = new ArrayList<>();
147+
if (isOracle()) {
148+
// As explained in
149+
// `com.scalar.db.storage.jdbc.RdbEngineOracle.bindBlobColumnToPreparedStatement()`,
150+
// handing a BLOB size bigger than 32,767 bytes requires a workaround so we particularly test
151+
// values around it.
152+
args.add(Arguments.of(32_766, "32.766 KB"));
153+
args.add(Arguments.of(32_767, "32.767 KB"));
154+
}
155+
args.add(Arguments.of(100_000_000, "100 MB"));
156+
return args.stream();
157+
}
158+
159+
@EnabledIf("isOracle")
160+
@Test
161+
public void put_largeBlobData_WithMultipleBlobColumnsShouldWorkCorrectly()
162+
throws ExecutionException {
163+
String tableName = TABLE + "_large_multiples_blob";
164+
try {
165+
// Arrange
166+
TableMetadata.Builder metadata =
167+
TableMetadata.newBuilder()
168+
.addColumn(COL_NAME1, DataType.INT)
169+
.addColumn(COL_NAME2, DataType.BLOB)
170+
.addColumn(COL_NAME3, DataType.BLOB)
171+
.addPartitionKey(COL_NAME1);
172+
173+
admin.createTable(namespace, tableName, metadata.build(), true, getCreationOptions());
174+
admin.truncateTable(namespace, tableName);
175+
byte[] blobDataCol2 = createLargeBlob(32_766);
176+
byte[] blobDataCol3 = createLargeBlob(5000);
177+
Put put =
178+
Put.newBuilder()
179+
.namespace(namespace)
180+
.table(tableName)
181+
.partitionKey(Key.ofInt(COL_NAME1, 1))
182+
.blobValue(COL_NAME2, blobDataCol2)
183+
.blobValue(COL_NAME3, blobDataCol3)
184+
.build();
185+
186+
// Act
187+
storage.put(put);
188+
189+
// Assert
190+
Optional<Result> optionalResult =
191+
storage.get(
192+
Get.newBuilder()
193+
.namespace(namespace)
194+
.table(tableName)
195+
.partitionKey(Key.ofInt(COL_NAME1, 1))
196+
.build());
197+
assertThat(optionalResult).isPresent();
198+
Result result = optionalResult.get();
199+
assertThat(result.getColumns().get(COL_NAME2).getBlobValueAsBytes()).isEqualTo(blobDataCol2);
200+
assertThat(result.getColumns().get(COL_NAME3).getBlobValueAsBytes()).isEqualTo(blobDataCol3);
201+
} finally {
202+
admin.dropTable(namespace, tableName, true);
203+
}
204+
}
205+
206+
@EnabledIf("isOracle")
207+
@Test
208+
public void put_largeBlobData_WithAllColumnsTypesShouldWorkCorrectly() throws ExecutionException {
209+
// Arrange
210+
IntColumn partitionKeyValue = (IntColumn) getColumnWithMaxValue(PARTITION_KEY, DataType.INT);
211+
BooleanColumn col1Value = (BooleanColumn) getColumnWithMaxValue(COL_NAME1, DataType.BOOLEAN);
212+
IntColumn col2Value = (IntColumn) getColumnWithMaxValue(COL_NAME2, DataType.INT);
213+
BigIntColumn col3Value = (BigIntColumn) getColumnWithMaxValue(COL_NAME3, DataType.BIGINT);
214+
FloatColumn col4Value = (FloatColumn) getColumnWithMaxValue(COL_NAME4, DataType.FLOAT);
215+
DoubleColumn col5Value = (DoubleColumn) getColumnWithMaxValue(COL_NAME5, DataType.DOUBLE);
216+
TextColumn col6Value = (TextColumn) getColumnWithMaxValue(COL_NAME6, DataType.TEXT);
217+
BlobColumn col7Value = BlobColumn.of(COL_NAME7, createLargeBlob(32_766));
218+
DateColumn col8Value = (DateColumn) getColumnWithMaxValue(COL_NAME8, DataType.DATE);
219+
TimeColumn col9Value = (TimeColumn) getColumnWithMaxValue(COL_NAME9, DataType.TIME);
220+
TimestampTZColumn col10Value =
221+
(TimestampTZColumn) getColumnWithMaxValue(COL_NAME10, DataType.TIMESTAMPTZ);
222+
TimestampColumn column11Value = null;
223+
if (isTimestampTypeSupported()) {
224+
column11Value = (TimestampColumn) getColumnWithMaxValue(COL_NAME11, DataType.TIMESTAMP);
225+
}
226+
227+
PutBuilder.Buildable put =
228+
Put.newBuilder()
229+
.namespace(namespace)
230+
.table(TABLE)
231+
.partitionKey(Key.newBuilder().add(partitionKeyValue).build())
232+
.value(col1Value)
233+
.value(col2Value)
234+
.value(col3Value)
235+
.value(col4Value)
236+
.value(col5Value)
237+
.value(col6Value)
238+
.value(col7Value)
239+
.value(col8Value)
240+
.value(col9Value)
241+
.value(col10Value);
242+
if (isTimestampTypeSupported()) {
243+
put.value(column11Value);
244+
}
245+
// Act
246+
storage.put(put.build());
247+
248+
// Assert
249+
assertResult(
250+
partitionKeyValue,
251+
col1Value,
252+
col2Value,
253+
col3Value,
254+
col4Value,
255+
col5Value,
256+
col6Value,
257+
col7Value,
258+
col8Value,
259+
col9Value,
260+
col10Value,
261+
column11Value);
262+
}
263+
264+
private byte[] createLargeBlob(int size) {
265+
byte[] blob = new byte[size];
266+
random.nextBytes(blob);
267+
return blob;
268+
}
269+
270+
@SuppressWarnings("unused")
271+
private boolean isDb2OrOracle() {
272+
return JdbcEnv.isOracle() || JdbcEnv.isDb2();
273+
}
274+
275+
@SuppressWarnings("unused")
276+
private boolean isOracle() {
277+
return JdbcEnv.isOracle();
278+
}
71279
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseConditionalMutationIntegrationTest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,9 @@ protected Column<?> getColumnWithRandomValue(
4242
}
4343
return super.getColumnWithRandomValue(random, columnName, dataType);
4444
}
45+
46+
@Override
47+
protected boolean isConditionOnBlobColumnSupported() {
48+
return !JdbcTestUtils.isOracle(rdbEngine);
49+
}
4550
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseCrossPartitionScanIntegrationTest.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,11 @@ protected Stream<Arguments> provideColumnsForCNFConditionsTest() {
8383

8484
@Override
8585
protected boolean isOrderingOnBlobColumnSupported() {
86-
return !JdbcTestUtils.isDb2(rdbEngine);
86+
return !(JdbcTestUtils.isDb2(rdbEngine) || JdbcTestUtils.isOracle(rdbEngine));
87+
}
88+
89+
@Override
90+
protected boolean isConditionOnBlobColumnSupported() {
91+
return !JdbcTestUtils.isOracle(rdbEngine);
8792
}
8893
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseMultipleClusteringKeyScanIntegrationTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ protected List<DataType> getDataTypes() {
100100
rdbEngine,
101101
ImmutableMap.of(
102102
RdbEngineOracle.class,
103-
ImmutableList.of(DataType.TIMESTAMPTZ),
103+
ImmutableList.of(DataType.TIMESTAMPTZ, DataType.BLOB),
104104
RdbEngineDb2.class,
105105
ImmutableList.of(DataType.BLOB)));
106106
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseMultiplePartitionKeyIntegrationTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ protected List<DataType> getDataTypes() {
9494
rdbEngine,
9595
ImmutableMap.of(
9696
RdbEngineOracle.class,
97-
ImmutableList.of(DataType.TIMESTAMPTZ),
97+
ImmutableList.of(DataType.TIMESTAMPTZ, DataType.BLOB),
9898
RdbEngineYugabyte.class,
9999
ImmutableList.of(DataType.FLOAT, DataType.DOUBLE),
100100
RdbEngineDb2.class,

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseSecondaryIndexIntegrationTest.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,10 @@ protected Set<DataType> getSecondaryIndexTypes() {
8181
JdbcTestUtils.filterDataTypes(
8282
Arrays.asList(DataType.values()),
8383
rdbEngine,
84-
ImmutableMap.of(RdbEngineDb2.class, ImmutableList.of(DataType.BLOB))));
84+
ImmutableMap.of(
85+
RdbEngineDb2.class,
86+
ImmutableList.of(DataType.BLOB),
87+
RdbEngineOracle.class,
88+
ImmutableList.of(DataType.BLOB))));
8589
}
8690
}

0 commit comments

Comments
 (0)