Skip to content
This repository was archived by the owner on Jun 18, 2020. It is now read-only.

Support on demand tables #17

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
target/
Original file line number Diff line number Diff line change
Expand Up @@ -62,16 +62,32 @@ public String getDestinationTable() {
return destinationTable;
}

public static final String READ_CAPACITY_UNITS = "--onDemandReadCapacityUnits";
@Parameter(names = READ_CAPACITY_UNITS, description = "The number of read capacity units used to scan the source table (only used if table is `on demand`; default = 50.0)", required = false)
private double onDemandReadCapacityUnits = 50.0;

public double getOnDemandReadCapacityUnits() {
return onDemandReadCapacityUnits;
}

public static final String WRITE_CAPACITY_UNITS = "--onDemandWriteCapacityUnits";
@Parameter(names = WRITE_CAPACITY_UNITS, description = "The number of write capacity units used to write the destination table (only used if table is `on demand`; default = 50.0)", required = false)
private double onDemandWriteCapacityUnits = 50.0;

public double getOnDemandWriteCapacityUnits() {
return onDemandWriteCapacityUnits;
}

public static final String READ_THROUGHPUT_RATIO = "--readThroughputRatio";
@Parameter(names = READ_THROUGHPUT_RATIO, description = "Percentage of total read throughput to scan the source table", required = true)
@Parameter(names = READ_THROUGHPUT_RATIO, description = "Percentage of total read throughput to scan the source table (> 0.0, <= 1.0)", required = true)
private double readThroughputRatio;

public double getReadThroughputRatio() {
return readThroughputRatio;
}

public static final String WRITE_THROUGHPUT_RATIO = "--writeThroughputRatio";
@Parameter(names = WRITE_THROUGHPUT_RATIO, description = "Percentage of total write throughput to write the destination table", required = true)
@Parameter(names = WRITE_THROUGHPUT_RATIO, description = "Percentage of total write throughput to write the destination table (> 0.0, <= 1.0)", required = true)
private double writeThroughputRatio;

public double getWriteThroughputRatio() {
Expand Down Expand Up @@ -101,7 +117,7 @@ public int getTotalSections() {
public int getSection() {
return section;
}

public static final String CONSISTENT_SCAN = "--consistentScan";
@Parameter(names = CONSISTENT_SCAN, description = "Use this flag to use strongly consistent scan. If the flag is not used it will default to eventually consistent scan")
private boolean consistentScan = false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public class CommandLineInterface {
/**
* Main class to begin transferring data from one DynamoDB table to another
* DynamoDB table.
*
*
* @param args
*/
public static void main(String[] args) {
Expand All @@ -74,6 +74,8 @@ public static void main(String[] args) {
final String destinationEndpoint = params.getDestinationEndpoint();
final String destinationTable = params.getDestinationTable();
final String sourceTable = params.getSourceTable();
final double onDemandReadCapacityUnits = params.getOnDemandReadCapacityUnits();
final double onDemandWriteCapacityUnits = params.getOnDemandWriteCapacityUnits();
final double readThroughputRatio = params.getReadThroughputRatio();
final double writeThroughputRatio = params.getWriteThroughputRatio();
final int maxWriteThreads = params.getMaxWriteThreads();
Expand Down Expand Up @@ -103,9 +105,9 @@ public static void main(String[] args) {
}

final double readThroughput = calculateThroughput(readTableDescription,
readThroughputRatio, true);
final double writeThroughput = calculateThroughput(
writeTableDescription, writeThroughputRatio, false);
onDemandReadCapacityUnits, readThroughputRatio, true);
final double writeThroughput = calculateThroughput(writeTableDescription,
onDemandWriteCapacityUnits, writeThroughputRatio, false);

try {
ExecutorService sourceExec = getSourceThreadPool(numSegments);
Expand Down Expand Up @@ -135,14 +137,24 @@ public static void main(String[] args) {
* specified DynamoDB table provisioned throughput.
*/
private static double calculateThroughput(
TableDescription tableDescription, double throughputRatio,
boolean read) {
TableDescription tableDescription, double onDemandCapacityUnits,
double throughputRatio, boolean read) {

double capacityUnits = 0.0;

if (read) {
return tableDescription.getProvisionedThroughput()
.getReadCapacityUnits() * throughputRatio;
capacityUnits = tableDescription.getProvisionedThroughput()
.getReadCapacityUnits();
} else {
capacityUnits = tableDescription.getProvisionedThroughput()
.getWriteCapacityUnits();
}
return tableDescription.getProvisionedThroughput()
.getWriteCapacityUnits() * throughputRatio;

if (capacityUnits < 1.0) {
capacityUnits = onDemandCapacityUnits;
}

return capacityUnits * throughputRatio;
}

/**
Expand Down