DBSync Initial Implemetation #737
Conversation
| try { | ||
| exec(session, "mkdir -p /tmp/dbsync/"); | ||
|
|
||
| ProductMetadata productMetadata = new ProductMetadata(); |
There was a problem hiding this comment.
Use this to get client jar
dbsync/client/src/main/java/com/google/edwmigration/dbsync/client/SshManager.java
Outdated
Show resolved
Hide resolved
dbsync/client/src/main/java/com/google/edwmigration/dbsync/impl/DbsyncTargetImpl.java
Outdated
Show resolved
Hide resolved
| } | ||
| } | ||
|
|
||
| message OpenRequest { |
| message OpenRequest { | ||
| } | ||
|
|
||
| message OpenResponse { |
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| public class ClientMain { |
| getChecksumURI(stagingBucket, targetUri) | ||
| ); | ||
| ChecksumGenerator generator = new ChecksumGenerator(CHECKSUM_BLOCK_SIZE); | ||
| List<Checksum> checksums = new ArrayList<>(); |
There was a problem hiding this comment.
Should go direct to output stream
| generator.generate(checksums::add, source); | ||
| try (OutputStream checksumStream = checksumSink.openBufferedStream()) { | ||
| for (Checksum checksum : checksums) { | ||
| checksum.writeDelimitedTo(checksumStream); |
There was a problem hiding this comment.
Try without delimited
| ); | ||
|
|
||
| String receiveCommand = | ||
| String.format( |
There was a problem hiding this comment.
Alternative to string format
samdgupi
changed the title
samdgupi
requested review from
vladislav-sidorovich and
shevek-google
as code owners
| @@ -0,0 +1,10 @@ | |||
| plugins { | |||
| id 'idea' | |||
There was a problem hiding this comment.
Probably don't want to commit the idea plugin
| ); | ||
| } | ||
|
|
||
| public static void runRsyncJob(String projectId, String location, Mode mode) |
There was a problem hiding this comment.
Can split into server and client classes as discussed
| } | ||
| } | ||
|
|
||
| private static final String PRODUCT_GROUP = "com.google.edwmigration.dumper"; |
| @SuppressWarnings("UnusedVariable") | ||
| private static final Logger LOG = LoggerFactory.getLogger(DefaultArguments.class); | ||
|
|
||
| public static class BooleanValueConverter implements ValueConverter<Boolean> { |
There was a problem hiding this comment.
Probably don't need this since we aren't accepting any boolean options
|
|
||
| public class RsyncClient { | ||
|
|
||
| private static final int MIN_SIZE_TO_RSYNC = 1; |
There was a problem hiding this comment.
Update with valid min size - I think we said ~10KB but I could be wrong
cc @shevek-google
| throws IOException, URISyntaxException { | ||
| // deploy all cloudrun jobs | ||
| try { | ||
| CloudRunHelper.deployRsyncJobs(projectId, stagingBucket, targetUri); |
There was a problem hiding this comment.
So IIUC for each file you are deploying and running 2 Rsync jobs (generate + receive). Could we deploy both jobs first, and then run them per file instead? i.e re-use the CR deployment but change the flags per execution with stagingBucket, targetURI, etc.
I think this would involve using ExecutionsClient and supplying the CLI args for targetUri, stagingUri per execution.
dbsync/settings.gradle
Outdated
| id 'org.gradle.toolchains.foojay-resolver-convention' version '0.4.0' | ||
| } | ||
|
|
||
| rootProject.name = 'dbsync' |
There was a problem hiding this comment.
Move to top level settings.gradle as discussed
No description provided.