From 533c7b6cd0ddbc925f43f1ea4a9b75b071802ac9 Mon Sep 17 00:00:00 2001 From: Zack Shoylev Date: Wed, 7 Sep 2016 15:28:10 -0500 Subject: [PATCH] Updates to the parallel examples, using the new code. --- rackspace/pom.xml | 17 ++- .../cloudfiles/UploadLargeObject.java | 110 +++++++++++++++++- 2 files changed, 121 insertions(+), 6 deletions(-) diff --git a/rackspace/pom.xml b/rackspace/pom.xml index e76fdba2..6cb84089 100644 --- a/rackspace/pom.xml +++ b/rackspace/pom.xml @@ -26,13 +26,28 @@ rackspace-examples - 1.9.1 + 2.0.0-SNAPSHOT + + apache-snapshots https://repository.apache.org/content/repositories/snapshots + + false + + + true + + + + sonatype-nexus-snapshots + https://oss.sonatype.org/content/repositories/snapshots + + false + true diff --git a/rackspace/src/main/java/org/jclouds/examples/rackspace/cloudfiles/UploadLargeObject.java b/rackspace/src/main/java/org/jclouds/examples/rackspace/cloudfiles/UploadLargeObject.java index 3b68f809..04079755 100644 --- a/rackspace/src/main/java/org/jclouds/examples/rackspace/cloudfiles/UploadLargeObject.java +++ b/rackspace/src/main/java/org/jclouds/examples/rackspace/cloudfiles/UploadLargeObject.java @@ -26,8 +26,15 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; import java.util.Properties; +import java.util.Random; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.jclouds.ContextBuilder; import org.jclouds.blobstore.BlobStore; @@ -36,6 +43,7 @@ import org.jclouds.io.Payloads; import org.jclouds.openstack.swift.v1.blobstore.RegionScopedBlobStoreContext; +import com.google.common.hash.Hashing; import com.google.common.io.ByteSource; import com.google.common.io.Closeables; import com.google.common.io.Files; @@ -51,34 +59,45 @@ public class UploadLargeObject implements Closeable { * * The first argument (args[0]) must be your username * The second argument (args[1]) must be your API key - * The third argument (args[2]) must be the absolute path to a large file */ public static void main(String[] args) throws IOException { UploadLargeObject createContainer = new UploadLargeObject(args[0], args[1]); + File largeFile = new File("largefile.dat"); + File downloadedFile = new File(largeFile.getName()+".downloaded"); try { - createContainer.uploadLargeObjectFromFile(new File(args[2])); + // Create a 200MB file for this example + createContainer.createRandomFile(200000000, largeFile); + createContainer.uploadLargeObjectFromFile(largeFile); + createContainer.downloadLargeObjectToFile(largeFile.getName()); + System.out.println("Random file hash: " + Files.hash(largeFile, Hashing.md5())); + System.out.println("Downloaded file hash: " + Files.hash(downloadedFile, Hashing.md5())); } catch (Exception e) { e.printStackTrace(); } finally { + createContainer.cleanup(); createContainer.close(); + if(largeFile.exists()) largeFile.delete(); + if(downloadedFile.exists()) downloadedFile.delete(); } } public UploadLargeObject(String username, String apiKey) { Properties overrides = new Properties(); // This property controls the number of parts being uploaded in parallel, the default is 4 - overrides.setProperty("jclouds.mpu.parallel.degree", "5"); + overrides.setProperty("jclouds.user-threads", "14"); // This property controls the size (in bytes) of parts being uploaded in parallel, the default is 33554432 bytes = 32 MB - overrides.setProperty("jclouds.mpu.parts.size", "67108864"); // 64 MB + overrides.setProperty("jclouds.mpu.parts.size", "1100000"); // 1 MB RegionScopedBlobStoreContext context = ContextBuilder.newBuilder(PROVIDER) .credentials(username, apiKey) .overrides(overrides) .buildView(RegionScopedBlobStoreContext.class); blobStore = context.getBlobStore(REGION); + + blobStore.createContainerInLocation(null, CONTAINER); } /** @@ -102,7 +121,24 @@ private void uploadLargeObjectFromFile(File largeFile) throws InterruptedExcepti // configure the blobstore to use multipart uploading of the file String eTag = blobStore.putBlob(CONTAINER, blob, multipart()); - System.out.format(" Uploaded %s eTag=%s", largeFile.getName(), eTag); + System.out.format(" Uploaded %s eTag=%s to %s in %s%n", largeFile.getName(), eTag, REGION, CONTAINER); + } + + /** + * Download a large object from a File using the BlobStore API. + * + * @throws ExecutionException + * @throws InterruptedException + */ + private void downloadLargeObjectToFile(String blobName) throws InterruptedException, ExecutionException { + System.out.format("Download large object to file%n"); + + blobStore.downloadBlob(CONTAINER, blobName, new File(blobName + ".downloaded")); + } + + private void cleanup() { + System.out.format("Cleaning up...%n"); + blobStore.clearContainer(CONTAINER); } /** @@ -111,4 +147,68 @@ private void uploadLargeObjectFromFile(File largeFile) throws InterruptedExcepti public void close() throws IOException { Closeables.close(blobStore.getContext(), true); } + + /** + * Helper method; so that we don't have to add a large file to the repo + * @param size File size + * @param file The new random file to generate (will overwrite if it exists) + * @throws IOException + * @throws InterruptedException + */ + private void createRandomFile(long size, File file) throws IOException, InterruptedException { + RandomAccessFile raf = null; + + // Reserve space for performance reasons + raf = new RandomAccessFile(file.getAbsoluteFile(), "rw"); + raf.seek(size - 1); + raf.write(0); + + // Loop through ranges within the file + long from; + long to; + long partSize = 1000000; + + ExecutorService threadPool = Executors.newFixedThreadPool(16); + + for (from = 0; from < size; from = from + partSize) { + to = (from + partSize >= size) ? size - 1 : from + partSize - 1; + RandomFileWriter writer = new RandomFileWriter(raf, from, to); + threadPool.submit(writer); + } + + threadPool.shutdown(); + threadPool.awaitTermination(1, TimeUnit.DAYS); + + raf.close(); + } + + /** + * Helper class that runs the random file generation + */ + private final class RandomFileWriter implements Runnable { + private final RandomAccessFile raf; + private final long begin; + private final long end; + + RandomFileWriter(RandomAccessFile raf, long begin, long end) { + this.raf = raf; + this.begin = begin; + this.end = end; + } + + @Override + public void run() { + try { + byte[] targetArray = new byte[(int) (end - begin + 1)]; + Random random = new Random(); + random.nextBytes(targetArray); + // Map file region + MappedByteBuffer out = raf.getChannel().map(FileChannel.MapMode.READ_WRITE, begin, end - begin + 1); + out.put(targetArray); + out.force(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } }