Examples with MultipartUpload com.amazonaws.services.s3.model.MultipartUpload used on opensource projects

Example 1 with MultipartUpload

use of com.amazonaws.services.s3.model.MultipartUpload in project deeplearning4j by deeplearning4j.

the class S3Uploader method multiPartUpload.

/**
     * Multi part upload for big files
     * @param file the file to upload
     * @param bucketName the bucket name to upload
     */
public void multiPartUpload(File file, String bucketName) {
    AmazonS3 client = new AmazonS3Client(creds);
    bucketName = ensureValidBucketName(bucketName);
    List<Bucket> buckets = client.listBuckets();
    for (Bucket b : buckets) if (b.getName().equals(bucketName)) {
        doMultiPart(client, bucketName, file);
        return;
    }
    //bucket didn't exist: create it
    client.createBucket(bucketName);
    doMultiPart(client, bucketName, file);
}

Example 2 with MultipartUpload

use of com.amazonaws.services.s3.model.MultipartUpload in project nifi by apache.

the class ITPutS3Object method testS3MultipartAgeoff.

@Ignore
@Test
public void testS3MultipartAgeoff() throws InterruptedException, IOException {
    final PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);
    final ProcessContext context = runner.getProcessContext();
    runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutS3Object.REGION, REGION);
    runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
    // set check interval and age off to minimum values
    runner.setProperty(PutS3Object.MULTIPART_S3_AGEOFF_INTERVAL, "1 milli");
    runner.setProperty(PutS3Object.MULTIPART_S3_MAX_AGE, "1 milli");
    // create some dummy uploads
    for (Integer i = 0; i < 3; i++) {
        final InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest(BUCKET_NAME, "file" + i.toString() + ".txt");
        try {
            client.initiateMultipartUpload(initiateRequest);
        } catch (AmazonClientException e) {
            Assert.fail("Failed to initiate upload: " + e.getMessage());
        }
    }
    // Age off is time dependent, so test has some timing constraints.  This
    // sleep() delays long enough to satisfy interval and age intervals.
    Thread.sleep(2000L);
    // System millis are used for timing, but it is incremented on each
    // call to circumvent what appears to be caching in the AWS library.
    // The increments are 1000 millis because AWS returns upload
    // initiation times in whole seconds.
    Long now = System.currentTimeMillis();
    MultipartUploadListing uploadList = processor.getS3AgeoffListAndAgeoffLocalState(context, client, now);
    Assert.assertEquals(3, uploadList.getMultipartUploads().size());
    MultipartUpload upload0 = uploadList.getMultipartUploads().get(0);
    processor.abortS3MultipartUpload(client, BUCKET_NAME, upload0);
    uploadList = processor.getS3AgeoffListAndAgeoffLocalState(context, client, now + 1000);
    Assert.assertEquals(2, uploadList.getMultipartUploads().size());
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "test-upload.txt");
    runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
    runner.run();
    uploadList = processor.getS3AgeoffListAndAgeoffLocalState(context, client, now + 2000);
    Assert.assertEquals(0, uploadList.getMultipartUploads().size());
}

Example 3 with MultipartUpload

use of com.amazonaws.services.s3.model.MultipartUpload in project nifi by apache.

the class ITPutS3Object method testStatePersistsETags.

@Test
public void testStatePersistsETags() throws IOException {
    final PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);
    final String bucket = runner.getProcessContext().getProperty(PutS3Object.BUCKET).getValue();
    final String key = runner.getProcessContext().getProperty(PutS3Object.KEY).getValue();
    final String cacheKey1 = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key + "-bv1";
    final String cacheKey2 = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key + "-bv2";
    final String cacheKey3 = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key + "-bv3";
    /*
         * store 3 versions of state
         */
    PutS3Object.MultipartState state1orig = new PutS3Object.MultipartState();
    processor.persistLocalState(cacheKey1, state1orig);
    PutS3Object.MultipartState state2orig = new PutS3Object.MultipartState();
    state2orig.setUploadId("1234");
    state2orig.setContentLength(1234L);
    processor.persistLocalState(cacheKey2, state2orig);
    PutS3Object.MultipartState state3orig = new PutS3Object.MultipartState();
    state3orig.setUploadId("5678");
    state3orig.setContentLength(5678L);
    processor.persistLocalState(cacheKey3, state3orig);
    /*
         * persist state to caches so that
         *      1. v2 has 2 and then 4 tags
         *      2. v3 has 4 and then 2 tags
         */
    state2orig.getPartETags().add(new PartETag(1, "state 2 tag one"));
    state2orig.getPartETags().add(new PartETag(2, "state 2 tag two"));
    processor.persistLocalState(cacheKey2, state2orig);
    state2orig.getPartETags().add(new PartETag(3, "state 2 tag three"));
    state2orig.getPartETags().add(new PartETag(4, "state 2 tag four"));
    processor.persistLocalState(cacheKey2, state2orig);
    state3orig.getPartETags().add(new PartETag(1, "state 3 tag one"));
    state3orig.getPartETags().add(new PartETag(2, "state 3 tag two"));
    state3orig.getPartETags().add(new PartETag(3, "state 3 tag three"));
    state3orig.getPartETags().add(new PartETag(4, "state 3 tag four"));
    processor.persistLocalState(cacheKey3, state3orig);
    state3orig.getPartETags().remove(state3orig.getPartETags().size() - 1);
    state3orig.getPartETags().remove(state3orig.getPartETags().size() - 1);
    processor.persistLocalState(cacheKey3, state3orig);
    final List<MultipartUpload> uploadList = new ArrayList<>();
    final MultipartUpload upload1 = new MultipartUpload();
    upload1.setKey(key + "-bv2");
    upload1.setUploadId("1234");
    uploadList.add(upload1);
    final MultipartUpload upload2 = new MultipartUpload();
    upload2.setKey(key + "-bv3");
    upload2.setUploadId("5678");
    uploadList.add(upload2);
    final MultipartUploadListing uploadListing = new MultipartUploadListing();
    uploadListing.setMultipartUploads(uploadList);
    final MockAmazonS3Client mockClient = new MockAmazonS3Client();
    mockClient.setListing(uploadListing);
    /*
         * load state and validate that
         *     1. v2 restore shows 4 tags
         *     2. v3 restore shows 2 tags
         */
    final PutS3Object.MultipartState state2new = processor.getLocalStateIfInS3(mockClient, bucket, cacheKey2);
    Assert.assertEquals("1234", state2new.getUploadId());
    Assert.assertEquals(4, state2new.getPartETags().size());
    final PutS3Object.MultipartState state3new = processor.getLocalStateIfInS3(mockClient, bucket, cacheKey3);
    Assert.assertEquals("5678", state3new.getUploadId());
    Assert.assertEquals(2, state3new.getPartETags().size());
}

Example 4 with MultipartUpload

use of com.amazonaws.services.s3.model.MultipartUpload in project nifi by apache.

the class ITPutS3Object method testLocalStatePersistence.

@Test
public void testLocalStatePersistence() throws IOException {
    final PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);
    final String bucket = runner.getProcessContext().getProperty(PutS3Object.BUCKET).getValue();
    final String key = runner.getProcessContext().getProperty(PutS3Object.KEY).getValue();
    final String cacheKey1 = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key;
    final String cacheKey2 = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key + "-v2";
    final String cacheKey3 = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key + "-v3";
    /*
         * store 3 versions of state
         */
    PutS3Object.MultipartState state1orig = new PutS3Object.MultipartState();
    processor.persistLocalState(cacheKey1, state1orig);
    PutS3Object.MultipartState state2orig = new PutS3Object.MultipartState();
    state2orig.setUploadId("1234");
    state2orig.setContentLength(1234L);
    processor.persistLocalState(cacheKey2, state2orig);
    PutS3Object.MultipartState state3orig = new PutS3Object.MultipartState();
    state3orig.setUploadId("5678");
    state3orig.setContentLength(5678L);
    processor.persistLocalState(cacheKey3, state3orig);
    final List<MultipartUpload> uploadList = new ArrayList<>();
    final MultipartUpload upload1 = new MultipartUpload();
    upload1.setKey(key);
    upload1.setUploadId("");
    uploadList.add(upload1);
    final MultipartUpload upload2 = new MultipartUpload();
    upload2.setKey(key + "-v2");
    upload2.setUploadId("1234");
    uploadList.add(upload2);
    final MultipartUpload upload3 = new MultipartUpload();
    upload3.setKey(key + "-v3");
    upload3.setUploadId("5678");
    uploadList.add(upload3);
    final MultipartUploadListing uploadListing = new MultipartUploadListing();
    uploadListing.setMultipartUploads(uploadList);
    final MockAmazonS3Client mockClient = new MockAmazonS3Client();
    mockClient.setListing(uploadListing);
    /*
         * reload and validate stored state
         */
    final PutS3Object.MultipartState state1new = processor.getLocalStateIfInS3(mockClient, bucket, cacheKey1);
    Assert.assertEquals("", state1new.getUploadId());
    Assert.assertEquals(0L, state1new.getFilePosition().longValue());
    Assert.assertEquals(new ArrayList<PartETag>(), state1new.getPartETags());
    Assert.assertEquals(0L, state1new.getPartSize().longValue());
    Assert.assertEquals(StorageClass.fromValue(StorageClass.Standard.toString()), state1new.getStorageClass());
    Assert.assertEquals(0L, state1new.getContentLength().longValue());
    final PutS3Object.MultipartState state2new = processor.getLocalStateIfInS3(mockClient, bucket, cacheKey2);
    Assert.assertEquals("1234", state2new.getUploadId());
    Assert.assertEquals(0L, state2new.getFilePosition().longValue());
    Assert.assertEquals(new ArrayList<PartETag>(), state2new.getPartETags());
    Assert.assertEquals(0L, state2new.getPartSize().longValue());
    Assert.assertEquals(StorageClass.fromValue(StorageClass.Standard.toString()), state2new.getStorageClass());
    Assert.assertEquals(1234L, state2new.getContentLength().longValue());
    final PutS3Object.MultipartState state3new = processor.getLocalStateIfInS3(mockClient, bucket, cacheKey3);
    Assert.assertEquals("5678", state3new.getUploadId());
    Assert.assertEquals(0L, state3new.getFilePosition().longValue());
    Assert.assertEquals(new ArrayList<PartETag>(), state3new.getPartETags());
    Assert.assertEquals(0L, state3new.getPartSize().longValue());
    Assert.assertEquals(StorageClass.fromValue(StorageClass.Standard.toString()), state3new.getStorageClass());
    Assert.assertEquals(5678L, state3new.getContentLength().longValue());
}

Example 5 with MultipartUpload

use of com.amazonaws.services.s3.model.MultipartUpload in project nifi by apache.

the class PutS3Object method localUploadExistsInS3.

protected boolean localUploadExistsInS3(final AmazonS3Client s3, final String bucket, final MultipartState localState) {
    ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest(bucket);
    MultipartUploadListing listing = s3.listMultipartUploads(listRequest);
    for (MultipartUpload upload : listing.getMultipartUploads()) {
        if (upload.getUploadId().equals(localState.getUploadId())) {
            return true;
        }
    }
    return false;
}