Examples with MultipartUploadListing com.amazonaws.services.s3.model.MultipartUploadListing used on opensource projects

Example 11 with MultipartUploadListing

use of com.amazonaws.services.s3.model.MultipartUploadListing in project nifi by apache.

the class TestPutS3Object method testPutSinglePart.

@Test
public void testPutSinglePart() {
    runner.setProperty(PutS3Object.REGION, "ap-northeast-1");
    runner.setProperty(PutS3Object.BUCKET, "test-bucket");
    runner.setProperty("x-custom-prop", "hello");
    final Map<String, String> ffAttributes = new HashMap<>();
    ffAttributes.put("filename", "testfile.txt");
    runner.enqueue("Test Content", ffAttributes);
    PutObjectResult putObjectResult = Mockito.spy(PutObjectResult.class);
    Date expiration = new Date();
    putObjectResult.setExpirationTime(expiration);
    putObjectResult.setMetadata(new ObjectMetadata());
    putObjectResult.setVersionId("test-version");
    Mockito.when(putObjectResult.getETag()).thenReturn("test-etag");
    Mockito.when(mockS3Client.putObject(Mockito.any(PutObjectRequest.class))).thenReturn(putObjectResult);
    MultipartUploadListing uploadListing = new MultipartUploadListing();
    Mockito.when(mockS3Client.listMultipartUploads(Mockito.any(ListMultipartUploadsRequest.class))).thenReturn(uploadListing);
    Mockito.when(mockS3Client.getResourceUrl(Mockito.anyString(), Mockito.anyString())).thenReturn("test-s3-url");
    runner.assertValid();
    runner.run(1);
    ArgumentCaptor<PutObjectRequest> captureRequest = ArgumentCaptor.forClass(PutObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).putObject(captureRequest.capture());
    PutObjectRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals(CoreAttributes.FILENAME.key(), "testfile.txt");
    ff0.assertAttributeEquals(PutS3Object.S3_ETAG_ATTR_KEY, "test-etag");
    ff0.assertAttributeEquals(PutS3Object.S3_VERSION_ATTR_KEY, "test-version");
}

Example 12 with MultipartUploadListing

use of com.amazonaws.services.s3.model.MultipartUploadListing in project herd by FINRAOS.

the class S3DaoImpl method abortMultipartUploads.

@Override
public int abortMultipartUploads(S3FileTransferRequestParamsDto params, Date thresholdDate) {
    // Create an Amazon S3 client.
    AmazonS3Client s3Client = getAmazonS3(params);
    int abortedMultipartUploadsCount = 0;
    try {
        // List upload markers. Null implies initial list request.
        String uploadIdMarker = null;
        String keyMarker = null;
        boolean truncated;
        do {
            // Create the list multipart request, optionally using the last markers.
            ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(params.getS3BucketName());
            request.setUploadIdMarker(uploadIdMarker);
            request.setKeyMarker(keyMarker);
            // Request the multipart upload listing.
            MultipartUploadListing uploadListing = s3Operations.listMultipartUploads(TransferManager.appendSingleObjectUserAgent(request), s3Client);
            for (MultipartUpload upload : uploadListing.getMultipartUploads()) {
                if (upload.getInitiated().compareTo(thresholdDate) < 0) {
                    // Abort the upload.
                    s3Operations.abortMultipartUpload(TransferManager.appendSingleObjectUserAgent(new AbortMultipartUploadRequest(params.getS3BucketName(), upload.getKey(), upload.getUploadId())), s3Client);
                    // Log the information about the aborted multipart upload.
                    LOGGER.info("Aborted S3 multipart upload. s3Key=\"{}\" s3BucketName=\"{}\" s3MultipartUploadInitiatedDate=\"{}\"", upload.getKey(), params.getS3BucketName(), upload.getInitiated());
                    // Increment the counter.
                    abortedMultipartUploadsCount++;
                }
            }
            // Determine whether there are more uploads to list.
            truncated = uploadListing.isTruncated();
            if (truncated) {
                // Record the list markers.
                uploadIdMarker = uploadListing.getNextUploadIdMarker();
                keyMarker = uploadListing.getNextKeyMarker();
            }
        } while (truncated);
    } finally {
        // Shutdown the Amazon S3 client instance to release resources.
        s3Client.shutdown();
    }
    return abortedMultipartUploadsCount;
}