use of com.amazonaws.services.s3.model.MultipartUploadListing in project nifi by apache.
the class TestPutS3Object method testPutSinglePart.
@Test
public void testPutSinglePart() {
runner.setProperty(PutS3Object.REGION, "ap-northeast-1");
runner.setProperty(PutS3Object.BUCKET, "test-bucket");
runner.setProperty("x-custom-prop", "hello");
final Map<String, String> ffAttributes = new HashMap<>();
ffAttributes.put("filename", "testfile.txt");
runner.enqueue("Test Content", ffAttributes);
PutObjectResult putObjectResult = Mockito.spy(PutObjectResult.class);
Date expiration = new Date();
putObjectResult.setExpirationTime(expiration);
putObjectResult.setMetadata(new ObjectMetadata());
putObjectResult.setVersionId("test-version");
Mockito.when(putObjectResult.getETag()).thenReturn("test-etag");
Mockito.when(mockS3Client.putObject(Mockito.any(PutObjectRequest.class))).thenReturn(putObjectResult);
MultipartUploadListing uploadListing = new MultipartUploadListing();
Mockito.when(mockS3Client.listMultipartUploads(Mockito.any(ListMultipartUploadsRequest.class))).thenReturn(uploadListing);
Mockito.when(mockS3Client.getResourceUrl(Mockito.anyString(), Mockito.anyString())).thenReturn("test-s3-url");
runner.assertValid();
runner.run(1);
ArgumentCaptor<PutObjectRequest> captureRequest = ArgumentCaptor.forClass(PutObjectRequest.class);
Mockito.verify(mockS3Client, Mockito.times(1)).putObject(captureRequest.capture());
PutObjectRequest request = captureRequest.getValue();
assertEquals("test-bucket", request.getBucketName());
runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS);
MockFlowFile ff0 = flowFiles.get(0);
ff0.assertAttributeEquals(CoreAttributes.FILENAME.key(), "testfile.txt");
ff0.assertAttributeEquals(PutS3Object.S3_ETAG_ATTR_KEY, "test-etag");
ff0.assertAttributeEquals(PutS3Object.S3_VERSION_ATTR_KEY, "test-version");
}
use of com.amazonaws.services.s3.model.MultipartUploadListing in project herd by FINRAOS.
the class S3DaoImpl method abortMultipartUploads.
@Override
public int abortMultipartUploads(S3FileTransferRequestParamsDto params, Date thresholdDate) {
// Create an Amazon S3 client.
AmazonS3Client s3Client = getAmazonS3(params);
int abortedMultipartUploadsCount = 0;
try {
// List upload markers. Null implies initial list request.
String uploadIdMarker = null;
String keyMarker = null;
boolean truncated;
do {
// Create the list multipart request, optionally using the last markers.
ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(params.getS3BucketName());
request.setUploadIdMarker(uploadIdMarker);
request.setKeyMarker(keyMarker);
// Request the multipart upload listing.
MultipartUploadListing uploadListing = s3Operations.listMultipartUploads(TransferManager.appendSingleObjectUserAgent(request), s3Client);
for (MultipartUpload upload : uploadListing.getMultipartUploads()) {
if (upload.getInitiated().compareTo(thresholdDate) < 0) {
// Abort the upload.
s3Operations.abortMultipartUpload(TransferManager.appendSingleObjectUserAgent(new AbortMultipartUploadRequest(params.getS3BucketName(), upload.getKey(), upload.getUploadId())), s3Client);
// Log the information about the aborted multipart upload.
LOGGER.info("Aborted S3 multipart upload. s3Key=\"{}\" s3BucketName=\"{}\" s3MultipartUploadInitiatedDate=\"{}\"", upload.getKey(), params.getS3BucketName(), upload.getInitiated());
// Increment the counter.
abortedMultipartUploadsCount++;
}
}
// Determine whether there are more uploads to list.
truncated = uploadListing.isTruncated();
if (truncated) {
// Record the list markers.
uploadIdMarker = uploadListing.getNextUploadIdMarker();
keyMarker = uploadListing.getNextKeyMarker();
}
} while (truncated);
} finally {
// Shutdown the Amazon S3 client instance to release resources.
s3Client.shutdown();
}
return abortedMultipartUploadsCount;
}
Aggregations