Examples with AmazonS3Client com.amazonaws.services.s3.AmazonS3Client used on opensource projects

Example 11 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project nifi by apache.

the class TestPutS3Object method testSignerOverrideOptions.

@Test
public void testSignerOverrideOptions() {
    final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
    final ClientConfiguration config = new ClientConfiguration();
    final PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);
    final List<AllowableValue> allowableSignerValues = PutS3Object.SIGNER_OVERRIDE.getAllowableValues();
    final String defaultSignerValue = PutS3Object.SIGNER_OVERRIDE.getDefaultValue();
    for (AllowableValue allowableSignerValue : allowableSignerValues) {
        String signerType = allowableSignerValue.getValue();
        if (!signerType.equals(defaultSignerValue)) {
            runner.setProperty(PutS3Object.SIGNER_OVERRIDE, signerType);
            ProcessContext context = runner.getProcessContext();
            try {
                AmazonS3Client s3Client = processor.createClient(context, credentialsProvider, config);
            } catch (IllegalArgumentException argEx) {
                Assert.fail(argEx.getMessage());
            }
        }
    }
}

Example 12 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project nifi by apache.

the class AbstractS3Processor method createClient.

/**
 * Create client using AWSCredentials
 *
 * @deprecated use {@link #createClient(ProcessContext, AWSCredentialsProvider, ClientConfiguration)} instead
 */
@Override
protected AmazonS3Client createClient(final ProcessContext context, final AWSCredentials credentials, final ClientConfiguration config) {
    getLogger().info("Creating client with AWS credentials");
    initializeSignerOverride(context, config);
    final AmazonS3Client s3 = new AmazonS3Client(credentials, config);
    initalizeEndpointOverride(context, s3);
    return s3;
}

Example 13 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project nifi by apache.

the class PutS3Object method localUploadExistsInS3.

protected boolean localUploadExistsInS3(final AmazonS3Client s3, final String bucket, final MultipartState localState) {
    ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest(bucket);
    MultipartUploadListing listing = s3.listMultipartUploads(listRequest);
    for (MultipartUpload upload : listing.getMultipartUploads()) {
        if (upload.getUploadId().equals(localState.getUploadId())) {
            return true;
        }
    }
    return false;
}

Example 14 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project nifi by apache.

the class PutS3Object method getS3AgeoffListAndAgeoffLocalState.

protected MultipartUploadListing getS3AgeoffListAndAgeoffLocalState(final ProcessContext context, final AmazonS3Client s3, final long now) {
    final long ageoff_interval = context.getProperty(MULTIPART_S3_AGEOFF_INTERVAL).asTimePeriod(TimeUnit.MILLISECONDS);
    final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions().getValue();
    final Long maxAge = context.getProperty(MULTIPART_S3_MAX_AGE).asTimePeriod(TimeUnit.MILLISECONDS);
    final long ageCutoff = now - maxAge;
    final List<MultipartUpload> ageoffList = new ArrayList<>();
    if ((lastS3AgeOff.get() < now - ageoff_interval) && s3BucketLock.tryLock()) {
        try {
            ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest(bucket);
            MultipartUploadListing listing = s3.listMultipartUploads(listRequest);
            for (MultipartUpload upload : listing.getMultipartUploads()) {
                long uploadTime = upload.getInitiated().getTime();
                if (uploadTime < ageCutoff) {
                    ageoffList.add(upload);
                }
            }
            // ageoff any local state
            ageoffLocalState(ageCutoff);
            lastS3AgeOff.set(System.currentTimeMillis());
        } catch (AmazonClientException e) {
            if (e instanceof AmazonS3Exception && ((AmazonS3Exception) e).getStatusCode() == 403 && ((AmazonS3Exception) e).getErrorCode().equals("AccessDenied")) {
                getLogger().warn("AccessDenied checking S3 Multipart Upload list for {}: {} " + "** The configured user does not have the s3:ListBucketMultipartUploads permission " + "for this bucket, S3 ageoff cannot occur without this permission.  Next ageoff check " + "time is being advanced by interval to prevent checking on every upload **", new Object[] { bucket, e.getMessage() });
                lastS3AgeOff.set(System.currentTimeMillis());
            } else {
                getLogger().error("Error checking S3 Multipart Upload list for {}: {}", new Object[] { bucket, e.getMessage() });
            }
        } finally {
            s3BucketLock.unlock();
        }
    }
    MultipartUploadListing result = new MultipartUploadListing();
    result.setBucketName(bucket);
    result.setMultipartUploads(ageoffList);
    return result;
}

Example 15 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project nifi by apache.

the class PutS3Object method abortS3MultipartUpload.

protected void abortS3MultipartUpload(final AmazonS3Client s3, final String bucket, final MultipartUpload upload) {
    final String uploadKey = upload.getKey();
    final String uploadId = upload.getUploadId();
    final AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest(bucket, uploadKey, uploadId);
    try {
        s3.abortMultipartUpload(abortRequest);
        getLogger().info("Aborting out of date multipart upload, bucket {} key {} ID {}, initiated {}", new Object[] { bucket, uploadKey, uploadId, logFormat.format(upload.getInitiated()) });
    } catch (AmazonClientException ace) {
        getLogger().info("Error trying to abort multipart upload from bucket {} with key {} and ID {}: {}", new Object[] { bucket, uploadKey, uploadId, ace.getMessage() });
    }
}