Examples with AmazonS3Client com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client used on opensource projects

Example 86 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project herd by FINRAOS.

the class S3DaoImpl method validateGlacierS3FilesRestored.

@Override
public void validateGlacierS3FilesRestored(S3FileTransferRequestParamsDto params) throws RuntimeException {
    LOGGER.info("Checking for already restored Glacier storage class objects... s3KeyPrefix=\"{}\" s3BucketName=\"{}\" s3KeyCount={}", params.getS3KeyPrefix(), params.getS3BucketName(), params.getFiles().size());
    if (!CollectionUtils.isEmpty(params.getFiles())) {
        // Initialize a key value pair for the error message in the catch block.
        String key = params.getFiles().get(0).getPath().replaceAll("\\\\", "/");
        try {
            // Create an S3 client.
            AmazonS3Client s3Client = getAmazonS3(params);
            try {
                for (File file : params.getFiles()) {
                    key = file.getPath().replaceAll("\\\\", "/");
                    ObjectMetadata objectMetadata = s3Operations.getObjectMetadata(params.getS3BucketName(), key, s3Client);
                    // Fail if a not already restored object is detected.
                    if (BooleanUtils.isNotFalse(objectMetadata.getOngoingRestore())) {
                        throw new IllegalArgumentException(String.format("Archived Glacier S3 file \"%s\" is not restored. StorageClass {%s}, OngoingRestore flag {%s}, S3 bucket name {%s}", key, objectMetadata.getStorageClass(), objectMetadata.getOngoingRestore(), params.getS3BucketName()));
                    }
                }
            } finally {
                s3Client.shutdown();
            }
        } catch (AmazonServiceException e) {
            throw new IllegalStateException(String.format("Fail to check restore status for \"%s\" key in \"%s\" bucket. Reason: %s", key, params.getS3BucketName(), e.getMessage()), e);
        }
    }
}

Example 87 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project herd by FINRAOS.

the class S3DaoImpl method listVersions.

@Override
public List<DeleteObjectsRequest.KeyVersion> listVersions(final S3FileTransferRequestParamsDto params) {
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 key versions from root directory is not allowed.");
    AmazonS3Client s3Client = getAmazonS3(params);
    List<DeleteObjectsRequest.KeyVersion> keyVersions = new ArrayList<>();
    try {
        ListVersionsRequest listVersionsRequest = new ListVersionsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        VersionListing versionListing;
        do {
            versionListing = s3Operations.listVersions(listVersionsRequest, s3Client);
            for (S3VersionSummary versionSummary : versionListing.getVersionSummaries()) {
                keyVersions.add(new DeleteObjectsRequest.KeyVersion(versionSummary.getKey(), versionSummary.getVersionId()));
            }
            listVersionsRequest.setKeyMarker(versionListing.getNextKeyMarker());
            listVersionsRequest.setVersionIdMarker(versionListing.getNextVersionIdMarker());
        } while (versionListing.isTruncated());
    } catch (AmazonS3Exception amazonS3Exception) {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode())) {
            throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    } catch (AmazonClientException e) {
        throw new IllegalStateException(String.format("Failed to list keys/key versions with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(), e.getMessage()), e);
    } finally {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }
    return keyVersions;
}

Example 88 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project herd by FINRAOS.

the class S3DaoImpl method listDirectory.

@Override
public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers) {
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 objects from root directory is not allowed.");
    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>();
    try {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        ObjectListing objectListing;
        do {
            objectListing = s3Operations.listObjects(listObjectsRequest, s3Client);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                // Ignore 0 byte objects that represent S3 directories.
                if (!(ignoreZeroByteDirectoryMarkers && objectSummary.getKey().endsWith("/") && objectSummary.getSize() == 0L)) {
                    s3ObjectSummaries.add(objectSummary);
                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonS3Exception amazonS3Exception) {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode())) {
            throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    } catch (AmazonClientException e) {
        throw new IllegalStateException(String.format("Failed to list keys with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(), e.getMessage()), e);
    } finally {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }
    return s3ObjectSummaries;
}

Example 89 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project herd by FINRAOS.

the class S3DaoImpl method tagObjects.

@Override
public void tagObjects(final S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto, final S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto, final Tag tag) {
    LOGGER.info("Tagging objects in S3... s3BucketName=\"{}\" s3KeyCount={} s3ObjectTagKey=\"{}\" s3ObjectTagValue=\"{}\"", s3FileTransferRequestParamsDto.getS3BucketName(), s3FileTransferRequestParamsDto.getFiles().size(), tag.getKey(), tag.getValue());
    if (!CollectionUtils.isEmpty(s3FileTransferRequestParamsDto.getFiles())) {
        // Initialize a key value pair for the error message in the catch block.
        String s3Key = s3FileTransferRequestParamsDto.getFiles().get(0).getPath().replaceAll("\\\\", "/");
        // Amazon S3 client to access S3 objects.
        AmazonS3Client s3Client = null;
        // Amazon S3 client for S3 object tagging.
        AmazonS3Client s3ObjectTaggerClient = null;
        try {
            // Create an S3 client to access S3 objects.
            s3Client = getAmazonS3(s3FileTransferRequestParamsDto);
            // Create an S3 client for S3 object tagging.
            s3ObjectTaggerClient = getAmazonS3(s3ObjectTaggerParamsDto);
            // Create a get object tagging request.
            GetObjectTaggingRequest getObjectTaggingRequest = new GetObjectTaggingRequest(s3FileTransferRequestParamsDto.getS3BucketName(), null);
            // Create a restore object request.
            SetObjectTaggingRequest setObjectTaggingRequest = new SetObjectTaggingRequest(s3FileTransferRequestParamsDto.getS3BucketName(), null, null);
            for (File file : s3FileTransferRequestParamsDto.getFiles()) {
                // Prepare an S3 key.
                s3Key = file.getPath().replaceAll("\\\\", "/");
                // Retrieve the current tagging information for the S3 key.
                getObjectTaggingRequest.setKey(s3Key);
                GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(getObjectTaggingRequest, s3Client);
                // Update the list of tags to include the specified S3 object tag.
                List<Tag> updatedTags = new ArrayList<>();
                updatedTags.add(tag);
                if (CollectionUtils.isNotEmpty(getObjectTaggingResult.getTagSet())) {
                    for (Tag currentTag : getObjectTaggingResult.getTagSet()) {
                        if (!StringUtils.equals(tag.getKey(), currentTag.getKey())) {
                            updatedTags.add(currentTag);
                        }
                    }
                }
                // Update the tagging information.
                setObjectTaggingRequest.setKey(s3Key);
                setObjectTaggingRequest.setTagging(new ObjectTagging(updatedTags));
                s3Operations.setObjectTagging(setObjectTaggingRequest, s3ObjectTaggerClient);
            }
        } catch (Exception e) {
            throw new IllegalStateException(String.format("Failed to tag S3 object with \"%s\" key in \"%s\" bucket. Reason: %s", s3Key, s3FileTransferRequestParamsDto.getS3BucketName(), e.getMessage()), e);
        } finally {
            if (s3Client != null) {
                s3Client.shutdown();
            }
            if (s3ObjectTaggerClient != null) {
                s3ObjectTaggerClient.shutdown();
            }
        }
    }
}

Example 90 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project herd by FINRAOS.

the class S3DaoImpl method getAmazonS3.

/**
 * Gets a new S3 client based on the specified parameters. The HTTP proxy information will be added if the host and port are specified in the parameters.
 *
 * @param params the parameters.
 *
 * @return the Amazon S3 client.
 */
private AmazonS3Client getAmazonS3(S3FileTransferRequestParamsDto params) {
    AmazonS3Client amazonS3Client;
    ClientConfiguration clientConfiguration = new ClientConfiguration().withRetryPolicy(retryPolicyFactory.getRetryPolicy());
    // Set the proxy configuration, if proxy is specified.
    if (StringUtils.isNotBlank(params.getHttpProxyHost()) && params.getHttpProxyPort() != null) {
        clientConfiguration.setProxyHost(params.getHttpProxyHost());
        clientConfiguration.setProxyPort(params.getHttpProxyPort());
    }
    // Sign all S3 API's with V4 signing.
    // AmazonS3Client.upgradeToSigV4 already has some scenarios where it will "upgrade" the signing approach to use V4 if not already present (e.g.
    // GetObjectRequest and KMS PutObjectRequest), but setting it here (especially when KMS is used) will ensure it isn't missed when required (e.g.
    // copying objects between KMS encrypted buckets). Otherwise, AWS will return a bad request error and retry which isn't desirable.
    clientConfiguration.setSignerOverride(SIGNER_OVERRIDE_V4);
    // Set the optional socket timeout, if configured.
    if (params.getSocketTimeout() != null) {
        clientConfiguration.setSocketTimeout(params.getSocketTimeout());
    }
    // Create an S3 client using passed in credentials and HTTP proxy information.
    if (StringUtils.isNotBlank(params.getAwsAccessKeyId()) && StringUtils.isNotBlank(params.getAwsSecretKey()) && StringUtils.isNotBlank(params.getSessionToken())) {
        // Create an S3 client using basic session credentials.
        amazonS3Client = new AmazonS3Client(new BasicSessionCredentials(params.getAwsAccessKeyId(), params.getAwsSecretKey(), params.getSessionToken()), clientConfiguration);
    } else {
        // Create an S3 client using AWS credentials provider.
        amazonS3Client = new AmazonS3Client(getAWSCredentialsProvider(params), clientConfiguration);
    }
    // Set the optional endpoint, if specified.
    if (StringUtils.isNotBlank(params.getS3Endpoint())) {
        LOGGER.info("Configured S3 Endpoint: " + params.getS3Endpoint());
        amazonS3Client.setEndpoint(params.getS3Endpoint());
    }
    // Return the newly created client.
    return amazonS3Client;
}