Examples with AmazonS3Client com.amazonaws.services.s3.AmazonS3Client used on opensource projects

Example 81 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project herd by FINRAOS.

the class S3DaoImpl method deleteKeyVersions.

/**
 * Deletes a list of keys/key versions from the specified S3 bucket.
 *
 * @param s3Client the S3 client
 * @param s3BucketName the S3 bucket name
 * @param keyVersions the list of S3 keys/key versions
 */
private void deleteKeyVersions(AmazonS3Client s3Client, String s3BucketName, List<DeleteObjectsRequest.KeyVersion> keyVersions) {
    // Create a request to delete multiple objects in the specified bucket.
    DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(s3BucketName);
    // The Multi-Object Delete request can contain a list of up to 1000 keys.
    for (int i = 0; i < keyVersions.size() / MAX_KEYS_PER_DELETE_REQUEST + 1; i++) {
        List<DeleteObjectsRequest.KeyVersion> keysSubList = keyVersions.subList(i * MAX_KEYS_PER_DELETE_REQUEST, Math.min(keyVersions.size(), (i + 1) * MAX_KEYS_PER_DELETE_REQUEST));
        multiObjectDeleteRequest.setKeys(keysSubList);
        try {
            s3Operations.deleteObjects(multiObjectDeleteRequest, s3Client);
        } catch (MultiObjectDeleteException multiObjectDeleteException) {
            logMultiObjectDeleteException(multiObjectDeleteException);
            throw multiObjectDeleteException;
        }
        LOGGER.info("Successfully requested the deletion of the listed below keys/key versions from the S3 bucket. s3KeyCount={} s3BucketName=\"{}\"", keysSubList.size(), s3BucketName);
        for (DeleteObjectsRequest.KeyVersion keyVersion : keysSubList) {
            LOGGER.info("s3Key=\"{}\" s3VersionId=\"{}\"", keyVersion.getKey(), keyVersion.getVersion());
        }
    }
}

Example 82 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project herd by FINRAOS.

the class S3DaoImpl method listVersions.

@Override
public List<DeleteObjectsRequest.KeyVersion> listVersions(final S3FileTransferRequestParamsDto params) {
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 key versions from root directory is not allowed.");
    AmazonS3Client s3Client = getAmazonS3(params);
    List<DeleteObjectsRequest.KeyVersion> keyVersions = new ArrayList<>();
    try {
        ListVersionsRequest listVersionsRequest = new ListVersionsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        VersionListing versionListing;
        do {
            versionListing = s3Operations.listVersions(listVersionsRequest, s3Client);
            for (S3VersionSummary versionSummary : versionListing.getVersionSummaries()) {
                keyVersions.add(new DeleteObjectsRequest.KeyVersion(versionSummary.getKey(), versionSummary.getVersionId()));
            }
            listVersionsRequest.setKeyMarker(versionListing.getNextKeyMarker());
            listVersionsRequest.setVersionIdMarker(versionListing.getNextVersionIdMarker());
        } while (versionListing.isTruncated());
    } catch (AmazonS3Exception amazonS3Exception) {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode())) {
            throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    } catch (AmazonClientException e) {
        throw new IllegalStateException(String.format("Failed to list keys/key versions with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(), e.getMessage()), e);
    } finally {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }
    return keyVersions;
}

Example 83 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project herd by FINRAOS.

the class S3DaoImpl method listDirectory.

@Override
public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers) {
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 objects from root directory is not allowed.");
    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>();
    try {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        ObjectListing objectListing;
        do {
            objectListing = s3Operations.listObjects(listObjectsRequest, s3Client);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                // Ignore 0 byte objects that represent S3 directories.
                if (!(ignoreZeroByteDirectoryMarkers && objectSummary.getKey().endsWith("/") && objectSummary.getSize() == 0L)) {
                    s3ObjectSummaries.add(objectSummary);
                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonS3Exception amazonS3Exception) {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode())) {
            throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    } catch (AmazonClientException e) {
        throw new IllegalStateException(String.format("Failed to list keys with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(), e.getMessage()), e);
    } finally {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }
    return s3ObjectSummaries;
}

Example 84 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project herd by FINRAOS.

the class S3DaoImpl method tagObjects.

@Override
public void tagObjects(final S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto, final S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto, final Tag tag) {
    LOGGER.info("Tagging objects in S3... s3BucketName=\"{}\" s3KeyCount={} s3ObjectTagKey=\"{}\" s3ObjectTagValue=\"{}\"", s3FileTransferRequestParamsDto.getS3BucketName(), s3FileTransferRequestParamsDto.getFiles().size(), tag.getKey(), tag.getValue());
    if (!CollectionUtils.isEmpty(s3FileTransferRequestParamsDto.getFiles())) {
        // Initialize a key value pair for the error message in the catch block.
        String s3Key = s3FileTransferRequestParamsDto.getFiles().get(0).getPath().replaceAll("\\\\", "/");
        // Amazon S3 client to access S3 objects.
        AmazonS3Client s3Client = null;
        // Amazon S3 client for S3 object tagging.
        AmazonS3Client s3ObjectTaggerClient = null;
        try {
            // Create an S3 client to access S3 objects.
            s3Client = getAmazonS3(s3FileTransferRequestParamsDto);
            // Create an S3 client for S3 object tagging.
            s3ObjectTaggerClient = getAmazonS3(s3ObjectTaggerParamsDto);
            // Create a get object tagging request.
            GetObjectTaggingRequest getObjectTaggingRequest = new GetObjectTaggingRequest(s3FileTransferRequestParamsDto.getS3BucketName(), null);
            // Create a restore object request.
            SetObjectTaggingRequest setObjectTaggingRequest = new SetObjectTaggingRequest(s3FileTransferRequestParamsDto.getS3BucketName(), null, null);
            for (File file : s3FileTransferRequestParamsDto.getFiles()) {
                // Prepare an S3 key.
                s3Key = file.getPath().replaceAll("\\\\", "/");
                // Retrieve the current tagging information for the S3 key.
                getObjectTaggingRequest.setKey(s3Key);
                GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(getObjectTaggingRequest, s3Client);
                // Update the list of tags to include the specified S3 object tag.
                List<Tag> updatedTags = new ArrayList<>();
                updatedTags.add(tag);
                if (CollectionUtils.isNotEmpty(getObjectTaggingResult.getTagSet())) {
                    for (Tag currentTag : getObjectTaggingResult.getTagSet()) {
                        if (!StringUtils.equals(tag.getKey(), currentTag.getKey())) {
                            updatedTags.add(currentTag);
                        }
                    }
                }
                // Update the tagging information.
                setObjectTaggingRequest.setKey(s3Key);
                setObjectTaggingRequest.setTagging(new ObjectTagging(updatedTags));
                s3Operations.setObjectTagging(setObjectTaggingRequest, s3ObjectTaggerClient);
            }
        } catch (Exception e) {
            throw new IllegalStateException(String.format("Failed to tag S3 object with \"%s\" key in \"%s\" bucket. Reason: %s", s3Key, s3FileTransferRequestParamsDto.getS3BucketName(), e.getMessage()), e);
        } finally {
            if (s3Client != null) {
                s3Client.shutdown();
            }
            if (s3ObjectTaggerClient != null) {
                s3ObjectTaggerClient.shutdown();
            }
        }
    }
}

Example 85 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project herd by FINRAOS.

the class S3DaoImpl method getAmazonS3.

/**
 * Gets a new S3 client based on the specified parameters. The HTTP proxy information will be added if the host and port are specified in the parameters.
 *
 * @param params the parameters.
 *
 * @return the Amazon S3 client.
 */
private AmazonS3Client getAmazonS3(S3FileTransferRequestParamsDto params) {
    AmazonS3Client amazonS3Client;
    ClientConfiguration clientConfiguration = new ClientConfiguration().withRetryPolicy(retryPolicyFactory.getRetryPolicy());
    // Set the proxy configuration, if proxy is specified.
    if (StringUtils.isNotBlank(params.getHttpProxyHost()) && params.getHttpProxyPort() != null) {
        clientConfiguration.setProxyHost(params.getHttpProxyHost());
        clientConfiguration.setProxyPort(params.getHttpProxyPort());
    }
    // Sign all S3 API's with V4 signing.
    // AmazonS3Client.upgradeToSigV4 already has some scenarios where it will "upgrade" the signing approach to use V4 if not already present (e.g.
    // GetObjectRequest and KMS PutObjectRequest), but setting it here (especially when KMS is used) will ensure it isn't missed when required (e.g.
    // copying objects between KMS encrypted buckets). Otherwise, AWS will return a bad request error and retry which isn't desirable.
    clientConfiguration.setSignerOverride(SIGNER_OVERRIDE_V4);
    // Set the optional socket timeout, if configured.
    if (params.getSocketTimeout() != null) {
        clientConfiguration.setSocketTimeout(params.getSocketTimeout());
    }
    // Create an S3 client using passed in credentials and HTTP proxy information.
    if (StringUtils.isNotBlank(params.getAwsAccessKeyId()) && StringUtils.isNotBlank(params.getAwsSecretKey()) && StringUtils.isNotBlank(params.getSessionToken())) {
        // Create an S3 client using basic session credentials.
        amazonS3Client = new AmazonS3Client(new BasicSessionCredentials(params.getAwsAccessKeyId(), params.getAwsSecretKey(), params.getSessionToken()), clientConfiguration);
    } else {
        // Create an S3 client using AWS credentials provider.
        amazonS3Client = new AmazonS3Client(getAWSCredentialsProvider(params), clientConfiguration);
    }
    // Set the optional endpoint, if specified.
    if (StringUtils.isNotBlank(params.getS3Endpoint())) {
        LOGGER.info("Configured S3 Endpoint: " + params.getS3Endpoint());
        amazonS3Client.setEndpoint(params.getS3Endpoint());
    }
    // Return the newly created client.
    return amazonS3Client;
}