Examples with AmazonS3 com.talend.shaded.com.amazonaws.services.s3.AmazonS3 used on opensource projects

Example 31 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project ORCID-Source by ORCID.

the class S3Utils method createBuckets.

// Create S3 buckets for a given prefix
public static void createBuckets(String bucketPrefix, String accessKey, String secretKey) {
    AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 s3 = new AmazonS3Client(credentials);
    String api12JsonPrefix = bucketPrefix + "-api-1-2-json-";
    String api12XMLPrefix = bucketPrefix + "-api-1-2-xml-";
    String api20JsonPrefix = bucketPrefix + "-api-2-0-json-";
    String api20XMLPrefix = bucketPrefix + "-api-2-0-xml-";
    for (int i = 0; i <= 10; i++) {
        char lastCharacter = (i == 10 ? 'x' : Character.forDigit(i, 10));
        if (!s3.doesBucketExist(api12JsonPrefix + lastCharacter)) {
            s3.createBucket((api12JsonPrefix + lastCharacter), Region.EU_Ireland);
        }
        if (!s3.doesBucketExist(api12XMLPrefix + lastCharacter)) {
            s3.createBucket((api12XMLPrefix + lastCharacter), Region.EU_Ireland);
        }
        if (!s3.doesBucketExist(api20JsonPrefix + lastCharacter)) {
            s3.createBucket((api20JsonPrefix + lastCharacter), Region.EU_Ireland);
        }
        if (!s3.doesBucketExist(api20XMLPrefix + lastCharacter)) {
            s3.createBucket((api20XMLPrefix + lastCharacter), Region.EU_Ireland);
        }
    }
}

Example 32 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project Singularity by HubSpot.

the class S3LogResource method getS3LogsWithExecutorService.

// Fetching logs
private List<SingularityS3LogMetadata> getS3LogsWithExecutorService(S3Configuration s3Configuration, ListeningExecutorService executorService, Map<SingularityS3Service, Set<String>> servicesToPrefixes, int totalPrefixCount, final SingularityS3SearchRequest search, final ConcurrentHashMap<String, ContinuationToken> continuationTokens, final boolean paginated) throws InterruptedException, ExecutionException, TimeoutException {
    List<ListenableFuture<List<S3ObjectSummaryHolder>>> futures = Lists.newArrayListWithCapacity(totalPrefixCount);
    final AtomicInteger resultCount = new AtomicInteger();
    for (final Map.Entry<SingularityS3Service, Set<String>> entry : servicesToPrefixes.entrySet()) {
        final String s3Bucket = entry.getKey().getBucket();
        final String group = entry.getKey().getGroup();
        final AmazonS3 s3Client = entry.getKey().getS3Client();
        for (final String s3Prefix : entry.getValue()) {
            final String key = String.format(CONTINUATION_TOKEN_KEY_FORMAT, group, s3Bucket, s3Prefix);
            if (search.getContinuationTokens().containsKey(key) && search.getContinuationTokens().get(key).isLastPage()) {
                LOG.trace("No further content for prefix {} in bucket {}, skipping", s3Prefix, s3Bucket);
                continuationTokens.putIfAbsent(key, search.getContinuationTokens().get(key));
                continue;
            }
            futures.add(executorService.submit(new Callable<List<S3ObjectSummaryHolder>>() {

                @Override
                public List<S3ObjectSummaryHolder> call() throws Exception {
                    ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3Prefix);
                    if (paginated) {
                        Optional<ContinuationToken> token = Optional.absent();
                        if (search.getContinuationTokens().containsKey(key) && !Strings.isNullOrEmpty(search.getContinuationTokens().get(key).getValue())) {
                            request.setContinuationToken(search.getContinuationTokens().get(key).getValue());
                            token = Optional.of(search.getContinuationTokens().get(key));
                        }
                        int targetResultCount = search.getMaxPerPage().or(DEFAULT_TARGET_MAX_RESULTS);
                        request.setMaxKeys(targetResultCount);
                        if (resultCount.get() < targetResultCount) {
                            ListObjectsV2Result result = s3Client.listObjectsV2(request);
                            if (result.getObjectSummaries().isEmpty()) {
                                continuationTokens.putIfAbsent(key, new ContinuationToken(result.getNextContinuationToken(), true));
                                return Collections.emptyList();
                            } else {
                                boolean addToList = incrementIfLessThan(resultCount, result.getObjectSummaries().size(), targetResultCount);
                                if (addToList) {
                                    continuationTokens.putIfAbsent(key, new ContinuationToken(result.getNextContinuationToken(), !result.isTruncated()));
                                    List<S3ObjectSummaryHolder> objectSummaryHolders = new ArrayList<>();
                                    for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                                        objectSummaryHolders.add(new S3ObjectSummaryHolder(group, objectSummary));
                                    }
                                    return objectSummaryHolders;
                                } else {
                                    continuationTokens.putIfAbsent(key, token.or(new ContinuationToken(null, false)));
                                    return Collections.emptyList();
                                }
                            }
                        } else {
                            continuationTokens.putIfAbsent(key, token.or(new ContinuationToken(null, false)));
                            return Collections.emptyList();
                        }
                    } else {
                        ListObjectsV2Result result = s3Client.listObjectsV2(request);
                        List<S3ObjectSummaryHolder> objectSummaryHolders = new ArrayList<>();
                        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                            objectSummaryHolders.add(new S3ObjectSummaryHolder(group, objectSummary));
                        }
                        return objectSummaryHolders;
                    }
                }
            }));
        }
    }
    final long start = System.currentTimeMillis();
    List<List<S3ObjectSummaryHolder>> results = Futures.allAsList(futures).get(s3Configuration.getWaitForS3ListSeconds(), TimeUnit.SECONDS);
    List<S3ObjectSummaryHolder> objects = Lists.newArrayListWithExpectedSize(results.size() * 2);
    for (List<S3ObjectSummaryHolder> s3ObjectSummaryHolders : results) {
        for (final S3ObjectSummaryHolder s3ObjectHolder : s3ObjectSummaryHolders) {
            objects.add(s3ObjectHolder);
        }
    }
    LOG.trace("Got {} objects from S3 after {}", objects.size(), JavaUtils.duration(start));
    List<ListenableFuture<SingularityS3LogMetadata>> logFutures = Lists.newArrayListWithCapacity(objects.size());
    final Date expireAt = new Date(System.currentTimeMillis() + s3Configuration.getExpireS3LinksAfterMillis());
    for (final S3ObjectSummaryHolder s3ObjectHolder : objects) {
        final S3ObjectSummary s3Object = s3ObjectHolder.getObjectSummary();
        final AmazonS3 s3Client = s3Services.getServiceByGroupAndBucketOrDefault(s3ObjectHolder.getGroup(), s3Object.getBucketName()).getS3Client();
        logFutures.add(executorService.submit(new Callable<SingularityS3LogMetadata>() {

            @Override
            public SingularityS3LogMetadata call() throws Exception {
                Optional<Long> maybeStartTime = Optional.absent();
                Optional<Long> maybeEndTime = Optional.absent();
                if (!search.isExcludeMetadata()) {
                    GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(s3Object.getBucketName(), s3Object.getKey());
                    Map<String, String> objectMetadata = s3Client.getObjectMetadata(metadataRequest).getUserMetadata();
                    maybeStartTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_START_S3_ATTR);
                    maybeEndTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_END_S3_ATTR);
                }
                if (search.isListOnly()) {
                    return new SingularityS3LogMetadata(s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), maybeStartTime, maybeEndTime);
                } else {
                    GeneratePresignedUrlRequest getUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()).withMethod(HttpMethod.GET).withExpiration(expireAt);
                    String getUrl = s3Client.generatePresignedUrl(getUrlRequest).toString();
                    ResponseHeaderOverrides downloadHeaders = new ResponseHeaderOverrides();
                    downloadHeaders.setContentDisposition(CONTENT_DISPOSITION_DOWNLOAD_HEADER);
                    downloadHeaders.setContentEncoding(CONTENT_ENCODING_DOWNLOAD_HEADER);
                    GeneratePresignedUrlRequest downloadUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()).withMethod(HttpMethod.GET).withExpiration(expireAt).withResponseHeaders(downloadHeaders);
                    String downloadUrl = s3Client.generatePresignedUrl(downloadUrlRequest).toString();
                    return new SingularityS3Log(getUrl, s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), downloadUrl, maybeStartTime, maybeEndTime);
                }
            }
        }));
    }
    return Futures.allAsList(logFutures).get(s3Configuration.getWaitForS3LinksSeconds(), TimeUnit.SECONDS);
}

Example 33 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project incubator-gobblin by apache.

the class AWSSdkClient method listS3Bucket.

/**
 * Get list of S3 objects within a S3 bucket qualified by prefix path
 *
 * @param bucketName S3 bucket name
 * @param prefix S3 prefix to object
 * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path
 */
public List<S3ObjectSummary> listS3Bucket(String bucketName, String prefix) {
    final AmazonS3 amazonS3 = getS3Client();
    final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
    final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest);
    LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing);
    return objectListing.getObjectSummaries();
}

Example 34 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project incubator-gobblin by apache.

the class AWSSdkClient method downloadS3Object.

/**
 * Download a S3 object to local directory
 *
 * @param s3ObjectSummary S3 object summary for the object to download
 * @param targetDirectory Local target directory to download the object to
 * @throws IOException If any errors were encountered in downloading the object
 */
public void downloadS3Object(S3ObjectSummary s3ObjectSummary, String targetDirectory) throws IOException {
    final AmazonS3 amazonS3 = getS3Client();
    final GetObjectRequest getObjectRequest = new GetObjectRequest(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey());
    final S3Object s3Object = amazonS3.getObject(getObjectRequest);
    final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey();
    FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile));
    LOGGER.info("S3 object downloaded to file: " + targetFile);
}

Example 35 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project tutorials by eugenp.

the class S3Application method main.

public static void main(String[] args) throws IOException {
    // set-up the client
    AmazonS3 s3client = AmazonS3ClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(credentials)).withRegion(Regions.US_EAST_2).build();
    AWSS3Service awsService = new AWSS3Service(s3client);
    bucketName = "baeldung-bucket";
    // creating a bucket
    if (awsService.doesBucketExist(bucketName)) {
        System.out.println("Bucket name is not available." + " Try again with a different Bucket name.");
        return;
    }
    awsService.createBucket(bucketName);
    // list all the buckets
    for (Bucket s : awsService.listBuckets()) {
        System.out.println(s.getName());
    }
    // deleting bucket
    awsService.deleteBucket("baeldung-bucket-test2");
    // uploading object
    awsService.putObject(bucketName, "Document/hello.txt", new File("/Users/user/Document/hello.txt"));
    // listing objects
    ObjectListing objectListing = awsService.listObjects(bucketName);
    for (S3ObjectSummary os : objectListing.getObjectSummaries()) {
        System.out.println(os.getKey());
    }
    // downloading an object
    S3Object s3object = awsService.getObject(bucketName, "Document/hello.txt");
    S3ObjectInputStream inputStream = s3object.getObjectContent();
    FileUtils.copyInputStreamToFile(inputStream, new File("/Users/user/Desktop/hello.txt"));
    // copying an object
    awsService.copyObject("baeldung-bucket", "picture/pic.png", "baeldung-bucket2", "Document/picture.png");
    // deleting an object
    awsService.deleteObject(bucketName, "Document/hello.txt");
    // deleting multiple objects
    String[] objkeyArr = { "Document/hello2.txt", "Document/picture.png" };
    DeleteObjectsRequest delObjReq = new DeleteObjectsRequest("baeldung-bucket").withKeys(objkeyArr);
    awsService.deleteObjects(delObjReq);
}