Search in sources :

Example 11 with ListObjectsV2Request

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Request in project aws-doc-sdk-examples by awsdocs.

the class ExportEndpoints method exportEndpointsToS3.

public static List<String> exportEndpointsToS3(PinpointClient pinpoint, S3Client s3Client, String s3BucketName, String iamExportRoleArn, String applicationId) {
    SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH_mm:ss.SSS_z");
    String endpointsKeyPrefix = "exports/" + applicationId + "_" + dateFormat.format(new Date());
    String s3UrlPrefix = "s3://" + s3BucketName + "/" + endpointsKeyPrefix + "/";
    List<String> objectKeys = new ArrayList<>();
    String key = "";
    try {
        // Defines the export job that Amazon Pinpoint runs
        ExportJobRequest jobRequest = ExportJobRequest.builder().roleArn(iamExportRoleArn).s3UrlPrefix(s3UrlPrefix).build();
        CreateExportJobRequest exportJobRequest = CreateExportJobRequest.builder().applicationId(applicationId).exportJobRequest(jobRequest).build();
        System.out.format("Exporting endpoints from Amazon Pinpoint application %s to Amazon S3 " + "bucket %s . . .\n", applicationId, s3BucketName);
        CreateExportJobResponse exportResult = pinpoint.createExportJob(exportJobRequest);
        String jobId = exportResult.exportJobResponse().id();
        System.out.println(jobId);
        printExportJobStatus(pinpoint, applicationId, jobId);
        ListObjectsV2Request v2Request = ListObjectsV2Request.builder().bucket(s3BucketName).prefix(endpointsKeyPrefix).build();
        // Create a list of object keys
        ListObjectsV2Response v2Response = s3Client.listObjectsV2(v2Request);
        List<S3Object> objects = v2Response.contents();
        for (S3Object object : objects) {
            key = object.key();
            objectKeys.add(key);
        }
        return objectKeys;
    } catch (PinpointException e) {
        System.err.println(e.awsErrorDetails().errorMessage());
        System.exit(1);
    }
    return null;
}
Also used : CreateExportJobRequest(software.amazon.awssdk.services.pinpoint.model.CreateExportJobRequest) ExportJobRequest(software.amazon.awssdk.services.pinpoint.model.ExportJobRequest) GetExportJobRequest(software.amazon.awssdk.services.pinpoint.model.GetExportJobRequest) CreateExportJobRequest(software.amazon.awssdk.services.pinpoint.model.CreateExportJobRequest) PinpointException(software.amazon.awssdk.services.pinpoint.model.PinpointException) ArrayList(java.util.ArrayList) Date(java.util.Date) ListObjectsV2Request(software.amazon.awssdk.services.s3.model.ListObjectsV2Request) CreateExportJobResponse(software.amazon.awssdk.services.pinpoint.model.CreateExportJobResponse) S3Object(software.amazon.awssdk.services.s3.model.S3Object) SimpleDateFormat(java.text.SimpleDateFormat) ListObjectsV2Response(software.amazon.awssdk.services.s3.model.ListObjectsV2Response)

Example 12 with ListObjectsV2Request

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Request in project Singularity by HubSpot.

the class S3LogResource method getS3LogsWithExecutorService.

// Fetching logs
private List<SingularityS3LogMetadata> getS3LogsWithExecutorService(S3Configuration s3Configuration, ListeningExecutorService executorService, Map<SingularityS3Service, Set<String>> servicesToPrefixes, int totalPrefixCount, final SingularityS3SearchRequest search, final ConcurrentHashMap<String, ContinuationToken> continuationTokens, final boolean paginated) throws InterruptedException, ExecutionException, TimeoutException {
    List<ListenableFuture<List<S3ObjectSummaryHolder>>> futures = Lists.newArrayListWithCapacity(totalPrefixCount);
    final AtomicInteger resultCount = new AtomicInteger();
    for (final Map.Entry<SingularityS3Service, Set<String>> entry : servicesToPrefixes.entrySet()) {
        final String s3Bucket = entry.getKey().getBucket();
        final String group = entry.getKey().getGroup();
        final AmazonS3 s3Client = entry.getKey().getS3Client();
        for (final String s3Prefix : entry.getValue()) {
            final String key = String.format(CONTINUATION_TOKEN_KEY_FORMAT, group, s3Bucket, s3Prefix);
            if (search.getContinuationTokens().containsKey(key) && search.getContinuationTokens().get(key).isLastPage()) {
                LOG.trace("No further content for prefix {} in bucket {}, skipping", s3Prefix, s3Bucket);
                continuationTokens.putIfAbsent(key, search.getContinuationTokens().get(key));
                continue;
            }
            futures.add(executorService.submit(new Callable<List<S3ObjectSummaryHolder>>() {

                @Override
                public List<S3ObjectSummaryHolder> call() throws Exception {
                    ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3Prefix);
                    if (paginated) {
                        Optional<ContinuationToken> token = Optional.empty();
                        if (search.getContinuationTokens().containsKey(key) && !Strings.isNullOrEmpty(search.getContinuationTokens().get(key).getValue())) {
                            request.setContinuationToken(search.getContinuationTokens().get(key).getValue());
                            token = Optional.of(search.getContinuationTokens().get(key));
                        }
                        int targetResultCount = search.getMaxPerPage().orElse(DEFAULT_TARGET_MAX_RESULTS);
                        request.setMaxKeys(targetResultCount);
                        if (resultCount.get() < targetResultCount) {
                            ListObjectsV2Result result = s3Client.listObjectsV2(request);
                            if (result.getObjectSummaries().isEmpty()) {
                                continuationTokens.putIfAbsent(key, new ContinuationToken(result.getNextContinuationToken(), true));
                                return Collections.emptyList();
                            } else {
                                boolean addToList = incrementIfLessThan(resultCount, result.getObjectSummaries().size(), targetResultCount);
                                if (addToList) {
                                    continuationTokens.putIfAbsent(key, new ContinuationToken(result.getNextContinuationToken(), !result.isTruncated()));
                                    List<S3ObjectSummaryHolder> objectSummaryHolders = new ArrayList<>();
                                    for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                                        if (search.getFileNamePrefixWhitelist().isEmpty() || search.getFileNamePrefixWhitelist().stream().anyMatch(whitelistedPrefix -> objectSummary.getKey().startsWith(whitelistedPrefix))) {
                                            objectSummaryHolders.add(new S3ObjectSummaryHolder(group, objectSummary));
                                        }
                                    }
                                    return objectSummaryHolders;
                                } else {
                                    continuationTokens.putIfAbsent(key, token.orElse(new ContinuationToken(null, false)));
                                    return Collections.emptyList();
                                }
                            }
                        } else {
                            continuationTokens.putIfAbsent(key, token.orElse(new ContinuationToken(null, false)));
                            return Collections.emptyList();
                        }
                    } else {
                        ListObjectsV2Result result = s3Client.listObjectsV2(request);
                        List<S3ObjectSummaryHolder> objectSummaryHolders = new ArrayList<>();
                        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                            if (search.getFileNamePrefixWhitelist().isEmpty() || search.getFileNamePrefixWhitelist().stream().anyMatch(whitelistedPrefix -> objectSummary.getKey().startsWith(whitelistedPrefix))) {
                                objectSummaryHolders.add(new S3ObjectSummaryHolder(group, objectSummary));
                            }
                        }
                        while (result.isTruncated() && result.getContinuationToken() != null) {
                            result = s3Client.listObjectsV2(new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3Prefix).withContinuationToken(result.getContinuationToken()));
                            for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                                objectSummaryHolders.add(new S3ObjectSummaryHolder(group, objectSummary));
                            }
                        }
                        return objectSummaryHolders;
                    }
                }
            }));
        }
    }
    final long start = System.currentTimeMillis();
    List<List<S3ObjectSummaryHolder>> results = Futures.allAsList(futures).get(s3Configuration.getWaitForS3ListSeconds(), TimeUnit.SECONDS);
    List<S3ObjectSummaryHolder> objects = Lists.newArrayListWithExpectedSize(results.size() * 2);
    for (List<S3ObjectSummaryHolder> s3ObjectSummaryHolders : results) {
        for (final S3ObjectSummaryHolder s3ObjectHolder : s3ObjectSummaryHolders) {
            objects.add(s3ObjectHolder);
        }
    }
    LOG.trace("Got {} objects from S3 after {}", objects.size(), JavaUtils.duration(start));
    List<ListenableFuture<SingularityS3LogMetadata>> logFutures = Lists.newArrayListWithCapacity(objects.size());
    final Date expireAt = new Date(System.currentTimeMillis() + s3Configuration.getExpireS3LinksAfterMillis());
    for (final S3ObjectSummaryHolder s3ObjectHolder : objects) {
        final S3ObjectSummary s3Object = s3ObjectHolder.getObjectSummary();
        final AmazonS3 s3Client = s3Services.getServiceByGroupAndBucketOrDefault(s3ObjectHolder.getGroup(), s3Object.getBucketName()).getS3Client();
        logFutures.add(executorService.submit(new Callable<SingularityS3LogMetadata>() {

            @Override
            public SingularityS3LogMetadata call() throws Exception {
                Optional<Long> maybeStartTime = Optional.empty();
                Optional<Long> maybeEndTime = Optional.empty();
                if (!search.isExcludeMetadata()) {
                    GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(s3Object.getBucketName(), s3Object.getKey());
                    Map<String, String> objectMetadata = s3Client.getObjectMetadata(metadataRequest).getUserMetadata();
                    maybeStartTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_START_S3_ATTR);
                    maybeEndTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_END_S3_ATTR);
                }
                if (search.isListOnly()) {
                    return new SingularityS3LogMetadata(s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), maybeStartTime, maybeEndTime);
                } else {
                    GeneratePresignedUrlRequest getUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()).withMethod(HttpMethod.GET).withExpiration(expireAt);
                    String getUrl = s3Client.generatePresignedUrl(getUrlRequest).toString();
                    ResponseHeaderOverrides downloadHeaders = new ResponseHeaderOverrides();
                    downloadHeaders.setContentDisposition(CONTENT_DISPOSITION_DOWNLOAD_HEADER);
                    downloadHeaders.setContentEncoding(CONTENT_ENCODING_DOWNLOAD_HEADER);
                    GeneratePresignedUrlRequest downloadUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()).withMethod(HttpMethod.GET).withExpiration(expireAt).withResponseHeaders(downloadHeaders);
                    String downloadUrl = s3Client.generatePresignedUrl(downloadUrlRequest).toString();
                    return new SingularityS3Log(getUrl, s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), downloadUrl, maybeStartTime, maybeEndTime);
                }
            }
        }));
    }
    return Futures.allAsList(logFutures).get(s3Configuration.getWaitForS3LinksSeconds(), TimeUnit.SECONDS);
}
Also used : WebExceptions.timeout(com.hubspot.singularity.WebExceptions.timeout) DeployManager(com.hubspot.singularity.data.DeployManager) WebExceptions.checkBadRequest(com.hubspot.singularity.WebExceptions.checkBadRequest) HistoryManager(com.hubspot.singularity.data.history.HistoryManager) Tags(io.swagger.v3.oas.annotations.tags.Tags) Produces(javax.ws.rs.Produces) Date(java.util.Date) Inject(com.google.inject.Inject) RequestManager(com.hubspot.singularity.data.RequestManager) Path(javax.ws.rs.Path) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) SingularityS3SearchRequest(com.hubspot.singularity.api.SingularityS3SearchRequest) LeaderLatch(org.apache.curator.framework.recipes.leader.LeaderLatch) ListObjectsV2Request(com.amazonaws.services.s3.model.ListObjectsV2Request) RequestHistoryType(com.hubspot.singularity.SingularityRequestHistory.RequestHistoryType) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) Consumes(javax.ws.rs.Consumes) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) SingularityS3SearchResult(com.hubspot.singularity.api.SingularityS3SearchResult) RequestBody(io.swagger.v3.oas.annotations.parameters.RequestBody) Map(java.util.Map) DefaultValue(javax.ws.rs.DefaultValue) ApiPaths(com.hubspot.singularity.config.ApiPaths) TaskManager(com.hubspot.singularity.data.TaskManager) HttpMethod(com.amazonaws.HttpMethod) SingularityTaskId(com.hubspot.singularity.SingularityTaskId) GeneratePresignedUrlRequest(com.amazonaws.services.s3.model.GeneratePresignedUrlRequest) Schema(io.swagger.v3.oas.annotations.media.Schema) Longs(com.google.common.primitives.Longs) S3ObjectSummaryHolder(com.hubspot.singularity.helpers.S3ObjectSummaryHolder) ImmutableMap(com.google.common.collect.ImmutableMap) ContinuationToken(com.hubspot.singularity.api.ContinuationToken) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Singularity(com.hubspot.singularity.Singularity) Set(java.util.Set) S3Configuration(com.hubspot.singularity.config.S3Configuration) SingularityS3LogMetadata(com.hubspot.singularity.SingularityS3LogMetadata) Executors(java.util.concurrent.Executors) Parameter(io.swagger.v3.oas.annotations.Parameter) JavaUtils(com.hubspot.mesos.JavaUtils) List(java.util.List) SingularityAuthorizationScope(com.hubspot.singularity.SingularityAuthorizationScope) Tag(io.swagger.v3.oas.annotations.tags.Tag) Optional(java.util.Optional) SingularityS3FormatHelper(com.hubspot.singularity.SingularityS3FormatHelper) SimplifiedTaskState(com.hubspot.singularity.SingularityTaskHistoryUpdate.SimplifiedTaskState) SingularityS3Log(com.hubspot.singularity.SingularityS3Log) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) SingularityRequestWithState(com.hubspot.singularity.SingularityRequestWithState) SingularityRequest(com.hubspot.singularity.SingularityRequest) SingularityTaskHistoryUpdate(com.hubspot.singularity.SingularityTaskHistoryUpdate) Iterables(com.google.common.collect.Iterables) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) PathParam(javax.ws.rs.PathParam) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) SingularityRequestHistory(com.hubspot.singularity.SingularityRequestHistory) GET(javax.ws.rs.GET) SingularityUser(com.hubspot.singularity.SingularityUser) Auth(io.dropwizard.auth.Auth) HashMap(java.util.HashMap) Callable(java.util.concurrent.Callable) ListObjectsV2Result(com.amazonaws.services.s3.model.ListObjectsV2Result) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Strings(com.google.common.base.Strings) SingularityS3UploaderFile(com.hubspot.singularity.SingularityS3UploaderFile) Content(io.swagger.v3.oas.annotations.media.Content) Operation(io.swagger.v3.oas.annotations.Operation) Lists(com.google.common.collect.Lists) ApiResponse(io.swagger.v3.oas.annotations.responses.ApiResponse) AsyncHttpClient(com.ning.http.client.AsyncHttpClient) AmazonS3(com.amazonaws.services.s3.AmazonS3) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) SingularityAuthorizer(com.hubspot.singularity.auth.SingularityAuthorizer) SingularityDeployHistory(com.hubspot.singularity.SingularityDeployHistory) SingularityTaskHistory(com.hubspot.singularity.SingularityTaskHistory) POST(javax.ws.rs.POST) Logger(org.slf4j.Logger) SingularityS3Services(com.hubspot.singularity.helpers.SingularityS3Services) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ResponseHeaderOverrides(com.amazonaws.services.s3.model.ResponseHeaderOverrides) WebExceptions.checkNotFound(com.hubspot.singularity.WebExceptions.checkNotFound) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Futures(com.google.common.util.concurrent.Futures) ArraySchema(io.swagger.v3.oas.annotations.media.ArraySchema) RequestHistoryHelper(com.hubspot.singularity.data.history.RequestHistoryHelper) GetObjectMetadataRequest(com.amazonaws.services.s3.model.GetObjectMetadataRequest) SingularityS3Service(com.hubspot.singularity.helpers.SingularityS3Service) Comparator(java.util.Comparator) Collections(java.util.Collections) AmazonS3(com.amazonaws.services.s3.AmazonS3) Set(java.util.Set) HashSet(java.util.HashSet) ContinuationToken(com.hubspot.singularity.api.ContinuationToken) ListObjectsV2Result(com.amazonaws.services.s3.model.ListObjectsV2Result) ArrayList(java.util.ArrayList) Callable(java.util.concurrent.Callable) S3ObjectSummaryHolder(com.hubspot.singularity.helpers.S3ObjectSummaryHolder) GetObjectMetadataRequest(com.amazonaws.services.s3.model.GetObjectMetadataRequest) ListObjectsV2Request(com.amazonaws.services.s3.model.ListObjectsV2Request) List(java.util.List) ArrayList(java.util.ArrayList) SingularityS3Service(com.hubspot.singularity.helpers.SingularityS3Service) SingularityS3LogMetadata(com.hubspot.singularity.SingularityS3LogMetadata) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) Date(java.util.Date) GeneratePresignedUrlRequest(com.amazonaws.services.s3.model.GeneratePresignedUrlRequest) SingularityS3Log(com.hubspot.singularity.SingularityS3Log) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) ResponseHeaderOverrides(com.amazonaws.services.s3.model.ResponseHeaderOverrides) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap)

Example 13 with ListObjectsV2Request

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Request in project quickutil by quickutil.

the class AWSS3Util method list.

/**
 * 获取文件列表
 *
 * @param s3Name-S3Name
 * @param prefix-文件前缀
 * @return
 */
public static List<String> list(String s3Name, String prefix) {
    List<String> filePaths = new ArrayList<String>();
    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketMap.get(s3Name).get("bucket")).withMaxKeys(100);
    req.setPrefix(prefix);
    ListObjectsV2Result result;
    do {
        result = buildClient(s3Name).listObjectsV2(req);
        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            filePaths.add(objectSummary.getKey());
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated());
    return filePaths;
}
Also used : ListObjectsV2Request(com.amazonaws.services.s3.model.ListObjectsV2Request) ListObjectsV2Result(com.amazonaws.services.s3.model.ListObjectsV2Result) ArrayList(java.util.ArrayList) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary)

Example 14 with ListObjectsV2Request

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Request in project nifi by apache.

the class TestListS3 method testListVersion2.

@Test
public void testListVersion2() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.LIST_TYPE, "2");
    Date lastModified = new Date();
    ListObjectsV2Result objectListing = new ListObjectsV2Result();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjectsV2(Mockito.any(ListObjectsV2Request.class))).thenReturn(objectListing);
    runner.run();
    ArgumentCaptor<ListObjectsV2Request> captureRequest = ArgumentCaptor.forClass(ListObjectsV2Request.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjectsV2(captureRequest.capture());
    ListObjectsV2Request request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}
Also used : MockFlowFile(org.apache.nifi.util.MockFlowFile) ListObjectsV2Result(com.amazonaws.services.s3.model.ListObjectsV2Result) ListObjectsV2Request(com.amazonaws.services.s3.model.ListObjectsV2Request) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) Date(java.util.Date) Test(org.junit.Test)

Example 15 with ListObjectsV2Request

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Request in project aws-doc-sdk-examples by awsdocs.

the class ListKeys method main.

public static void main(String[] args) throws IOException {
    Regions clientRegion = Regions.DEFAULT_REGION;
    String bucketName = "*** Bucket name ***";
    try {
        AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(new ProfileCredentialsProvider()).withRegion(clientRegion).build();
        System.out.println("Listing objects");
        // maxKeys is set to 2 to demonstrate the use of
        // ListObjectsV2Result.getNextContinuationToken()
        ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName).withMaxKeys(2);
        ListObjectsV2Result result;
        do {
            result = s3Client.listObjectsV2(req);
            for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                System.out.printf(" - %s (size: %d)\n", objectSummary.getKey(), objectSummary.getSize());
            }
            // If there are more than maxKeys keys in the bucket, get a continuation token
            // and list the next objects.
            String token = result.getNextContinuationToken();
            System.out.println("Next Continuation Token: " + token);
            req.setContinuationToken(token);
        } while (result.isTruncated());
    } catch (AmazonServiceException e) {
        // The call was transmitted successfully, but Amazon S3 couldn't process
        // it, so it returned an error response.
        e.printStackTrace();
    } catch (SdkClientException e) {
        // Amazon S3 couldn't be contacted for a response, or the client
        // couldn't parse the response from Amazon S3.
        e.printStackTrace();
    }
}
Also used : AmazonS3(com.amazonaws.services.s3.AmazonS3) ListObjectsV2Request(com.amazonaws.services.s3.model.ListObjectsV2Request) ListObjectsV2Result(com.amazonaws.services.s3.model.ListObjectsV2Result) SdkClientException(com.amazonaws.SdkClientException) AmazonServiceException(com.amazonaws.AmazonServiceException) ProfileCredentialsProvider(com.amazonaws.auth.profile.ProfileCredentialsProvider) Regions(com.amazonaws.regions.Regions) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary)

Aggregations

ListObjectsV2Request (com.amazonaws.services.s3.model.ListObjectsV2Request)15 ListObjectsV2Result (com.amazonaws.services.s3.model.ListObjectsV2Result)14 S3ObjectSummary (com.amazonaws.services.s3.model.S3ObjectSummary)12 ListObjectsV2Response (software.amazon.awssdk.services.s3.model.ListObjectsV2Response)9 Date (java.util.Date)8 Test (org.junit.Test)8 ListObjectsV2Request (software.amazon.awssdk.services.s3.model.ListObjectsV2Request)8 S3Object (software.amazon.awssdk.services.s3.model.S3Object)8 ArrayList (java.util.ArrayList)6 ObjectMetadata (com.amazonaws.services.s3.model.ObjectMetadata)4 IOException (java.io.IOException)4 S3TestUtils.buildMockedS3FileSystem (org.apache.beam.sdk.io.aws.s3.S3TestUtils.buildMockedS3FileSystem)4 AmazonS3 (com.amazonaws.services.s3.AmazonS3)3 Collection (java.util.Collection)3 List (java.util.List)3 Pattern (java.util.regex.Pattern)3 S3TestUtils.buildMockedS3FileSystem (org.apache.beam.sdk.io.aws2.s3.S3TestUtils.buildMockedS3FileSystem)3 S3Client (software.amazon.awssdk.services.s3.S3Client)3 HeadObjectRequest (software.amazon.awssdk.services.s3.model.HeadObjectRequest)3 HeadObjectResponse (software.amazon.awssdk.services.s3.model.HeadObjectResponse)3