Examples with ListObjectsV2Request com.amazonaws.services.s3.model.ListObjectsV2Request used on opensource projects

Example 6 with ListObjectsV2Request

use of com.amazonaws.services.s3.model.ListObjectsV2Request in project druid by druid-io.

the class ObjectSummaryIteratorTest method makeMockClient.

/**
 * Makes a mock S3 client that handles enough of "listObjectsV2" to test the functionality of the
 * {@link ObjectSummaryIterator} class.
 */
private static ServerSideEncryptingAmazonS3 makeMockClient(final List<S3ObjectSummary> objects) {
    return new ServerSideEncryptingAmazonS3(null, null) {

        @Override
        public ListObjectsV2Result listObjectsV2(final ListObjectsV2Request request) {
            // Continuation token is an index in the "objects" list.
            final String continuationToken = request.getContinuationToken();
            final int startIndex = continuationToken == null ? 0 : Integer.parseInt(continuationToken);
            // Find matching objects.
            final List<S3ObjectSummary> summaries = new ArrayList<>();
            int nextIndex = -1;
            for (int i = startIndex; i < objects.size(); i++) {
                final S3ObjectSummary summary = objects.get(i);
                if (summary.getBucketName().equals(request.getBucketName()) && summary.getKey().startsWith(request.getPrefix())) {
                    if (summaries.size() == request.getMaxKeys()) {
                        // We reached our max key limit; set nextIndex (which will lead to a result with truncated = true).
                        nextIndex = i;
                        break;
                    }
                    // Generate a summary.
                    summaries.add(summary);
                }
            }
            // Generate the result.
            final ListObjectsV2Result retVal = new ListObjectsV2Result();
            retVal.setContinuationToken(continuationToken);
            retVal.getObjectSummaries().addAll(summaries);
            if (nextIndex >= 0) {
                retVal.setTruncated(true);
                retVal.setNextContinuationToken(String.valueOf(nextIndex));
            }
            return retVal;
        }
    };
}

Example 7 with ListObjectsV2Request

use of com.amazonaws.services.s3.model.ListObjectsV2Request in project druid by druid-io.

the class S3Utils method getSingleObjectSummary.

/**
 * Gets a single {@link S3ObjectSummary} from s3. Since this method might return a wrong object if there are multiple
 * objects that match the given key, this method should be used only when it's guaranteed that the given key is unique
 * in the given bucket.
 *
 * @param s3Client s3 client
 * @param bucket   s3 bucket
 * @param key      unique key for the object to be retrieved
 */
public static S3ObjectSummary getSingleObjectSummary(ServerSideEncryptingAmazonS3 s3Client, String bucket, String key) {
    final ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucket).withPrefix(key).withMaxKeys(1);
    final ListObjectsV2Result result = s3Client.listObjectsV2(request);
    // keyCount is still zero.
    if (result.getObjectSummaries().size() == 0) {
        throw new ISE("Cannot find object for bucket[%s] and key[%s]", bucket, key);
    }
    final S3ObjectSummary objectSummary = result.getObjectSummaries().get(0);
    if (!objectSummary.getBucketName().equals(bucket) || !objectSummary.getKey().equals(key)) {
        throw new ISE("Wrong object[%s] for bucket[%s] and key[%s]", objectSummary, bucket, key);
    }
    return objectSummary;
}

Example 8 with ListObjectsV2Request

use of com.amazonaws.services.s3.model.ListObjectsV2Request in project aws-doc-sdk-examples by awsdocs.

the class S3BucketDeletion method listAllObjects.

// snippet-start:[s3.java2.s3_bucket_ops.delete_bucket]
public static void listAllObjects(S3Client s3, String bucket) {
    try {
        // To delete a bucket, all the objects in the bucket must be deleted first
        ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucket).build();
        ListObjectsV2Response listObjectsV2Response;
        do {
            listObjectsV2Response = s3.listObjectsV2(listObjectsV2Request);
            for (S3Object s3Object : listObjectsV2Response.contents()) {
                s3.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(s3Object.key()).build());
            }
            listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucket).continuationToken(listObjectsV2Response.nextContinuationToken()).build();
        } while (listObjectsV2Response.isTruncated());
        // snippet-end:[s3.java2.s3_bucket_ops.delete_bucket]
        DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder().bucket(bucket).build();
        s3.deleteBucket(deleteBucketRequest);
    } catch (S3Exception e) {
        System.err.println(e.awsErrorDetails().errorMessage());
        System.exit(1);
    }
}

Example 9 with ListObjectsV2Request

use of com.amazonaws.services.s3.model.ListObjectsV2Request in project aws-doc-sdk-examples by awsdocs.

the class S3ObjectOperations method main.

public static void main(String[] args) throws IOException {
    final String USAGE = "\n" + "Usage:\n" + "    <bucketName> <key>\n\n" + "Where:\n" + "    bucketName - the Amazon S3 bucket to create.\n\n" + "    key - the key to use.\n\n";
    if (args.length != 2) {
        System.out.println(USAGE);
        System.exit(1);
    }
    String bucketName = args[0];
    String key = args[1];
    // snippet-start:[s3.java2.s3_object_operations.upload]
    Region region = Region.US_WEST_2;
    s3 = S3Client.builder().region(region).build();
    createBucket(s3, bucketName, region);
    PutObjectRequest objectRequest = PutObjectRequest.builder().bucket(bucketName).key(key).build();
    s3.putObject(objectRequest, RequestBody.fromByteBuffer(getRandomByteBuffer(10_000)));
    // snippet-end:[s3.java2.s3_object_operations.upload]
    // Multipart upload example
    String multipartKey = "multiPartKey";
    multipartUpload(bucketName, multipartKey);
    // snippet-start:[s3.java2.s3_object_operations.pagination]
    ListObjectsV2Request listObjectsReqManual = ListObjectsV2Request.builder().bucket(bucketName).maxKeys(1).build();
    boolean done = false;
    while (!done) {
        ListObjectsV2Response listObjResponse = s3.listObjectsV2(listObjectsReqManual);
        for (S3Object content : listObjResponse.contents()) {
            System.out.println(content.key());
        }
        if (listObjResponse.nextContinuationToken() == null) {
            done = true;
        }
        listObjectsReqManual = listObjectsReqManual.toBuilder().continuationToken(listObjResponse.nextContinuationToken()).build();
    }
    // snippet-end:[s3.java2.s3_object_operations.pagination]
    // snippet-start:[s3.java2.s3_object_operations.iterative]
    ListObjectsV2Request listReq = ListObjectsV2Request.builder().bucket(bucketName).maxKeys(1).build();
    ListObjectsV2Iterable listRes = s3.listObjectsV2Paginator(listReq);
    // Process response pages
    listRes.stream().flatMap(r -> r.contents().stream()).forEach(content -> System.out.println(" Key: " + content.key() + " size = " + content.size()));
    // snippet-end:[s3.java2.s3_object_operations.iterative]
    // snippet-start:[s3.java2.s3_object_operations.stream]
    // Helper method to work with paginated collection of items directly
    listRes.contents().stream().forEach(content -> System.out.println(" Key: " + content.key() + " size = " + content.size()));
    // snippet-start:[s3.java2.s3_object_operations.forloop]
    for (S3Object content : listRes.contents()) {
        System.out.println(" Key: " + content.key() + " size = " + content.size());
    }
    // snippet-end:[s3.java2.s3_object_operations.forloop]
    // snippet-start:[s3.java2.s3_object_operations.download]
    GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucketName).key(key).build();
    s3.getObject(getObjectRequest);
    // snippet-end:[s3.java2.s3_object_operations.download]
    // snippet-start:[s3.java2.s3_object_operations.delete]
    DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(bucketName).key(key).build();
    s3.deleteObject(deleteObjectRequest);
    // snippet-end:[s3.java2.s3_object_operations.delete]
    // Delete an object
    deleteObjectRequest = DeleteObjectRequest.builder().bucket(bucketName).key(multipartKey).build();
    s3.deleteObject(deleteObjectRequest);
    deleteBucket(s3, bucketName);
    System.out.println("Done");
}

Example 10 with ListObjectsV2Request

use of com.amazonaws.services.s3.model.ListObjectsV2Request in project aws-doc-sdk-examples by awsdocs.

the class ExportEndpoints method exportEndpointsToS3.

public static List<String> exportEndpointsToS3(PinpointClient pinpoint, S3Client s3Client, String s3BucketName, String iamExportRoleArn, String applicationId) {
    SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH_mm:ss.SSS_z");
    String endpointsKeyPrefix = "exports/" + applicationId + "_" + dateFormat.format(new Date());
    String s3UrlPrefix = "s3://" + s3BucketName + "/" + endpointsKeyPrefix + "/";
    List<String> objectKeys = new ArrayList<>();
    String key = "";
    try {
        // Defines the export job that Amazon Pinpoint runs
        ExportJobRequest jobRequest = ExportJobRequest.builder().roleArn(iamExportRoleArn).s3UrlPrefix(s3UrlPrefix).build();
        CreateExportJobRequest exportJobRequest = CreateExportJobRequest.builder().applicationId(applicationId).exportJobRequest(jobRequest).build();
        System.out.format("Exporting endpoints from Amazon Pinpoint application %s to Amazon S3 " + "bucket %s . . .\n", applicationId, s3BucketName);
        CreateExportJobResponse exportResult = pinpoint.createExportJob(exportJobRequest);
        String jobId = exportResult.exportJobResponse().id();
        System.out.println(jobId);
        printExportJobStatus(pinpoint, applicationId, jobId);
        ListObjectsV2Request v2Request = ListObjectsV2Request.builder().bucket(s3BucketName).prefix(endpointsKeyPrefix).build();
        // Create a list of object keys
        ListObjectsV2Response v2Response = s3Client.listObjectsV2(v2Request);
        List<S3Object> objects = v2Response.contents();
        for (S3Object object : objects) {
            key = object.key();
            objectKeys.add(key);
        }
        return objectKeys;
    } catch (PinpointException e) {
        System.err.println(e.awsErrorDetails().errorMessage());
        System.exit(1);
    }
    return null;
}