Examples with S3ObjectSummary com.amazonaws.services.s3.model.S3ObjectSummary used on opensource projects

Example 76 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project druid by druid-io.

the class S3TimestampVersionedDataFinderTest method testFindExact.

@Test
public void testFindExact() {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    ServerSideEncryptingAmazonS3 s3Client = EasyMock.createStrictMock(ServerSideEncryptingAmazonS3.class);
    S3ObjectSummary object0 = new S3ObjectSummary();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModified(new Date(0));
    object0.setSize(10);
    final ListObjectsV2Result result = new ListObjectsV2Result();
    result.getObjectSummaries().add(object0);
    result.setKeyCount(1);
    result.setTruncated(false);
    EasyMock.expect(s3Client.listObjectsV2(EasyMock.anyObject(ListObjectsV2Request.class))).andReturn(result).once();
    S3TimestampVersionedDataFinder finder = new S3TimestampVersionedDataFinder(s3Client);
    EasyMock.replay(s3Client);
    URI latest = finder.getLatestVersion(URI.create(StringUtils.format("s3://%s/%s", bucket, object0.getKey())), null);
    EasyMock.verify(s3Client);
    URI expected = URI.create(StringUtils.format("s3://%s/%s", bucket, object0.getKey()));
    Assert.assertEquals(expected, latest);
}

Example 77 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project druid by druid-io.

the class S3TimestampVersionedDataFinder method getLatestVersion.

/**
 * Gets the key with the most recently modified timestamp.
 * `pattern` is evaluated against the entire key AFTER the path given in `uri`.
 * The substring `pattern` is matched against will have a leading `/` removed.
 * For example `s3://some_bucket/some_prefix/some_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key`.
 * `s3://some_bucket/some_prefixsome_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key`
 * `s3://some_bucket/some_prefix//some_key` with a URI of `s3://some_bucket/some_prefix` will match against `/some_key`
 *
 * @param uri     The URI of in the form of `s3://some_bucket/some_key`
 * @param pattern The pattern matcher to determine if a *key* is of interest, or `null` to match everything.
 *
 * @return A URI to the most recently modified object which matched the pattern.
 */
@Override
public URI getLatestVersion(final URI uri, @Nullable final Pattern pattern) {
    try {
        final CloudObjectLocation coords = new CloudObjectLocation(S3Utils.checkURI(uri));
        long mostRecent = Long.MIN_VALUE;
        URI latest = null;
        final Iterator<S3ObjectSummary> objectSummaryIterator = S3Utils.objectSummaryIterator(s3Client, Collections.singletonList(uri), MAX_LISTING_KEYS);
        while (objectSummaryIterator.hasNext()) {
            final S3ObjectSummary objectSummary = objectSummaryIterator.next();
            final CloudObjectLocation objectLocation = S3Utils.summaryToCloudObjectLocation(objectSummary);
            // remove coords path prefix from object path
            String keyString = StringUtils.maybeRemoveLeadingSlash(objectLocation.getPath().substring(coords.getPath().length()));
            if (pattern != null && !pattern.matcher(keyString).matches()) {
                continue;
            }
            final long latestModified = objectSummary.getLastModified().getTime();
            if (latestModified >= mostRecent) {
                mostRecent = latestModified;
                latest = objectLocation.toUri(S3StorageDruidModule.SCHEME);
            }
        }
        return latest;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 78 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project druid by druid-io.

the class S3Utils method getSingleObjectSummary.

/**
 * Gets a single {@link S3ObjectSummary} from s3. Since this method might return a wrong object if there are multiple
 * objects that match the given key, this method should be used only when it's guaranteed that the given key is unique
 * in the given bucket.
 *
 * @param s3Client s3 client
 * @param bucket   s3 bucket
 * @param key      unique key for the object to be retrieved
 */
public static S3ObjectSummary getSingleObjectSummary(ServerSideEncryptingAmazonS3 s3Client, String bucket, String key) {
    final ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucket).withPrefix(key).withMaxKeys(1);
    final ListObjectsV2Result result = s3Client.listObjectsV2(request);
    // keyCount is still zero.
    if (result.getObjectSummaries().size() == 0) {
        throw new ISE("Cannot find object for bucket[%s] and key[%s]", bucket, key);
    }
    final S3ObjectSummary objectSummary = result.getObjectSummaries().get(0);
    if (!objectSummary.getBucketName().equals(bucket) || !objectSummary.getKey().equals(key)) {
        throw new ISE("Wrong object[%s] for bucket[%s] and key[%s]", objectSummary, bucket, key);
    }
    return objectSummary;
}

Example 79 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project aws-doc-sdk-examples by awsdocs.

the class DeleteBucket method main.

public static void main(String[] args) {
    final String USAGE = "\n" + "To run this example, supply the name of an S3 bucket\n" + "\n" + "Ex: DeleteBucket <bucketname>\n";
    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }
    String bucket_name = args[0];
    System.out.println("Deleting S3 bucket: " + bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    try {
        System.out.println(" - removing objects from bucket");
        ObjectListing object_listing = s3.listObjects(bucket_name);
        while (true) {
            for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext(); ) {
                S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
                s3.deleteObject(bucket_name, summary.getKey());
            }
            // more object_listing to retrieve?
            if (object_listing.isTruncated()) {
                object_listing = s3.listNextBatchOfObjects(object_listing);
            } else {
                break;
            }
        }
        System.out.println(" - removing versions from bucket");
        VersionListing version_listing = s3.listVersions(new ListVersionsRequest().withBucketName(bucket_name));
        while (true) {
            for (Iterator<?> iterator = version_listing.getVersionSummaries().iterator(); iterator.hasNext(); ) {
                S3VersionSummary vs = (S3VersionSummary) iterator.next();
                s3.deleteVersion(bucket_name, vs.getKey(), vs.getVersionId());
            }
            if (version_listing.isTruncated()) {
                version_listing = s3.listNextBatchOfVersions(version_listing);
            } else {
                break;
            }
        }
        System.out.println(" OK, bucket ready to delete!");
        s3.deleteBucket(bucket_name);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    System.out.println("Done!");
}

Example 80 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project aws-doc-sdk-examples by awsdocs.

the class S3EncryptV2 method main.

public static void main(String[] args) throws NoSuchAlgorithmException {
    AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    // of this example
    try {
        s3Client.createBucket(bucketName);
    } catch (AmazonS3Exception e) {
        System.err.println(e.getErrorMessage());
    }
    putEncryptedData1();
    putEncryptedData2();
    putEncryptedData3_Kms();
    try {
        ObjectListing objectListing = s3Client.listObjects(bucketName);
        while (true) {
            for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext(); ) {
                S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
                s3Client.deleteObject(bucketName, summary.getKey());
            }
            if (objectListing.isTruncated()) {
                objectListing = s3Client.listNextBatchOfObjects(objectListing);
            } else {
                break;
            }
        }
    } catch (AmazonS3Exception e) {
        System.err.println(e.getErrorMessage());
    }
    // delete test bucket
    try {
        s3Client.deleteBucket(bucketName);
    } catch (AmazonS3Exception e) {
        System.err.println(e.getErrorMessage());
    }
    s3Client.shutdown();
    System.out.println("Done");
}