Examples with S3ObjectSummary com.amazonaws.services.s3.model.S3ObjectSummary used on opensource projects

Example 71 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project druid by druid-io.

the class S3TaskLogsTest method test_killAll_nonrecoverableExceptionWhenListingObjects_doesntDeleteAnyTaskLogs.

@Test
public void test_killAll_nonrecoverableExceptionWhenListingObjects_doesntDeleteAnyTaskLogs() {
    boolean ioExceptionThrown = false;
    try {
        S3ObjectSummary objectSummary1 = S3TestUtils.newS3ObjectSummary(TEST_BUCKET, KEY_1, TIME_0);
        EasyMock.expect(timeSupplier.getAsLong()).andReturn(TIME_NOW);
        S3TestUtils.expectListObjects(s3Client, PREFIX_URI, ImmutableList.of(objectSummary1));
        DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET).withBucketName(TEST_BUCKET).withKeys(ImmutableList.of(new DeleteObjectsRequest.KeyVersion(KEY_1)));
        S3TestUtils.mockS3ClientDeleteObjects(s3Client, ImmutableList.of(), ImmutableMap.of(deleteRequest1, NON_RECOVERABLE_EXCEPTION));
        EasyMock.replay(s3Client, timeSupplier);
        S3TaskLogsConfig config = new S3TaskLogsConfig();
        config.setS3Bucket(TEST_BUCKET);
        config.setS3Prefix(TEST_PREFIX);
        S3InputDataConfig inputDataConfig = new S3InputDataConfig();
        inputDataConfig.setMaxListingLength(MAX_KEYS);
        S3TaskLogs s3TaskLogs = new S3TaskLogs(s3Client, config, inputDataConfig, timeSupplier);
        s3TaskLogs.killAll();
    } catch (IOException e) {
        ioExceptionThrown = true;
    }
    Assert.assertTrue(ioExceptionThrown);
    EasyMock.verify(s3Client, timeSupplier);
}

Example 72 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project druid by druid-io.

the class S3TaskLogsTest method test_killOlderThan_noException_deletesOnlyTaskLogsOlderThan.

@Test
public void test_killOlderThan_noException_deletesOnlyTaskLogsOlderThan() throws IOException {
    S3ObjectSummary objectSummary1 = S3TestUtils.newS3ObjectSummary(TEST_BUCKET, KEY_1, TIME_0);
    S3ObjectSummary objectSummary2 = S3TestUtils.newS3ObjectSummary(TEST_BUCKET, KEY_2, TIME_FUTURE);
    S3TestUtils.expectListObjects(s3Client, PREFIX_URI, ImmutableList.of(objectSummary1, objectSummary2));
    DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET).withBucketName(TEST_BUCKET).withKeys(ImmutableList.of(new DeleteObjectsRequest.KeyVersion(KEY_1)));
    S3TestUtils.mockS3ClientDeleteObjects(s3Client, ImmutableList.of(deleteRequest1), ImmutableMap.of());
    EasyMock.replay(s3Client, timeSupplier);
    S3TaskLogsConfig config = new S3TaskLogsConfig();
    config.setS3Bucket(TEST_BUCKET);
    config.setS3Prefix(TEST_PREFIX);
    S3InputDataConfig inputDataConfig = new S3InputDataConfig();
    inputDataConfig.setMaxListingLength(MAX_KEYS);
    S3TaskLogs s3TaskLogs = new S3TaskLogs(s3Client, config, inputDataConfig, timeSupplier);
    s3TaskLogs.killOlderThan(TIME_NOW);
    EasyMock.verify(s3Client, timeSupplier);
}

Example 73 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project druid by druid-io.

the class S3TaskLogsTest method test_killAll_recoverableExceptionWhenDeletingObjects_deletesAllTaskLogs.

@Test
public void test_killAll_recoverableExceptionWhenDeletingObjects_deletesAllTaskLogs() throws IOException {
    S3ObjectSummary objectSummary1 = S3TestUtils.newS3ObjectSummary(TEST_BUCKET, KEY_1, TIME_0);
    EasyMock.expect(timeSupplier.getAsLong()).andReturn(TIME_NOW);
    S3TestUtils.expectListObjects(s3Client, PREFIX_URI, ImmutableList.of(objectSummary1));
    DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET).withBucketName(TEST_BUCKET).withKeys(ImmutableList.of(new DeleteObjectsRequest.KeyVersion(KEY_1)));
    S3TestUtils.mockS3ClientDeleteObjects(s3Client, ImmutableList.of(deleteRequest1), ImmutableMap.of(deleteRequest1, RECOVERABLE_EXCEPTION));
    EasyMock.replay(s3Client, timeSupplier);
    S3TaskLogsConfig config = new S3TaskLogsConfig();
    config.setS3Bucket(TEST_BUCKET);
    config.setS3Prefix(TEST_PREFIX);
    S3InputDataConfig inputDataConfig = new S3InputDataConfig();
    inputDataConfig.setMaxListingLength(MAX_KEYS);
    S3TaskLogs s3TaskLogs = new S3TaskLogs(s3Client, config, inputDataConfig, timeSupplier);
    s3TaskLogs.killAll();
    EasyMock.verify(s3Client, timeSupplier);
}

Example 74 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project druid by druid-io.

the class S3TaskLogsTest method test_killOlderThan_nonrecoverableExceptionWhenListingObjects_doesntDeleteAnyTaskLogs.

@Test
public void test_killOlderThan_nonrecoverableExceptionWhenListingObjects_doesntDeleteAnyTaskLogs() {
    boolean ioExceptionThrown = false;
    try {
        S3ObjectSummary objectSummary1 = S3TestUtils.newS3ObjectSummary(TEST_BUCKET, KEY_1, TIME_0);
        S3TestUtils.expectListObjects(s3Client, PREFIX_URI, ImmutableList.of(objectSummary1));
        DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET).withBucketName(TEST_BUCKET).withKeys(ImmutableList.of(new DeleteObjectsRequest.KeyVersion(KEY_1)));
        S3TestUtils.mockS3ClientDeleteObjects(s3Client, ImmutableList.of(), ImmutableMap.of(deleteRequest1, NON_RECOVERABLE_EXCEPTION));
        EasyMock.replay(s3Client, timeSupplier);
        S3TaskLogsConfig config = new S3TaskLogsConfig();
        config.setS3Bucket(TEST_BUCKET);
        config.setS3Prefix(TEST_PREFIX);
        S3InputDataConfig inputDataConfig = new S3InputDataConfig();
        inputDataConfig.setMaxListingLength(MAX_KEYS);
        S3TaskLogs s3TaskLogs = new S3TaskLogs(s3Client, config, inputDataConfig, timeSupplier);
        s3TaskLogs.killOlderThan(TIME_NOW);
    } catch (IOException e) {
        ioExceptionThrown = true;
    }
    Assert.assertTrue(ioExceptionThrown);
    EasyMock.verify(s3Client, timeSupplier);
}

Example 75 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project druid by druid-io.

the class S3TimestampVersionedDataFinderTest method testSimpleLatestVersion.

@Test
public void testSimpleLatestVersion() {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    ServerSideEncryptingAmazonS3 s3Client = EasyMock.createStrictMock(ServerSideEncryptingAmazonS3.class);
    S3ObjectSummary object0 = new S3ObjectSummary(), object1 = new S3ObjectSummary();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModified(new Date(0));
    object0.setSize(10);
    object1.setBucketName(bucket);
    object1.setKey(keyPrefix + "/renames-1.gz");
    object1.setLastModified(new Date(1));
    object1.setSize(10);
    final ListObjectsV2Result result = new ListObjectsV2Result();
    result.getObjectSummaries().add(object0);
    result.getObjectSummaries().add(object1);
    result.setKeyCount(2);
    result.setTruncated(false);
    EasyMock.expect(s3Client.listObjectsV2(EasyMock.anyObject(ListObjectsV2Request.class))).andReturn(result).once();
    S3TimestampVersionedDataFinder finder = new S3TimestampVersionedDataFinder(s3Client);
    Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz");
    EasyMock.replay(s3Client);
    URI latest = finder.getLatestVersion(URI.create(StringUtils.format("s3://%s/%s", bucket, keyPrefix)), pattern);
    EasyMock.verify(s3Client);
    URI expected = URI.create(StringUtils.format("s3://%s/%s", bucket, object1.getKey()));
    Assert.assertEquals(expected, latest);
}