Examples with CloudObjectLocation org.apache.druid.data.input.impl.CloudObjectLocation used on opensource projects

Example 31 with CloudObjectLocation

use of org.apache.druid.data.input.impl.CloudObjectLocation in project druid by druid-io.

the class S3InputSourceTest method testS3InputSourceUseDefaultPasswordWhenCloudConfigPropertiesWithoutCrediential.

@Test
public void testS3InputSourceUseDefaultPasswordWhenCloudConfigPropertiesWithoutCrediential() {
    S3InputSourceConfig mockConfigPropertiesWithoutKeyAndSecret = EasyMock.createMock(S3InputSourceConfig.class);
    EasyMock.reset(mockConfigPropertiesWithoutKeyAndSecret);
    EasyMock.expect(mockConfigPropertiesWithoutKeyAndSecret.getAssumeRoleArn()).andStubReturn(null);
    EasyMock.expect(mockConfigPropertiesWithoutKeyAndSecret.isCredentialsConfigured()).andStubReturn(false);
    EasyMock.replay(mockConfigPropertiesWithoutKeyAndSecret);
    EasyMock.reset(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER);
    EasyMock.expect(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER.build()).andReturn(SERVICE);
    EasyMock.replay(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER);
    final S3InputSource withPrefixes = new S3InputSource(SERVICE, SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER, INPUT_DATA_CONFIG, null, null, EXPECTED_LOCATION, mockConfigPropertiesWithoutKeyAndSecret);
    Assert.assertNotNull(withPrefixes);
    // This is to force the s3ClientSupplier to initialize the ServerSideEncryptingAmazonS3
    withPrefixes.createEntity(new CloudObjectLocation("bucket", "path"));
    EasyMock.verify(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER);
    EasyMock.verify(mockConfigPropertiesWithoutKeyAndSecret);
}

Example 32 with CloudObjectLocation

use of org.apache.druid.data.input.impl.CloudObjectLocation in project druid by druid-io.

the class S3InputSourceTest method testSerdeWithCloudConfigPropertiesWithKeyAndSecret.

@Test
public void testSerdeWithCloudConfigPropertiesWithKeyAndSecret() throws Exception {
    EasyMock.reset(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER);
    EasyMock.expect(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER.getAmazonS3ClientBuilder()).andStubReturn(AMAZON_S3_CLIENT_BUILDER);
    EasyMock.expect(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER.build()).andReturn(SERVICE);
    EasyMock.replay(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER);
    final S3InputSource withPrefixes = new S3InputSource(SERVICE, SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER, INPUT_DATA_CONFIG, null, null, EXPECTED_LOCATION, CLOUD_CONFIG_PROPERTIES);
    final S3InputSource serdeWithPrefixes = MAPPER.readValue(MAPPER.writeValueAsString(withPrefixes), S3InputSource.class);
    // This is to force the s3ClientSupplier to initialize the ServerSideEncryptingAmazonS3
    serdeWithPrefixes.createEntity(new CloudObjectLocation("bucket", "path"));
    Assert.assertEquals(withPrefixes, serdeWithPrefixes);
    EasyMock.verify(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER);
}

Example 33 with CloudObjectLocation

use of org.apache.druid.data.input.impl.CloudObjectLocation in project druid by druid-io.

the class S3InputSourceTest method testCreateSplitsWithEmptyObjectsIteratingOnlyNonEmptyObjects.

@Test
public void testCreateSplitsWithEmptyObjectsIteratingOnlyNonEmptyObjects() {
    EasyMock.reset(S3_CLIENT);
    expectListObjects(PREFIXES.get(0), ImmutableList.of(EXPECTED_URIS.get(0)), CONTENT);
    expectListObjects(PREFIXES.get(1), ImmutableList.of(EXPECTED_URIS.get(1)), new byte[0]);
    EasyMock.replay(S3_CLIENT);
    S3InputSource inputSource = new S3InputSource(SERVICE, SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER, INPUT_DATA_CONFIG, null, PREFIXES, null, null);
    Stream<InputSplit<List<CloudObjectLocation>>> splits = inputSource.createSplits(new JsonInputFormat(JSONPathSpec.DEFAULT, null, null), null);
    Assert.assertEquals(ImmutableList.of(ImmutableList.of(new CloudObjectLocation(EXPECTED_URIS.get(0)))), splits.map(InputSplit::get).collect(Collectors.toList()));
    EasyMock.verify(S3_CLIENT);
}

Example 34 with CloudObjectLocation

use of org.apache.druid.data.input.impl.CloudObjectLocation in project druid by druid-io.

the class S3InputSourceTest method testWithUrisSplit.

@Test
public void testWithUrisSplit() {
    S3InputSource inputSource = new S3InputSource(SERVICE, SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER, INPUT_DATA_CONFIG, EXPECTED_URIS, null, null, null);
    Stream<InputSplit<List<CloudObjectLocation>>> splits = inputSource.createSplits(new JsonInputFormat(JSONPathSpec.DEFAULT, null, null), null);
    Assert.assertEquals(EXPECTED_COORDS, splits.map(InputSplit::get).collect(Collectors.toList()));
}