Examples with CloudObjectLocation org.apache.druid.data.input.impl.CloudObjectLocation used on opensource projects

Example 11 with CloudObjectLocation

use of org.apache.druid.data.input.impl.CloudObjectLocation in project druid by druid-io.

the class OssInputSourceTest method testWithPrefixesSplit.

@Test
public void testWithPrefixesSplit() {
    EasyMock.reset(OSSCLIENT);
    expectListObjects(PREFIXES.get(0), ImmutableList.of(EXPECTED_URIS.get(0)), CONTENT);
    expectListObjects(PREFIXES.get(1), ImmutableList.of(EXPECTED_URIS.get(1)), CONTENT);
    EasyMock.replay(OSSCLIENT);
    OssInputSource inputSource = new OssInputSource(OSSCLIENT, INPUT_DATA_CONFIG, null, PREFIXES, null, null);
    Stream<InputSplit<List<CloudObjectLocation>>> splits = inputSource.createSplits(new JsonInputFormat(JSONPathSpec.DEFAULT, null, null), new MaxSizeSplitHintSpec(null, 1));
    Assert.assertEquals(EXPECTED_COORDS, splits.map(InputSplit::get).collect(Collectors.toList()));
    EasyMock.verify(OSSCLIENT);
}

Example 12 with CloudObjectLocation

use of org.apache.druid.data.input.impl.CloudObjectLocation in project druid by druid-io.

the class OssDataSegmentPullerTest method testGZUncompressRetries.

@Test
public void testGZUncompressRetries() throws IOException, SegmentLoadingException {
    final String bucket = "bucket";
    final String keyPrefix = "prefix/dir/0";
    final OSS ossClient = EasyMock.createStrictMock(OSS.class);
    final byte[] value = bucket.getBytes(StandardCharsets.UTF_8);
    final File tmpFile = temporaryFolder.newFile("gzTest.gz");
    try (OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(tmpFile))) {
        outputStream.write(value);
    }
    OSSObject object0 = new OSSObject();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.getObjectMetadata().setLastModified(new Date(0));
    object0.setObjectContent(new FileInputStream(tmpFile));
    final OSSObjectSummary objectSummary = new OSSObjectSummary();
    objectSummary.setBucketName(bucket);
    objectSummary.setKey(keyPrefix + "/renames-0.gz");
    objectSummary.setLastModified(new Date(0));
    final ObjectListing listObjectsResult = new ObjectListing();
    listObjectsResult.getObjectSummaries().add(objectSummary);
    File tmpDir = temporaryFolder.newFolder("gzTestDir");
    OSSException exception = new OSSException("OssDataSegmentPullerTest", "NoSuchKey", null, null, null, null, null);
    EasyMock.expect(ossClient.doesObjectExist(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(true).once();
    EasyMock.expect(ossClient.listObjects(EasyMock.anyObject(ListObjectsRequest.class))).andReturn(listObjectsResult).once();
    EasyMock.expect(ossClient.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))).andThrow(exception).once();
    EasyMock.expect(ossClient.listObjects(EasyMock.anyObject(ListObjectsRequest.class))).andReturn(listObjectsResult).once();
    EasyMock.expect(ossClient.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    OssDataSegmentPuller puller = new OssDataSegmentPuller(ossClient);
    EasyMock.replay(ossClient);
    FileUtils.FileCopyResult result = puller.getSegmentFiles(new CloudObjectLocation(bucket, object0.getKey()), tmpDir);
    EasyMock.verify(ossClient);
    Assert.assertEquals(value.length, result.size());
    File expected = new File(tmpDir, "renames-0");
    Assert.assertTrue(expected.exists());
    Assert.assertEquals(value.length, expected.length());
}

Example 13 with CloudObjectLocation

use of org.apache.druid.data.input.impl.CloudObjectLocation in project druid by druid-io.

the class OssDataSegmentPullerTest method testGZUncompress.

@Test
public void testGZUncompress() throws IOException, SegmentLoadingException {
    final String bucket = "bucket";
    final String keyPrefix = "prefix/dir/0";
    final OSS ossClient = EasyMock.createStrictMock(OSS.class);
    final byte[] value = bucket.getBytes(StandardCharsets.UTF_8);
    final File tmpFile = temporaryFolder.newFile("gzTest.gz");
    try (OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(tmpFile))) {
        outputStream.write(value);
    }
    final OSSObject object0 = new OSSObject();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.getObjectMetadata().setLastModified(new Date(0));
    object0.setObjectContent(new FileInputStream(tmpFile));
    final OSSObjectSummary objectSummary = new OSSObjectSummary();
    objectSummary.setBucketName(bucket);
    objectSummary.setKey(keyPrefix + "/renames-0.gz");
    objectSummary.setLastModified(new Date(0));
    final ObjectListing listObjectsResult = new ObjectListing();
    listObjectsResult.getObjectSummaries().add(objectSummary);
    final File tmpDir = temporaryFolder.newFolder("gzTestDir");
    EasyMock.expect(ossClient.doesObjectExist(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(true).once();
    EasyMock.expect(ossClient.listObjects(EasyMock.anyObject(ListObjectsRequest.class))).andReturn(listObjectsResult).once();
    EasyMock.expect(ossClient.getObject(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    OssDataSegmentPuller puller = new OssDataSegmentPuller(ossClient);
    EasyMock.replay(ossClient);
    FileUtils.FileCopyResult result = puller.getSegmentFiles(new CloudObjectLocation(bucket, object0.getKey()), tmpDir);
    EasyMock.verify(ossClient);
    Assert.assertEquals(value.length, result.size());
    File expected = new File(tmpDir, "renames-0");
    Assert.assertTrue(expected.exists());
    Assert.assertEquals(value.length, expected.length());
}

Example 14 with CloudObjectLocation

use of org.apache.druid.data.input.impl.CloudObjectLocation in project druid by druid-io.

the class AzureCloudBlobHolderToCloudObjectLocationConverterTest method test_createCloudObjectLocation_returnsExpectedLocation.

@Test
public void test_createCloudObjectLocation_returnsExpectedLocation() throws Exception {
    EasyMock.expect(cloudBlob.getContainerName()).andReturn(CONTAINER1);
    EasyMock.expect(cloudBlob.getName()).andReturn(BLOB1);
    replayAll();
    CloudObjectLocation expectedLocation = new CloudObjectLocation(CONTAINER1, BLOB1);
    converter = new AzureCloudBlobHolderToCloudObjectLocationConverter();
    CloudObjectLocation actualLocation = converter.createCloudObjectLocation(cloudBlob);
    Assert.assertEquals(expectedLocation, actualLocation);
    verifyAll();
}

Example 15 with CloudObjectLocation

use of org.apache.druid.data.input.impl.CloudObjectLocation in project druid by druid-io.

the class GoogleCloudStorageInputSourceTest method addExpectedGetObjectMock.

private static void addExpectedGetObjectMock(URI uri) throws IOException {
    CloudObjectLocation location = new CloudObjectLocation(uri);
    EasyMock.expect(STORAGE.get(EasyMock.eq(location.getBucket()), EasyMock.eq(location.getPath()), EasyMock.eq(0L))).andReturn(new ByteArrayInputStream(CONTENT)).once();
}