Examples with CloudBlobHolder org.apache.druid.storage.azure.blob.CloudBlobHolder used on opensource projects

Example 11 with CloudBlobHolder

use of org.apache.druid.storage.azure.blob.CloudBlobHolder in project druid by druid-io.

the class AzureCloudBlobIteratorTest method test_next_emptyObjects_skipEmptyObjects.

@Test
public void test_next_emptyObjects_skipEmptyObjects() throws URISyntaxException, StorageException {
    EasyMock.expect(config.getMaxTries()).andReturn(MAX_TRIES).atLeastOnce();
    EasyMock.expect(cloudBlobItemPrefixWithOnlyCloudBlobs1.isCloudBlob()).andReturn(true);
    EasyMock.expect(cloudBlobItemPrefixWithOnlyCloudBlobs1.getCloudBlob()).andReturn(cloudBlobDruidPrefixWithOnlyCloudBlobs1).anyTimes();
    EasyMock.expect(blobItemDruidFactory.create(blobItemPrefixWithOnlyCloudBlobs1)).andReturn(cloudBlobItemPrefixWithOnlyCloudBlobs1);
    ListBlobItem emptyBlobItem = createMock(ListBlobItem.class);
    ListBlobItemHolder emptyBlobItemHolder = createMock(ListBlobItemHolder.class);
    CloudBlobHolder emptyBlobHolder = createMock(CloudBlobHolder.class);
    EasyMock.expect(emptyBlobHolder.getBlobLength()).andReturn(0L).anyTimes();
    EasyMock.expect(emptyBlobItemHolder.isCloudBlob()).andReturn(true);
    EasyMock.expect(emptyBlobItemHolder.getCloudBlob()).andReturn(emptyBlobHolder).anyTimes();
    EasyMock.expect(blobItemDruidFactory.create(emptyBlobItem)).andReturn(emptyBlobItemHolder);
    EasyMock.expect(storage.listBlobsWithPrefixInContainerSegmented(CONTAINER1, PREFIX_ONLY_CLOUD_BLOBS, nullResultContinuationToken, MAX_LISTING_LENGTH)).andReturn(resultSegmentPrefixOnlyAndFailLessThanMaxTriesCloudBlobs1);
    EasyMock.expect(resultSegmentPrefixOnlyAndFailLessThanMaxTriesCloudBlobs1.getContinuationToken()).andReturn(nullResultContinuationToken);
    ArrayList<ListBlobItem> resultBlobItemsPrefixWithOnlyCloudBlobs1 = new ArrayList<>();
    resultBlobItemsPrefixWithOnlyCloudBlobs1.add(blobItemPrefixWithOnlyCloudBlobs1);
    resultBlobItemsPrefixWithOnlyCloudBlobs1.add(emptyBlobItem);
    EasyMock.expect(resultSegmentPrefixOnlyAndFailLessThanMaxTriesCloudBlobs1.getResults()).andReturn(resultBlobItemsPrefixWithOnlyCloudBlobs1);
    replayAll();
    azureCloudBlobIterator = new AzureCloudBlobIterator(storage, blobItemDruidFactory, config, ImmutableList.of(PREFIX_ONLY_CLOUD_BLOBS_URI), MAX_LISTING_LENGTH);
    List<CloudBlobHolder> expectedBlobItems = ImmutableList.of(cloudBlobDruidPrefixWithOnlyCloudBlobs1);
    List<CloudBlobHolder> actualBlobItems = Lists.newArrayList(azureCloudBlobIterator);
    Assert.assertEquals(expectedBlobItems.size(), actualBlobItems.size());
    Assert.assertTrue(expectedBlobItems.containsAll(actualBlobItems));
    verifyAll();
}

Example 12 with CloudBlobHolder

use of org.apache.druid.storage.azure.blob.CloudBlobHolder in project druid by druid-io.

the class AzureInputSourceTest method test_getPrefixesSplitStream_successfullyCreatesCloudLocation_returnsExpectedLocations.

@Test
public void test_getPrefixesSplitStream_successfullyCreatesCloudLocation_returnsExpectedLocations() {
    List<URI> prefixes = ImmutableList.of(PREFIX_URI);
    List<List<CloudObjectLocation>> expectedCloudLocations = ImmutableList.of(ImmutableList.of(CLOUD_OBJECT_LOCATION_1));
    List<CloudBlobHolder> expectedCloudBlobs = ImmutableList.of(cloudBlobDruid1);
    Iterator<CloudBlobHolder> expectedCloudBlobsIterator = expectedCloudBlobs.iterator();
    EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_LISTING_LENGTH);
    EasyMock.expect(azureCloudBlobIterableFactory.create(prefixes, MAX_LISTING_LENGTH)).andReturn(azureCloudBlobIterable);
    EasyMock.expect(azureCloudBlobIterable.iterator()).andReturn(expectedCloudBlobsIterator);
    EasyMock.expect(azureCloudBlobToLocationConverter.createCloudObjectLocation(cloudBlobDruid1)).andReturn(CLOUD_OBJECT_LOCATION_1);
    EasyMock.expect(cloudBlobDruid1.getBlobLength()).andReturn(100L).anyTimes();
    replayAll();
    azureInputSource = new AzureInputSource(storage, entityFactory, azureCloudBlobIterableFactory, azureCloudBlobToLocationConverter, inputDataConfig, EMPTY_URIS, prefixes, EMPTY_OBJECTS);
    Stream<InputSplit<List<CloudObjectLocation>>> cloudObjectStream = azureInputSource.getPrefixesSplitStream(new MaxSizeSplitHintSpec(null, 1));
    List<List<CloudObjectLocation>> actualCloudLocationList = cloudObjectStream.map(InputSplit::get).collect(Collectors.toList());
    verifyAll();
    Assert.assertEquals(expectedCloudLocations, actualCloudLocationList);
}

Example 13 with CloudBlobHolder

use of org.apache.druid.storage.azure.blob.CloudBlobHolder in project druid by druid-io.

the class AzureDataSegmentKillerTest method test_killAll_nonrecoverableExceptionWhenListingObjects_deletesAllSegments.

@Test
public void test_killAll_nonrecoverableExceptionWhenListingObjects_deletesAllSegments() throws Exception {
    boolean ioExceptionThrown = false;
    CloudBlobHolder object1 = null;
    AzureCloudBlobIterable azureCloudBlobIterable = null;
    try {
        EasyMock.expect(segmentConfig.getContainer()).andReturn(CONTAINER).atLeastOnce();
        EasyMock.expect(segmentConfig.getPrefix()).andReturn(PREFIX).atLeastOnce();
        EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS);
        EasyMock.expect(accountConfig.getMaxTries()).andReturn(MAX_TRIES).atLeastOnce();
        object1 = AzureTestUtils.newCloudBlobHolder(CONTAINER, KEY_1, TIME_0);
        azureCloudBlobIterable = AzureTestUtils.expectListObjects(azureCloudBlobIterableFactory, MAX_KEYS, PREFIX_URI, ImmutableList.of(object1));
        EasyMock.replay(object1);
        AzureTestUtils.expectDeleteObjects(azureStorage, ImmutableList.of(), ImmutableMap.of(object1, NON_RECOVERABLE_EXCEPTION));
        EasyMock.replay(segmentConfig, inputDataConfig, accountConfig, azureCloudBlobIterable, azureCloudBlobIterableFactory, azureStorage);
        AzureDataSegmentKiller killer = new AzureDataSegmentKiller(segmentConfig, inputDataConfig, accountConfig, azureStorage, azureCloudBlobIterableFactory);
        killer.killAll();
    } catch (IOException e) {
        ioExceptionThrown = true;
    }
    Assert.assertTrue(ioExceptionThrown);
    EasyMock.verify(segmentConfig, inputDataConfig, accountConfig, object1, azureCloudBlobIterable, azureCloudBlobIterableFactory, azureStorage);
}

Example 14 with CloudBlobHolder

use of org.apache.druid.storage.azure.blob.CloudBlobHolder in project druid by druid-io.

the class AzureCloudBlobIterator method next.

@Override
public CloudBlobHolder next() {
    if (currentBlobItem == null) {
        throw new NoSuchElementException();
    }
    final CloudBlobHolder retVal = currentBlobItem;
    advanceBlobItem();
    return retVal;
}

Example 15 with CloudBlobHolder

use of org.apache.druid.storage.azure.blob.CloudBlobHolder in project druid by druid-io.

the class AzureUtils method deleteObjectsInPath.

/**
 * Delete the files from Azure Storage in a specified bucket, matching a specified prefix and filter
 *
 * @param storage Azure Storage client
 * @param config  specifies the configuration to use when finding matching files in Azure Storage to delete
 * @param bucket  Azure Storage bucket
 * @param prefix  the file prefix
 * @param filter  function which returns true if the prefix file found should be deleted and false otherwise.
 * @throws Exception
 */
public static void deleteObjectsInPath(AzureStorage storage, AzureInputDataConfig config, AzureAccountConfig accountConfig, AzureCloudBlobIterableFactory azureCloudBlobIterableFactory, String bucket, String prefix, Predicate<CloudBlobHolder> filter) throws Exception {
    AzureCloudBlobIterable azureCloudBlobIterable = azureCloudBlobIterableFactory.create(ImmutableList.of(new CloudObjectLocation(bucket, prefix).toUri("azure")), config.getMaxListingLength());
    Iterator<CloudBlobHolder> iterator = azureCloudBlobIterable.iterator();
    while (iterator.hasNext()) {
        final CloudBlobHolder nextObject = iterator.next();
        if (filter.apply(nextObject)) {
            deleteBucketKeys(storage, accountConfig.getMaxTries(), nextObject.getContainerName(), nextObject.getName());
        }
    }
}