Examples with CloudBlobHolder org.apache.druid.storage.azure.blob.CloudBlobHolder used on opensource projects

Example 1 with CloudBlobHolder

use of org.apache.druid.storage.azure.blob.CloudBlobHolder in project druid by druid-io.

the class AzureDataSegmentKillerTest method test_killAll_noException_deletesAllSegments.

@Test
public void test_killAll_noException_deletesAllSegments() throws Exception {
    EasyMock.expect(segmentConfig.getContainer()).andReturn(CONTAINER).atLeastOnce();
    EasyMock.expect(segmentConfig.getPrefix()).andReturn(PREFIX).atLeastOnce();
    EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS);
    EasyMock.expect(accountConfig.getMaxTries()).andReturn(MAX_TRIES).atLeastOnce();
    CloudBlobHolder object1 = AzureTestUtils.newCloudBlobHolder(CONTAINER, KEY_1, TIME_0);
    CloudBlobHolder object2 = AzureTestUtils.newCloudBlobHolder(CONTAINER, KEY_2, TIME_1);
    AzureCloudBlobIterable azureCloudBlobIterable = AzureTestUtils.expectListObjects(azureCloudBlobIterableFactory, MAX_KEYS, PREFIX_URI, ImmutableList.of(object1, object2));
    EasyMock.replay(object1, object2);
    AzureTestUtils.expectDeleteObjects(azureStorage, ImmutableList.of(object1, object2), ImmutableMap.of());
    EasyMock.replay(segmentConfig, inputDataConfig, accountConfig, azureCloudBlobIterable, azureCloudBlobIterableFactory, azureStorage);
    AzureDataSegmentKiller killer = new AzureDataSegmentKiller(segmentConfig, inputDataConfig, accountConfig, azureStorage, azureCloudBlobIterableFactory);
    killer.killAll();
    EasyMock.verify(segmentConfig, inputDataConfig, accountConfig, object1, object2, azureCloudBlobIterable, azureCloudBlobIterableFactory, azureStorage);
}

Example 2 with CloudBlobHolder

use of org.apache.druid.storage.azure.blob.CloudBlobHolder in project druid by druid-io.

the class AzureDataSegmentKillerTest method test_killAll_recoverableExceptionWhenListingObjects_deletesAllSegments.

@Test
public void test_killAll_recoverableExceptionWhenListingObjects_deletesAllSegments() throws Exception {
    EasyMock.expect(segmentConfig.getContainer()).andReturn(CONTAINER).atLeastOnce();
    EasyMock.expect(segmentConfig.getPrefix()).andReturn(PREFIX).atLeastOnce();
    EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS);
    EasyMock.expect(accountConfig.getMaxTries()).andReturn(MAX_TRIES).atLeastOnce();
    CloudBlobHolder object1 = AzureTestUtils.newCloudBlobHolder(CONTAINER, KEY_1, TIME_0);
    AzureCloudBlobIterable azureCloudBlobIterable = AzureTestUtils.expectListObjects(azureCloudBlobIterableFactory, MAX_KEYS, PREFIX_URI, ImmutableList.of(object1));
    EasyMock.replay(object1);
    AzureTestUtils.expectDeleteObjects(azureStorage, ImmutableList.of(object1), ImmutableMap.of(object1, RECOVERABLE_EXCEPTION));
    EasyMock.replay(segmentConfig, inputDataConfig, accountConfig, azureCloudBlobIterable, azureCloudBlobIterableFactory, azureStorage);
    AzureDataSegmentKiller killer = new AzureDataSegmentKiller(segmentConfig, inputDataConfig, accountConfig, azureStorage, azureCloudBlobIterableFactory);
    killer.killAll();
    EasyMock.verify(segmentConfig, inputDataConfig, accountConfig, object1, azureCloudBlobIterable, azureCloudBlobIterableFactory, azureStorage);
}

Example 3 with CloudBlobHolder

use of org.apache.druid.storage.azure.blob.CloudBlobHolder in project druid by druid-io.

the class AzureTaskLogsTest method test_killAll_nonrecoverableExceptionWhenListingObjects_doesntDeleteAnyTaskLogs.

@Test
public void test_killAll_nonrecoverableExceptionWhenListingObjects_doesntDeleteAnyTaskLogs() throws Exception {
    boolean ioExceptionThrown = false;
    CloudBlobHolder object1 = null;
    AzureCloudBlobIterable azureCloudBlobIterable = null;
    try {
        EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS);
        EasyMock.expect(accountConfig.getMaxTries()).andReturn(MAX_TRIES).atLeastOnce();
        EasyMock.expect(timeSupplier.getAsLong()).andReturn(TIME_NOW);
        object1 = AzureTestUtils.newCloudBlobHolder(CONTAINER, KEY_1, TIME_0);
        azureCloudBlobIterable = AzureTestUtils.expectListObjects(azureCloudBlobIterableFactory, MAX_KEYS, PREFIX_URI, ImmutableList.of(object1));
        EasyMock.replay(object1);
        AzureTestUtils.expectDeleteObjects(azureStorage, ImmutableList.of(), ImmutableMap.of(object1, NON_RECOVERABLE_EXCEPTION));
        EasyMock.replay(inputDataConfig, accountConfig, timeSupplier, azureCloudBlobIterable, azureCloudBlobIterableFactory, azureStorage);
        azureTaskLogs.killAll();
    } catch (IOException e) {
        ioExceptionThrown = true;
    }
    Assert.assertTrue(ioExceptionThrown);
    EasyMock.verify(inputDataConfig, accountConfig, timeSupplier, object1, azureCloudBlobIterable, azureCloudBlobIterableFactory, azureStorage);
}

Example 4 with CloudBlobHolder

use of org.apache.druid.storage.azure.blob.CloudBlobHolder in project druid by druid-io.

the class AzureTaskLogsTest method test_killOlderThan_recoverableExceptionWhenDeletingObjects_deletesOnlyTaskLogsOlderThan.

@Test
public void test_killOlderThan_recoverableExceptionWhenDeletingObjects_deletesOnlyTaskLogsOlderThan() throws Exception {
    EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS);
    EasyMock.expect(accountConfig.getMaxTries()).andReturn(MAX_TRIES).atLeastOnce();
    CloudBlobHolder object1 = AzureTestUtils.newCloudBlobHolder(CONTAINER, KEY_1, TIME_0);
    AzureCloudBlobIterable azureCloudBlobIterable = AzureTestUtils.expectListObjects(azureCloudBlobIterableFactory, MAX_KEYS, PREFIX_URI, ImmutableList.of(object1));
    EasyMock.replay(object1);
    AzureTestUtils.expectDeleteObjects(azureStorage, ImmutableList.of(object1), ImmutableMap.of(object1, RECOVERABLE_EXCEPTION));
    EasyMock.replay(inputDataConfig, accountConfig, timeSupplier, azureCloudBlobIterable, azureCloudBlobIterableFactory, azureStorage);
    azureTaskLogs.killOlderThan(TIME_NOW);
    EasyMock.verify(inputDataConfig, accountConfig, timeSupplier, object1, azureCloudBlobIterable, azureCloudBlobIterableFactory, azureStorage);
}

Example 5 with CloudBlobHolder

use of org.apache.druid.storage.azure.blob.CloudBlobHolder in project druid by druid-io.

the class AzureTaskLogsTest method test_killAll_recoverableExceptionWhenDeletingObjects_deletesAllTaskLogs.

@Test
public void test_killAll_recoverableExceptionWhenDeletingObjects_deletesAllTaskLogs() throws Exception {
    EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS);
    EasyMock.expect(accountConfig.getMaxTries()).andReturn(MAX_TRIES).atLeastOnce();
    EasyMock.expect(timeSupplier.getAsLong()).andReturn(TIME_NOW);
    CloudBlobHolder object1 = AzureTestUtils.newCloudBlobHolder(CONTAINER, KEY_1, TIME_0);
    AzureCloudBlobIterable azureCloudBlobIterable = AzureTestUtils.expectListObjects(azureCloudBlobIterableFactory, MAX_KEYS, PREFIX_URI, ImmutableList.of(object1));
    EasyMock.replay(object1);
    AzureTestUtils.expectDeleteObjects(azureStorage, ImmutableList.of(object1), ImmutableMap.of(object1, RECOVERABLE_EXCEPTION));
    EasyMock.replay(inputDataConfig, accountConfig, timeSupplier, azureCloudBlobIterable, azureCloudBlobIterableFactory, azureStorage);
    azureTaskLogs.killAll();
    EasyMock.verify(inputDataConfig, accountConfig, timeSupplier, object1, azureCloudBlobIterable, azureCloudBlobIterableFactory, azureStorage);
}