Search in sources :

Example 16 with SegmentPublishResult

use of org.apache.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinatorTest method testTransactionalAnnounceRetryAndSuccess.

@Test
public void testTransactionalAnnounceRetryAndSuccess() throws IOException {
    final AtomicLong attemptCounter = new AtomicLong();
    final IndexerSQLMetadataStorageCoordinator failOnceCoordinator = new IndexerSQLMetadataStorageCoordinator(mapper, derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector) {

        @Override
        protected DataStoreMetadataUpdateResult updateDataSourceMetadataWithHandle(Handle handle, String dataSource, DataSourceMetadata startMetadata, DataSourceMetadata endMetadata) throws IOException {
            metadataUpdateCounter.getAndIncrement();
            if (attemptCounter.getAndIncrement() == 0) {
                return DataStoreMetadataUpdateResult.TRY_AGAIN;
            } else {
                return super.updateDataSourceMetadataWithHandle(handle, dataSource, startMetadata, endMetadata);
            }
        }
    };
    // Insert first segment.
    final SegmentPublishResult result1 = failOnceCoordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment), ImmutableSet.of(), new ObjectMetadata(null), new ObjectMetadata(ImmutableMap.of("foo", "bar")));
    Assert.assertEquals(SegmentPublishResult.ok(ImmutableSet.of(defaultSegment)), result1);
    Assert.assertArrayEquals(mapper.writeValueAsString(defaultSegment).getBytes(StandardCharsets.UTF_8), derbyConnector.lookup(derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(), "id", "payload", defaultSegment.getId().toString()));
    // Reset attempt counter to induce another failure.
    attemptCounter.set(0);
    // Insert second segment.
    final SegmentPublishResult result2 = failOnceCoordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment2), ImmutableSet.of(), new ObjectMetadata(ImmutableMap.of("foo", "bar")), new ObjectMetadata(ImmutableMap.of("foo", "baz")));
    Assert.assertEquals(SegmentPublishResult.ok(ImmutableSet.of(defaultSegment2)), result2);
    Assert.assertArrayEquals(mapper.writeValueAsString(defaultSegment2).getBytes(StandardCharsets.UTF_8), derbyConnector.lookup(derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(), "id", "payload", defaultSegment2.getId().toString()));
    // Examine metadata.
    Assert.assertEquals(new ObjectMetadata(ImmutableMap.of("foo", "baz")), failOnceCoordinator.retrieveDataSourceMetadata("fooDataSource"));
    // Should be tried twice per call.
    Assert.assertEquals(4, metadataUpdateCounter.get());
}
Also used : SegmentPublishResult(org.apache.druid.indexing.overlord.SegmentPublishResult) AtomicLong(java.util.concurrent.atomic.AtomicLong) DataSourceMetadata(org.apache.druid.indexing.overlord.DataSourceMetadata) ObjectMetadata(org.apache.druid.indexing.overlord.ObjectMetadata) Handle(org.skife.jdbi.v2.Handle) Test(org.junit.Test)

Example 17 with SegmentPublishResult

use of org.apache.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinatorTest method testTransactionalAnnounceFailDbNullWantNotNull.

@Test
public void testTransactionalAnnounceFailDbNullWantNotNull() throws IOException {
    final SegmentPublishResult result1 = coordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment), ImmutableSet.of(), new ObjectMetadata(ImmutableMap.of("foo", "bar")), new ObjectMetadata(ImmutableMap.of("foo", "baz")));
    Assert.assertEquals(SegmentPublishResult.fail("java.lang.RuntimeException: Aborting transaction!"), result1);
    // Should only be tried once.
    Assert.assertEquals(1, metadataUpdateCounter.get());
}
Also used : SegmentPublishResult(org.apache.druid.indexing.overlord.SegmentPublishResult) ObjectMetadata(org.apache.druid.indexing.overlord.ObjectMetadata) Test(org.junit.Test)

Aggregations

SegmentPublishResult (org.apache.druid.indexing.overlord.SegmentPublishResult)17 Test (org.junit.Test)14 ObjectMetadata (org.apache.druid.indexing.overlord.ObjectMetadata)7 DataSegment (org.apache.druid.timeline.DataSegment)7 NoopTask (org.apache.druid.indexing.common.task.NoopTask)5 Task (org.apache.druid.indexing.common.task.Task)5 IOException (java.io.IOException)3 HashSet (java.util.HashSet)2 Set (java.util.Set)2 LinearShardSpec (org.apache.druid.timeline.partition.LinearShardSpec)2 Handle (org.skife.jdbi.v2.Handle)2 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)1 TypeReference (com.fasterxml.jackson.core.type.TypeReference)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Iterables (com.google.common.collect.Iterables)1 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)1 ListeningExecutorService (com.google.common.util.concurrent.ListeningExecutorService)1