Examples with SegmentPublishResult io.druid.indexing.overlord.SegmentPublishResult used on opensource projects

Example 6 with SegmentPublishResult

use of io.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class SegmentTransactionalInsertAction method perform.

/**
   * Behaves similarly to
   * {@link io.druid.indexing.overlord.IndexerMetadataStorageCoordinator#announceHistoricalSegments(Set, DataSourceMetadata, DataSourceMetadata)}.
   */
@Override
public SegmentPublishResult perform(Task task, TaskActionToolbox toolbox) throws IOException {
    toolbox.verifyTaskLocks(task, segments);
    final SegmentPublishResult retVal = toolbox.getIndexerMetadataStorageCoordinator().announceHistoricalSegments(segments, startMetadata, endMetadata);
    // Emit metrics
    final ServiceMetricEvent.Builder metricBuilder = new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, task.getDataSource()).setDimension(DruidMetrics.TASK_TYPE, task.getType());
    if (retVal.isSuccess()) {
        toolbox.getEmitter().emit(metricBuilder.build("segment/txn/success", 1));
    } else {
        toolbox.getEmitter().emit(metricBuilder.build("segment/txn/failure", 1));
    }
    for (DataSegment segment : retVal.getSegments()) {
        metricBuilder.setDimension(DruidMetrics.INTERVAL, segment.getInterval().toString());
        toolbox.getEmitter().emit(metricBuilder.build("segment/added/bytes", segment.getSize()));
    }
    return retVal;
}

Example 7 with SegmentPublishResult

use of io.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class SegmentTransactionalInsertActionTest method testTransactional.

@Test
public void testTransactional() throws Exception {
    final Task task = new NoopTask(null, 0, 0, null, null, null);
    actionTestKit.getTaskLockbox().add(task);
    actionTestKit.getTaskLockbox().lock(task, new Interval(INTERVAL));
    SegmentPublishResult result1 = new SegmentTransactionalInsertAction(ImmutableSet.of(SEGMENT1), new ObjectMetadata(null), new ObjectMetadata(ImmutableList.of(1))).perform(task, actionTestKit.getTaskActionToolbox());
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(SEGMENT1), true), result1);
    SegmentPublishResult result2 = new SegmentTransactionalInsertAction(ImmutableSet.of(SEGMENT2), new ObjectMetadata(ImmutableList.of(1)), new ObjectMetadata(ImmutableList.of(2))).perform(task, actionTestKit.getTaskActionToolbox());
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(SEGMENT2), true), result2);
    Assert.assertEquals(ImmutableSet.of(SEGMENT1, SEGMENT2), ImmutableSet.copyOf(actionTestKit.getMetadataStorageCoordinator().getUsedSegmentsForInterval(DATA_SOURCE, INTERVAL)));
    Assert.assertEquals(new ObjectMetadata(ImmutableList.of(2)), actionTestKit.getMetadataStorageCoordinator().getDataSourceMetadata(DATA_SOURCE));
}

Example 8 with SegmentPublishResult

use of io.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class SegmentTransactionalInsertActionTest method testFailBadVersion.

@Test
public void testFailBadVersion() throws Exception {
    final Task task = new NoopTask(null, 0, 0, null, null, null);
    final SegmentTransactionalInsertAction action = new SegmentTransactionalInsertAction(ImmutableSet.of(SEGMENT3));
    actionTestKit.getTaskLockbox().add(task);
    actionTestKit.getTaskLockbox().lock(task, new Interval(INTERVAL));
    thrown.expect(IllegalStateException.class);
    thrown.expectMessage(CoreMatchers.startsWith("Segments not covered by locks for task"));
    SegmentPublishResult result = action.perform(task, actionTestKit.getTaskActionToolbox());
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(SEGMENT3), true), result);
}

Example 9 with SegmentPublishResult

use of io.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinatorTest method testTransactionalAnnounceSuccess.

@Test
public void testTransactionalAnnounceSuccess() throws IOException {
    // Insert first segment.
    final SegmentPublishResult result1 = coordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment), new ObjectMetadata(null), new ObjectMetadata(ImmutableMap.of("foo", "bar")));
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(defaultSegment), true), result1);
    Assert.assertArrayEquals(mapper.writeValueAsString(defaultSegment).getBytes("UTF-8"), derbyConnector.lookup(derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(), "id", "payload", defaultSegment.getIdentifier()));
    // Insert second segment.
    final SegmentPublishResult result2 = coordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment2), new ObjectMetadata(ImmutableMap.of("foo", "bar")), new ObjectMetadata(ImmutableMap.of("foo", "baz")));
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(defaultSegment2), true), result2);
    Assert.assertArrayEquals(mapper.writeValueAsString(defaultSegment2).getBytes("UTF-8"), derbyConnector.lookup(derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(), "id", "payload", defaultSegment2.getIdentifier()));
    // Examine metadata.
    Assert.assertEquals(new ObjectMetadata(ImmutableMap.of("foo", "baz")), coordinator.getDataSourceMetadata("fooDataSource"));
    // Should only be tried once per call.
    Assert.assertEquals(2, metadataUpdateCounter.get());
}

Example 10 with SegmentPublishResult

use of io.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinatorTest method testTransactionalAnnounceRetryAndSuccess.

@Test
public void testTransactionalAnnounceRetryAndSuccess() throws IOException {
    final AtomicLong attemptCounter = new AtomicLong();
    final IndexerSQLMetadataStorageCoordinator failOnceCoordinator = new IndexerSQLMetadataStorageCoordinator(mapper, derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector) {

        @Override
        protected DataSourceMetadataUpdateResult updateDataSourceMetadataWithHandle(Handle handle, String dataSource, DataSourceMetadata startMetadata, DataSourceMetadata endMetadata) throws IOException {
            metadataUpdateCounter.getAndIncrement();
            if (attemptCounter.getAndIncrement() == 0) {
                return DataSourceMetadataUpdateResult.TRY_AGAIN;
            } else {
                return super.updateDataSourceMetadataWithHandle(handle, dataSource, startMetadata, endMetadata);
            }
        }
    };
    // Insert first segment.
    final SegmentPublishResult result1 = failOnceCoordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment), new ObjectMetadata(null), new ObjectMetadata(ImmutableMap.of("foo", "bar")));
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(defaultSegment), true), result1);
    Assert.assertArrayEquals(mapper.writeValueAsString(defaultSegment).getBytes("UTF-8"), derbyConnector.lookup(derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(), "id", "payload", defaultSegment.getIdentifier()));
    // Reset attempt counter to induce another failure.
    attemptCounter.set(0);
    // Insert second segment.
    final SegmentPublishResult result2 = failOnceCoordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment2), new ObjectMetadata(ImmutableMap.of("foo", "bar")), new ObjectMetadata(ImmutableMap.of("foo", "baz")));
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(defaultSegment2), true), result2);
    Assert.assertArrayEquals(mapper.writeValueAsString(defaultSegment2).getBytes("UTF-8"), derbyConnector.lookup(derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(), "id", "payload", defaultSegment2.getIdentifier()));
    // Examine metadata.
    Assert.assertEquals(new ObjectMetadata(ImmutableMap.of("foo", "baz")), failOnceCoordinator.getDataSourceMetadata("fooDataSource"));
    // Should be tried twice per call.
    Assert.assertEquals(4, metadataUpdateCounter.get());
}