Examples with SegmentPublishResult io.druid.indexing.overlord.SegmentPublishResult used on opensource projects

Example 1 with SegmentPublishResult

use of io.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinator method announceHistoricalSegments.

/**
   * {@inheritDoc}
   */
@Override
public SegmentPublishResult announceHistoricalSegments(final Set<DataSegment> segments, final DataSourceMetadata startMetadata, final DataSourceMetadata endMetadata) throws IOException {
    if (segments.isEmpty()) {
        throw new IllegalArgumentException("segment set must not be empty");
    }
    final String dataSource = segments.iterator().next().getDataSource();
    for (DataSegment segment : segments) {
        if (!dataSource.equals(segment.getDataSource())) {
            throw new IllegalArgumentException("segments must all be from the same dataSource");
        }
    }
    if ((startMetadata == null && endMetadata != null) || (startMetadata != null && endMetadata == null)) {
        throw new IllegalArgumentException("start/end metadata pair must be either null or non-null");
    }
    // Find which segments are used (i.e. not overshadowed).
    final Set<DataSegment> usedSegments = Sets.newHashSet();
    for (TimelineObjectHolder<String, DataSegment> holder : VersionedIntervalTimeline.forSegments(segments).lookup(JodaUtils.ETERNITY)) {
        for (PartitionChunk<DataSegment> chunk : holder.getObject()) {
            usedSegments.add(chunk.getObject());
        }
    }
    final AtomicBoolean txnFailure = new AtomicBoolean(false);
    try {
        return connector.retryTransaction(new TransactionCallback<SegmentPublishResult>() {

            @Override
            public SegmentPublishResult inTransaction(final Handle handle, final TransactionStatus transactionStatus) throws Exception {
                final Set<DataSegment> inserted = Sets.newHashSet();
                if (startMetadata != null) {
                    final DataSourceMetadataUpdateResult result = updateDataSourceMetadataWithHandle(handle, dataSource, startMetadata, endMetadata);
                    if (result != DataSourceMetadataUpdateResult.SUCCESS) {
                        transactionStatus.setRollbackOnly();
                        txnFailure.set(true);
                        if (result == DataSourceMetadataUpdateResult.FAILURE) {
                            throw new RuntimeException("Aborting transaction!");
                        } else if (result == DataSourceMetadataUpdateResult.TRY_AGAIN) {
                            throw new RetryTransactionException("Aborting transaction!");
                        }
                    }
                }
                for (final DataSegment segment : segments) {
                    if (announceHistoricalSegment(handle, segment, usedSegments.contains(segment))) {
                        inserted.add(segment);
                    }
                }
                return new SegmentPublishResult(ImmutableSet.copyOf(inserted), true);
            }
        }, 3, SQLMetadataConnector.DEFAULT_MAX_TRIES);
    } catch (CallbackFailedException e) {
        if (txnFailure.get()) {
            return new SegmentPublishResult(ImmutableSet.<DataSegment>of(), false);
        } else {
            throw e;
        }
    }
}

Example 2 with SegmentPublishResult

use of io.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinatorTest method testTransactionalAnnounceFailDbNotNullWantNull.

@Test
public void testTransactionalAnnounceFailDbNotNullWantNull() throws IOException {
    final SegmentPublishResult result1 = coordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment), new ObjectMetadata(null), new ObjectMetadata(ImmutableMap.of("foo", "baz")));
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(defaultSegment), true), result1);
    final SegmentPublishResult result2 = coordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment2), new ObjectMetadata(null), new ObjectMetadata(ImmutableMap.of("foo", "baz")));
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.<DataSegment>of(), false), result2);
    // Should only be tried once per call.
    Assert.assertEquals(2, metadataUpdateCounter.get());
}

Example 3 with SegmentPublishResult

use of io.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinatorTest method testTransactionalAnnounceFailDbNullWantNotNull.

@Test
public void testTransactionalAnnounceFailDbNullWantNotNull() throws IOException {
    final SegmentPublishResult result1 = coordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment), new ObjectMetadata(ImmutableMap.of("foo", "bar")), new ObjectMetadata(ImmutableMap.of("foo", "baz")));
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.<DataSegment>of(), false), result1);
    // Should only be tried once.
    Assert.assertEquals(1, metadataUpdateCounter.get());
}

Example 4 with SegmentPublishResult

use of io.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class IndexTaskTest method runTask.

private final List<DataSegment> runTask(final IndexTask indexTask) throws Exception {
    final List<DataSegment> segments = Lists.newArrayList();
    indexTask.run(new TaskToolbox(null, null, new TaskActionClient() {

        @Override
        public <RetType> RetType submit(TaskAction<RetType> taskAction) throws IOException {
            if (taskAction instanceof LockListAction) {
                return (RetType) Arrays.asList(new TaskLock("", "", null, new DateTime().toString()));
            }
            if (taskAction instanceof LockAcquireAction) {
                return (RetType) new TaskLock("groupId", "test", ((LockAcquireAction) taskAction).getInterval(), new DateTime().toString());
            }
            if (taskAction instanceof SegmentTransactionalInsertAction) {
                return (RetType) new SegmentPublishResult(((SegmentTransactionalInsertAction) taskAction).getSegments(), true);
            }
            if (taskAction instanceof SegmentAllocateAction) {
                SegmentAllocateAction action = (SegmentAllocateAction) taskAction;
                Interval interval = action.getPreferredSegmentGranularity().bucket(action.getTimestamp());
                ShardSpec shardSpec = new NumberedShardSpec(segmentAllocatePartitionCounter++, 0);
                return (RetType) new SegmentIdentifier(action.getDataSource(), interval, "latestVersion", shardSpec);
            }
            return null;
        }
    }, null, new DataSegmentPusher() {

        @Deprecated
        @Override
        public String getPathForHadoop(String dataSource) {
            return getPathForHadoop();
        }

        @Override
        public String getPathForHadoop() {
            return null;
        }

        @Override
        public DataSegment push(File file, DataSegment segment) throws IOException {
            segments.add(segment);
            return segment;
        }
    }, null, null, null, null, null, null, null, null, null, jsonMapper, temporaryFolder.newFolder(), indexMerger, indexIO, null, null, indexMergerV9));
    Collections.sort(segments);
    return segments;
}

Example 5 with SegmentPublishResult

use of io.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class SegmentTransactionalInsertActionTest method testFailTransactional.

@Test
public void testFailTransactional() throws Exception {
    final Task task = new NoopTask(null, 0, 0, null, null, null);
    actionTestKit.getTaskLockbox().add(task);
    actionTestKit.getTaskLockbox().lock(task, new Interval(INTERVAL));
    SegmentPublishResult result = new SegmentTransactionalInsertAction(ImmutableSet.of(SEGMENT1), new ObjectMetadata(ImmutableList.of(1)), new ObjectMetadata(ImmutableList.of(2))).perform(task, actionTestKit.getTaskActionToolbox());
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.<DataSegment>of(), false), result);
}