Examples with SegmentPublishResult org.apache.druid.indexing.overlord.SegmentPublishResult used on opensource projects

Example 11 with SegmentPublishResult

use of org.apache.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class SegmentTransactionalInsertActionTest method testFailTransactionalUpdateDataSourceMetadata.

@Test
public void testFailTransactionalUpdateDataSourceMetadata() throws Exception {
    final Task task = NoopTask.create();
    actionTestKit.getTaskLockbox().add(task);
    acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task, INTERVAL, 5000);
    SegmentPublishResult result = SegmentTransactionalInsertAction.appendAction(ImmutableSet.of(SEGMENT1), new ObjectMetadata(ImmutableList.of(1)), new ObjectMetadata(ImmutableList.of(2))).perform(task, actionTestKit.getTaskActionToolbox());
    Assert.assertEquals(SegmentPublishResult.fail("java.lang.RuntimeException: Aborting transaction!"), result);
}

Example 12 with SegmentPublishResult

use of org.apache.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class SegmentTransactionalInsertActionTest method testTransactionalDropSegments.

@Test
public void testTransactionalDropSegments() throws Exception {
    final Task task = NoopTask.create();
    actionTestKit.getTaskLockbox().add(task);
    acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task, INTERVAL, 5000);
    SegmentPublishResult result1 = SegmentTransactionalInsertAction.overwriteAction(null, null, ImmutableSet.of(SEGMENT1)).perform(task, actionTestKit.getTaskActionToolbox());
    Assert.assertEquals(SegmentPublishResult.ok(ImmutableSet.of(SEGMENT1)), result1);
    SegmentPublishResult result2 = SegmentTransactionalInsertAction.overwriteAction(null, ImmutableSet.of(SEGMENT1), ImmutableSet.of(SEGMENT2)).perform(task, actionTestKit.getTaskActionToolbox());
    Assert.assertEquals(SegmentPublishResult.ok(ImmutableSet.of(SEGMENT2)), result2);
    Assertions.assertThat(actionTestKit.getMetadataStorageCoordinator().retrieveUsedSegmentsForInterval(DATA_SOURCE, INTERVAL, Segments.ONLY_VISIBLE)).containsExactlyInAnyOrder(SEGMENT2);
}

Example 13 with SegmentPublishResult

use of org.apache.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class BaseAppenderatorDriver method publishInBackground.

/**
 * Publish segments in background. The segments should be dropped (in batch ingestion) or pushed (in streaming
 * ingestion) before being published.
 *
 * @param segmentsAndCommitMetadata result of dropping or pushing
 * @param publisher transactional segment publisher
 *
 * @return a future for publishing segments
 */
ListenableFuture<SegmentsAndCommitMetadata> publishInBackground(@Nullable Set<DataSegment> segmentsToBeOverwritten, @Nullable Set<DataSegment> segmentsToBeDropped, SegmentsAndCommitMetadata segmentsAndCommitMetadata, TransactionalSegmentPublisher publisher, java.util.function.Function<Set<DataSegment>, Set<DataSegment>> outputSegmentsAnnotateFunction) {
    if (segmentsAndCommitMetadata.getSegments().isEmpty()) {
        if (!publisher.supportsEmptyPublish()) {
            log.info("Nothing to publish, skipping publish step.");
            final SettableFuture<SegmentsAndCommitMetadata> retVal = SettableFuture.create();
            retVal.set(segmentsAndCommitMetadata);
            return retVal;
        } else {
            // supportsEmptyPublish, but is kept limited for now until further testing.
            if (appenderator.getTotalRowCount() != 0) {
                throw new ISE("Attempting to publish with empty segment set, but total row count was not 0: [%s].", appenderator.getTotalRowCount());
            }
        }
    }
    final Object metadata = segmentsAndCommitMetadata.getCommitMetadata();
    final Object callerMetadata = metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata();
    return executor.submit(() -> {
        try {
            final ImmutableSet<DataSegment> ourSegments = ImmutableSet.copyOf(segmentsAndCommitMetadata.getSegments());
            final SegmentPublishResult publishResult = publisher.publishSegments(segmentsToBeOverwritten, segmentsToBeDropped, ourSegments, outputSegmentsAnnotateFunction, callerMetadata);
            if (publishResult.isSuccess()) {
                log.info("Published [%s] segments with commit metadata [%s]", segmentsAndCommitMetadata.getSegments().size(), callerMetadata);
                log.infoSegments(segmentsAndCommitMetadata.getSegments(), "Published segments");
            } else {
                // Publishing didn't affirmatively succeed. However, segments with our identifiers may still be active
                // now after all, for two possible reasons:
                // 
                // 1) A replica may have beat us to publishing these segments. In this case we want to delete the
                // segments we pushed (if they had unique paths) to avoid wasting space on deep storage.
                // 2) We may have actually succeeded, but not realized it due to missing the confirmation response
                // from the overlord. In this case we do not want to delete the segments we pushed, since they are
                // now live!
                final Set<SegmentIdWithShardSpec> segmentsIdentifiers = segmentsAndCommitMetadata.getSegments().stream().map(SegmentIdWithShardSpec::fromDataSegment).collect(Collectors.toSet());
                final Set<DataSegment> activeSegments = usedSegmentChecker.findUsedSegments(segmentsIdentifiers);
                if (activeSegments.equals(ourSegments)) {
                    log.info("Could not publish [%s] segments, but checked and found them already published; continuing.", ourSegments.size());
                    log.infoSegments(segmentsAndCommitMetadata.getSegments(), "Could not publish segments");
                    // Clean up pushed segments if they are physically disjoint from the published ones (this means
                    // they were probably pushed by a replica, and with the unique paths option).
                    final boolean physicallyDisjoint = Sets.intersection(activeSegments.stream().map(DataSegment::getLoadSpec).collect(Collectors.toSet()), ourSegments.stream().map(DataSegment::getLoadSpec).collect(Collectors.toSet())).isEmpty();
                    if (physicallyDisjoint) {
                        segmentsAndCommitMetadata.getSegments().forEach(dataSegmentKiller::killQuietly);
                    }
                } else {
                    // Our segments aren't active. Publish failed for some reason. Clean them up and then throw an error.
                    segmentsAndCommitMetadata.getSegments().forEach(dataSegmentKiller::killQuietly);
                    if (publishResult.getErrorMsg() != null) {
                        log.errorSegments(ourSegments, "Failed to publish segments");
                        throw new ISE("Failed to publish segments because of [%s]", publishResult.getErrorMsg());
                    } else {
                        log.errorSegments(ourSegments, "Failed to publish segments");
                        throw new ISE("Failed to publish segments");
                    }
                }
            }
        } catch (Exception e) {
            // Must not remove segments here, we aren't sure if our transaction succeeded or not.
            log.noStackTrace().warn(e, "Failed publish");
            log.warnSegments(segmentsAndCommitMetadata.getSegments(), "Failed publish, not removing segments");
            Throwables.propagateIfPossible(e);
            throw new RuntimeException(e);
        }
        return segmentsAndCommitMetadata;
    });
}

Example 14 with SegmentPublishResult

use of org.apache.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinator method announceHistoricalSegments.

@Override
public SegmentPublishResult announceHistoricalSegments(final Set<DataSegment> segments, final Set<DataSegment> segmentsToDrop, @Nullable final DataSourceMetadata startMetadata, @Nullable final DataSourceMetadata endMetadata) throws IOException {
    if (segments.isEmpty()) {
        throw new IllegalArgumentException("segment set must not be empty");
    }
    final String dataSource = segments.iterator().next().getDataSource();
    for (DataSegment segment : segments) {
        if (!dataSource.equals(segment.getDataSource())) {
            throw new IllegalArgumentException("segments must all be from the same dataSource");
        }
    }
    if ((startMetadata == null && endMetadata != null) || (startMetadata != null && endMetadata == null)) {
        throw new IllegalArgumentException("start/end metadata pair must be either null or non-null");
    }
    // Find which segments are used (i.e. not overshadowed).
    final Set<DataSegment> usedSegments = new HashSet<>();
    List<TimelineObjectHolder<String, DataSegment>> segmentHolders = VersionedIntervalTimeline.forSegments(segments).lookupWithIncompletePartitions(Intervals.ETERNITY);
    for (TimelineObjectHolder<String, DataSegment> holder : segmentHolders) {
        for (PartitionChunk<DataSegment> chunk : holder.getObject()) {
            usedSegments.add(chunk.getObject());
        }
    }
    final AtomicBoolean definitelyNotUpdated = new AtomicBoolean(false);
    try {
        return connector.retryTransaction(new TransactionCallback<SegmentPublishResult>() {

            @Override
            public SegmentPublishResult inTransaction(final Handle handle, final TransactionStatus transactionStatus) throws Exception {
                // Set definitelyNotUpdated back to false upon retrying.
                definitelyNotUpdated.set(false);
                if (startMetadata != null) {
                    final DataStoreMetadataUpdateResult result = updateDataSourceMetadataWithHandle(handle, dataSource, startMetadata, endMetadata);
                    if (result != DataStoreMetadataUpdateResult.SUCCESS) {
                        // Metadata was definitely not updated.
                        transactionStatus.setRollbackOnly();
                        definitelyNotUpdated.set(true);
                        if (result == DataStoreMetadataUpdateResult.FAILURE) {
                            throw new RuntimeException("Aborting transaction!");
                        } else if (result == DataStoreMetadataUpdateResult.TRY_AGAIN) {
                            throw new RetryTransactionException("Aborting transaction!");
                        }
                    }
                }
                if (segmentsToDrop != null && !segmentsToDrop.isEmpty()) {
                    final DataStoreMetadataUpdateResult result = dropSegmentsWithHandle(handle, segmentsToDrop, dataSource);
                    if (result != DataStoreMetadataUpdateResult.SUCCESS) {
                        // Metadata store was definitely not updated.
                        transactionStatus.setRollbackOnly();
                        definitelyNotUpdated.set(true);
                        if (result == DataStoreMetadataUpdateResult.FAILURE) {
                            throw new RuntimeException("Aborting transaction!");
                        } else if (result == DataStoreMetadataUpdateResult.TRY_AGAIN) {
                            throw new RetryTransactionException("Aborting transaction!");
                        }
                    }
                }
                final Set<DataSegment> inserted = announceHistoricalSegmentBatch(handle, segments, usedSegments);
                return SegmentPublishResult.ok(ImmutableSet.copyOf(inserted));
            }
        }, 3, getSqlMetadataMaxRetry());
    } catch (CallbackFailedException e) {
        if (definitelyNotUpdated.get()) {
            return SegmentPublishResult.fail(e.getMessage());
        } else {
            // Must throw exception if we are not sure if we updated or not.
            throw e;
        }
    }
}

Example 15 with SegmentPublishResult

use of org.apache.druid.indexing.overlord.SegmentPublishResult in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinatorTest method testTransactionalAnnounceFailDbNotNullWantDifferent.

@Test
public void testTransactionalAnnounceFailDbNotNullWantDifferent() throws IOException {
    final SegmentPublishResult result1 = coordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment), ImmutableSet.of(), new ObjectMetadata(null), new ObjectMetadata(ImmutableMap.of("foo", "baz")));
    Assert.assertEquals(SegmentPublishResult.ok(ImmutableSet.of(defaultSegment)), result1);
    final SegmentPublishResult result2 = coordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment2), ImmutableSet.of(), new ObjectMetadata(ImmutableMap.of("foo", "qux")), new ObjectMetadata(ImmutableMap.of("foo", "baz")));
    Assert.assertEquals(SegmentPublishResult.fail("java.lang.RuntimeException: Aborting transaction!"), result2);
    // Should only be tried once per call.
    Assert.assertEquals(2, metadataUpdateCounter.get());
}