Examples with SegmentsAndCommitMetadata org.apache.druid.segment.realtime.appenderator.SegmentsAndCommitMetadata used on opensource projects

Example 6 with SegmentsAndCommitMetadata

use of org.apache.druid.segment.realtime.appenderator.SegmentsAndCommitMetadata in project druid by druid-io.

the class SeekableStreamIndexTaskRunner method publishAndRegisterHandoff.

private void publishAndRegisterHandoff(SequenceMetadata<PartitionIdType, SequenceOffsetType> sequenceMetadata) {
    log.debug("Publishing segments for sequence [%s].", sequenceMetadata);
    final ListenableFuture<SegmentsAndCommitMetadata> publishFuture = Futures.transform(driver.publish(sequenceMetadata.createPublisher(this, toolbox, ioConfig.isUseTransaction()), sequenceMetadata.getCommitterSupplier(this, stream, lastPersistedOffsets).get(), Collections.singletonList(sequenceMetadata.getSequenceName())), (Function<SegmentsAndCommitMetadata, SegmentsAndCommitMetadata>) publishedSegmentsAndMetadata -> {
        if (publishedSegmentsAndMetadata == null) {
            throw new ISE("Transaction failure publishing segments for sequence [%s]", sequenceMetadata);
        } else {
            return publishedSegmentsAndMetadata;
        }
    });
    publishWaitList.add(publishFuture);
    // Create a handoffFuture for every publishFuture. The created handoffFuture must fail if publishFuture fails.
    final SettableFuture<SegmentsAndCommitMetadata> handoffFuture = SettableFuture.create();
    handOffWaitList.add(handoffFuture);
    Futures.addCallback(publishFuture, new FutureCallback<SegmentsAndCommitMetadata>() {

        @Override
        public void onSuccess(SegmentsAndCommitMetadata publishedSegmentsAndCommitMetadata) {
            log.info("Published %s segments for sequence [%s] with metadata [%s].", publishedSegmentsAndCommitMetadata.getSegments().size(), sequenceMetadata.getSequenceName(), Preconditions.checkNotNull(publishedSegmentsAndCommitMetadata.getCommitMetadata(), "commitMetadata"));
            log.infoSegments(publishedSegmentsAndCommitMetadata.getSegments(), "Published segments");
            sequences.remove(sequenceMetadata);
            publishingSequences.remove(sequenceMetadata.getSequenceName());
            try {
                persistSequences();
            } catch (IOException e) {
                log.error(e, "Unable to persist state, dying");
                handoffFuture.setException(e);
                throw new RuntimeException(e);
            }
            Futures.transform(driver.registerHandoff(publishedSegmentsAndCommitMetadata), new Function<SegmentsAndCommitMetadata, Void>() {

                @Nullable
                @Override
                public Void apply(@Nullable SegmentsAndCommitMetadata handoffSegmentsAndCommitMetadata) {
                    if (handoffSegmentsAndCommitMetadata == null) {
                        log.warn("Failed to hand off %s segments", publishedSegmentsAndCommitMetadata.getSegments().size());
                        log.warnSegments(publishedSegmentsAndCommitMetadata.getSegments(), "Failed to hand off segments");
                    }
                    handoffFuture.set(handoffSegmentsAndCommitMetadata);
                    return null;
                }
            });
        }

        @Override
        public void onFailure(Throwable t) {
            log.error(t, "Error while publishing segments for sequenceNumber[%s]", sequenceMetadata);
            handoffFuture.setException(t);
        }
    });
}