use of org.apache.druid.indexing.seekablestream.common.StreamException in project druid by druid-io.
the class SeekableStreamSupervisor method getOffsetFromStorageForPartition.
/**
* Queries the dataSource metadata table to see if there is a previous ending sequence for this partition. If it
* doesn't find any data, it will retrieve the latest or earliest Kafka/Kinesis sequence depending on the
* {@link SeekableStreamSupervisorIOConfig#useEarliestSequenceNumber}.
*/
private OrderedSequenceNumber<SequenceOffsetType> getOffsetFromStorageForPartition(PartitionIdType partition) {
final Map<PartitionIdType, SequenceOffsetType> metadataOffsets = getOffsetsFromMetadataStorage();
SequenceOffsetType sequence = metadataOffsets.get(partition);
if (sequence != null) {
log.debug("Getting sequence [%s] from metadata storage for partition [%s]", sequence, partition);
if (!taskTuningConfig.isSkipSequenceNumberAvailabilityCheck()) {
if (!checkOffsetAvailability(partition, sequence)) {
if (taskTuningConfig.isResetOffsetAutomatically()) {
resetInternal(createDataSourceMetaDataForReset(ioConfig.getStream(), ImmutableMap.of(partition, sequence)));
throw new StreamException(new ISE("Previous sequenceNumber [%s] is no longer available for partition [%s] - automatically resetting" + " sequence", sequence, partition));
} else {
throw new StreamException(new ISE("Previous sequenceNumber [%s] is no longer available for partition [%s]. You can clear the previous" + " sequenceNumber and start reading from a valid message by using the supervisor's reset API.", sequence, partition));
}
}
}
return makeSequenceNumber(sequence, useExclusiveStartSequenceNumberForNonFirstSequence());
} else {
boolean useEarliestSequenceNumber = ioConfig.isUseEarliestSequenceNumber();
if (subsequentlyDiscoveredPartitions.contains(partition)) {
log.info("Overriding useEarliestSequenceNumber and starting from beginning of newly discovered partition [%s] (which is probably from a split or merge)", partition);
useEarliestSequenceNumber = true;
}
sequence = getOffsetFromStreamForPartition(partition, useEarliestSequenceNumber);
if (sequence == null) {
throw new ISE("unable to fetch sequence number for partition[%s] from stream", partition);
}
log.debug("Getting sequence number [%s] for partition [%s]", sequence, partition);
return makeSequenceNumber(sequence, false);
}
}
use of org.apache.druid.indexing.seekablestream.common.StreamException in project druid by druid-io.
the class SeekableStreamSupervisorStateManagerTest method testStreamFailureUnableToConnect.
@Test
public void testStreamFailureUnableToConnect() {
stateManager.maybeSetState(SeekableStreamState.CONNECTING_TO_STREAM);
for (int i = 0; i < config.getUnhealthinessThreshold(); i++) {
Assert.assertEquals(SeekableStreamState.CONNECTING_TO_STREAM, stateManager.getSupervisorState());
stateManager.recordThrowableEvent(new StreamException(new IllegalStateException("DOH!")));
stateManager.markRunFinished();
}
Assert.assertEquals(SeekableStreamState.UNABLE_TO_CONNECT_TO_STREAM, stateManager.getSupervisorState());
Assert.assertEquals(BasicState.UNHEALTHY_SUPERVISOR, stateManager.getSupervisorState().getBasicState());
Assert.assertEquals(config.getUnhealthinessThreshold(), stateManager.getExceptionEvents().size());
stateManager.getExceptionEvents().forEach(x -> {
Assert.assertTrue(((SeekableStreamExceptionEvent) x).isStreamException());
Assert.assertEquals(IllegalStateException.class.getName(), x.getExceptionClass());
});
}
Aggregations