Examples with StreamException org.apache.druid.indexing.seekablestream.common.StreamException used on opensource projects

Example 1 with StreamException

use of org.apache.druid.indexing.seekablestream.common.StreamException in project druid by druid-io.

the class SeekableStreamSupervisor method updatePartitionDataFromStream.

private boolean updatePartitionDataFromStream() {
    List<PartitionIdType> previousPartitionIds = new ArrayList<>(partitionIds);
    Set<PartitionIdType> partitionIdsFromSupplier;
    recordSupplierLock.lock();
    try {
        partitionIdsFromSupplier = recordSupplier.getPartitionIds(ioConfig.getStream());
        if (shouldSkipIgnorablePartitions()) {
            partitionIdsFromSupplier.removeAll(computeIgnorablePartitionIds());
        }
    } catch (Exception e) {
        stateManager.recordThrowableEvent(e);
        log.warn("Could not fetch partitions for topic/stream [%s]: %s", ioConfig.getStream(), e.getMessage());
        log.debug(e, "full stack trace");
        return false;
    } finally {
        recordSupplierLock.unlock();
    }
    if (partitionIdsFromSupplier == null || partitionIdsFromSupplier.size() == 0) {
        String errMsg = StringUtils.format("No partitions found for stream [%s]", ioConfig.getStream());
        stateManager.recordThrowableEvent(new StreamException(new ISE(errMsg)));
        log.warn(errMsg);
        return false;
    }
    log.debug("Found [%d] partitions for stream [%s]", partitionIdsFromSupplier.size(), ioConfig.getStream());
    Map<PartitionIdType, SequenceOffsetType> storedMetadata = getOffsetsFromMetadataStorage();
    Set<PartitionIdType> storedPartitions = storedMetadata.keySet();
    Set<PartitionIdType> closedPartitions = storedMetadata.entrySet().stream().filter(x -> isEndOfShard(x.getValue())).map(Entry::getKey).collect(Collectors.toSet());
    Set<PartitionIdType> previouslyExpiredPartitions = storedMetadata.entrySet().stream().filter(x -> isShardExpirationMarker(x.getValue())).map(Entry::getKey).collect(Collectors.toSet());
    Set<PartitionIdType> partitionIdsFromSupplierWithoutPreviouslyExpiredPartitions = Sets.difference(partitionIdsFromSupplier, previouslyExpiredPartitions);
    Set<PartitionIdType> activePartitionsIdsFromSupplier = Sets.difference(partitionIdsFromSupplierWithoutPreviouslyExpiredPartitions, closedPartitions);
    Set<PartitionIdType> newlyClosedPartitions = Sets.intersection(closedPartitions, new HashSet<>(previousPartitionIds));
    log.debug("active partitions from supplier: " + activePartitionsIdsFromSupplier);
    if (partitionIdsFromSupplierWithoutPreviouslyExpiredPartitions.size() != partitionIdsFromSupplier.size()) {
        // this should never happen, but we check for it and exclude the expired partitions if they somehow reappear
        log.warn("Previously expired partitions [%s] were present in the current list [%s] from the record supplier.", previouslyExpiredPartitions, partitionIdsFromSupplier);
    }
    if (activePartitionsIdsFromSupplier.size() == 0) {
        String errMsg = StringUtils.format("No active partitions found for stream [%s] after removing closed and previously expired partitions", ioConfig.getStream());
        stateManager.recordThrowableEvent(new StreamException(new ISE(errMsg)));
        log.warn(errMsg);
        return false;
    }
    boolean initialPartitionDiscovery = this.partitionIds.isEmpty();
    for (PartitionIdType partitionId : partitionIdsFromSupplierWithoutPreviouslyExpiredPartitions) {
        if (closedPartitions.contains(partitionId)) {
            log.info("partition [%s] is closed and has no more data, skipping.", partitionId);
            continue;
        }
        if (!this.partitionIds.contains(partitionId)) {
            partitionIds.add(partitionId);
            if (!initialPartitionDiscovery) {
                subsequentlyDiscoveredPartitions.add(partitionId);
            }
        }
    }
    // partitions across tasks.
    if (supportsPartitionExpiration()) {
        cleanupClosedAndExpiredPartitions(storedPartitions, newlyClosedPartitions, activePartitionsIdsFromSupplier, previouslyExpiredPartitions, partitionIdsFromSupplier);
    }
    Int2ObjectMap<List<PartitionIdType>> newlyDiscovered = new Int2ObjectLinkedOpenHashMap<>();
    for (PartitionIdType partitionId : activePartitionsIdsFromSupplier) {
        int taskGroupId = getTaskGroupIdForPartition(partitionId);
        Set<PartitionIdType> partitionGroup = partitionGroups.computeIfAbsent(taskGroupId, k -> new HashSet<>());
        partitionGroup.add(partitionId);
        if (partitionOffsets.putIfAbsent(partitionId, getNotSetMarker()) == null) {
            log.debug("New partition [%s] discovered for stream [%s], added to task group [%d]", partitionId, ioConfig.getStream(), taskGroupId);
            newlyDiscovered.computeIfAbsent(taskGroupId, k -> new ArrayList<>()).add(partitionId);
        }
    }
    if (newlyDiscovered.size() > 0) {
        for (Int2ObjectMap.Entry<List<PartitionIdType>> taskGroupPartitions : newlyDiscovered.int2ObjectEntrySet()) {
            log.info("New partitions %s discovered for stream [%s], added to task group [%s]", taskGroupPartitions.getValue(), ioConfig.getStream(), taskGroupPartitions.getIntKey());
        }
    }
    if (!partitionIds.equals(previousPartitionIds)) {
        assignRecordSupplierToPartitionIds();
        // repartitioning quickly by creating new tasks
        for (TaskGroup taskGroup : activelyReadingTaskGroups.values()) {
            if (!taskGroup.taskIds().isEmpty()) {
                // Partitions have changed and we are managing active tasks - set an early publish time
                // at the current time + repartitionTransitionDuration.
                // This allows time for the stream to start writing to the new partitions after repartitioning.
                // For Kinesis ingestion, this cooldown time is particularly useful, lowering the possibility of
                // the new shards being empty, which can cause issues presently
                // (see https://github.com/apache/druid/issues/7600)
                earlyStopTime = DateTimes.nowUtc().plus(tuningConfig.getRepartitionTransitionDuration());
                log.info("Previous partition set [%s] has changed to [%s] - requesting that tasks stop after [%s] at [%s]", previousPartitionIds, partitionIds, tuningConfig.getRepartitionTransitionDuration(), earlyStopTime);
                break;
            }
        }
    }
    return true;
}

Example 2 with StreamException

use of org.apache.druid.indexing.seekablestream.common.StreamException in project druid by druid-io.

the class SeekableStreamSupervisorStateManagerTest method testStreamFailureLostContact.

@Test
public void testStreamFailureLostContact() {
    // clean run without errors
    stateManager.markRunFinished();
    Assert.assertEquals(BasicState.RUNNING, stateManager.getSupervisorState());
    for (int i = 0; i < config.getUnhealthinessThreshold(); i++) {
        Assert.assertEquals(BasicState.RUNNING, stateManager.getSupervisorState());
        stateManager.recordThrowableEvent(new StreamException(new IllegalStateException("DOH!")));
        stateManager.markRunFinished();
    }
    Assert.assertEquals(SeekableStreamState.LOST_CONTACT_WITH_STREAM, stateManager.getSupervisorState());
    Assert.assertEquals(BasicState.UNHEALTHY_SUPERVISOR, stateManager.getSupervisorState().getBasicState());
    Assert.assertEquals(config.getUnhealthinessThreshold(), stateManager.getExceptionEvents().size());
    stateManager.getExceptionEvents().forEach(x -> {
        Assert.assertTrue(((SeekableStreamExceptionEvent) x).isStreamException());
        Assert.assertEquals(IllegalStateException.class.getName(), x.getExceptionClass());
    });
}

Example 3 with StreamException

use of org.apache.druid.indexing.seekablestream.common.StreamException in project druid by druid-io.

the class SeekableStreamSupervisorStateManagerTest method testGetThrowableEvents.

@Test
public void testGetThrowableEvents() {
    List<Exception> exceptions = ImmutableList.of(new StreamException(new UnsupportedOperationException("oof")), new NullPointerException("oof"), new RuntimeException(new StreamException(new Exception("oof"))), new RuntimeException(new IllegalArgumentException("oof")));
    for (Exception exception : exceptions) {
        stateManager.recordThrowableEvent(exception);
        stateManager.markRunFinished();
    }
    Assert.assertEquals(BasicState.UNHEALTHY_SUPERVISOR, stateManager.getSupervisorState());
    List<Pair<String, Boolean>> expected = ImmutableList.of(Pair.of("java.lang.UnsupportedOperationException", true), Pair.of("java.lang.NullPointerException", false), Pair.of("java.lang.Exception", true), Pair.of("java.lang.IllegalArgumentException", false));
    Iterator<SupervisorStateManager.ExceptionEvent> it = stateManager.getExceptionEvents().iterator();
    expected.forEach(x -> {
        SupervisorStateManager.ExceptionEvent event = it.next();
        Assert.assertNotNull(event.getMessage());
        Assert.assertEquals(x.lhs, event.getExceptionClass());
        Assert.assertEquals(x.rhs, ((SeekableStreamExceptionEvent) event).isStreamException());
    });
    Assert.assertFalse(it.hasNext());
}

Example 4 with StreamException

use of org.apache.druid.indexing.seekablestream.common.StreamException in project druid by druid-io.

the class KafkaSupervisor method updatePartitionLagFromStream.

@Override
protected void updatePartitionLagFromStream() {
    getRecordSupplierLock().lock();
    try {
        Set<Integer> partitionIds;
        try {
            partitionIds = recordSupplier.getPartitionIds(getIoConfig().getStream());
        } catch (Exception e) {
            log.warn("Could not fetch partitions for topic/stream [%s]", getIoConfig().getStream());
            throw new StreamException(e);
        }
        Set<StreamPartition<Integer>> partitions = partitionIds.stream().map(e -> new StreamPartition<>(getIoConfig().getStream(), e)).collect(Collectors.toSet());
        recordSupplier.seekToLatest(partitions);
        // this method isn't actually computing the lag, just fetching the latests offsets from the stream. This is
        // because we currently only have record lag for kafka, which can be lazily computed by subtracting the highest
        // task offsets from the latest offsets from the stream when it is needed
        latestSequenceFromStream = partitions.stream().collect(Collectors.toMap(StreamPartition::getPartitionId, recordSupplier::getPosition));
    } catch (InterruptedException e) {
        throw new StreamException(e);
    } finally {
        getRecordSupplierLock().unlock();
    }
}

Example 5 with StreamException

use of org.apache.druid.indexing.seekablestream.common.StreamException in project druid by druid-io.

the class KafkaRecordSupplier method getKafkaDeserializer.

private static Deserializer getKafkaDeserializer(Properties properties, String kafkaConfigKey) {
    Deserializer deserializerObject;
    try {
        Class deserializerClass = Class.forName(properties.getProperty(kafkaConfigKey, ByteArrayDeserializer.class.getTypeName()));
        Method deserializerMethod = deserializerClass.getMethod("deserialize", String.class, byte[].class);
        Type deserializerReturnType = deserializerMethod.getGenericReturnType();
        if (deserializerReturnType == byte[].class) {
            deserializerObject = (Deserializer) deserializerClass.getConstructor().newInstance();
        } else {
            throw new IllegalArgumentException("Kafka deserializers must return a byte array (byte[]), " + deserializerClass.getName() + " returns " + deserializerReturnType.getTypeName());
        }
    } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) {
        throw new StreamException(e);
    }
    return deserializerObject;
}