Examples with TaskMetadata org.apache.kafka.streams.TaskMetadata used on opensource projects

Example 1 with TaskMetadata

use of org.apache.kafka.streams.TaskMetadata in project kafka by apache.

the class TaskMetadataIntegrationTest method shouldReportCorrectEndOffsetInformation.

@Test
public void shouldReportCorrectEndOffsetInformation() {
    try (final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), properties)) {
        IntegrationTestUtils.startApplicationAndWaitUntilRunning(Collections.singletonList(kafkaStreams), DEFAULT_DURATION);
        final TaskMetadata taskMetadata = getTaskMetadata(kafkaStreams);
        assertThat(taskMetadata.endOffsets().size(), equalTo(1));
        final TopicPartition topicPartition = new TopicPartition(inputTopic, 0);
        commit.set(false);
        for (int i = 0; i < 10; i++) {
            produceMessages(0L, inputTopic, "test");
            TestUtils.waitForCondition(() -> !process.get(), "The record was not processed");
            process.set(true);
        }
        assertThat(taskMetadata.endOffsets().get(topicPartition), equalTo(9L));
    } catch (final Exception e) {
        e.printStackTrace();
    }
}

Example 2 with TaskMetadata

use of org.apache.kafka.streams.TaskMetadata in project kafka by apache.

the class TaskMetadataIntegrationTest method shouldReportCorrectCommittedOffsetInformation.

@Test
public void shouldReportCorrectCommittedOffsetInformation() {
    try (final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), properties)) {
        IntegrationTestUtils.startApplicationAndWaitUntilRunning(Collections.singletonList(kafkaStreams), DEFAULT_DURATION);
        final TaskMetadata taskMetadata = getTaskMetadata(kafkaStreams);
        assertThat(taskMetadata.committedOffsets().size(), equalTo(1));
        final TopicPartition topicPartition = new TopicPartition(inputTopic, 0);
        produceMessages(0L, inputTopic, "test");
        TestUtils.waitForCondition(() -> !process.get(), "The record was not processed");
        TestUtils.waitForCondition(() -> taskMetadata.committedOffsets().get(topicPartition) == 1L, "the record was processed");
        process.set(true);
        produceMessages(0L, inputTopic, "test1");
        TestUtils.waitForCondition(() -> !process.get(), "The record was not processed");
        TestUtils.waitForCondition(() -> taskMetadata.committedOffsets().get(topicPartition) == 2L, "the record was processed");
        process.set(true);
        produceMessages(0L, inputTopic, "test1");
        TestUtils.waitForCondition(() -> !process.get(), "The record was not processed");
        TestUtils.waitForCondition(() -> taskMetadata.committedOffsets().get(topicPartition) == 3L, "the record was processed");
    } catch (final Exception e) {
        e.printStackTrace();
    }
}

Example 3 with TaskMetadata

use of org.apache.kafka.streams.TaskMetadata in project kafka by apache.

the class KafkaStreamsNamedTopologyWrapper method removeNamedTopology.

/**
 * Remove an existing NamedTopology from a running Kafka Streams app. If multiple instances of the application are
 * running, you should inform all of them by calling {@code #removeNamedTopology(String)} on each client to ensure
 * it stops processing the old topology.
 * This method is not purely Async.
 *
 * @param topologyToRemove          name of the topology to be removed
 * @param resetOffsets              whether to reset the committed offsets for any source topics
 *
 * May complete exceptionally (does not actually directly throw) with:
 *
 * @throws IllegalArgumentException if this topology name cannot be found
 * @throws IllegalStateException    if streams has not been started or has already shut down
 */
public RemoveNamedTopologyResult removeNamedTopology(final String topologyToRemove, final boolean resetOffsets) {
    log.info("Informed to remove topology {} with resetOffsets={} ", topologyToRemove, resetOffsets);
    final KafkaFutureImpl<Void> removeTopologyFuture = new KafkaFutureImpl<>();
    if (hasStartedOrFinishedShuttingDown()) {
        log.error("Attempted to remove topology {} from while the Kafka Streams was in state {}, " + "application must be started first.", topologyToRemove, state);
        removeTopologyFuture.completeExceptionally(new IllegalStateException("Cannot remove a NamedTopology while the state is " + super.state));
    } else if (!getTopologyByName(topologyToRemove).isPresent()) {
        log.error("Attempted to remove unknown topology {}. This application currently contains the" + "following topologies: {}.", topologyToRemove, topologyMetadata.namedTopologiesView());
        removeTopologyFuture.completeExceptionally(new UnknownTopologyException("Unable to remove topology", topologyToRemove));
    }
    final Set<TopicPartition> partitionsToReset = metadataForLocalThreads().stream().flatMap(t -> {
        final Set<TaskMetadata> tasks = new HashSet<>(t.activeTasks());
        return tasks.stream();
    }).flatMap(t -> t.topicPartitions().stream()).filter(t -> topologyMetadata.sourceTopicsForTopology(topologyToRemove).contains(t.topic())).collect(Collectors.toSet());
    topologyMetadata.unregisterTopology(removeTopologyFuture, topologyToRemove);
    if (resetOffsets) {
        log.info("Resetting offsets for the following partitions of {} removed NamedTopology {}: {}", removeTopologyFuture.isCompletedExceptionally() ? "unsuccessfully" : "successfully", topologyToRemove, partitionsToReset);
        if (!partitionsToReset.isEmpty()) {
            removeTopologyFuture.whenComplete((v, throwable) -> {
                if (throwable != null) {
                    removeTopologyFuture.completeExceptionally(throwable);
                }
                DeleteConsumerGroupOffsetsResult deleteOffsetsResult = null;
                while (deleteOffsetsResult == null) {
                    try {
                        deleteOffsetsResult = adminClient.deleteConsumerGroupOffsets(applicationConfigs.getString(StreamsConfig.APPLICATION_ID_CONFIG), partitionsToReset);
                        deleteOffsetsResult.all().get();
                    } catch (final InterruptedException ex) {
                        ex.printStackTrace();
                        break;
                    } catch (final ExecutionException ex) {
                        if (ex.getCause() != null && ex.getCause() instanceof GroupSubscribedToTopicException && ex.getCause().getMessage().equals("Deleting offsets of a topic is forbidden while the consumer group is actively subscribed to it.")) {
                            ex.printStackTrace();
                        } else if (ex.getCause() != null && ex.getCause() instanceof GroupIdNotFoundException) {
                            log.debug("The offsets have been reset by another client or the group has been deleted, no need to retry further.");
                            break;
                        } else {
                            removeTopologyFuture.completeExceptionally(ex);
                        }
                        deleteOffsetsResult = null;
                    }
                    try {
                        Thread.sleep(100);
                    } catch (final InterruptedException ex) {
                        ex.printStackTrace();
                    }
                }
                removeTopologyFuture.complete(null);
            });
            return new RemoveNamedTopologyResult(removeTopologyFuture, removeTopologyFuture);
        }
    }
    return new RemoveNamedTopologyResult(removeTopologyFuture);
}