Examples with LagInfo org.apache.kafka.streams.LagInfo used on opensource projects

Example 1 with LagInfo

use of org.apache.kafka.streams.LagInfo in project kafka by apache.

the class LagFetchIntegrationTest method shouldFetchLagsDuringRestoration.

@Test
public void shouldFetchLagsDuringRestoration() throws Exception {
    IntegrationTestUtils.produceKeyValuesSynchronously(inputTopicName, mkSet(new KeyValue<>("k1", 1L), new KeyValue<>("k2", 2L), new KeyValue<>("k3", 3L), new KeyValue<>("k4", 4L), new KeyValue<>("k5", 5L)), TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, LongSerializer.class, new Properties()), mockTime);
    // create stream threads
    final Properties props = (Properties) streamsConfiguration.clone();
    final File stateDir = TestUtils.tempDirectory(stateStoreName + "0");
    props.put(StreamsConfig.APPLICATION_SERVER_CONFIG, "localhost:0");
    props.put(StreamsConfig.CLIENT_ID_CONFIG, "instance-0");
    props.put(StreamsConfig.STATE_DIR_CONFIG, stateDir.getAbsolutePath());
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<String, Long> t1 = builder.table(inputTopicName, Materialized.as(stateStoreName));
    t1.toStream().to(outputTopicName);
    final KafkaStreams streams = new KafkaStreams(builder.build(), props);
    try {
        // First start up the active.
        TestUtils.waitForCondition(() -> streams.allLocalStorePartitionLags().size() == 0, WAIT_TIMEOUT_MS, "Should see empty lag map before streams is started.");
        // Get the instance to fully catch up and reach RUNNING state
        startApplicationAndWaitUntilRunning(Collections.singletonList(streams), Duration.ofSeconds(60));
        IntegrationTestUtils.waitUntilMinValuesRecordsReceived(consumerConfiguration, outputTopicName, 5, WAIT_TIMEOUT_MS);
        // check for proper lag values.
        final AtomicReference<LagInfo> zeroLagRef = new AtomicReference<>();
        TestUtils.waitForCondition(() -> {
            final Map<String, Map<Integer, LagInfo>> offsetLagInfoMap = streams.allLocalStorePartitionLags();
            assertThat(offsetLagInfoMap.size(), equalTo(1));
            assertThat(offsetLagInfoMap.keySet(), equalTo(mkSet(stateStoreName)));
            assertThat(offsetLagInfoMap.get(stateStoreName).size(), equalTo(1));
            final LagInfo zeroLagInfo = offsetLagInfoMap.get(stateStoreName).get(0);
            assertThat(zeroLagInfo.currentOffsetPosition(), equalTo(5L));
            assertThat(zeroLagInfo.endOffsetPosition(), equalTo(5L));
            assertThat(zeroLagInfo.offsetLag(), equalTo(0L));
            zeroLagRef.set(zeroLagInfo);
            return true;
        }, WAIT_TIMEOUT_MS, "Eventually should reach zero lag.");
        // Kill instance, delete state to force restoration.
        assertThat("Streams instance did not close within timeout", streams.close(Duration.ofSeconds(60)));
        IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
        Files.walk(stateDir.toPath()).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(f -> assertTrue("Some state " + f + " could not be deleted", f.delete()));
        // wait till the lag goes down to 0
        final KafkaStreams restartedStreams = new KafkaStreams(builder.build(), props);
        // set a state restoration listener to track progress of restoration
        final CountDownLatch restorationEndLatch = new CountDownLatch(1);
        final Map<String, Map<Integer, LagInfo>> restoreStartLagInfo = new HashMap<>();
        final Map<String, Map<Integer, LagInfo>> restoreEndLagInfo = new HashMap<>();
        restartedStreams.setGlobalStateRestoreListener(new StateRestoreListener() {

            @Override
            public void onRestoreStart(final TopicPartition topicPartition, final String storeName, final long startingOffset, final long endingOffset) {
                try {
                    restoreStartLagInfo.putAll(getFirstNonEmptyLagMap(restartedStreams));
                } catch (final Exception e) {
                    LOG.error("Exception while trying to obtain lag map", e);
                }
            }

            @Override
            public void onBatchRestored(final TopicPartition topicPartition, final String storeName, final long batchEndOffset, final long numRestored) {
            }

            @Override
            public void onRestoreEnd(final TopicPartition topicPartition, final String storeName, final long totalRestored) {
                try {
                    restoreEndLagInfo.putAll(getFirstNonEmptyLagMap(restartedStreams));
                } catch (final Exception e) {
                    LOG.error("Exception while trying to obtain lag map", e);
                }
                restorationEndLatch.countDown();
            }
        });
        restartedStreams.start();
        restorationEndLatch.await(WAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
        TestUtils.waitForCondition(() -> restartedStreams.allLocalStorePartitionLags().get(stateStoreName).get(0).offsetLag() == 0, WAIT_TIMEOUT_MS, "Standby should eventually catchup and have zero lag.");
        final LagInfo fullLagInfo = restoreStartLagInfo.get(stateStoreName).get(0);
        assertThat(fullLagInfo.currentOffsetPosition(), equalTo(0L));
        assertThat(fullLagInfo.endOffsetPosition(), equalTo(5L));
        assertThat(fullLagInfo.offsetLag(), equalTo(5L));
        assertThat(restoreEndLagInfo.get(stateStoreName).get(0), equalTo(zeroLagRef.get()));
    } finally {
        streams.close();
        streams.cleanUp();
    }
}

Example 2 with LagInfo

use of org.apache.kafka.streams.LagInfo in project kafka by apache.

the class LagFetchIntegrationTest method shouldFetchLagsDuringRebalancing.

private void shouldFetchLagsDuringRebalancing(final String optimization) throws Exception {
    final CountDownLatch latchTillActiveIsRunning = new CountDownLatch(1);
    final CountDownLatch latchTillStandbyIsRunning = new CountDownLatch(1);
    final CountDownLatch latchTillStandbyHasPartitionsAssigned = new CountDownLatch(1);
    final CyclicBarrier lagCheckBarrier = new CyclicBarrier(2);
    final List<KafkaStreamsWrapper> streamsList = new ArrayList<>();
    IntegrationTestUtils.produceKeyValuesSynchronously(inputTopicName, mkSet(new KeyValue<>("k1", 1L), new KeyValue<>("k2", 2L), new KeyValue<>("k3", 3L), new KeyValue<>("k4", 4L), new KeyValue<>("k5", 5L)), TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, LongSerializer.class, new Properties()), mockTime);
    // create stream threads
    for (int i = 0; i < 2; i++) {
        final Properties props = (Properties) streamsConfiguration.clone();
        // this test relies on the second instance getting the standby, so we specify
        // an assignor with this contract.
        props.put(StreamsConfig.InternalConfig.INTERNAL_TASK_ASSIGNOR_CLASS, FallbackPriorTaskAssignor.class.getName());
        props.put(StreamsConfig.APPLICATION_SERVER_CONFIG, "localhost:" + i);
        props.put(StreamsConfig.CLIENT_ID_CONFIG, "instance-" + i);
        props.put(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, optimization);
        props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
        props.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory(stateStoreName + i).getAbsolutePath());
        final StreamsBuilder builder = new StreamsBuilder();
        final KTable<String, Long> t1 = builder.table(inputTopicName, Materialized.as(stateStoreName));
        t1.toStream().to(outputTopicName);
        final KafkaStreamsWrapper streams = new KafkaStreamsWrapper(builder.build(props), props);
        streamsList.add(streams);
    }
    final KafkaStreamsWrapper activeStreams = streamsList.get(0);
    final KafkaStreamsWrapper standbyStreams = streamsList.get(1);
    activeStreams.setStreamThreadStateListener((thread, newState, oldState) -> {
        if (newState == StreamThread.State.RUNNING) {
            latchTillActiveIsRunning.countDown();
        }
    });
    standbyStreams.setStreamThreadStateListener((thread, newState, oldState) -> {
        if (oldState == StreamThread.State.PARTITIONS_ASSIGNED && newState == StreamThread.State.RUNNING) {
            latchTillStandbyHasPartitionsAssigned.countDown();
            try {
                lagCheckBarrier.await(60, TimeUnit.SECONDS);
            } catch (final Exception e) {
                throw new RuntimeException(e);
            }
        } else if (newState == StreamThread.State.RUNNING) {
            latchTillStandbyIsRunning.countDown();
        }
    });
    try {
        // First start up the active.
        TestUtils.waitForCondition(() -> activeStreams.allLocalStorePartitionLags().size() == 0, WAIT_TIMEOUT_MS, "Should see empty lag map before streams is started.");
        activeStreams.start();
        latchTillActiveIsRunning.await(60, TimeUnit.SECONDS);
        IntegrationTestUtils.waitUntilMinValuesRecordsReceived(consumerConfiguration, outputTopicName, 5, WAIT_TIMEOUT_MS);
        // Check the active reports proper lag values.
        Map<String, Map<Integer, LagInfo>> offsetLagInfoMap = getFirstNonEmptyLagMap(activeStreams);
        assertThat(offsetLagInfoMap.size(), equalTo(1));
        assertThat(offsetLagInfoMap.keySet(), equalTo(mkSet(stateStoreName)));
        assertThat(offsetLagInfoMap.get(stateStoreName).size(), equalTo(1));
        LagInfo lagInfo = offsetLagInfoMap.get(stateStoreName).get(0);
        assertThat(lagInfo.currentOffsetPosition(), equalTo(5L));
        assertThat(lagInfo.endOffsetPosition(), equalTo(5L));
        assertThat(lagInfo.offsetLag(), equalTo(0L));
        // start up the standby & make it pause right after it has partition assigned
        standbyStreams.start();
        latchTillStandbyHasPartitionsAssigned.await(60, TimeUnit.SECONDS);
        offsetLagInfoMap = getFirstNonEmptyLagMap(standbyStreams);
        assertThat(offsetLagInfoMap.size(), equalTo(1));
        assertThat(offsetLagInfoMap.keySet(), equalTo(mkSet(stateStoreName)));
        assertThat(offsetLagInfoMap.get(stateStoreName).size(), equalTo(1));
        lagInfo = offsetLagInfoMap.get(stateStoreName).get(0);
        assertThat(lagInfo.currentOffsetPosition(), equalTo(0L));
        assertThat(lagInfo.endOffsetPosition(), equalTo(5L));
        assertThat(lagInfo.offsetLag(), equalTo(5L));
        // standby thread wont proceed to RUNNING before this barrier is crossed
        lagCheckBarrier.await(60, TimeUnit.SECONDS);
        // wait till the lag goes down to 0, on the standby
        TestUtils.waitForCondition(() -> standbyStreams.allLocalStorePartitionLags().get(stateStoreName).get(0).offsetLag() == 0, WAIT_TIMEOUT_MS, "Standby should eventually catchup and have zero lag.");
    } finally {
        for (final KafkaStreams streams : streamsList) {
            streams.close();
        }
    }
}

Example 3 with LagInfo

use of org.apache.kafka.streams.LagInfo in project kafka by apache.

the class NamedTopologyIntegrationTest method shouldAddAndRemoveNamedTopologiesBeforeStartingAndRouteQueriesToCorrectTopology.

@Test
public void shouldAddAndRemoveNamedTopologiesBeforeStartingAndRouteQueriesToCorrectTopology() throws Exception {
    try {
        // for this test we have one of the topologies read from an input topic with just one partition so
        // that there's only one instance of that topology's store and thus should always have exactly one
        // StreamsMetadata returned by any of the methods that look up all hosts with a specific store and topology
        CLUSTER.createTopic(SINGLE_PARTITION_INPUT_STREAM, 1, 1);
        CLUSTER.createTopic(SINGLE_PARTITION_OUTPUT_STREAM, 1, 1);
        produceToInputTopics(SINGLE_PARTITION_INPUT_STREAM, STANDARD_INPUT_DATA);
        final String topology1Store = "store-" + TOPOLOGY_1;
        final String topology2Store = "store-" + TOPOLOGY_2;
        topology1Builder.stream(INPUT_STREAM_1).groupBy((k, v) -> k).count(Materialized.as(topology1Store)).toStream().to(OUTPUT_STREAM_1);
        topology2Builder.stream(SINGLE_PARTITION_INPUT_STREAM).groupByKey().count(Materialized.as(topology2Store)).toStream().to(SINGLE_PARTITION_OUTPUT_STREAM);
        streams.addNamedTopology(topology1Builder.build());
        streams.removeNamedTopology(TOPOLOGY_1);
        assertThat(streams.getTopologyByName(TOPOLOGY_1), is(Optional.empty()));
        streams.addNamedTopology(topology1Builder.build());
        streams.addNamedTopology(topology2Builder.build());
        IntegrationTestUtils.startApplicationAndWaitUntilRunning(singletonList(streams), Duration.ofSeconds(15));
        assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, OUTPUT_STREAM_1, 3), equalTo(COUNT_OUTPUT_DATA));
        assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, SINGLE_PARTITION_OUTPUT_STREAM, 3), equalTo(COUNT_OUTPUT_DATA));
        final ReadOnlyKeyValueStore<String, Long> store = streams.store(NamedTopologyStoreQueryParameters.fromNamedTopologyAndStoreNameAndType(TOPOLOGY_1, topology1Store, QueryableStoreTypes.keyValueStore()));
        assertThat(store.get("A"), equalTo(2L));
        final Collection<StreamsMetadata> streamsMetadata = streams.streamsMetadataForStore(topology1Store, TOPOLOGY_1);
        final Collection<StreamsMetadata> streamsMetadata2 = streams.streamsMetadataForStore(topology2Store, TOPOLOGY_2);
        assertThat(streamsMetadata.size(), equalTo(1));
        assertThat(streamsMetadata2.size(), equalTo(1));
        final KeyQueryMetadata keyMetadata = streams.queryMetadataForKey(topology1Store, "A", new StringSerializer(), TOPOLOGY_1);
        final KeyQueryMetadata keyMetadata2 = streams.queryMetadataForKey(topology2Store, "A", new StringSerializer(), TOPOLOGY_2);
        assertThat(keyMetadata, not(NOT_AVAILABLE));
        assertThat(keyMetadata, equalTo(keyMetadata2));
        final Map<String, Map<Integer, LagInfo>> partitionLags1 = streams.allLocalStorePartitionLagsForTopology(TOPOLOGY_1);
        final Map<String, Map<Integer, LagInfo>> partitionLags2 = streams.allLocalStorePartitionLagsForTopology(TOPOLOGY_2);
        assertThat(partitionLags1.keySet(), equalTo(singleton(topology1Store)));
        assertThat(partitionLags1.get(topology1Store).keySet(), equalTo(mkSet(0, 1)));
        assertThat(partitionLags2.keySet(), equalTo(singleton(topology2Store)));
        // only one copy of the store in topology-2
        assertThat(partitionLags2.get(topology2Store).keySet(), equalTo(singleton(0)));
        // Start up a second node with both topologies
        setupSecondKafkaStreams();
        topology1Builder2.stream(INPUT_STREAM_1).groupBy((k, v) -> k).count(Materialized.as(topology1Store)).toStream().to(OUTPUT_STREAM_1);
        topology2Builder2.stream(SINGLE_PARTITION_INPUT_STREAM).groupByKey().count(Materialized.as(topology2Store)).toStream().to(SINGLE_PARTITION_OUTPUT_STREAM);
        streams2.start(asList(topology1Builder2.build(), topology2Builder2.build()));
        waitForApplicationState(asList(streams, streams2), State.RUNNING, Duration.ofSeconds(30));
        verifyMetadataForTopology(TOPOLOGY_1, streams.streamsMetadataForStore(topology1Store, TOPOLOGY_1), streams2.streamsMetadataForStore(topology1Store, TOPOLOGY_1));
        verifyMetadataForTopology(TOPOLOGY_2, streams.streamsMetadataForStore(topology2Store, TOPOLOGY_2), streams2.streamsMetadataForStore(topology2Store, TOPOLOGY_2));
        verifyMetadataForTopology(TOPOLOGY_1, streams.allStreamsClientsMetadataForTopology(TOPOLOGY_1), streams2.allStreamsClientsMetadataForTopology(TOPOLOGY_1));
        verifyMetadataForTopology(TOPOLOGY_2, streams.allStreamsClientsMetadataForTopology(TOPOLOGY_2), streams2.allStreamsClientsMetadataForTopology(TOPOLOGY_2));
    } finally {
        CLUSTER.deleteTopics(SINGLE_PARTITION_INPUT_STREAM, SINGLE_PARTITION_OUTPUT_STREAM);
    }
}