Examples with KeyValue org.apache.kafka.streams.KeyValue used on opensource projects

Example 11 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class AbstractResetIntegrationTest method testReprocessingFromFileAfterResetWithoutIntermediateUserTopic.

void testReprocessingFromFileAfterResetWithoutIntermediateUserTopic() throws Exception {
    appID = testId + "-from-file";
    streamsConfig.put(StreamsConfig.APPLICATION_ID_CONFIG, appID);
    // RUN
    streams = new KafkaStreams(setupTopologyWithoutIntermediateUserTopic(), streamsConfig);
    streams.start();
    final List<KeyValue<Long, Long>> result = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(resultConsumerConfig, OUTPUT_TOPIC, 10);
    streams.close();
    TestUtils.waitForCondition(new ConsumerGroupInactiveCondition(), TIMEOUT_MULTIPLIER * STREAMS_CONSUMER_TIMEOUT, "Streams Application consumer group " + appID + " did not time out after " + (TIMEOUT_MULTIPLIER * STREAMS_CONSUMER_TIMEOUT) + " ms.");
    // RESET
    final File resetFile = File.createTempFile("reset", ".csv");
    try (BufferedWriter writer = new BufferedWriter(new FileWriter(resetFile))) {
        writer.write(INPUT_TOPIC + ",0,1");
        writer.close();
    }
    streams = new KafkaStreams(setupTopologyWithoutIntermediateUserTopic(), streamsConfig);
    streams.cleanUp();
    cleanGlobal(false, "--from-file", resetFile.getAbsolutePath());
    TestUtils.waitForCondition(new ConsumerGroupInactiveCondition(), TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT, "Reset Tool consumer group " + appID + " did not time out after " + (TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT) + " ms.");
    assertInternalTopicsGotDeleted(null);
    resetFile.deleteOnExit();
    // RE-RUN
    streams.start();
    final List<KeyValue<Long, Long>> resultRerun = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(resultConsumerConfig, OUTPUT_TOPIC, 5);
    streams.close();
    result.remove(0);
    assertThat(resultRerun, equalTo(result));
    TestUtils.waitForCondition(new ConsumerGroupInactiveCondition(), TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT, "Reset Tool consumer group " + appID + " did not time out after " + (TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT) + " ms.");
    cleanGlobal(false, null, null);
}

Example 12 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class AbstractResetIntegrationTest method testReprocessingFromScratchAfterResetWithIntermediateUserTopic.

void testReprocessingFromScratchAfterResetWithIntermediateUserTopic() throws Exception {
    cluster.createTopic(INTERMEDIATE_USER_TOPIC);
    appID = testId + "-from-scratch-with-intermediate-topic";
    streamsConfig.put(StreamsConfig.APPLICATION_ID_CONFIG, appID);
    // RUN
    streams = new KafkaStreams(setupTopologyWithIntermediateUserTopic(OUTPUT_TOPIC_2), streamsConfig);
    streams.start();
    final List<KeyValue<Long, Long>> result = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(resultConsumerConfig, OUTPUT_TOPIC, 10);
    // receive only first values to make sure intermediate user topic is not consumed completely
    // => required to test "seekToEnd" for intermediate topics
    final List<KeyValue<Long, Long>> result2 = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(resultConsumerConfig, OUTPUT_TOPIC_2, 40);
    streams.close();
    TestUtils.waitForCondition(new ConsumerGroupInactiveCondition(), TIMEOUT_MULTIPLIER * STREAMS_CONSUMER_TIMEOUT, "Streams Application consumer group " + appID + " did not time out after " + (TIMEOUT_MULTIPLIER * STREAMS_CONSUMER_TIMEOUT) + " ms.");
    // insert bad record to make sure intermediate user topic gets seekToEnd()
    mockTime.sleep(1);
    KeyValue<Long, String> badMessage = new KeyValue<>(-1L, "badRecord-ShouldBeSkipped");
    IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(INTERMEDIATE_USER_TOPIC, Collections.singleton(badMessage), producerConfig, mockTime.milliseconds());
    // RESET
    streams = new KafkaStreams(setupTopologyWithIntermediateUserTopic(OUTPUT_TOPIC_2_RERUN), streamsConfig);
    streams.cleanUp();
    cleanGlobal(true, null, null);
    TestUtils.waitForCondition(new ConsumerGroupInactiveCondition(), TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT, "Reset Tool consumer group " + appID + " did not time out after " + (TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT) + " ms.");
    assertInternalTopicsGotDeleted(INTERMEDIATE_USER_TOPIC);
    // RE-RUN
    streams.start();
    final List<KeyValue<Long, Long>> resultRerun = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(resultConsumerConfig, OUTPUT_TOPIC, 10);
    final List<KeyValue<Long, Long>> resultRerun2 = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(resultConsumerConfig, OUTPUT_TOPIC_2_RERUN, 40);
    streams.close();
    assertThat(resultRerun, equalTo(result));
    assertThat(resultRerun2, equalTo(result2));
    final Properties props = TestUtils.consumerConfig(cluster.bootstrapServers(), testId + "-result-consumer", LongDeserializer.class, StringDeserializer.class, commonClientConfig);
    final List<KeyValue<Long, String>> resultIntermediate = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(props, INTERMEDIATE_USER_TOPIC, 21);
    for (int i = 0; i < 10; i++) {
        assertThat(resultIntermediate.get(i), equalTo(resultIntermediate.get(i + 11)));
    }
    assertThat(resultIntermediate.get(10), equalTo(badMessage));
    TestUtils.waitForCondition(new ConsumerGroupInactiveCondition(), TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT, "Reset Tool consumer group " + appID + " did not time out after " + (TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT) + " ms.");
    cleanGlobal(true, null, null);
    cluster.deleteTopicAndWait(INTERMEDIATE_USER_TOPIC);
}

Example 13 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class AbstractResetIntegrationTest method testReprocessingFromScratchAfterResetWithoutIntermediateUserTopic.

void testReprocessingFromScratchAfterResetWithoutIntermediateUserTopic() throws Exception {
    appID = testId + "-from-scratch";
    streamsConfig.put(StreamsConfig.APPLICATION_ID_CONFIG, appID);
    // RUN
    streams = new KafkaStreams(setupTopologyWithoutIntermediateUserTopic(), streamsConfig);
    streams.start();
    final List<KeyValue<Long, Long>> result = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(resultConsumerConfig, OUTPUT_TOPIC, 10);
    streams.close();
    TestUtils.waitForCondition(new ConsumerGroupInactiveCondition(), TIMEOUT_MULTIPLIER * STREAMS_CONSUMER_TIMEOUT, "Streams Application consumer group " + appID + " did not time out after " + (TIMEOUT_MULTIPLIER * STREAMS_CONSUMER_TIMEOUT) + " ms.");
    // RESET
    streams = new KafkaStreams(setupTopologyWithoutIntermediateUserTopic(), streamsConfig);
    streams.cleanUp();
    cleanGlobal(false, null, null);
    TestUtils.waitForCondition(new ConsumerGroupInactiveCondition(), TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT, "Reset Tool consumer group " + appID + " did not time out after " + (TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT) + " ms.");
    assertInternalTopicsGotDeleted(null);
    // RE-RUN
    streams.start();
    final List<KeyValue<Long, Long>> resultRerun = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(resultConsumerConfig, OUTPUT_TOPIC, 10);
    streams.close();
    assertThat(resultRerun, equalTo(result));
    TestUtils.waitForCondition(new ConsumerGroupInactiveCondition(), TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT, "Reset Tool consumer group " + appID + " did not time out after " + (TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT) + " ms.");
    cleanGlobal(false, null, null);
}

Example 14 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class EosIntegrationTest method runSimpleCopyTest.

private void runSimpleCopyTest(final int numberOfRestarts, final String inputTopic, final String throughTopic, final String outputTopic) throws Exception {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Long, Long> input = builder.stream(inputTopic);
    KStream<Long, Long> output = input;
    if (throughTopic != null) {
        output = input.through(throughTopic);
    }
    output.to(outputTopic);
    for (int i = 0; i < numberOfRestarts; ++i) {
        final KafkaStreams streams = new KafkaStreams(builder.build(), StreamsTestUtils.getStreamsConfig(applicationId, CLUSTER.bootstrapServers(), Serdes.LongSerde.class.getName(), Serdes.LongSerde.class.getName(), new Properties() {

            {
                put(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), 1);
                put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
            }
        }));
        try {
            streams.start();
            final List<KeyValue<Long, Long>> inputData = prepareData(i * 100, i * 100 + 10L, 0L, 1L);
            IntegrationTestUtils.produceKeyValuesSynchronously(inputTopic, inputData, TestUtils.producerConfig(CLUSTER.bootstrapServers(), LongSerializer.class, LongSerializer.class), CLUSTER.time);
            final List<KeyValue<Long, Long>> committedRecords = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(TestUtils.consumerConfig(CLUSTER.bootstrapServers(), CONSUMER_GROUP_ID, LongDeserializer.class, LongDeserializer.class, new Properties() {

                {
                    put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT));
                }
            }), outputTopic, inputData.size());
            checkResultPerKey(committedRecords, inputData);
        } finally {
            streams.close();
        }
    }
}

Example 15 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class GlobalKTableIntegrationTest method produceInitialGlobalTableValues.

private void produceInitialGlobalTableValues(final boolean enableTransactions) throws Exception {
    Properties properties = new Properties();
    if (enableTransactions) {
        properties.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "someid");
        properties.put(ProducerConfig.RETRIES_CONFIG, 1);
    }
    IntegrationTestUtils.produceKeyValuesSynchronously(globalTableTopic, Arrays.asList(new KeyValue<>(1L, "A"), new KeyValue<>(2L, "B"), new KeyValue<>(3L, "C"), new KeyValue<>(4L, "D")), TestUtils.producerConfig(CLUSTER.bootstrapServers(), LongSerializer.class, StringSerializer.class, properties), mockTime, enableTransactions);
}