Examples with KafkaStreams org.apache.kafka.streams.KafkaStreams used on opensource projects

Example 11 with KafkaStreams

use of org.apache.kafka.streams.KafkaStreams in project kafka by apache.

the class KStreamAggregationIntegrationTest method startStreams.

private void startStreams() {
    kafkaStreams = new KafkaStreams(builder, streamsConfiguration);
    kafkaStreams.start();
}

Example 12 with KafkaStreams

use of org.apache.kafka.streams.KafkaStreams in project kafka by apache.

the class KStreamsFineGrainedAutoResetIntegrationTest method shouldOnlyReadRecordsWhereEarliestSpecified.

@Test
public void shouldOnlyReadRecordsWhereEarliestSpecified() throws Exception {
    final KStreamBuilder builder = new KStreamBuilder();
    final KStream<String, String> pattern1Stream = builder.stream(KStreamBuilder.AutoOffsetReset.EARLIEST, Pattern.compile("topic-\\d"));
    final KStream<String, String> pattern2Stream = builder.stream(KStreamBuilder.AutoOffsetReset.LATEST, Pattern.compile("topic-[A-D]"));
    final KStream<String, String> namedTopicsStream = builder.stream(TOPIC_Y, TOPIC_Z);
    pattern1Stream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC);
    pattern2Stream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC);
    namedTopicsStream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC);
    final Properties producerConfig = TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class);
    final String topic1TestMessage = "topic-1 test";
    final String topic2TestMessage = "topic-2 test";
    final String topicATestMessage = "topic-A test";
    final String topicCTestMessage = "topic-C test";
    final String topicYTestMessage = "topic-Y test";
    final String topicZTestMessage = "topic-Z test";
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_1, Collections.singletonList(topic1TestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_2, Collections.singletonList(topic2TestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_A, Collections.singletonList(topicATestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_C, Collections.singletonList(topicCTestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_Y, Collections.singletonList(topicYTestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_Z, Collections.singletonList(topicZTestMessage), producerConfig, mockTime);
    final Properties consumerConfig = TestUtils.consumerConfig(CLUSTER.bootstrapServers(), StringDeserializer.class, StringDeserializer.class);
    final KafkaStreams streams = new KafkaStreams(builder, streamsConfiguration);
    streams.start();
    final List<String> expectedReceivedValues = Arrays.asList(topic1TestMessage, topic2TestMessage, topicYTestMessage, topicZTestMessage);
    final List<KeyValue<String, String>> receivedKeyValues = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(consumerConfig, DEFAULT_OUTPUT_TOPIC, 4);
    final List<String> actualValues = new ArrayList<>(4);
    for (final KeyValue<String, String> receivedKeyValue : receivedKeyValues) {
        actualValues.add(receivedKeyValue.value);
    }
    streams.close();
    Collections.sort(actualValues);
    Collections.sort(expectedReceivedValues);
    assertThat(actualValues, equalTo(expectedReceivedValues));
}

Example 13 with KafkaStreams

use of org.apache.kafka.streams.KafkaStreams in project kafka by apache.

the class KTableKTableJoinIntegrationTest method prepareTopology.

private KafkaStreams prepareTopology() {
    final KStreamBuilder builder = new KStreamBuilder();
    final KTable<String, String> table1 = builder.table(TABLE_1, TABLE_1);
    final KTable<String, String> table2 = builder.table(TABLE_2, TABLE_2);
    final KTable<String, String> table3 = builder.table(TABLE_3, TABLE_3);
    join(join(table1, table2, joinType1), table3, joinType2).to(OUTPUT);
    return new KafkaStreams(builder, new StreamsConfig(streamsConfig));
}

Example 14 with KafkaStreams

use of org.apache.kafka.streams.KafkaStreams in project kafka by apache.

the class SimpleBenchmark method processStream.

public void processStream(final String topic) throws Exception {
    if (maybeSetupPhase(topic, "simple-benchmark-process-stream-load", true)) {
        return;
    }
    CountDownLatch latch = new CountDownLatch(1);
    final KafkaStreams streams = createKafkaStreams(topic, latch);
    long latency = startStreamsThread(streams, latch);
    printResults("Streams Performance [records/latency/rec-sec/MB-sec source]: ", latency);
}

Example 15 with KafkaStreams

use of org.apache.kafka.streams.KafkaStreams in project kafka by apache.

the class SimpleBenchmark method processStreamWithStateStore.

public void processStreamWithStateStore(String topic) throws Exception {
    if (maybeSetupPhase(topic, "simple-benchmark-process-stream-with-state-store-load", true)) {
        return;
    }
    CountDownLatch latch = new CountDownLatch(1);
    final KafkaStreams streams = createKafkaStreamsWithStateStore(topic, latch, false);
    long latency = startStreamsThread(streams, latch);
    printResults("Streams Performance [records/latency/rec-sec/MB-sec source+store]: ", latency);
}