Examples with KStreamBuilder org.apache.kafka.streams.kstream.KStreamBuilder used on opensource projects

Example 16 with KStreamBuilder

use of org.apache.kafka.streams.kstream.KStreamBuilder in project kafka by apache.

the class KStreamsFineGrainedAutoResetIntegrationTest method shouldOnlyReadRecordsWhereEarliestSpecified.

@Test
public void shouldOnlyReadRecordsWhereEarliestSpecified() throws Exception {
    final KStreamBuilder builder = new KStreamBuilder();
    final KStream<String, String> pattern1Stream = builder.stream(KStreamBuilder.AutoOffsetReset.EARLIEST, Pattern.compile("topic-\\d"));
    final KStream<String, String> pattern2Stream = builder.stream(KStreamBuilder.AutoOffsetReset.LATEST, Pattern.compile("topic-[A-D]"));
    final KStream<String, String> namedTopicsStream = builder.stream(TOPIC_Y, TOPIC_Z);
    pattern1Stream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC);
    pattern2Stream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC);
    namedTopicsStream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC);
    final Properties producerConfig = TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class);
    final String topic1TestMessage = "topic-1 test";
    final String topic2TestMessage = "topic-2 test";
    final String topicATestMessage = "topic-A test";
    final String topicCTestMessage = "topic-C test";
    final String topicYTestMessage = "topic-Y test";
    final String topicZTestMessage = "topic-Z test";
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_1, Collections.singletonList(topic1TestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_2, Collections.singletonList(topic2TestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_A, Collections.singletonList(topicATestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_C, Collections.singletonList(topicCTestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_Y, Collections.singletonList(topicYTestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_Z, Collections.singletonList(topicZTestMessage), producerConfig, mockTime);
    final Properties consumerConfig = TestUtils.consumerConfig(CLUSTER.bootstrapServers(), StringDeserializer.class, StringDeserializer.class);
    final KafkaStreams streams = new KafkaStreams(builder, streamsConfiguration);
    streams.start();
    final List<String> expectedReceivedValues = Arrays.asList(topic1TestMessage, topic2TestMessage, topicYTestMessage, topicZTestMessage);
    final List<KeyValue<String, String>> receivedKeyValues = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(consumerConfig, DEFAULT_OUTPUT_TOPIC, 4);
    final List<String> actualValues = new ArrayList<>(4);
    for (final KeyValue<String, String> receivedKeyValue : receivedKeyValues) {
        actualValues.add(receivedKeyValue.value);
    }
    streams.close();
    Collections.sort(actualValues);
    Collections.sort(expectedReceivedValues);
    assertThat(actualValues, equalTo(expectedReceivedValues));
}

Example 17 with KStreamBuilder

use of org.apache.kafka.streams.kstream.KStreamBuilder in project kafka by apache.

the class KTableKTableJoinIntegrationTest method prepareTopology.

private KafkaStreams prepareTopology() {
    final KStreamBuilder builder = new KStreamBuilder();
    final KTable<String, String> table1 = builder.table(TABLE_1, TABLE_1);
    final KTable<String, String> table2 = builder.table(TABLE_2, TABLE_2);
    final KTable<String, String> table3 = builder.table(TABLE_3, TABLE_3);
    join(join(table1, table2, joinType1), table3, joinType2).to(OUTPUT);
    return new KafkaStreams(builder, new StreamsConfig(streamsConfig));
}

Example 18 with KStreamBuilder

use of org.apache.kafka.streams.kstream.KStreamBuilder in project kafka by apache.

the class KStreamFilterTest method testFilterNot.

@Test
public void testFilterNot() {
    KStreamBuilder builder = new KStreamBuilder();
    final int[] expectedKeys = new int[] { 1, 2, 3, 4, 5, 6, 7 };
    KStream<Integer, String> stream;
    MockProcessorSupplier<Integer, String> processor;
    processor = new MockProcessorSupplier<>();
    stream = builder.stream(Serdes.Integer(), Serdes.String(), topicName);
    stream.filterNot(isMultipleOfThree).process(processor);
    driver = new KStreamTestDriver(builder);
    for (int expectedKey : expectedKeys) {
        driver.process(topicName, expectedKey, "V" + expectedKey);
    }
    assertEquals(5, processor.processed.size());
}

Example 19 with KStreamBuilder

use of org.apache.kafka.streams.kstream.KStreamBuilder in project kafka by apache.

the class KStreamFlatMapTest method testFlatMap.

@Test
public void testFlatMap() {
    KStreamBuilder builder = new KStreamBuilder();
    KeyValueMapper<Number, Object, Iterable<KeyValue<String, String>>> mapper = new KeyValueMapper<Number, Object, Iterable<KeyValue<String, String>>>() {

        @Override
        public Iterable<KeyValue<String, String>> apply(Number key, Object value) {
            ArrayList<KeyValue<String, String>> result = new ArrayList<>();
            for (int i = 0; i < key.intValue(); i++) {
                result.add(KeyValue.pair(Integer.toString(key.intValue() * 10 + i), value.toString()));
            }
            return result;
        }
    };
    final int[] expectedKeys = { 0, 1, 2, 3 };
    KStream<Integer, String> stream;
    MockProcessorSupplier<String, String> processor;
    processor = new MockProcessorSupplier<>();
    stream = builder.stream(Serdes.Integer(), Serdes.String(), topicName);
    stream.flatMap(mapper).process(processor);
    driver = new KStreamTestDriver(builder);
    for (int expectedKey : expectedKeys) {
        driver.process(topicName, expectedKey, "V" + expectedKey);
    }
    assertEquals(6, processor.processed.size());
    String[] expected = { "10:V1", "20:V2", "21:V2", "30:V3", "31:V3", "32:V3" };
    for (int i = 0; i < expected.length; i++) {
        assertEquals(expected[i], processor.processed.get(i));
    }
}

Example 20 with KStreamBuilder

use of org.apache.kafka.streams.kstream.KStreamBuilder in project kafka by apache.

the class KStreamImplTest method before.

@Before
public void before() {
    builder = new KStreamBuilder();
    testStream = builder.stream("source");
}