Examples with StreamsBuilder org.apache.kafka.streams.StreamsBuilder used on opensource projects

Example 21 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class RestoreIntegrationTest method shouldProcessDataFromStoresWithLoggingDisabled.

@Test
public void shouldProcessDataFromStoresWithLoggingDisabled() throws InterruptedException, ExecutionException {
    IntegrationTestUtils.produceKeyValuesSynchronously(INPUT_STREAM_2, Arrays.asList(KeyValue.pair(1, 1), KeyValue.pair(2, 2), KeyValue.pair(3, 3)), TestUtils.producerConfig(CLUSTER.bootstrapServers(), IntegerSerializer.class, IntegerSerializer.class), CLUSTER.time);
    final KeyValueBytesStoreSupplier lruMapSupplier = Stores.lruMap(INPUT_STREAM_2, 10);
    final StoreBuilder<KeyValueStore<Integer, Integer>> storeBuilder = new KeyValueStoreBuilder<>(lruMapSupplier, Serdes.Integer(), Serdes.Integer(), CLUSTER.time).withLoggingDisabled();
    final StreamsBuilder streamsBuilder = new StreamsBuilder();
    streamsBuilder.addStateStore(storeBuilder);
    final KStream<Integer, Integer> stream = streamsBuilder.stream(INPUT_STREAM_2);
    final CountDownLatch processorLatch = new CountDownLatch(3);
    stream.process(new ProcessorSupplier<Integer, Integer>() {

        @Override
        public Processor<Integer, Integer> get() {
            return new KeyValueStoreProcessor(INPUT_STREAM_2, processorLatch);
        }
    }, INPUT_STREAM_2);
    final Topology topology = streamsBuilder.build();
    kafkaStreams = new KafkaStreams(topology, props(applicationId + "-logging-disabled"));
    final CountDownLatch latch = new CountDownLatch(1);
    kafkaStreams.setStateListener(new KafkaStreams.StateListener() {

        @Override
        public void onChange(final KafkaStreams.State newState, final KafkaStreams.State oldState) {
            if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
                latch.countDown();
            }
        }
    });
    kafkaStreams.start();
    latch.await(30, TimeUnit.SECONDS);
    assertTrue(processorLatch.await(30, TimeUnit.SECONDS));
}

Example 22 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class StreamTableJoinIntegrationTest method prepareTopology.

@Before
public void prepareTopology() throws InterruptedException {
    super.prepareEnvironment();
    appID = "stream-table-join-integration-test";
    builder = new StreamsBuilder();
    rightTable = builder.table(INPUT_TOPIC_RIGHT);
    leftStream = builder.stream(INPUT_TOPIC_LEFT);
}

Example 23 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class KTableAggregateTest method testRemoveOldBeforeAddNew.

@Test
public void testRemoveOldBeforeAddNew() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String input = "count-test-input";
    final MockProcessorSupplier<String, String> proc = new MockProcessorSupplier<>();
    builder.table(input, consumed).groupBy(new KeyValueMapper<String, String, KeyValue<String, String>>() {

        @Override
        public KeyValue<String, String> apply(String key, String value) {
            return KeyValue.pair(String.valueOf(key.charAt(0)), String.valueOf(key.charAt(1)));
        }
    }, stringSerialzied).aggregate(new Initializer<String>() {

        @Override
        public String apply() {
            return "";
        }
    }, new Aggregator<String, String, String>() {

        @Override
        public String apply(String aggKey, String value, String aggregate) {
            return aggregate + value;
        }
    }, new Aggregator<String, String, String>() {

        @Override
        public String apply(String key, String value, String aggregate) {
            return aggregate.replaceAll(value, "");
        }
    }, Serdes.String(), "someStore").toStream().process(proc);
    driver.setUp(builder, stateDir);
    driver.process(input, "11", "A");
    driver.flushState();
    driver.process(input, "12", "B");
    driver.flushState();
    driver.process(input, "11", null);
    driver.flushState();
    driver.process(input, "12", "C");
    driver.flushState();
    assertEquals(Utils.mkList("1:1", "1:12", "1:2", "1:2"), proc.processed);
}

Example 24 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class KTableAggregateTest method testCount.

@Test
public void testCount() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String input = "count-test-input";
    final MockProcessorSupplier<String, Long> proc = new MockProcessorSupplier<>();
    builder.table(input, consumed).groupBy(MockMapper.<String, String>selectValueKeyValueMapper(), stringSerialzied).count("count").toStream().process(proc);
    testCountHelper(builder, input, proc);
}

Example 25 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class KTableAggregateTest method testAggBasic.

@Test
public void testAggBasic() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final MockProcessorSupplier<String, String> proc = new MockProcessorSupplier<>();
    KTable<String, String> table1 = builder.table(topic1, consumed);
    KTable<String, String> table2 = table1.groupBy(MockMapper.<String, String>noOpKeyValueMapper(), stringSerialzied).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, stringSerde, "topic1-Canonized");
    table2.toStream().process(proc);
    driver.setUp(builder, stateDir, Serdes.String(), Serdes.String());
    driver.process(topic1, "A", "1");
    driver.flushState();
    driver.process(topic1, "B", "2");
    driver.flushState();
    driver.process(topic1, "A", "3");
    driver.flushState();
    driver.process(topic1, "B", "4");
    driver.flushState();
    driver.process(topic1, "C", "5");
    driver.flushState();
    driver.process(topic1, "D", "6");
    driver.flushState();
    driver.process(topic1, "B", "7");
    driver.flushState();
    driver.process(topic1, "C", "8");
    driver.flushState();
    assertEquals(Utils.mkList("A:0+1", "B:0+2", "A:0+1-1+3", "B:0+2-2+4", "C:0+5", "D:0+6", "B:0+2-2+4-4+7", "C:0+5-5+8"), proc.processed);
}