Examples with KTable org.apache.kafka.streams.kstream.KTable used on opensource projects

Example 41 with KTable

use of org.apache.kafka.streams.kstream.KTable in project kafka by apache.

the class KTableImplTest method shouldPreserveSerdesForOperators.

@Test
public void shouldPreserveSerdesForOperators() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<String, String> table1 = builder.table("topic-2", stringConsumed);
    final ConsumedInternal<String, String> consumedInternal = new ConsumedInternal<>(stringConsumed);
    final KeyValueMapper<String, String, String> selector = (key, value) -> key;
    final ValueMapper<String, String> mapper = value -> value;
    final ValueJoiner<String, String, String> joiner = (value1, value2) -> value1;
    final ValueTransformerWithKeySupplier<String, String, String> valueTransformerWithKeySupplier = () -> new ValueTransformerWithKey<String, String, String>() {

        @Override
        public void init(final ProcessorContext context) {
        }

        @Override
        public String transform(final String key, final String value) {
            return value;
        }

        @Override
        public void close() {
        }
    };
    assertEquals(((AbstractStream) table1.filter((key, value) -> false)).keySerde(), consumedInternal.keySerde());
    assertEquals(((AbstractStream) table1.filter((key, value) -> false)).valueSerde(), consumedInternal.valueSerde());
    assertEquals(((AbstractStream) table1.filter((key, value) -> false, Materialized.with(mySerde, mySerde))).keySerde(), mySerde);
    assertEquals(((AbstractStream) table1.filter((key, value) -> false, Materialized.with(mySerde, mySerde))).valueSerde(), mySerde);
    assertEquals(((AbstractStream) table1.filterNot((key, value) -> false)).keySerde(), consumedInternal.keySerde());
    assertEquals(((AbstractStream) table1.filterNot((key, value) -> false)).valueSerde(), consumedInternal.valueSerde());
    assertEquals(((AbstractStream) table1.filterNot((key, value) -> false, Materialized.with(mySerde, mySerde))).keySerde(), mySerde);
    assertEquals(((AbstractStream) table1.filterNot((key, value) -> false, Materialized.with(mySerde, mySerde))).valueSerde(), mySerde);
    assertEquals(((AbstractStream) table1.mapValues(mapper)).keySerde(), consumedInternal.keySerde());
    assertNull(((AbstractStream) table1.mapValues(mapper)).valueSerde());
    assertEquals(((AbstractStream) table1.mapValues(mapper, Materialized.with(mySerde, mySerde))).keySerde(), mySerde);
    assertEquals(((AbstractStream) table1.mapValues(mapper, Materialized.with(mySerde, mySerde))).valueSerde(), mySerde);
    assertEquals(((AbstractStream) table1.toStream()).keySerde(), consumedInternal.keySerde());
    assertEquals(((AbstractStream) table1.toStream()).valueSerde(), consumedInternal.valueSerde());
    assertNull(((AbstractStream) table1.toStream(selector)).keySerde());
    assertEquals(((AbstractStream) table1.toStream(selector)).valueSerde(), consumedInternal.valueSerde());
    assertEquals(((AbstractStream) table1.transformValues(valueTransformerWithKeySupplier)).keySerde(), consumedInternal.keySerde());
    assertNull(((AbstractStream) table1.transformValues(valueTransformerWithKeySupplier)).valueSerde());
    assertEquals(((AbstractStream) table1.transformValues(valueTransformerWithKeySupplier, Materialized.with(mySerde, mySerde))).keySerde(), mySerde);
    assertEquals(((AbstractStream) table1.transformValues(valueTransformerWithKeySupplier, Materialized.with(mySerde, mySerde))).valueSerde(), mySerde);
    assertNull(((AbstractStream) table1.groupBy(KeyValue::new)).keySerde());
    assertNull(((AbstractStream) table1.groupBy(KeyValue::new)).valueSerde());
    assertEquals(((AbstractStream) table1.groupBy(KeyValue::new, Grouped.with(mySerde, mySerde))).keySerde(), mySerde);
    assertEquals(((AbstractStream) table1.groupBy(KeyValue::new, Grouped.with(mySerde, mySerde))).valueSerde(), mySerde);
    assertEquals(((AbstractStream) table1.join(table1, joiner)).keySerde(), consumedInternal.keySerde());
    assertNull(((AbstractStream) table1.join(table1, joiner)).valueSerde());
    assertEquals(((AbstractStream) table1.join(table1, joiner, Materialized.with(mySerde, mySerde))).keySerde(), mySerde);
    assertEquals(((AbstractStream) table1.join(table1, joiner, Materialized.with(mySerde, mySerde))).valueSerde(), mySerde);
    assertEquals(((AbstractStream) table1.leftJoin(table1, joiner)).keySerde(), consumedInternal.keySerde());
    assertNull(((AbstractStream) table1.leftJoin(table1, joiner)).valueSerde());
    assertEquals(((AbstractStream) table1.leftJoin(table1, joiner, Materialized.with(mySerde, mySerde))).keySerde(), mySerde);
    assertEquals(((AbstractStream) table1.leftJoin(table1, joiner, Materialized.with(mySerde, mySerde))).valueSerde(), mySerde);
    assertEquals(((AbstractStream) table1.outerJoin(table1, joiner)).keySerde(), consumedInternal.keySerde());
    assertNull(((AbstractStream) table1.outerJoin(table1, joiner)).valueSerde());
    assertEquals(((AbstractStream) table1.outerJoin(table1, joiner, Materialized.with(mySerde, mySerde))).keySerde(), mySerde);
    assertEquals(((AbstractStream) table1.outerJoin(table1, joiner, Materialized.with(mySerde, mySerde))).valueSerde(), mySerde);
}

Example 42 with KTable

use of org.apache.kafka.streams.kstream.KTable in project kafka by apache.

the class KStreamKTableJoinTest method shouldCreateRepartitionTopicsWithUserProvidedName.

@Test
public void shouldCreateRepartitionTopicsWithUserProvidedName() {
    final StreamsBuilder builder = new StreamsBuilder();
    final Properties props = new Properties();
    props.put(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, StreamsConfig.NO_OPTIMIZATION);
    final KStream<String, String> streamA = builder.stream("topic", Consumed.with(Serdes.String(), Serdes.String()));
    final KTable<String, String> tableB = builder.table("topic2", Consumed.with(Serdes.String(), Serdes.String()));
    final KTable<String, String> tableC = builder.table("topic3", Consumed.with(Serdes.String(), Serdes.String()));
    final KStream<String, String> rekeyedStream = streamA.map((k, v) -> new KeyValue<>(v, k));
    rekeyedStream.join(tableB, (value1, value2) -> value1 + value2, Joined.with(Serdes.String(), Serdes.String(), Serdes.String(), "first-join")).to("out-one");
    rekeyedStream.join(tableC, (value1, value2) -> value1 + value2, Joined.with(Serdes.String(), Serdes.String(), Serdes.String(), "second-join")).to("out-two");
    final Topology topology = builder.build(props);
    System.out.println(topology.describe().toString());
    assertEquals(expectedTopologyWithUserProvidedRepartitionTopicNames, topology.describe().toString());
}

Example 43 with KTable

use of org.apache.kafka.streams.kstream.KTable in project kafka by apache.

the class KStreamKTableJoinTest method shouldReuseRepartitionTopicWithGeneratedName.

@Test
public void shouldReuseRepartitionTopicWithGeneratedName() {
    final StreamsBuilder builder = new StreamsBuilder();
    final Properties props = new Properties();
    props.put(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, StreamsConfig.NO_OPTIMIZATION);
    final KStream<String, String> streamA = builder.stream("topic", Consumed.with(Serdes.String(), Serdes.String()));
    final KTable<String, String> tableB = builder.table("topic2", Consumed.with(Serdes.String(), Serdes.String()));
    final KTable<String, String> tableC = builder.table("topic3", Consumed.with(Serdes.String(), Serdes.String()));
    final KStream<String, String> rekeyedStream = streamA.map((k, v) -> new KeyValue<>(v, k));
    rekeyedStream.join(tableB, (value1, value2) -> value1 + value2).to("out-one");
    rekeyedStream.join(tableC, (value1, value2) -> value1 + value2).to("out-two");
    final Topology topology = builder.build(props);
    assertEquals(expectedTopologyWithGeneratedRepartitionTopicNames, topology.describe().toString());
}

Example 44 with KTable

use of org.apache.kafka.streams.kstream.KTable in project kafka by apache.

the class QueryableStateIntegrationTest method shouldBeAbleToQueryFilterState.

@Test
public void shouldBeAbleToQueryFilterState() throws Exception {
    streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass());
    final StreamsBuilder builder = new StreamsBuilder();
    final String[] keys = { "hello", "goodbye", "welcome", "go", "kafka" };
    final Set<KeyValue<String, Long>> batch1 = new HashSet<>(Arrays.asList(new KeyValue<>(keys[0], 1L), new KeyValue<>(keys[1], 1L), new KeyValue<>(keys[2], 3L), new KeyValue<>(keys[3], 5L), new KeyValue<>(keys[4], 2L)));
    final Set<KeyValue<String, Long>> expectedBatch1 = new HashSet<>(Collections.singleton(new KeyValue<>(keys[4], 2L)));
    IntegrationTestUtils.produceKeyValuesSynchronously(streamOne, batch1, TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, LongSerializer.class, new Properties()), mockTime);
    final Predicate<String, Long> filterPredicate = (key, value) -> key.contains("kafka");
    final KTable<String, Long> t1 = builder.table(streamOne);
    final KTable<String, Long> t2 = t1.filter(filterPredicate, Materialized.as("queryFilter"));
    t1.filterNot(filterPredicate, Materialized.as("queryFilterNot"));
    t2.toStream().to(outputTopic);
    kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
    startKafkaStreamsAndWaitForRunningState(kafkaStreams);
    waitUntilAtLeastNumRecordProcessed(outputTopic, 1);
    final ReadOnlyKeyValueStore<String, Long> myFilterStore = IntegrationTestUtils.getStore("queryFilter", kafkaStreams, keyValueStore());
    final ReadOnlyKeyValueStore<String, Long> myFilterNotStore = IntegrationTestUtils.getStore("queryFilterNot", kafkaStreams, keyValueStore());
    for (final KeyValue<String, Long> expectedEntry : expectedBatch1) {
        TestUtils.waitForCondition(() -> expectedEntry.value.equals(myFilterStore.get(expectedEntry.key)), "Cannot get expected result");
    }
    for (final KeyValue<String, Long> batchEntry : batch1) {
        if (!expectedBatch1.contains(batchEntry)) {
            TestUtils.waitForCondition(() -> myFilterStore.get(batchEntry.key) == null, "Cannot get null result");
        }
    }
    for (final KeyValue<String, Long> expectedEntry : expectedBatch1) {
        TestUtils.waitForCondition(() -> myFilterNotStore.get(expectedEntry.key) == null, "Cannot get null result");
    }
    for (final KeyValue<String, Long> batchEntry : batch1) {
        if (!expectedBatch1.contains(batchEntry)) {
            TestUtils.waitForCondition(() -> batchEntry.value.equals(myFilterNotStore.get(batchEntry.key)), "Cannot get expected result");
        }
    }
}

Example 45 with KTable

use of org.apache.kafka.streams.kstream.KTable in project kafka by apache.

the class NamedTopologyIntegrationTest method shouldPrefixAllInternalTopicNamesWithNamedTopology.

@Test
public void shouldPrefixAllInternalTopicNamesWithNamedTopology() throws Exception {
    final String countTopologyName = "count-topology";
    final String fkjTopologyName = "FKJ-topology";
    final NamedTopologyBuilder countBuilder = streams.newNamedTopologyBuilder(countTopologyName);
    countBuilder.stream(INPUT_STREAM_1).groupBy((k, v) -> k).count();
    final NamedTopologyBuilder fkjBuilder = streams.newNamedTopologyBuilder(fkjTopologyName);
    final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
    final KTable<String, Long> left = fkjBuilder.table(INPUT_STREAM_2, Consumed.with(serdeScope.decorateSerde(Serdes.String(), props, true), serdeScope.decorateSerde(Serdes.Long(), props, false)));
    final KTable<String, Long> right = fkjBuilder.table(INPUT_STREAM_3, Consumed.with(serdeScope.decorateSerde(Serdes.String(), props, true), serdeScope.decorateSerde(Serdes.Long(), props, false)));
    left.join(right, Object::toString, (value1, value2) -> String.valueOf(value1 + value2), Materialized.with(null, serdeScope.decorateSerde(Serdes.String(), props, false)));
    streams.start(asList(fkjBuilder.build(), countBuilder.build()));
    waitForApplicationState(singletonList(streams), State.RUNNING, Duration.ofSeconds(60));
    final String countTopicPrefix = TOPIC_PREFIX + "-" + countTopologyName;
    final String fkjTopicPrefix = TOPIC_PREFIX + "-" + fkjTopologyName;
    final Set<String> internalTopics = CLUSTER.getAllTopicsInCluster().stream().filter(t -> t.contains(TOPIC_PREFIX)).filter(t -> t.endsWith("-repartition") || t.endsWith("-changelog") || t.endsWith("-topic")).collect(Collectors.toSet());
    assertThat(internalTopics, is(mkSet(countTopicPrefix + "-KSTREAM-AGGREGATE-STATE-STORE-0000000002-repartition", countTopicPrefix + "-KSTREAM-AGGREGATE-STATE-STORE-0000000002-changelog", fkjTopicPrefix + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic", fkjTopicPrefix + "-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic", fkjTopicPrefix + "-KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000010-changelog", fkjTopicPrefix + "-" + INPUT_STREAM_2 + "-STATE-STORE-0000000000-changelog", fkjTopicPrefix + "-" + INPUT_STREAM_3 + "-STATE-STORE-0000000003-changelog")));
}