Examples with KTable org.apache.kafka.streams.kstream.KTable used on opensource projects

Example 36 with KTable

use of org.apache.kafka.streams.kstream.KTable in project ksql by confluentinc.

the class SchemaKTable method filter.

@SuppressWarnings("unchecked")
@Override
public SchemaKTable filter(final Expression filterExpression) {
    SqlPredicate predicate = new SqlPredicate(filterExpression, schema, isWindowed, functionRegistry);
    KTable filteredKTable = ktable.filter(predicate.getPredicate());
    return new SchemaKTable(schema, filteredKTable, keyField, Arrays.asList(this), isWindowed, Type.FILTER, functionRegistry, schemaRegistryClient);
}

Example 37 with KTable

use of org.apache.kafka.streams.kstream.KTable in project ksql by confluentinc.

the class SchemaKTable method select.

@SuppressWarnings("unchecked")
@Override
public SchemaKTable select(final List<Pair<String, Expression>> expressionPairList) {
    final Pair<Schema, SelectValueMapper> schemaAndMapper = createSelectValueMapperAndSchema(expressionPairList);
    KTable projectedKTable = ktable.mapValues(schemaAndMapper.right);
    return new SchemaKTable(schemaAndMapper.left, projectedKTable, keyField, Collections.singletonList(this), isWindowed, Type.PROJECT, functionRegistry, schemaRegistryClient);
}

Example 38 with KTable

use of org.apache.kafka.streams.kstream.KTable in project tutorials by eugenp.

the class KafkaStreamsLiveTest method shouldTestKafkaStreams.

@Test
@Ignore("it needs to have kafka broker running on local")
public void shouldTestKafkaStreams() throws InterruptedException {
    // given
    String inputTopic = "inputTopic";
    Properties streamsConfiguration = new Properties();
    streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, "wordcount-live-test");
    streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
    streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    // Use a temporary directory for storing state, which will be automatically removed after the test.
    streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
    // when
    KStreamBuilder builder = new KStreamBuilder();
    KStream<String, String> textLines = builder.stream(inputTopic);
    Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS);
    KTable<String, Long> wordCounts = textLines.flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase()))).groupBy((key, word) -> word).count();
    wordCounts.foreach((word, count) -> System.out.println("word: " + word + " -> " + count));
    String outputTopic = "outputTopic";
    final Serde<String> stringSerde = Serdes.String();
    final Serde<Long> longSerde = Serdes.Long();
    wordCounts.to(stringSerde, longSerde, outputTopic);
    KafkaStreams streams = new KafkaStreams(builder, streamsConfiguration);
    streams.start();
    // then
    Thread.sleep(30000);
    streams.close();
}

Example 39 with KTable

use of org.apache.kafka.streams.kstream.KTable in project kafka by apache.

the class KTableKTableForeignKeyInnerJoinCustomPartitionerIntegrationTest method prepareTopology.

private static KafkaStreams prepareTopology(final String queryableName, final Properties streamsConfig) {
    final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<String, String> table1 = builder.stream(TABLE_1, Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))).repartition(repartitionA()).toTable(Named.as("table.a"));
    final KTable<String, String> table2 = builder.stream(TABLE_2, Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))).repartition(repartitionB()).toTable(Named.as("table.b"));
    final Materialized<String, String, KeyValueStore<Bytes, byte[]>> materialized;
    if (queryableName != null) {
        materialized = Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as(queryableName).withKeySerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true)).withValueSerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)).withCachingDisabled();
    } else {
        throw new RuntimeException("Current implementation of joinOnForeignKey requires a materialized store");
    }
    final ValueJoiner<String, String, String> joiner = (value1, value2) -> "value1=" + value1 + ",value2=" + value2;
    final TableJoined<String, String> tableJoined = TableJoined.with((topic, key, value, numPartitions) -> Math.abs(getKeyB(key).hashCode()) % numPartitions, (topic, key, value, numPartitions) -> Math.abs(key.hashCode()) % numPartitions);
    table1.join(table2, KTableKTableForeignKeyInnerJoinCustomPartitionerIntegrationTest::getKeyB, joiner, tableJoined, materialized).toStream().to(OUTPUT, Produced.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)));
    return new KafkaStreams(builder.build(streamsConfig), streamsConfig);
}

Example 40 with KTable

use of org.apache.kafka.streams.kstream.KTable in project kafka by apache.

the class JoinWithIncompleteMetadataIntegrationTest method testShouldAutoShutdownOnJoinWithIncompleteMetadata.

@Test
public void testShouldAutoShutdownOnJoinWithIncompleteMetadata() throws InterruptedException {
    STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, APP_ID);
    STREAMS_CONFIG.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
    final KStream<Long, String> notExistStream = builder.stream(NON_EXISTENT_INPUT_TOPIC_LEFT);
    final KTable<Long, String> aggregatedTable = notExistStream.leftJoin(rightTable, valueJoiner).groupBy((key, value) -> key).reduce((value1, value2) -> value1 + value2);
    // Write the (continuously updating) results to the output topic.
    aggregatedTable.toStream().to(OUTPUT_TOPIC);
    final KafkaStreamsWrapper streams = new KafkaStreamsWrapper(builder.build(), STREAMS_CONFIG);
    final IntegrationTestUtils.StateListenerStub listener = new IntegrationTestUtils.StateListenerStub();
    streams.setStreamThreadStateListener(listener);
    streams.start();
    TestUtils.waitForCondition(listener::transitToPendingShutdownSeen, "Did not seen thread state transited to PENDING_SHUTDOWN");
    streams.close();
    assertTrue(listener.transitToPendingShutdownSeen());
}