Examples with KStream org.apache.kafka.streams.kstream.KStream used on opensource projects

Example 86 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class StreamsBuilderTest method shouldUseSpecifiedNameForSplitOperation.

@Test
public void shouldUseSpecifiedNameForSplitOperation() {
    builder.stream(STREAM_TOPIC).split(Named.as("branch-processor")).branch((k, v) -> true, Branched.as("-1")).branch((k, v) -> false, Branched.as("-2"));
    builder.build();
    final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).buildTopology();
    assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", "branch-processor", "branch-processor-1", "branch-processor-2");
}

Example 87 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class StreamsNamedRepartitionTest method main.

public static void main(final String[] args) throws Exception {
    if (args.length < 1) {
        System.err.println("StreamsNamedRepartitionTest requires one argument (properties-file) but none provided: ");
    }
    final String propFileName = args[0];
    final Properties streamsProperties = Utils.loadProps(propFileName);
    System.out.println("StreamsTest instance started NAMED_REPARTITION_TEST");
    System.out.println("props=" + streamsProperties);
    final String inputTopic = (String) (Objects.requireNonNull(streamsProperties.remove("input.topic")));
    final String aggregationTopic = (String) (Objects.requireNonNull(streamsProperties.remove("aggregation.topic")));
    final boolean addOperators = Boolean.valueOf(Objects.requireNonNull((String) streamsProperties.remove("add.operations")));
    final Initializer<Integer> initializer = () -> 0;
    final Aggregator<String, String, Integer> aggregator = (k, v, agg) -> agg + Integer.parseInt(v);
    final Function<String, String> keyFunction = s -> Integer.toString(Integer.parseInt(s) % 9);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> sourceStream = builder.stream(inputTopic, Consumed.with(Serdes.String(), Serdes.String()));
    sourceStream.peek((k, v) -> System.out.println(String.format("input data key=%s, value=%s", k, v)));
    final KStream<String, String> mappedStream = sourceStream.selectKey((k, v) -> keyFunction.apply(v));
    final KStream<String, String> maybeUpdatedStream;
    if (addOperators) {
        maybeUpdatedStream = mappedStream.filter((k, v) -> true).mapValues(v -> Integer.toString(Integer.parseInt(v) + 1));
    } else {
        maybeUpdatedStream = mappedStream;
    }
    maybeUpdatedStream.groupByKey(Grouped.with("grouped-stream", Serdes.String(), Serdes.String())).aggregate(initializer, aggregator, Materialized.<String, Integer, KeyValueStore<Bytes, byte[]>>as("count-store").withKeySerde(Serdes.String()).withValueSerde(Serdes.Integer())).toStream().peek((k, v) -> System.out.println(String.format("AGGREGATED key=%s value=%s", k, v))).to(aggregationTopic, Produced.with(Serdes.String(), Serdes.Integer()));
    final Properties config = new Properties();
    config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "StreamsNamedRepartitionTest");
    config.setProperty(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, "0");
    config.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    config.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    config.putAll(streamsProperties);
    final Topology topology = builder.build(config);
    final KafkaStreams streams = new KafkaStreams(topology, config);
    streams.setStateListener((newState, oldState) -> {
        if (oldState == State.REBALANCING && newState == State.RUNNING) {
            if (addOperators) {
                System.out.println("UPDATED Topology");
            } else {
                System.out.println("REBALANCING -> RUNNING");
            }
            System.out.flush();
        }
    });
    streams.start();
    Exit.addShutdownHook("streams-shutdown-hook", () -> {
        System.out.println("closing Kafka Streams instance");
        System.out.flush();
        streams.close(Duration.ofMillis(5000));
        System.out.println("NAMED_REPARTITION_TEST Streams Stopped");
        System.out.flush();
    });
}

Example 88 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldGenerateTasksForAllCreatedPartitions.

@Test
public void shouldGenerateTasksForAllCreatedPartitions() {
    final StreamsBuilder streamsBuilder = new StreamsBuilder();
    // KStream with 3 partitions
    final KStream<Object, Object> stream1 = streamsBuilder.stream("topic1").map((KeyValueMapper<Object, Object, KeyValue<Object, Object>>) KeyValue::new);
    // KTable with 4 partitions
    final KTable<Object, Long> table1 = streamsBuilder.table("topic3").groupBy(KeyValue::new).count();
    // joining the stream and the table
    // this triggers the enforceCopartitioning() routine in the StreamsPartitionAssignor,
    // forcing the stream.map to get repartitioned to a topic with four partitions.
    stream1.join(table1, (ValueJoiner<Object, Object, Void>) (value1, value2) -> null);
    final String client = "client1";
    builder = TopologyWrapper.getInternalTopologyBuilder(streamsBuilder.build());
    topologyMetadata = new TopologyMetadata(builder, new StreamsConfig(configProps()));
    adminClient = createMockAdminClientForAssignor(getTopicPartitionOffsetsMap(asList(APPLICATION_ID + "-topic3-STATE-STORE-0000000002-changelog", APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-changelog"), asList(4, 4)));
    final MockInternalTopicManager mockInternalTopicManager = configureDefault();
    subscriptions.put(client, new Subscription(asList("topic1", "topic3"), defaultSubscriptionInfo.encode()));
    final Map<String, Assignment> assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    final Map<String, Integer> expectedCreatedInternalTopics = new HashMap<>();
    expectedCreatedInternalTopics.put(APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 4);
    expectedCreatedInternalTopics.put(APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-changelog", 4);
    expectedCreatedInternalTopics.put(APPLICATION_ID + "-topic3-STATE-STORE-0000000002-changelog", 4);
    expectedCreatedInternalTopics.put(APPLICATION_ID + "-KSTREAM-MAP-0000000001-repartition", 4);
    // check if all internal topics were created as expected
    assertThat(mockInternalTopicManager.readyTopics, equalTo(expectedCreatedInternalTopics));
    final List<TopicPartition> expectedAssignment = asList(new TopicPartition("topic1", 0), new TopicPartition("topic1", 1), new TopicPartition("topic1", 2), new TopicPartition("topic3", 0), new TopicPartition("topic3", 1), new TopicPartition("topic3", 2), new TopicPartition("topic3", 3), new TopicPartition(APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 0), new TopicPartition(APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 1), new TopicPartition(APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 2), new TopicPartition(APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 3), new TopicPartition(APPLICATION_ID + "-KSTREAM-MAP-0000000001-repartition", 0), new TopicPartition(APPLICATION_ID + "-KSTREAM-MAP-0000000001-repartition", 1), new TopicPartition(APPLICATION_ID + "-KSTREAM-MAP-0000000001-repartition", 2), new TopicPartition(APPLICATION_ID + "-KSTREAM-MAP-0000000001-repartition", 3));
    // check if we created a task for all expected topicPartitions.
    assertThat(new HashSet<>(assignment.get(client).partitions()), equalTo(new HashSet<>(expectedAssignment)));
}

Example 89 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldNotLoopInfinitelyOnMissingMetadataAndShouldNotCreateRelatedTasks.

@Test
public void shouldNotLoopInfinitelyOnMissingMetadataAndShouldNotCreateRelatedTasks() {
    final StreamsBuilder streamsBuilder = new StreamsBuilder();
    final KStream<Object, Object> stream1 = streamsBuilder.stream("topic1").selectKey((key, value) -> null).groupByKey().count(Materialized.as("count")).toStream().map((KeyValueMapper<Object, Long, KeyValue<Object, Object>>) (key, value) -> null);
    streamsBuilder.stream("unknownTopic").selectKey((key, value) -> null).join(stream1, (ValueJoiner<Object, Object, Void>) (value1, value2) -> null, JoinWindows.of(ofMillis(0)));
    final String client = "client1";
    builder = TopologyWrapper.getInternalTopologyBuilder(streamsBuilder.build());
    final MockInternalTopicManager mockInternalTopicManager = configureDefault();
    subscriptions.put(client, new Subscription(Collections.singletonList("unknownTopic"), defaultSubscriptionInfo.encode()));
    final Map<String, Assignment> assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    assertThat(mockInternalTopicManager.readyTopics.isEmpty(), equalTo(true));
    assertThat(assignment.get(client).partitions().isEmpty(), equalTo(true));
}