Examples with KStream org.apache.kafka.streams.kstream.KStream used on opensource projects

Example 31 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class AdjustStreamThreadCountTest method shouldResizeCacheAfterThreadReplacement.

@Test
public void shouldResizeCacheAfterThreadReplacement() throws InterruptedException {
    final long totalCacheBytes = 10L;
    final Properties props = new Properties();
    props.putAll(properties);
    props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 2);
    props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, totalCacheBytes);
    final AtomicBoolean injectError = new AtomicBoolean(false);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> stream = builder.stream(inputTopic);
    stream.transform(() -> new Transformer<String, String, KeyValue<String, String>>() {

        @Override
        public void init(final ProcessorContext context) {
            context.schedule(Duration.ofSeconds(1), PunctuationType.WALL_CLOCK_TIME, timestamp -> {
                if (Thread.currentThread().getName().endsWith("StreamThread-1") && injectError.get()) {
                    injectError.set(false);
                    throw new RuntimeException("BOOM");
                }
            });
        }

        @Override
        public KeyValue<String, String> transform(final String key, final String value) {
            return new KeyValue<>(key, value);
        }

        @Override
        public void close() {
        }
    });
    try (final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), props)) {
        addStreamStateChangeListener(kafkaStreams);
        kafkaStreams.setUncaughtExceptionHandler(e -> StreamThreadExceptionResponse.REPLACE_THREAD);
        startStreamsAndWaitForRunning(kafkaStreams);
        stateTransitionHistory.clear();
        try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister()) {
            injectError.set(true);
            waitForCondition(() -> !injectError.get(), "StreamThread did not hit and reset the injected error");
            waitForTransitionFromRebalancingToRunning();
            for (final String log : appender.getMessages()) {
                // after we replace the thread there should be two remaining threads with 5 bytes each
                if (log.endsWith("Adding StreamThread-3, there will now be 2 live threads and the new cache size per thread is 5")) {
                    return;
                }
            }
        }
    }
    fail();
}

Example 32 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldThrowTaskAssignmentExceptionWhenUnableToResolvePartitionCount.

@Test
public void shouldThrowTaskAssignmentExceptionWhenUnableToResolvePartitionCount() {
    builder = new CorruptedInternalTopologyBuilder();
    topologyMetadata = new TopologyMetadata(builder, new StreamsConfig(configProps()));
    final InternalStreamsBuilder streamsBuilder = new InternalStreamsBuilder(builder);
    final KStream<String, String> inputTopic = streamsBuilder.stream(singleton("topic1"), new ConsumedInternal<>());
    final KTable<String, String> inputTable = streamsBuilder.table("topic2", new ConsumedInternal<>(), new MaterializedInternal<>(Materialized.as("store")));
    inputTopic.groupBy((k, v) -> k, Grouped.with("GroupName", Serdes.String(), Serdes.String())).windowedBy(TimeWindows.of(Duration.ofMinutes(10))).aggregate(() -> "", (k, v, a) -> a + k).leftJoin(inputTable, v -> v, (x, y) -> x + y);
    streamsBuilder.buildAndOptimizeTopology();
    configureDefault();
    subscriptions.put("consumer", new Subscription(singletonList("topic"), defaultSubscriptionInfo.encode()));
    final Map<String, Assignment> assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    assertThat(AssignmentInfo.decode(assignments.get("consumer").userData()).errCode(), equalTo(AssignorError.ASSIGNMENT_ERROR.code()));
}

Example 33 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class PageViewUntypedDemo method main.

public static void main(final String[] args) throws Exception {
    final Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pageview-untyped");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, JsonTimestampExtractor.class);
    props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    final StreamsBuilder builder = new StreamsBuilder();
    final Serializer<JsonNode> jsonSerializer = new JsonSerializer();
    final Deserializer<JsonNode> jsonDeserializer = new JsonDeserializer();
    final Serde<JsonNode> jsonSerde = Serdes.serdeFrom(jsonSerializer, jsonDeserializer);
    final Consumed<String, JsonNode> consumed = Consumed.with(Serdes.String(), jsonSerde);
    final KStream<String, JsonNode> views = builder.stream("streams-pageview-input", consumed);
    final KTable<String, JsonNode> users = builder.table("streams-userprofile-input", consumed);
    final KTable<String, String> userRegions = users.mapValues(record -> record.get("region").textValue());
    final Duration duration24Hours = Duration.ofHours(24);
    final KStream<JsonNode, JsonNode> regionCount = views.leftJoin(userRegions, (view, region) -> {
        final ObjectNode jNode = JsonNodeFactory.instance.objectNode();
        return (JsonNode) jNode.put("user", view.get("user").textValue()).put("page", view.get("page").textValue()).put("region", region == null ? "UNKNOWN" : region);
    }).map((user, viewRegion) -> new KeyValue<>(viewRegion.get("region").textValue(), viewRegion)).groupByKey(Grouped.with(Serdes.String(), jsonSerde)).windowedBy(TimeWindows.ofSizeAndGrace(Duration.ofDays(7), duration24Hours).advanceBy(Duration.ofSeconds(1))).count().toStream().map((key, value) -> {
        final ObjectNode keyNode = JsonNodeFactory.instance.objectNode();
        keyNode.put("window-start", key.window().start()).put("region", key.key());
        final ObjectNode valueNode = JsonNodeFactory.instance.objectNode();
        valueNode.put("count", value);
        return new KeyValue<>((JsonNode) keyNode, (JsonNode) valueNode);
    });
    // write to the result topic
    regionCount.to("streams-pageviewstats-untyped-output", Produced.with(jsonSerde, jsonSerde));
    final KafkaStreams streams = new KafkaStreams(builder.build(), props);
    streams.start();
    // usually the stream application would be running forever,
    // in this example we just let it run for some time and stop since the input data is finite.
    Thread.sleep(5000L);
    streams.close();
}

Example 34 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class TemperatureDemo method main.

public static void main(final String[] args) {
    final Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-temperature");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    final Duration duration24Hours = Duration.ofHours(24);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> source = builder.stream("iot-temperature");
    final KStream<Windowed<String>, String> max = source.selectKey((key, value) -> "temp").groupByKey().windowedBy(TimeWindows.ofSizeAndGrace(Duration.ofSeconds(TEMPERATURE_WINDOW_SIZE), duration24Hours)).reduce((value1, value2) -> {
        if (Integer.parseInt(value1) > Integer.parseInt(value2)) {
            return value1;
        } else {
            return value2;
        }
    }).toStream().filter((key, value) -> Integer.parseInt(value) > TEMPERATURE_THRESHOLD);
    final Serde<Windowed<String>> windowedSerde = WindowedSerdes.timeWindowedSerdeFrom(String.class, TEMPERATURE_WINDOW_SIZE);
    // need to override key serde to Windowed<String> type
    max.to("iot-temperature-max", Produced.with(windowedSerde, Serdes.String()));
    final KafkaStreams streams = new KafkaStreams(builder.build(), props);
    final CountDownLatch latch = new CountDownLatch(1);
    // attach shutdown handler to catch control-c
    Runtime.getRuntime().addShutdownHook(new Thread("streams-temperature-shutdown-hook") {

        @Override
        public void run() {
            streams.close();
            latch.countDown();
        }
    });
    try {
        streams.start();
        latch.await();
    } catch (final Throwable e) {
        System.exit(1);
    }
    System.exit(0);
}

Example 35 with KStream

use of org.apache.kafka.streams.kstream.KStream in project kafka by apache.

the class JoinStoreIntegrationTest method streamJoinChangelogTopicShouldBeConfiguredWithDeleteOnlyCleanupPolicy.

@Test
public void streamJoinChangelogTopicShouldBeConfiguredWithDeleteOnlyCleanupPolicy() throws Exception {
    STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, APP_ID + "-changelog-cleanup-policy");
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, Integer> left = builder.stream(INPUT_TOPIC_LEFT, Consumed.with(Serdes.String(), Serdes.Integer()));
    final KStream<String, Integer> right = builder.stream(INPUT_TOPIC_RIGHT, Consumed.with(Serdes.String(), Serdes.Integer()));
    final CountDownLatch latch = new CountDownLatch(1);
    left.join(right, Integer::sum, JoinWindows.of(ofMillis(100)), StreamJoined.with(Serdes.String(), Serdes.Integer(), Serdes.Integer()).withStoreName("join-store"));
    try (final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), STREAMS_CONFIG);
        final Admin admin = Admin.create(ADMIN_CONFIG)) {
        kafkaStreams.setStateListener((newState, oldState) -> {
            if (newState == KafkaStreams.State.RUNNING) {
                latch.countDown();
            }
        });
        kafkaStreams.start();
        latch.await();
        final Collection<ConfigResource> changelogTopics = Stream.of("join-store-integration-test-changelog-cleanup-policy-join-store-this-join-store-changelog", "join-store-integration-test-changelog-cleanup-policy-join-store-other-join-store-changelog").map(name -> new ConfigResource(Type.TOPIC, name)).collect(Collectors.toList());
        final Map<ConfigResource, org.apache.kafka.clients.admin.Config> topicConfig = admin.describeConfigs(changelogTopics).all().get();
        topicConfig.values().forEach(tc -> assertThat(tc.get("cleanup.policy").value(), is("delete")));
    }
}