Examples with StreamsBuilder org.apache.kafka.streams.StreamsBuilder used on opensource projects

Example 61 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class KStreamWindowAggregateTest method testAggBasic.

@Test
public void testAggBasic() {
    final StreamsBuilder builder = new StreamsBuilder();
    String topic1 = "topic1";
    KStream<String, String> stream1 = builder.stream(topic1, Consumed.with(strSerde, strSerde));
    KTable<Windowed<String>, String> table2 = stream1.groupByKey(Serialized.with(strSerde, strSerde)).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, TimeWindows.of(10).advanceBy(5), strSerde, "topic1-Canonized");
    MockProcessorSupplier<Windowed<String>, String> proc2 = new MockProcessorSupplier<>();
    table2.toStream().process(proc2);
    driver.setUp(builder, stateDir);
    setRecordContext(0, topic1);
    driver.process(topic1, "A", "1");
    driver.flushState();
    setRecordContext(1, topic1);
    driver.process(topic1, "B", "2");
    driver.flushState();
    setRecordContext(2, topic1);
    driver.process(topic1, "C", "3");
    driver.flushState();
    setRecordContext(3, topic1);
    driver.process(topic1, "D", "4");
    driver.flushState();
    setRecordContext(4, topic1);
    driver.process(topic1, "A", "1");
    driver.flushState();
    setRecordContext(5, topic1);
    driver.process(topic1, "A", "1");
    driver.flushState();
    setRecordContext(6, topic1);
    driver.process(topic1, "B", "2");
    driver.flushState();
    setRecordContext(7, topic1);
    driver.process(topic1, "D", "4");
    driver.flushState();
    setRecordContext(8, topic1);
    driver.process(topic1, "B", "2");
    driver.flushState();
    setRecordContext(9, topic1);
    driver.process(topic1, "C", "3");
    driver.flushState();
    setRecordContext(10, topic1);
    driver.process(topic1, "A", "1");
    driver.flushState();
    setRecordContext(11, topic1);
    driver.process(topic1, "B", "2");
    driver.flushState();
    setRecordContext(12, topic1);
    driver.flushState();
    driver.process(topic1, "D", "4");
    driver.flushState();
    setRecordContext(13, topic1);
    driver.process(topic1, "B", "2");
    driver.flushState();
    setRecordContext(14, topic1);
    driver.process(topic1, "C", "3");
    driver.flushState();
    assertEquals(Utils.mkList("[A@0/10]:0+1", "[B@0/10]:0+2", "[C@0/10]:0+3", "[D@0/10]:0+4", "[A@0/10]:0+1+1", "[A@0/10]:0+1+1+1", "[A@5/15]:0+1", "[B@0/10]:0+2+2", "[B@5/15]:0+2", "[D@0/10]:0+4+4", "[D@5/15]:0+4", "[B@0/10]:0+2+2+2", "[B@5/15]:0+2+2", "[C@0/10]:0+3+3", "[C@5/15]:0+3", "[A@5/15]:0+1+1", "[A@10/20]:0+1", "[B@5/15]:0+2+2+2", "[B@10/20]:0+2", "[D@5/15]:0+4+4", "[D@10/20]:0+4", "[B@5/15]:0+2+2+2+2", "[B@10/20]:0+2+2", "[C@5/15]:0+3+3", "[C@10/20]:0+3"), proc2.processed);
}

Example 62 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class PageViewTypedDemo method main.

public static void main(String[] args) throws Exception {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pageview-typed");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, JsonTimestampExtractor.class);
    props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    StreamsBuilder builder = new StreamsBuilder();
    // TODO: the following can be removed with a serialization factory
    Map<String, Object> serdeProps = new HashMap<>();
    final Serializer<PageView> pageViewSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", PageView.class);
    pageViewSerializer.configure(serdeProps, false);
    final Deserializer<PageView> pageViewDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", PageView.class);
    pageViewDeserializer.configure(serdeProps, false);
    final Serde<PageView> pageViewSerde = Serdes.serdeFrom(pageViewSerializer, pageViewDeserializer);
    final Serializer<UserProfile> userProfileSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", UserProfile.class);
    userProfileSerializer.configure(serdeProps, false);
    final Deserializer<UserProfile> userProfileDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", UserProfile.class);
    userProfileDeserializer.configure(serdeProps, false);
    final Serde<UserProfile> userProfileSerde = Serdes.serdeFrom(userProfileSerializer, userProfileDeserializer);
    final Serializer<WindowedPageViewByRegion> wPageViewByRegionSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", WindowedPageViewByRegion.class);
    wPageViewByRegionSerializer.configure(serdeProps, false);
    final Deserializer<WindowedPageViewByRegion> wPageViewByRegionDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", WindowedPageViewByRegion.class);
    wPageViewByRegionDeserializer.configure(serdeProps, false);
    final Serde<WindowedPageViewByRegion> wPageViewByRegionSerde = Serdes.serdeFrom(wPageViewByRegionSerializer, wPageViewByRegionDeserializer);
    final Serializer<RegionCount> regionCountSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", RegionCount.class);
    regionCountSerializer.configure(serdeProps, false);
    final Deserializer<RegionCount> regionCountDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", RegionCount.class);
    regionCountDeserializer.configure(serdeProps, false);
    final Serde<RegionCount> regionCountSerde = Serdes.serdeFrom(regionCountSerializer, regionCountDeserializer);
    final Serializer<PageViewByRegion> pageViewByRegionSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", PageViewByRegion.class);
    pageViewByRegionSerializer.configure(serdeProps, false);
    final Deserializer<PageViewByRegion> pageViewByRegionDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", PageViewByRegion.class);
    pageViewByRegionDeserializer.configure(serdeProps, false);
    final Serde<PageViewByRegion> pageViewByRegionSerde = Serdes.serdeFrom(pageViewByRegionSerializer, pageViewByRegionDeserializer);
    KStream<String, PageView> views = builder.stream("streams-pageview-input", Consumed.with(Serdes.String(), pageViewSerde));
    KTable<String, UserProfile> users = builder.table("streams-userprofile-input", Consumed.with(Serdes.String(), userProfileSerde));
    KStream<WindowedPageViewByRegion, RegionCount> regionCount = views.leftJoin(users, new ValueJoiner<PageView, UserProfile, PageViewByRegion>() {

        @Override
        public PageViewByRegion apply(PageView view, UserProfile profile) {
            PageViewByRegion viewByRegion = new PageViewByRegion();
            viewByRegion.user = view.user;
            viewByRegion.page = view.page;
            if (profile != null) {
                viewByRegion.region = profile.region;
            } else {
                viewByRegion.region = "UNKNOWN";
            }
            return viewByRegion;
        }
    }).map(new KeyValueMapper<String, PageViewByRegion, KeyValue<String, PageViewByRegion>>() {

        @Override
        public KeyValue<String, PageViewByRegion> apply(String user, PageViewByRegion viewRegion) {
            return new KeyValue<>(viewRegion.region, viewRegion);
        }
    }).groupByKey(Serialized.with(Serdes.String(), pageViewByRegionSerde)).windowedBy(TimeWindows.of(TimeUnit.DAYS.toMillis(7)).advanceBy(TimeUnit.SECONDS.toMillis(1))).count().toStream().map(new KeyValueMapper<Windowed<String>, Long, KeyValue<WindowedPageViewByRegion, RegionCount>>() {

        @Override
        public KeyValue<WindowedPageViewByRegion, RegionCount> apply(Windowed<String> key, Long value) {
            WindowedPageViewByRegion wViewByRegion = new WindowedPageViewByRegion();
            wViewByRegion.windowStart = key.window().start();
            wViewByRegion.region = key.key();
            RegionCount rCount = new RegionCount();
            rCount.region = key.key();
            rCount.count = value;
            return new KeyValue<>(wViewByRegion, rCount);
        }
    });
    // write to the result topic
    regionCount.to("streams-pageviewstats-typed-output", Produced.with(wPageViewByRegionSerde, regionCountSerde));
    KafkaStreams streams = new KafkaStreams(builder.build(), props);
    streams.start();
    // usually the stream application would be running forever,
    // in this example we just let it run for some time and stop since the input data is finite.
    Thread.sleep(5000L);
    streams.close();
}

Example 63 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class PageViewUntypedDemo method main.

public static void main(String[] args) throws Exception {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pageview-untyped");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, JsonTimestampExtractor.class);
    props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    StreamsBuilder builder = new StreamsBuilder();
    final Serializer<JsonNode> jsonSerializer = new JsonSerializer();
    final Deserializer<JsonNode> jsonDeserializer = new JsonDeserializer();
    final Serde<JsonNode> jsonSerde = Serdes.serdeFrom(jsonSerializer, jsonDeserializer);
    final Consumed<String, JsonNode> consumed = Consumed.with(Serdes.String(), jsonSerde);
    KStream<String, JsonNode> views = builder.stream("streams-pageview-input", consumed);
    KTable<String, JsonNode> users = builder.table("streams-userprofile-input", consumed);
    KTable<String, String> userRegions = users.mapValues(new ValueMapper<JsonNode, String>() {

        @Override
        public String apply(JsonNode record) {
            return record.get("region").textValue();
        }
    });
    KStream<JsonNode, JsonNode> regionCount = views.leftJoin(userRegions, new ValueJoiner<JsonNode, String, JsonNode>() {

        @Override
        public JsonNode apply(JsonNode view, String region) {
            ObjectNode jNode = JsonNodeFactory.instance.objectNode();
            return jNode.put("user", view.get("user").textValue()).put("page", view.get("page").textValue()).put("region", region == null ? "UNKNOWN" : region);
        }
    }).map(new KeyValueMapper<String, JsonNode, KeyValue<String, JsonNode>>() {

        @Override
        public KeyValue<String, JsonNode> apply(String user, JsonNode viewRegion) {
            return new KeyValue<>(viewRegion.get("region").textValue(), viewRegion);
        }
    }).groupByKey(Serialized.with(Serdes.String(), jsonSerde)).windowedBy(TimeWindows.of(7 * 24 * 60 * 60 * 1000L).advanceBy(1000)).count().toStream().map(new KeyValueMapper<Windowed<String>, Long, KeyValue<JsonNode, JsonNode>>() {

        @Override
        public KeyValue<JsonNode, JsonNode> apply(Windowed<String> key, Long value) {
            ObjectNode keyNode = JsonNodeFactory.instance.objectNode();
            keyNode.put("window-start", key.window().start()).put("region", key.key());
            ObjectNode valueNode = JsonNodeFactory.instance.objectNode();
            valueNode.put("count", value);
            return new KeyValue<>((JsonNode) keyNode, (JsonNode) valueNode);
        }
    });
    // write to the result topic
    regionCount.to("streams-pageviewstats-untyped-output", Produced.with(jsonSerde, jsonSerde));
    KafkaStreams streams = new KafkaStreams(builder.build(), props);
    streams.start();
    // usually the stream application would be running forever,
    // in this example we just let it run for some time and stop since the input data is finite.
    Thread.sleep(5000L);
    streams.close();
}

Example 64 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class ShutdownDeadlockTest method start.

public void start() {
    final String topic = "source";
    final Properties props = new Properties();
    props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "shouldNotDeadlock");
    props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> source = builder.stream(topic, Consumed.with(Serdes.String(), Serdes.String()));
    source.foreach(new ForeachAction<String, String>() {

        @Override
        public void apply(final String key, final String value) {
            throw new RuntimeException("KABOOM!");
        }
    });
    final KafkaStreams streams = new KafkaStreams(builder.build(), props);
    streams.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {

        @Override
        public void uncaughtException(final Thread t, final Throwable e) {
            Exit.exit(1);
        }
    });
    Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {

        @Override
        public void run() {
            streams.close(5, TimeUnit.SECONDS);
        }
    }));
    final Properties producerProps = new Properties();
    producerProps.put(ProducerConfig.CLIENT_ID_CONFIG, "SmokeTest");
    producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka);
    producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    final KafkaProducer<String, String> producer = new KafkaProducer<>(producerProps);
    producer.send(new ProducerRecord<>(topic, "a", "a"));
    producer.flush();
    streams.start();
    synchronized (this) {
        try {
            wait();
        } catch (InterruptedException e) {
        // ignored
        }
    }
}

Example 65 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class StreamsStandByReplicaTest method main.

public static void main(final String[] args) throws IOException {
    System.out.println("StreamsTest instance started");
    final String kafka = args.length > 0 ? args[0] : "localhost:9092";
    final String propFileName = args.length > 1 ? args[1] : null;
    final String additionalConfigs = args.length > 2 ? args[2] : null;
    final Serde<String> stringSerde = Serdes.String();
    final Properties streamsProperties = Utils.loadProps(propFileName);
    streamsProperties.put(StreamsConfig.APPLICATION_ID_CONFIG, "kafka-streams-standby-tasks");
    streamsProperties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka);
    streamsProperties.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100);
    streamsProperties.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
    streamsProperties.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    streamsProperties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    streamsProperties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    streamsProperties.put(StreamsConfig.producerPrefix(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG), true);
    if (additionalConfigs == null) {
        System.err.println("additional configs are not provided");
        System.err.flush();
        System.exit(1);
    }
    final Map<String, String> updated = SystemTestUtil.parseConfigs(additionalConfigs);
    System.out.println("Updating configs with " + updated);
    final String sourceTopic = updated.remove("sourceTopic");
    final String sinkTopic1 = updated.remove("sinkTopic1");
    final String sinkTopic2 = updated.remove("sinkTopic2");
    if (sourceTopic == null || sinkTopic1 == null || sinkTopic2 == null) {
        System.err.println(String.format("one or more required topics null sourceTopic[%s], sinkTopic1[%s], sinkTopic2[%s]", sourceTopic, sinkTopic1, sinkTopic2));
        System.err.flush();
        System.exit(1);
    }
    streamsProperties.putAll(updated);
    if (!confirmCorrectConfigs(streamsProperties)) {
        System.err.println(String.format("ERROR: Did not have all required configs expected  to contain %s, %s,  %s,  %s", StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), StreamsConfig.producerPrefix(ProducerConfig.RETRIES_CONFIG), StreamsConfig.producerPrefix(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG), StreamsConfig.producerPrefix(ProducerConfig.MAX_BLOCK_MS_CONFIG)));
        System.exit(1);
    }
    final StreamsBuilder builder = new StreamsBuilder();
    String inMemoryStoreName = "in-memory-store";
    String persistentMemoryStoreName = "persistent-memory-store";
    KeyValueBytesStoreSupplier inMemoryStoreSupplier = Stores.inMemoryKeyValueStore(inMemoryStoreName);
    KeyValueBytesStoreSupplier persistentStoreSupplier = Stores.persistentKeyValueStore(persistentMemoryStoreName);
    KStream<String, String> inputStream = builder.stream(sourceTopic, Consumed.with(stringSerde, stringSerde));
    ValueMapper<Long, String> countMapper = new ValueMapper<Long, String>() {

        @Override
        public String apply(final Long value) {
            return value.toString();
        }
    };
    inputStream.groupByKey().count(Materialized.<String, Long>as(inMemoryStoreSupplier)).toStream().mapValues(countMapper).to(sinkTopic1, Produced.with(stringSerde, stringSerde));
    inputStream.groupByKey().count(Materialized.<String, Long>as(persistentStoreSupplier)).toStream().mapValues(countMapper).to(sinkTopic2, Produced.with(stringSerde, stringSerde));
    final KafkaStreams streams = new KafkaStreams(builder.build(), streamsProperties);
    streams.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {

        @Override
        public void uncaughtException(final Thread t, final Throwable e) {
            System.err.println("FATAL: An unexpected exception " + e);
            e.printStackTrace(System.err);
            System.err.flush();
            shutdown(streams);
        }
    });
    streams.setStateListener(new KafkaStreams.StateListener() {

        @Override
        public void onChange(final KafkaStreams.State newState, final KafkaStreams.State oldState) {
            if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
                final Set<ThreadMetadata> threadMetadata = streams.localThreadsMetadata();
                for (final ThreadMetadata threadMetadatum : threadMetadata) {
                    System.out.println("ACTIVE_TASKS:" + threadMetadatum.activeTasks().size() + " STANDBY_TASKS:" + threadMetadatum.standbyTasks().size());
                }
            }
        }
    });
    System.out.println("Start Kafka Streams");
    streams.start();
    Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {

        @Override
        public void run() {
            shutdown(streams);
            System.out.println("Shut down streams now");
        }
    }));
}