Examples with KeyValue org.apache.kafka.streams.KeyValue used on opensource projects

Example 31 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class KStreamImplTest method shouldUseRecordMetadataTimestampExtractorWhenInternalRepartitioningTopicCreated.

@Test
public // TODO: this test should be refactored when we removed KStreamBuilder so that the created Topology contains internal topics as well
void shouldUseRecordMetadataTimestampExtractorWhenInternalRepartitioningTopicCreated() {
    final KStreamBuilder builder = new KStreamBuilder();
    KStream<String, String> kStream = builder.stream(stringSerde, stringSerde, "topic-1");
    ValueJoiner<String, String, String> valueJoiner = MockValueJoiner.instance(":");
    long windowSize = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS);
    final KStream<String, String> stream = kStream.map(new KeyValueMapper<String, String, KeyValue<? extends String, ? extends String>>() {

        @Override
        public KeyValue<? extends String, ? extends String> apply(String key, String value) {
            return KeyValue.pair(value, value);
        }
    });
    stream.join(kStream, valueJoiner, JoinWindows.of(windowSize).until(3 * windowSize), Joined.with(Serdes.String(), Serdes.String(), Serdes.String())).to(Serdes.String(), Serdes.String(), "output-topic");
    ProcessorTopology processorTopology = builder.setApplicationId("X").build(null);
    SourceNode originalSourceNode = processorTopology.source("topic-1");
    for (SourceNode sourceNode : processorTopology.sources()) {
        if (sourceNode.name().equals(originalSourceNode.name())) {
            assertEquals(sourceNode.getTimestampExtractor(), null);
        } else {
            assertThat(sourceNode.getTimestampExtractor(), instanceOf(FailOnInvalidTimestamp.class));
        }
    }
}

Example 32 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class KStreamPeekTest method shouldObserveStreamElements.

@Test
public void shouldObserveStreamElements() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Integer, String> stream = builder.stream(topicName, Consumed.with(intSerd, stringSerd));
    final List<KeyValue<Integer, String>> peekObserved = new ArrayList<>(), streamObserved = new ArrayList<>();
    stream.peek(collect(peekObserved)).foreach(collect(streamObserved));
    driver.setUp(builder);
    final List<KeyValue<Integer, String>> expected = new ArrayList<>();
    for (int key = 0; key < 32; key++) {
        final String value = "V" + key;
        driver.process(topicName, key, value);
        expected.add(new KeyValue<>(key, value));
    }
    assertEquals(expected, peekObserved);
    assertEquals(expected, streamObserved);
}

Example 33 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class PageViewTypedDemo method main.

public static void main(String[] args) throws Exception {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pageview-typed");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, JsonTimestampExtractor.class);
    props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    StreamsBuilder builder = new StreamsBuilder();
    // TODO: the following can be removed with a serialization factory
    Map<String, Object> serdeProps = new HashMap<>();
    final Serializer<PageView> pageViewSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", PageView.class);
    pageViewSerializer.configure(serdeProps, false);
    final Deserializer<PageView> pageViewDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", PageView.class);
    pageViewDeserializer.configure(serdeProps, false);
    final Serde<PageView> pageViewSerde = Serdes.serdeFrom(pageViewSerializer, pageViewDeserializer);
    final Serializer<UserProfile> userProfileSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", UserProfile.class);
    userProfileSerializer.configure(serdeProps, false);
    final Deserializer<UserProfile> userProfileDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", UserProfile.class);
    userProfileDeserializer.configure(serdeProps, false);
    final Serde<UserProfile> userProfileSerde = Serdes.serdeFrom(userProfileSerializer, userProfileDeserializer);
    final Serializer<WindowedPageViewByRegion> wPageViewByRegionSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", WindowedPageViewByRegion.class);
    wPageViewByRegionSerializer.configure(serdeProps, false);
    final Deserializer<WindowedPageViewByRegion> wPageViewByRegionDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", WindowedPageViewByRegion.class);
    wPageViewByRegionDeserializer.configure(serdeProps, false);
    final Serde<WindowedPageViewByRegion> wPageViewByRegionSerde = Serdes.serdeFrom(wPageViewByRegionSerializer, wPageViewByRegionDeserializer);
    final Serializer<RegionCount> regionCountSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", RegionCount.class);
    regionCountSerializer.configure(serdeProps, false);
    final Deserializer<RegionCount> regionCountDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", RegionCount.class);
    regionCountDeserializer.configure(serdeProps, false);
    final Serde<RegionCount> regionCountSerde = Serdes.serdeFrom(regionCountSerializer, regionCountDeserializer);
    final Serializer<PageViewByRegion> pageViewByRegionSerializer = new JsonPOJOSerializer<>();
    serdeProps.put("JsonPOJOClass", PageViewByRegion.class);
    pageViewByRegionSerializer.configure(serdeProps, false);
    final Deserializer<PageViewByRegion> pageViewByRegionDeserializer = new JsonPOJODeserializer<>();
    serdeProps.put("JsonPOJOClass", PageViewByRegion.class);
    pageViewByRegionDeserializer.configure(serdeProps, false);
    final Serde<PageViewByRegion> pageViewByRegionSerde = Serdes.serdeFrom(pageViewByRegionSerializer, pageViewByRegionDeserializer);
    KStream<String, PageView> views = builder.stream("streams-pageview-input", Consumed.with(Serdes.String(), pageViewSerde));
    KTable<String, UserProfile> users = builder.table("streams-userprofile-input", Consumed.with(Serdes.String(), userProfileSerde));
    KStream<WindowedPageViewByRegion, RegionCount> regionCount = views.leftJoin(users, new ValueJoiner<PageView, UserProfile, PageViewByRegion>() {

        @Override
        public PageViewByRegion apply(PageView view, UserProfile profile) {
            PageViewByRegion viewByRegion = new PageViewByRegion();
            viewByRegion.user = view.user;
            viewByRegion.page = view.page;
            if (profile != null) {
                viewByRegion.region = profile.region;
            } else {
                viewByRegion.region = "UNKNOWN";
            }
            return viewByRegion;
        }
    }).map(new KeyValueMapper<String, PageViewByRegion, KeyValue<String, PageViewByRegion>>() {

        @Override
        public KeyValue<String, PageViewByRegion> apply(String user, PageViewByRegion viewRegion) {
            return new KeyValue<>(viewRegion.region, viewRegion);
        }
    }).groupByKey(Serialized.with(Serdes.String(), pageViewByRegionSerde)).windowedBy(TimeWindows.of(TimeUnit.DAYS.toMillis(7)).advanceBy(TimeUnit.SECONDS.toMillis(1))).count().toStream().map(new KeyValueMapper<Windowed<String>, Long, KeyValue<WindowedPageViewByRegion, RegionCount>>() {

        @Override
        public KeyValue<WindowedPageViewByRegion, RegionCount> apply(Windowed<String> key, Long value) {
            WindowedPageViewByRegion wViewByRegion = new WindowedPageViewByRegion();
            wViewByRegion.windowStart = key.window().start();
            wViewByRegion.region = key.key();
            RegionCount rCount = new RegionCount();
            rCount.region = key.key();
            rCount.count = value;
            return new KeyValue<>(wViewByRegion, rCount);
        }
    });
    // write to the result topic
    regionCount.to("streams-pageviewstats-typed-output", Produced.with(wPageViewByRegionSerde, regionCountSerde));
    KafkaStreams streams = new KafkaStreams(builder.build(), props);
    streams.start();
    // usually the stream application would be running forever,
    // in this example we just let it run for some time and stop since the input data is finite.
    Thread.sleep(5000L);
    streams.close();
}

Example 34 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class PageViewUntypedDemo method main.

public static void main(String[] args) throws Exception {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pageview-untyped");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, JsonTimestampExtractor.class);
    props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    StreamsBuilder builder = new StreamsBuilder();
    final Serializer<JsonNode> jsonSerializer = new JsonSerializer();
    final Deserializer<JsonNode> jsonDeserializer = new JsonDeserializer();
    final Serde<JsonNode> jsonSerde = Serdes.serdeFrom(jsonSerializer, jsonDeserializer);
    final Consumed<String, JsonNode> consumed = Consumed.with(Serdes.String(), jsonSerde);
    KStream<String, JsonNode> views = builder.stream("streams-pageview-input", consumed);
    KTable<String, JsonNode> users = builder.table("streams-userprofile-input", consumed);
    KTable<String, String> userRegions = users.mapValues(new ValueMapper<JsonNode, String>() {

        @Override
        public String apply(JsonNode record) {
            return record.get("region").textValue();
        }
    });
    KStream<JsonNode, JsonNode> regionCount = views.leftJoin(userRegions, new ValueJoiner<JsonNode, String, JsonNode>() {

        @Override
        public JsonNode apply(JsonNode view, String region) {
            ObjectNode jNode = JsonNodeFactory.instance.objectNode();
            return jNode.put("user", view.get("user").textValue()).put("page", view.get("page").textValue()).put("region", region == null ? "UNKNOWN" : region);
        }
    }).map(new KeyValueMapper<String, JsonNode, KeyValue<String, JsonNode>>() {

        @Override
        public KeyValue<String, JsonNode> apply(String user, JsonNode viewRegion) {
            return new KeyValue<>(viewRegion.get("region").textValue(), viewRegion);
        }
    }).groupByKey(Serialized.with(Serdes.String(), jsonSerde)).windowedBy(TimeWindows.of(7 * 24 * 60 * 60 * 1000L).advanceBy(1000)).count().toStream().map(new KeyValueMapper<Windowed<String>, Long, KeyValue<JsonNode, JsonNode>>() {

        @Override
        public KeyValue<JsonNode, JsonNode> apply(Windowed<String> key, Long value) {
            ObjectNode keyNode = JsonNodeFactory.instance.objectNode();
            keyNode.put("window-start", key.window().start()).put("region", key.key());
            ObjectNode valueNode = JsonNodeFactory.instance.objectNode();
            valueNode.put("count", value);
            return new KeyValue<>((JsonNode) keyNode, (JsonNode) valueNode);
        }
    });
    // write to the result topic
    regionCount.to("streams-pageviewstats-untyped-output", Produced.with(jsonSerde, jsonSerde));
    KafkaStreams streams = new KafkaStreams(builder.build(), props);
    streams.start();
    // usually the stream application would be running forever,
    // in this example we just let it run for some time and stop since the input data is finite.
    Thread.sleep(5000L);
    streams.close();
}

Example 35 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class ThreadCacheTest method basicPutGet.

@Test
public void basicPutGet() throws IOException {
    List<KeyValue<String, String>> toInsert = Arrays.asList(new KeyValue<>("K1", "V1"), new KeyValue<>("K2", "V2"), new KeyValue<>("K3", "V3"), new KeyValue<>("K4", "V4"), new KeyValue<>("K5", "V5"));
    final KeyValue<String, String> kv = toInsert.get(0);
    ThreadCache cache = new ThreadCache(logContext, toInsert.size() * memoryCacheEntrySize(kv.key.getBytes(), kv.value.getBytes(), ""), new MockStreamsMetrics(new Metrics()));
    for (KeyValue<String, String> kvToInsert : toInsert) {
        Bytes key = Bytes.wrap(kvToInsert.key.getBytes());
        byte[] value = kvToInsert.value.getBytes();
        cache.put(namespace, key, new LRUCacheEntry(value, true, 1L, 1L, 1, ""));
    }
    for (KeyValue<String, String> kvToInsert : toInsert) {
        Bytes key = Bytes.wrap(kvToInsert.key.getBytes());
        LRUCacheEntry entry = cache.get(namespace, key);
        assertEquals(entry.isDirty(), true);
        assertEquals(new String(entry.value), kvToInsert.value);
    }
    assertEquals(cache.gets(), 5);
    assertEquals(cache.puts(), 5);
    assertEquals(cache.evicts(), 0);
    assertEquals(cache.flushes(), 0);
}