use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.
the class PageViewTypedDemo method main.
public static void main(String[] args) throws Exception {
Properties props = new Properties();
props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pageview-typed");
props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, JsonTimestampExtractor.class);
// setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
KStreamBuilder builder = new KStreamBuilder();
// TODO: the following can be removed with a serialization factory
Map<String, Object> serdeProps = new HashMap<>();
final Serializer<PageView> pageViewSerializer = new JsonPOJOSerializer<>();
serdeProps.put("JsonPOJOClass", PageView.class);
pageViewSerializer.configure(serdeProps, false);
final Deserializer<PageView> pageViewDeserializer = new JsonPOJODeserializer<>();
serdeProps.put("JsonPOJOClass", PageView.class);
pageViewDeserializer.configure(serdeProps, false);
final Serde<PageView> pageViewSerde = Serdes.serdeFrom(pageViewSerializer, pageViewDeserializer);
final Serializer<UserProfile> userProfileSerializer = new JsonPOJOSerializer<>();
serdeProps.put("JsonPOJOClass", UserProfile.class);
userProfileSerializer.configure(serdeProps, false);
final Deserializer<UserProfile> userProfileDeserializer = new JsonPOJODeserializer<>();
serdeProps.put("JsonPOJOClass", UserProfile.class);
userProfileDeserializer.configure(serdeProps, false);
final Serde<UserProfile> userProfileSerde = Serdes.serdeFrom(userProfileSerializer, userProfileDeserializer);
final Serializer<WindowedPageViewByRegion> wPageViewByRegionSerializer = new JsonPOJOSerializer<>();
serdeProps.put("JsonPOJOClass", WindowedPageViewByRegion.class);
wPageViewByRegionSerializer.configure(serdeProps, false);
final Deserializer<WindowedPageViewByRegion> wPageViewByRegionDeserializer = new JsonPOJODeserializer<>();
serdeProps.put("JsonPOJOClass", WindowedPageViewByRegion.class);
wPageViewByRegionDeserializer.configure(serdeProps, false);
final Serde<WindowedPageViewByRegion> wPageViewByRegionSerde = Serdes.serdeFrom(wPageViewByRegionSerializer, wPageViewByRegionDeserializer);
final Serializer<RegionCount> regionCountSerializer = new JsonPOJOSerializer<>();
serdeProps.put("JsonPOJOClass", RegionCount.class);
regionCountSerializer.configure(serdeProps, false);
final Deserializer<RegionCount> regionCountDeserializer = new JsonPOJODeserializer<>();
serdeProps.put("JsonPOJOClass", RegionCount.class);
regionCountDeserializer.configure(serdeProps, false);
final Serde<RegionCount> regionCountSerde = Serdes.serdeFrom(regionCountSerializer, regionCountDeserializer);
final Serializer<PageViewByRegion> pageViewByRegionSerializer = new JsonPOJOSerializer<>();
serdeProps.put("JsonPOJOClass", PageViewByRegion.class);
pageViewByRegionSerializer.configure(serdeProps, false);
final Deserializer<PageViewByRegion> pageViewByRegionDeserializer = new JsonPOJODeserializer<>();
serdeProps.put("JsonPOJOClass", PageViewByRegion.class);
pageViewByRegionDeserializer.configure(serdeProps, false);
final Serde<PageViewByRegion> pageViewByRegionSerde = Serdes.serdeFrom(pageViewByRegionSerializer, pageViewByRegionDeserializer);
KStream<String, PageView> views = builder.stream(Serdes.String(), pageViewSerde, "streams-pageview-input");
KTable<String, UserProfile> users = builder.table(Serdes.String(), userProfileSerde, "streams-userprofile-input", "streams-userprofile-store-name");
KStream<WindowedPageViewByRegion, RegionCount> regionCount = views.leftJoin(users, new ValueJoiner<PageView, UserProfile, PageViewByRegion>() {
@Override
public PageViewByRegion apply(PageView view, UserProfile profile) {
PageViewByRegion viewByRegion = new PageViewByRegion();
viewByRegion.user = view.user;
viewByRegion.page = view.page;
if (profile != null) {
viewByRegion.region = profile.region;
} else {
viewByRegion.region = "UNKNOWN";
}
return viewByRegion;
}
}).map(new KeyValueMapper<String, PageViewByRegion, KeyValue<String, PageViewByRegion>>() {
@Override
public KeyValue<String, PageViewByRegion> apply(String user, PageViewByRegion viewRegion) {
return new KeyValue<>(viewRegion.region, viewRegion);
}
}).groupByKey(Serdes.String(), pageViewByRegionSerde).count(TimeWindows.of(7 * 24 * 60 * 60 * 1000L).advanceBy(1000), "RollingSevenDaysOfPageViewsByRegion").toStream().map(new KeyValueMapper<Windowed<String>, Long, KeyValue<WindowedPageViewByRegion, RegionCount>>() {
@Override
public KeyValue<WindowedPageViewByRegion, RegionCount> apply(Windowed<String> key, Long value) {
WindowedPageViewByRegion wViewByRegion = new WindowedPageViewByRegion();
wViewByRegion.windowStart = key.window().start();
wViewByRegion.region = key.key();
RegionCount rCount = new RegionCount();
rCount.region = key.key();
rCount.count = value;
return new KeyValue<>(wViewByRegion, rCount);
}
});
// write to the result topic
regionCount.to(wPageViewByRegionSerde, regionCountSerde, "streams-pageviewstats-typed-output");
KafkaStreams streams = new KafkaStreams(builder, props);
streams.start();
// usually the stream application would be running forever,
// in this example we just let it run for some time and stop since the input data is finite.
Thread.sleep(5000L);
streams.close();
}
use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.
the class PageViewUntypedDemo method main.
public static void main(String[] args) throws Exception {
Properties props = new Properties();
props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pageview-untyped");
props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, JsonTimestampExtractor.class);
// setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
KStreamBuilder builder = new KStreamBuilder();
final Serializer<JsonNode> jsonSerializer = new JsonSerializer();
final Deserializer<JsonNode> jsonDeserializer = new JsonDeserializer();
final Serde<JsonNode> jsonSerde = Serdes.serdeFrom(jsonSerializer, jsonDeserializer);
KStream<String, JsonNode> views = builder.stream(Serdes.String(), jsonSerde, "streams-pageview-input");
KTable<String, JsonNode> users = builder.table(Serdes.String(), jsonSerde, "streams-userprofile-input", "streams-userprofile-store-name");
KTable<String, String> userRegions = users.mapValues(new ValueMapper<JsonNode, String>() {
@Override
public String apply(JsonNode record) {
return record.get("region").textValue();
}
});
KStream<JsonNode, JsonNode> regionCount = views.leftJoin(userRegions, new ValueJoiner<JsonNode, String, JsonNode>() {
@Override
public JsonNode apply(JsonNode view, String region) {
ObjectNode jNode = JsonNodeFactory.instance.objectNode();
return jNode.put("user", view.get("user").textValue()).put("page", view.get("page").textValue()).put("region", region == null ? "UNKNOWN" : region);
}
}).map(new KeyValueMapper<String, JsonNode, KeyValue<String, JsonNode>>() {
@Override
public KeyValue<String, JsonNode> apply(String user, JsonNode viewRegion) {
return new KeyValue<>(viewRegion.get("region").textValue(), viewRegion);
}
}).groupByKey(Serdes.String(), jsonSerde).count(TimeWindows.of(7 * 24 * 60 * 60 * 1000L).advanceBy(1000), "RollingSevenDaysOfPageViewsByRegion").toStream().map(new KeyValueMapper<Windowed<String>, Long, KeyValue<JsonNode, JsonNode>>() {
@Override
public KeyValue<JsonNode, JsonNode> apply(Windowed<String> key, Long value) {
ObjectNode keyNode = JsonNodeFactory.instance.objectNode();
keyNode.put("window-start", key.window().start()).put("region", key.key());
ObjectNode valueNode = JsonNodeFactory.instance.objectNode();
valueNode.put("count", value);
return new KeyValue<>((JsonNode) keyNode, (JsonNode) valueNode);
}
});
// write to the result topic
regionCount.to(jsonSerde, jsonSerde, "streams-pageviewstats-untyped-output");
KafkaStreams streams = new KafkaStreams(builder, props);
streams.start();
// usually the stream application would be running forever,
// in this example we just let it run for some time and stop since the input data is finite.
Thread.sleep(5000L);
streams.close();
}
use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.
the class SessionKeySerde method fromBytes.
public static Windowed<Bytes> fromBytes(Bytes bytesKey) {
final byte[] binaryKey = bytesKey.get();
final ByteBuffer buffer = ByteBuffer.wrap(binaryKey);
final long start = buffer.getLong(binaryKey.length - TIMESTAMP_SIZE);
final long end = buffer.getLong(binaryKey.length - 2 * TIMESTAMP_SIZE);
return new Windowed<>(Bytes.wrap(extractKeyBytes(binaryKey)), new SessionWindow(start, end));
}
use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.
the class ResetIntegrationTest method setupTopologyWithIntermediateUserTopic.
private KStreamBuilder setupTopologyWithIntermediateUserTopic(final String outputTopic2) {
final KStreamBuilder builder = new KStreamBuilder();
final KStream<Long, String> input = builder.stream(INPUT_TOPIC);
// use map to trigger internal re-partitioning before groupByKey
input.map(new KeyValueMapper<Long, String, KeyValue<Long, String>>() {
@Override
public KeyValue<Long, String> apply(final Long key, final String value) {
return new KeyValue<>(key, value);
}
}).groupByKey().count("global-count").to(Serdes.Long(), Serdes.Long(), OUTPUT_TOPIC);
input.through(INTERMEDIATE_USER_TOPIC).groupByKey().count(TimeWindows.of(35).advanceBy(10), "count").toStream().map(new KeyValueMapper<Windowed<Long>, Long, KeyValue<Long, Long>>() {
@Override
public KeyValue<Long, Long> apply(final Windowed<Long> key, final Long value) {
return new KeyValue<>(key.window().start() + key.window().end(), value);
}
}).to(Serdes.Long(), Serdes.Long(), outputTopic2);
return builder;
}
use of org.apache.kafka.streams.kstream.Windowed in project kafka by apache.
the class KGroupedStreamImplTest method shouldAggregateSessionWindows.
@Test
public void shouldAggregateSessionWindows() throws Exception {
final Map<Windowed<String>, Integer> results = new HashMap<>();
groupedStream.aggregate(new Initializer<Integer>() {
@Override
public Integer apply() {
return 0;
}
}, new Aggregator<String, String, Integer>() {
@Override
public Integer apply(final String aggKey, final String value, final Integer aggregate) {
return aggregate + 1;
}
}, new Merger<String, Integer>() {
@Override
public Integer apply(final String aggKey, final Integer aggOne, final Integer aggTwo) {
return aggOne + aggTwo;
}
}, SessionWindows.with(30), Serdes.Integer(), "session-store").foreach(new ForeachAction<Windowed<String>, Integer>() {
@Override
public void apply(final Windowed<String> key, final Integer value) {
results.put(key, value);
}
});
driver = new KStreamTestDriver(builder, TestUtils.tempDirectory());
driver.setTime(10);
driver.process(TOPIC, "1", "1");
driver.setTime(15);
driver.process(TOPIC, "2", "2");
driver.setTime(30);
driver.process(TOPIC, "1", "1");
driver.setTime(70);
driver.process(TOPIC, "1", "1");
driver.setTime(90);
driver.process(TOPIC, "1", "1");
driver.setTime(100);
driver.process(TOPIC, "1", "1");
driver.flushState();
assertEquals(Integer.valueOf(2), results.get(new Windowed<>("1", new SessionWindow(10, 30))));
assertEquals(Integer.valueOf(1), results.get(new Windowed<>("2", new SessionWindow(15, 15))));
assertEquals(Integer.valueOf(3), results.get(new Windowed<>("1", new SessionWindow(70, 100))));
}
Aggregations