Search in sources :

Example 1 with Rating

use of io.confluent.demo.Rating in project kafka-tutorials by confluentinc.

the class RunningAverage method getRatingAverageTable.

protected static KTable<Long, Double> getRatingAverageTable(KStream<Long, Rating> ratings, String avgRatingsTopicName, SpecificAvroSerde<CountAndSum> countAndSumSerde) {
    // Grouping Ratings
    KGroupedStream<Long, Double> ratingsById = ratings.map((key, rating) -> new KeyValue<>(rating.getMovieId(), rating.getRating())).groupByKey(with(Long(), Double()));
    final KTable<Long, CountAndSum> ratingCountAndSum = ratingsById.aggregate(() -> new CountAndSum(0L, 0.0), (key, value, aggregate) -> {
        aggregate.setCount(aggregate.getCount() + 1);
        aggregate.setSum(aggregate.getSum() + value);
        return aggregate;
    }, Materialized.with(Long(), countAndSumSerde));
    final KTable<Long, Double> ratingAverage = ratingCountAndSum.mapValues(value -> value.getSum() / value.getCount(), Materialized.as("average-ratings"));
    // persist the result in topic
    ratingAverage.toStream().to(avgRatingsTopicName);
    return ratingAverage;
}
Also used : StreamsConfig(org.apache.kafka.streams.StreamsConfig) CountAndSum(io.confluent.demo.CountAndSum) KGroupedStream(org.apache.kafka.streams.kstream.KGroupedStream) SCHEMA_REGISTRY_URL_CONFIG(io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG) HashMap(java.util.HashMap) KStream(org.apache.kafka.streams.kstream.KStream) Short.parseShort(java.lang.Short.parseShort) ArrayList(java.util.ArrayList) AdminClient(org.apache.kafka.clients.admin.AdminClient) Rating(io.confluent.demo.Rating) Collectors.toMap(java.util.stream.Collectors.toMap) REPLICATION_FACTOR_CONFIG(org.apache.kafka.streams.StreamsConfig.REPLICATION_FACTOR_CONFIG) Duration(java.time.Duration) Map(java.util.Map) APPLICATION_ID_CONFIG(org.apache.kafka.streams.StreamsConfig.APPLICATION_ID_CONFIG) ConfigFactory(com.typesafe.config.ConfigFactory) Serdes(org.apache.kafka.common.serialization.Serdes) Grouped.with(org.apache.kafka.streams.kstream.Grouped.with) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) KTable(org.apache.kafka.streams.kstream.KTable) Properties(java.util.Properties) Config(com.typesafe.config.Config) Consumed(org.apache.kafka.streams.kstream.Consumed) Optional.ofNullable(java.util.Optional.ofNullable) NewTopic(org.apache.kafka.clients.admin.NewTopic) KeyValue(org.apache.kafka.streams.KeyValue) Double(org.apache.kafka.common.serialization.Serdes.Double) Long(org.apache.kafka.common.serialization.Serdes.Long) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) Integer.parseInt(java.lang.Integer.parseInt) BOOTSTRAP_SERVERS_CONFIG(org.apache.kafka.streams.StreamsConfig.BOOTSTRAP_SERVERS_CONFIG) SpecificAvroSerde(io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Stream(java.util.stream.Stream) DEFAULT_VALUE_SERDE_CLASS_CONFIG(org.apache.kafka.streams.StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG) DEFAULT_KEY_SERDE_CLASS_CONFIG(org.apache.kafka.streams.StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG) Materialized(org.apache.kafka.streams.kstream.Materialized) KafkaStreams(org.apache.kafka.streams.KafkaStreams) Topology(org.apache.kafka.streams.Topology) KeyValue(org.apache.kafka.streams.KeyValue) CountAndSum(io.confluent.demo.CountAndSum) Long(org.apache.kafka.common.serialization.Serdes.Long) Double(org.apache.kafka.common.serialization.Serdes.Double)

Example 2 with Rating

use of io.confluent.demo.Rating in project kafka-tutorials by confluentinc.

the class RunningAverageTest method validateAverageRating.

@Test
public void validateAverageRating() {
    TestInputTopic<Long, Rating> inputTopic = testDriver.createInputTopic(RATINGS_TOPIC_NAME, new LongSerializer(), ratingSpecificAvroSerde.serializer());
    inputTopic.pipeKeyValueList(asList(new KeyValue<>(LETHAL_WEAPON_RATING_8.getMovieId(), LETHAL_WEAPON_RATING_8), new KeyValue<>(LETHAL_WEAPON_RATING_10.getMovieId(), LETHAL_WEAPON_RATING_10)));
    final TestOutputTopic<Long, Double> outputTopic = testDriver.createOutputTopic(AVERAGE_RATINGS_TOPIC_NAME, new LongDeserializer(), new DoubleDeserializer());
    final List<KeyValue<Long, Double>> keyValues = outputTopic.readKeyValuesToList();
    // I sent two records to input topic
    // I expect second record in topic will contain correct result
    final KeyValue<Long, Double> longDoubleKeyValue = keyValues.get(1);
    System.out.println("longDoubleKeyValue = " + longDoubleKeyValue);
    MatcherAssert.assertThat(longDoubleKeyValue, equalTo(new KeyValue<>(362L, 9.0)));
    final KeyValueStore<Long, Double> keyValueStore = testDriver.getKeyValueStore("average-ratings");
    final Double expected = keyValueStore.get(362L);
    Assert.assertEquals("Message", expected, 9.0, 0.0);
}
Also used : LongSerializer(org.apache.kafka.common.serialization.LongSerializer) KeyValue(org.apache.kafka.streams.KeyValue) LongDeserializer(org.apache.kafka.common.serialization.LongDeserializer) Rating(io.confluent.demo.Rating) DoubleDeserializer(org.apache.kafka.common.serialization.DoubleDeserializer) Test(org.junit.Test)

Example 3 with Rating

use of io.confluent.demo.Rating in project kafka-tutorials by confluentinc.

the class RunningAverageTest method setUp.

@Before
public void setUp() {
    final Properties mockProps = new Properties();
    mockProps.put("application.id", "kafka-movies-test");
    mockProps.put("bootstrap.servers", "DUMMY_KAFKA_CONFLUENT_CLOUD_9092");
    mockProps.put("schema.registry.url", "mock://DUMMY_SR_CONFLUENT_CLOUD_8080");
    mockProps.put("default.topic.replication.factor", "1");
    mockProps.put("offset.reset.policy", "latest");
    mockProps.put("specific.avro.reader", true);
    final RunningAverage streamsApp = new RunningAverage();
    final Properties streamsConfig = streamsApp.buildStreamsProperties(mockProps);
    StreamsBuilder builder = new StreamsBuilder();
    SpecificAvroSerde<CountAndSum> countAndSumSerde = RunningAverage.getCountAndSumSerde(mockProps);
    ratingSpecificAvroSerde = RunningAverage.getRatingSerde(mockProps);
    KStream<Long, Rating> ratingStream = builder.stream(RATINGS_TOPIC_NAME, Consumed.with(Serdes.Long(), ratingSpecificAvroSerde));
    RunningAverage.getRatingAverageTable(ratingStream, AVERAGE_RATINGS_TOPIC_NAME, countAndSumSerde);
    final Topology topology = builder.build();
    testDriver = new TopologyTestDriver(topology, streamsConfig);
}
Also used : StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) CountAndSum(io.confluent.demo.CountAndSum) Rating(io.confluent.demo.Rating) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) Topology(org.apache.kafka.streams.Topology) Properties(java.util.Properties) Before(org.junit.Before)

Aggregations

Rating (io.confluent.demo.Rating)3 CountAndSum (io.confluent.demo.CountAndSum)2 Properties (java.util.Properties)2 KeyValue (org.apache.kafka.streams.KeyValue)2 StreamsBuilder (org.apache.kafka.streams.StreamsBuilder)2 Topology (org.apache.kafka.streams.Topology)2 Config (com.typesafe.config.Config)1 ConfigFactory (com.typesafe.config.ConfigFactory)1 SCHEMA_REGISTRY_URL_CONFIG (io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG)1 SpecificAvroSerde (io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde)1 Integer.parseInt (java.lang.Integer.parseInt)1 Short.parseShort (java.lang.Short.parseShort)1 Duration (java.time.Duration)1 ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Map (java.util.Map)1 Optional.ofNullable (java.util.Optional.ofNullable)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 Collectors.toMap (java.util.stream.Collectors.toMap)1