Examples with StringDeserializer org.apache.kafka.common.serialization.StringDeserializer used on opensource projects

Example 46 with StringDeserializer

use of org.apache.kafka.common.serialization.StringDeserializer in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldGenerateRepartitionTopicWhenNameIsNotSpecified.

@Test
public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws Exception {
    final long timestamp = System.currentTimeMillis();
    sendEvents(timestamp, Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B")));
    final StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).selectKey((key, value) -> key.toString()).repartition(Repartitioned.with(Serdes.String(), Serdes.String())).to(outputTopic);
    startStreams(builder);
    validateReceivedMessages(new StringDeserializer(), new StringDeserializer(), Arrays.asList(new KeyValue<>("1", "A"), new KeyValue<>("2", "B")));
    final String topology = builder.build().describe().toString();
    assertEquals(1, countOccurrencesInTopology(topology, "Sink: .*-repartition"));
}

Example 47 with StringDeserializer

use of org.apache.kafka.common.serialization.StringDeserializer in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldReduceSlidingWindows.

@SuppressWarnings("deprecation")
@Test
public void shouldReduceSlidingWindows() throws Exception {
    final long firstBatchTimestamp = mockTime.milliseconds();
    final long timeDifference = 500L;
    produceMessages(firstBatchTimestamp);
    final long secondBatchTimestamp = firstBatchTimestamp + timeDifference / 2;
    produceMessages(secondBatchTimestamp);
    final long thirdBatchTimestamp = firstBatchTimestamp + timeDifference - 100L;
    produceMessages(thirdBatchTimestamp);
    final Serde<Windowed<String>> windowedSerde = WindowedSerdes.timeWindowedSerdeFrom(String.class, timeDifference);
    // noinspection deprecation
    groupedStream.windowedBy(SlidingWindows.withTimeDifferenceAndGrace(ofMillis(timeDifference), ofMillis(2000L))).reduce(reducer).toStream().to(outputTopic, Produced.with(windowedSerde, Serdes.String()));
    startStreams();
    final List<KeyValueTimestamp<Windowed<String>, String>> windowedOutput = receiveMessages(new TimeWindowedDeserializer<>(new StringDeserializer(), 500L), new StringDeserializer(), String.class, 30);
    final String resultFromConsoleConsumer = readWindowedKeyedMessagesViaConsoleConsumer(new TimeWindowedDeserializer<String>(), new StringDeserializer(), String.class, 30, true);
    final Comparator<KeyValueTimestamp<Windowed<String>, String>> comparator = Comparator.comparing((KeyValueTimestamp<Windowed<String>, String> o) -> o.key().key()).thenComparing(KeyValueTimestamp::value);
    windowedOutput.sort(comparator);
    final long firstBatchLeftWindowStart = firstBatchTimestamp - timeDifference;
    final long firstBatchLeftWindowEnd = firstBatchLeftWindowStart + timeDifference;
    final long firstBatchRightWindowStart = firstBatchTimestamp + 1;
    final long firstBatchRightWindowEnd = firstBatchRightWindowStart + timeDifference;
    final long secondBatchLeftWindowStart = secondBatchTimestamp - timeDifference;
    final long secondBatchLeftWindowEnd = secondBatchLeftWindowStart + timeDifference;
    final long secondBatchRightWindowStart = secondBatchTimestamp + 1;
    final long secondBatchRightWindowEnd = secondBatchRightWindowStart + timeDifference;
    final long thirdBatchLeftWindowStart = thirdBatchTimestamp - timeDifference;
    final long thirdBatchLeftWindowEnd = thirdBatchLeftWindowStart + timeDifference;
    final List<KeyValueTimestamp<Windowed<String>, String>> expectResult = Arrays.asList(// A @ firstBatchTimestamp left window created when A @ firstBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), "A", firstBatchTimestamp), // A @ firstBatchTimestamp right window created when A @ secondBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "A", secondBatchTimestamp), // A @ secondBatchTimestamp right window created when A @ thirdBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), "A", thirdBatchTimestamp), // A @ secondBatchTimestamp left window created when A @ secondBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), "A:A", secondBatchTimestamp), // A @ firstBatchTimestamp right window updated when A @ thirdBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "A:A", thirdBatchTimestamp), // A @ thirdBatchTimestamp left window created when A @ thirdBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), "A:A:A", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), "B", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "B", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), "B", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), "B:B", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "B:B", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), "B:B:B", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), "C", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "C", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), "C", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), "C:C", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "C:C", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), "C:C:C", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), "D", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "D", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), "D", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), "D:D", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "D:D", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), "D:D:D", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), "E", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "E", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), "E", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), "E:E", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "E:E", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), "E:E:E", thirdBatchTimestamp));
    assertThat(windowedOutput, is(expectResult));
    final Set<String> expectResultString = new HashSet<>(expectResult.size());
    for (final KeyValueTimestamp<Windowed<String>, String> eachRecord : expectResult) {
        expectResultString.add("CreateTime:" + eachRecord.timestamp() + ", " + eachRecord.key() + ", " + eachRecord.value());
    }
    // check every message is contained in the expect result
    final String[] allRecords = resultFromConsoleConsumer.split("\n");
    for (final String record : allRecords) {
        assertTrue(expectResultString.contains(record));
    }
}

Example 48 with StringDeserializer

use of org.apache.kafka.common.serialization.StringDeserializer in project kafka by apache.

the class KStreamRepartitionIntegrationTest method shouldDeductNumberOfPartitionsFromRepartitionOperation.

@Test
public void shouldDeductNumberOfPartitionsFromRepartitionOperation() throws Exception {
    final String topicBMapperName = "topic-b-mapper";
    final int topicBNumberOfPartitions = 6;
    final String inputTopicRepartitionName = "join-repartition-test";
    final int inputTopicRepartitionedNumOfPartitions = 3;
    final long timestamp = System.currentTimeMillis();
    CLUSTER.createTopic(topicB, topicBNumberOfPartitions, 1);
    final List<KeyValue<Integer, String>> expectedRecords = Arrays.asList(new KeyValue<>(1, "A"), new KeyValue<>(2, "B"));
    sendEvents(timestamp, expectedRecords);
    sendEvents(topicB, timestamp, expectedRecords);
    final StreamsBuilder builder = new StreamsBuilder();
    final Repartitioned<Integer, String> inputTopicRepartitioned = Repartitioned.<Integer, String>as(inputTopicRepartitionName).withNumberOfPartitions(inputTopicRepartitionedNumOfPartitions);
    final KStream<Integer, String> topicBStream = builder.stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())).map(KeyValue::new, Named.as(topicBMapperName));
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())).repartition(inputTopicRepartitioned).join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))).to(outputTopic);
    builder.build(streamsConfiguration);
    startStreams(builder);
    assertEquals(inputTopicRepartitionedNumOfPartitions, getNumberOfPartitionsForTopic(toRepartitionTopicName(inputTopicRepartitionName)));
    assertEquals(inputTopicRepartitionedNumOfPartitions, getNumberOfPartitionsForTopic(toRepartitionTopicName(topicBMapperName)));
    validateReceivedMessages(new IntegerDeserializer(), new StringDeserializer(), expectedRecords);
}

Example 49 with StringDeserializer

use of org.apache.kafka.common.serialization.StringDeserializer in project kafka by apache.

the class KTableKTableForeignKeyJoinIntegrationTest method shouldNotEmitTombstonesWhenDeletingNonExistingRecords.

@Test
public void shouldNotEmitTombstonesWhenDeletingNonExistingRecords() {
    final Topology topology = getTopology(streamsConfig, materialized ? "store" : null, leftJoin, rejoin);
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
        final TestInputTopic<String, String> left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
        final TestOutputTopic<String, String> outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
        final KeyValueStore<String, String> store = driver.getKeyValueStore("store");
        // Deleting a record that never existed doesn't need to emit tombstones.
        left.pipeInput("lhs1", (String) null);
        {
            assertThat(outputTopic.readKeyValuesToMap(), is(emptyMap()));
            if (materialized) {
                assertThat(asMap(store), is(emptyMap()));
            }
        }
    }
}

Example 50 with StringDeserializer

use of org.apache.kafka.common.serialization.StringDeserializer in project kafka by apache.

the class KTableKTableForeignKeyJoinIntegrationTest method joinShouldProduceNullsWhenValueHasNonMatchingForeignKey.

@Test
public void joinShouldProduceNullsWhenValueHasNonMatchingForeignKey() {
    final Topology topology = getTopology(streamsConfig, materialized ? "store" : null, leftJoin, rejoin);
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
        final TestInputTopic<String, String> right = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
        final TestInputTopic<String, String> left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
        final TestOutputTopic<String, String> outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
        final KeyValueStore<String, String> store = driver.getKeyValueStore("store");
        left.pipeInput("lhs1", "lhsValue1|rhs1");
        // no output for a new inner join on a non-existent FK
        // the left join of course emits the half-joined output
        assertThat(outputTopic.readKeyValuesToMap(), is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)")) : emptyMap()));
        if (materialized) {
            assertThat(asMap(store), is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)")) : emptyMap()));
        }
        // "moving" our subscription to another non-existent FK results in an unnecessary tombstone for inner join,
        // since it impossible to know whether the prior FK existed or not (and thus whether any results have
        // previously been emitted)
        // The left join emits a _necessary_ update (since the lhs record has actually changed)
        left.pipeInput("lhs1", "lhsValue1|rhs2");
        assertThat(outputTopic.readKeyValuesToMap(), is(mkMap(mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs2,null)" : null))));
        if (materialized) {
            assertThat(asMap(store), is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs2,null)")) : emptyMap()));
        }
        // of course, moving it again to yet another non-existent FK has the same effect
        left.pipeInput("lhs1", "lhsValue1|rhs3");
        assertThat(outputTopic.readKeyValuesToMap(), is(mkMap(mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs3,null)" : null))));
        if (materialized) {
            assertThat(asMap(store), is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs3,null)")) : emptyMap()));
        }
        // Adding an RHS record now, so that we can demonstrate "moving" from a non-existent FK to an existent one
        // This RHS key was previously referenced, but it's not referenced now, so adding this record should
        // result in no changes whatsoever.
        right.pipeInput("rhs1", "rhsValue1");
        assertThat(outputTopic.readKeyValuesToMap(), is(emptyMap()));
        if (materialized) {
            assertThat(asMap(store), is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs3,null)")) : emptyMap()));
        }
        // now, we change to a FK that exists, and see the join completes
        left.pipeInput("lhs1", "lhsValue1|rhs1");
        assertThat(outputTopic.readKeyValuesToMap(), is(mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"))));
        if (materialized) {
            assertThat(asMap(store), is(mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"))));
        }
        // but if we update it again to a non-existent one, we'll get a tombstone for the inner join, and the
        // left join updates appropriately.
        left.pipeInput("lhs1", "lhsValue1|rhs2");
        assertThat(outputTopic.readKeyValuesToMap(), is(mkMap(mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs2,null)" : null))));
        if (materialized) {
            assertThat(asMap(store), is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs2,null)")) : emptyMap()));
        }
    }
}