use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.
the class KTableKTableLeftJoinTest method testSendingOldValue.
@Test
public void testSendingOldValue() {
final StreamsBuilder builder = new StreamsBuilder();
final int[] expectedKeys = new int[] { 0, 1, 2, 3 };
final KTable<Integer, String> table1;
final KTable<Integer, String> table2;
final KTable<Integer, String> joined;
final MockApiProcessorSupplier<Integer, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
table1 = builder.table(topic1, consumed);
table2 = builder.table(topic2, consumed);
joined = table1.leftJoin(table2, MockValueJoiner.TOSTRING_JOINER);
((KTableImpl<?, ?, ?>) joined).enableSendingOldValues(true);
assertThat(((KTableImpl<?, ?, ?>) table1).sendingOldValueEnabled(), is(true));
assertThat(((KTableImpl<?, ?, ?>) table2).sendingOldValueEnabled(), is(true));
assertThat(((KTableImpl<?, ?, ?>) joined).sendingOldValueEnabled(), is(true));
final Topology topology = builder.build().addProcessor("proc", supplier, ((KTableImpl<?, ?, ?>) joined).name);
try (final TopologyTestDriver driver = new TopologyTestDriverWrapper(topology, props)) {
final TestInputTopic<Integer, String> inputTopic1 = driver.createInputTopic(topic1, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
final TestInputTopic<Integer, String> inputTopic2 = driver.createInputTopic(topic2, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
final MockApiProcessor<Integer, String, Void, Void> proc = supplier.theCapturedProcessor();
assertTrue(((KTableImpl<?, ?, ?>) table1).sendingOldValueEnabled());
assertTrue(((KTableImpl<?, ?, ?>) table2).sendingOldValueEnabled());
assertTrue(((KTableImpl<?, ?, ?>) joined).sendingOldValueEnabled());
// push two items to the primary stream. the other table is empty
for (int i = 0; i < 2; i++) {
inputTopic1.pipeInput(expectedKeys[i], "X" + expectedKeys[i], 5L + i);
}
// pass tuple with null key, it will be discarded in join process
inputTopic1.pipeInput(null, "SomeVal", 42L);
// left: X0:0 (ts: 5), X1:1 (ts: 6)
// right:
proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+null", null), 5), new KeyValueTimestamp<>(1, new Change<>("X1+null", null), 6));
// push two items to the other stream. this should produce two items.
for (int i = 0; i < 2; i++) {
inputTopic2.pipeInput(expectedKeys[i], "Y" + expectedKeys[i], 10L * i);
}
// pass tuple with null key, it will be discarded in join process
inputTopic2.pipeInput(null, "AnotherVal", 73L);
// left: X0:0 (ts: 5), X1:1 (ts: 6)
// right: Y0:0 (ts: 0), Y1:1 (ts: 10)
proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+Y0", "X0+null"), 5), new KeyValueTimestamp<>(1, new Change<>("X1+Y1", "X1+null"), 10));
// push all four items to the primary stream. this should produce four items.
for (final int expectedKey : expectedKeys) {
inputTopic1.pipeInput(expectedKey, "XX" + expectedKey, 7L);
}
// left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
// right: Y0:0 (ts: 0), Y1:1 (ts: 10)
proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XX0+Y0", "X0+Y0"), 7), new KeyValueTimestamp<>(1, new Change<>("XX1+Y1", "X1+Y1"), 10), new KeyValueTimestamp<>(2, new Change<>("XX2+null", null), 7), new KeyValueTimestamp<>(3, new Change<>("XX3+null", null), 7));
// push all items to the other stream. this should produce four items.
for (final int expectedKey : expectedKeys) {
inputTopic2.pipeInput(expectedKey, "YY" + expectedKey, expectedKey * 5L);
}
// left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
// right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XX0+YY0", "XX0+Y0"), 7), new KeyValueTimestamp<>(1, new Change<>("XX1+YY1", "XX1+Y1"), 7), new KeyValueTimestamp<>(2, new Change<>("XX2+YY2", "XX2+null"), 10), new KeyValueTimestamp<>(3, new Change<>("XX3+YY3", "XX3+null"), 15));
// push all four items to the primary stream. this should produce four items.
for (final int expectedKey : expectedKeys) {
inputTopic1.pipeInput(expectedKey, "XXX" + expectedKey, 6L);
}
// left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
// right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XXX0+YY0", "XX0+YY0"), 6), new KeyValueTimestamp<>(1, new Change<>("XXX1+YY1", "XX1+YY1"), 6), new KeyValueTimestamp<>(2, new Change<>("XXX2+YY2", "XX2+YY2"), 10), new KeyValueTimestamp<>(3, new Change<>("XXX3+YY3", "XX3+YY3"), 15));
// push two items with null to the other stream as deletes. this should produce two item.
inputTopic2.pipeInput(expectedKeys[0], null, 5L);
inputTopic2.pipeInput(expectedKeys[1], null, 7L);
// left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
// right: YY2:2 (ts: 10), YY3:3 (ts: 15)
proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XXX0+null", "XXX0+YY0"), 6), new KeyValueTimestamp<>(1, new Change<>("XXX1+null", "XXX1+YY1"), 7));
// push all four items to the primary stream. this should produce four items.
for (final int expectedKey : expectedKeys) {
inputTopic1.pipeInput(expectedKey, "XXXX" + expectedKey, 13L);
}
// left: XXXX0:0 (ts: 13), XXXX1:1 (ts: 13), XXXX2:2 (ts: 13), XXXX3:3 (ts: 13)
// right: YY2:2 (ts: 10), YY3:3 (ts: 15)
proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XXXX0+null", "XXX0+null"), 13), new KeyValueTimestamp<>(1, new Change<>("XXXX1+null", "XXX1+null"), 13), new KeyValueTimestamp<>(2, new Change<>("XXXX2+YY2", "XXX2+YY2"), 13), new KeyValueTimestamp<>(3, new Change<>("XXXX3+YY3", "XXX3+YY3"), 15));
// push four items to the primary stream with null. this should produce four items.
inputTopic1.pipeInput(expectedKeys[0], null, 0L);
inputTopic1.pipeInput(expectedKeys[1], null, 42L);
inputTopic1.pipeInput(expectedKeys[2], null, 5L);
inputTopic1.pipeInput(expectedKeys[3], null, 20L);
// left:
// right: YY2:2 (ts: 10), YY3:3 (ts: 15)
proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>(null, "XXXX0+null"), 0), new KeyValueTimestamp<>(1, new Change<>(null, "XXXX1+null"), 42), new KeyValueTimestamp<>(2, new Change<>(null, "XXXX2+YY2"), 10), new KeyValueTimestamp<>(3, new Change<>(null, "XXXX3+YY3"), 20));
}
}
use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.
the class KTableKTableLeftJoinTest method shouldNotThrowIllegalStateExceptionWhenMultiCacheEvictions.
/**
* This test was written to reproduce https://issues.apache.org/jira/browse/KAFKA-4492
* It is based on a fairly complicated join used by the developer that reported the bug.
* Before the fix this would trigger an IllegalStateException.
*/
@Test
public void shouldNotThrowIllegalStateExceptionWhenMultiCacheEvictions() {
final String agg = "agg";
final String tableOne = "tableOne";
final String tableTwo = "tableTwo";
final String tableThree = "tableThree";
final String tableFour = "tableFour";
final String tableFive = "tableFive";
final String tableSix = "tableSix";
final String[] inputs = { agg, tableOne, tableTwo, tableThree, tableFour, tableFive, tableSix };
final StreamsBuilder builder = new StreamsBuilder();
final Consumed<Long, String> consumed = Consumed.with(Serdes.Long(), Serdes.String());
final KTable<Long, String> aggTable = builder.table(agg, consumed, Materialized.as(Stores.inMemoryKeyValueStore("agg-base-store"))).groupBy(KeyValue::new, Grouped.with(Serdes.Long(), Serdes.String())).reduce(MockReducer.STRING_ADDER, MockReducer.STRING_ADDER, Materialized.as(Stores.inMemoryKeyValueStore("agg-store")));
final KTable<Long, String> one = builder.table(tableOne, consumed, Materialized.as(Stores.inMemoryKeyValueStore("tableOne-base-store")));
final KTable<Long, String> two = builder.table(tableTwo, consumed, Materialized.as(Stores.inMemoryKeyValueStore("tableTwo-base-store")));
final KTable<Long, String> three = builder.table(tableThree, consumed, Materialized.as(Stores.inMemoryKeyValueStore("tableThree-base-store")));
final KTable<Long, String> four = builder.table(tableFour, consumed, Materialized.as(Stores.inMemoryKeyValueStore("tableFour-base-store")));
final KTable<Long, String> five = builder.table(tableFive, consumed, Materialized.as(Stores.inMemoryKeyValueStore("tableFive-base-store")));
final KTable<Long, String> six = builder.table(tableSix, consumed, Materialized.as(Stores.inMemoryKeyValueStore("tableSix-base-store")));
final ValueMapper<String, String> mapper = value -> value.toUpperCase(Locale.ROOT);
final KTable<Long, String> seven = one.mapValues(mapper);
final KTable<Long, String> eight = six.leftJoin(seven, MockValueJoiner.TOSTRING_JOINER);
aggTable.leftJoin(one, MockValueJoiner.TOSTRING_JOINER).leftJoin(two, MockValueJoiner.TOSTRING_JOINER).leftJoin(three, MockValueJoiner.TOSTRING_JOINER).leftJoin(four, MockValueJoiner.TOSTRING_JOINER).leftJoin(five, MockValueJoiner.TOSTRING_JOINER).leftJoin(eight, MockValueJoiner.TOSTRING_JOINER).mapValues(mapper);
try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
final String[] values = { "a", "AA", "BBB", "CCCC", "DD", "EEEEEEEE", "F", "GGGGGGGGGGGGGGG", "HHH", "IIIIIIIIII", "J", "KK", "LLLL", "MMMMMMMMMMMMMMMMMMMMMM", "NNNNN", "O", "P", "QQQQQ", "R", "SSSS", "T", "UU", "VVVVVVVVVVVVVVVVVVV" };
TestInputTopic<Long, String> inputTopic;
final Random random = new Random();
for (int i = 0; i < 1000; i++) {
for (final String input : inputs) {
final Long key = (long) random.nextInt(1000);
final String value = values[random.nextInt(values.length)];
inputTopic = driver.createInputTopic(input, Serdes.Long().serializer(), Serdes.String().serializer());
inputTopic.pipeInput(key, value);
}
}
}
}
use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.
the class KTableMapValuesTest method doTestKTable.
private void doTestKTable(final StreamsBuilder builder, final String topic1, final MockApiProcessorSupplier<String, Integer, Void, Void> supplier) {
try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
inputTopic1.pipeInput("A", "1", 5L);
inputTopic1.pipeInput("B", "2", 25L);
inputTopic1.pipeInput("C", "3", 20L);
inputTopic1.pipeInput("D", "4", 10L);
assertEquals(asList(new KeyValueTimestamp<>("A", 1, 5), new KeyValueTimestamp<>("B", 2, 25), new KeyValueTimestamp<>("C", 3, 20), new KeyValueTimestamp<>("D", 4, 10)), supplier.theCapturedProcessor().processed());
}
}
use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.
the class KTableMapValuesTest method testNotSendingOldValue.
@Test
public void testNotSendingOldValue() {
final StreamsBuilder builder = new StreamsBuilder();
final String topic1 = "topic1";
final KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, consumed);
final KTableImpl<String, String, Integer> table2 = (KTableImpl<String, String, Integer>) table1.mapValues(s -> Integer.valueOf(s));
final MockApiProcessorSupplier<String, Integer, Void, Void> supplier = new MockApiProcessorSupplier<>();
final Topology topology = builder.build().addProcessor("proc", supplier, table2.name);
try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
final MockApiProcessor<String, Integer, Void, Void> proc = supplier.theCapturedProcessor();
assertFalse(table1.sendingOldValueEnabled());
assertFalse(table2.sendingOldValueEnabled());
inputTopic1.pipeInput("A", "01", 5L);
inputTopic1.pipeInput("B", "01", 10L);
inputTopic1.pipeInput("C", "01", 15L);
proc.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(1, null), 5), new KeyValueTimestamp<>("B", new Change<>(1, null), 10), new KeyValueTimestamp<>("C", new Change<>(1, null), 15));
inputTopic1.pipeInput("A", "02", 10L);
inputTopic1.pipeInput("B", "02", 8L);
proc.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(2, null), 10), new KeyValueTimestamp<>("B", new Change<>(2, null), 8));
inputTopic1.pipeInput("A", "03", 20L);
proc.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(3, null), 20));
inputTopic1.pipeInput("A", (String) null, 30L);
proc.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, null), 30));
}
}
use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.
the class KTableSourceTest method testNotSendingOldValue.
@Test
public void testNotSendingOldValue() {
final StreamsBuilder builder = new StreamsBuilder();
final String topic1 = "topic1";
@SuppressWarnings("unchecked") final KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, stringConsumed);
final MockApiProcessorSupplier<String, Integer, Void, Void> supplier = new MockApiProcessorSupplier<>();
final Topology topology = builder.build().addProcessor("proc1", supplier, table1.name);
try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
final MockApiProcessor<String, Integer, Void, Void> proc1 = supplier.theCapturedProcessor();
inputTopic1.pipeInput("A", "01", 10L);
inputTopic1.pipeInput("B", "01", 20L);
inputTopic1.pipeInput("C", "01", 15L);
proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("01", null), 10), new KeyValueTimestamp<>("B", new Change<>("01", null), 20), new KeyValueTimestamp<>("C", new Change<>("01", null), 15));
inputTopic1.pipeInput("A", "02", 8L);
inputTopic1.pipeInput("B", "02", 22L);
proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("02", null), 8), new KeyValueTimestamp<>("B", new Change<>("02", null), 22));
inputTopic1.pipeInput("A", "03", 12L);
proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("03", null), 12));
inputTopic1.pipeInput("A", null, 15L);
inputTopic1.pipeInput("B", null, 20L);
proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, null), 15), new KeyValueTimestamp<>("B", new Change<>(null, null), 20));
}
}
Aggregations