Examples with Topology org.apache.kafka.streams.Topology used on opensource projects

Example 86 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class HighAvailabilityTaskAssignorIntegrationTest method shouldScaleOutWithWarmupTasks.

private void shouldScaleOutWithWarmupTasks(final Function<String, Materialized<Object, Object, KeyValueStore<Bytes, byte[]>>> materializedFunction) throws InterruptedException {
    final String testId = safeUniqueTestName(getClass(), testName);
    final String appId = "appId_" + System.currentTimeMillis() + "_" + testId;
    final String inputTopic = "input" + testId;
    final Set<TopicPartition> inputTopicPartitions = mkSet(new TopicPartition(inputTopic, 0), new TopicPartition(inputTopic, 1));
    final String storeName = "store" + testId;
    final String storeChangelog = appId + "-store" + testId + "-changelog";
    final Set<TopicPartition> changelogTopicPartitions = mkSet(new TopicPartition(storeChangelog, 0), new TopicPartition(storeChangelog, 1));
    IntegrationTestUtils.cleanStateBeforeTest(CLUSTER, 2, inputTopic, storeChangelog);
    final ReentrantLock assignmentLock = new ReentrantLock();
    final AtomicInteger assignmentsCompleted = new AtomicInteger(0);
    final Map<Integer, Boolean> assignmentsStable = new ConcurrentHashMap<>();
    final AtomicBoolean assignmentStable = new AtomicBoolean(false);
    final AssignmentListener assignmentListener = stable -> {
        assignmentLock.lock();
        try {
            final int thisAssignmentIndex = assignmentsCompleted.incrementAndGet();
            assignmentsStable.put(thisAssignmentIndex, stable);
            assignmentStable.set(stable);
        } finally {
            assignmentLock.unlock();
        }
    };
    final StreamsBuilder builder = new StreamsBuilder();
    builder.table(inputTopic, materializedFunction.apply(storeName));
    final Topology topology = builder.build();
    final int numberOfRecords = 500;
    produceTestData(inputTopic, numberOfRecords);
    try (final KafkaStreams kafkaStreams0 = new KafkaStreams(topology, streamsProperties(appId, assignmentListener));
        final KafkaStreams kafkaStreams1 = new KafkaStreams(topology, streamsProperties(appId, assignmentListener));
        final Consumer<String, String> consumer = new KafkaConsumer<>(getConsumerProperties())) {
        kafkaStreams0.start();
        // sanity check: just make sure we actually wrote all the input records
        TestUtils.waitForCondition(() -> getEndOffsetSum(inputTopicPartitions, consumer) == numberOfRecords, 120_000L, () -> "Input records haven't all been written to the input topic: " + getEndOffsetSum(inputTopicPartitions, consumer));
        // wait until all the input records are in the changelog
        TestUtils.waitForCondition(() -> getEndOffsetSum(changelogTopicPartitions, consumer) == numberOfRecords, 120_000L, () -> "Input records haven't all been written to the changelog: " + getEndOffsetSum(changelogTopicPartitions, consumer));
        final AtomicLong instance1TotalRestored = new AtomicLong(-1);
        final AtomicLong instance1NumRestored = new AtomicLong(-1);
        final CountDownLatch restoreCompleteLatch = new CountDownLatch(1);
        kafkaStreams1.setGlobalStateRestoreListener(new StateRestoreListener() {

            @Override
            public void onRestoreStart(final TopicPartition topicPartition, final String storeName, final long startingOffset, final long endingOffset) {
            }

            @Override
            public void onBatchRestored(final TopicPartition topicPartition, final String storeName, final long batchEndOffset, final long numRestored) {
                instance1NumRestored.accumulateAndGet(numRestored, (prev, restored) -> prev == -1 ? restored : prev + restored);
            }

            @Override
            public void onRestoreEnd(final TopicPartition topicPartition, final String storeName, final long totalRestored) {
                instance1TotalRestored.accumulateAndGet(totalRestored, (prev, restored) -> prev == -1 ? restored : prev + restored);
                restoreCompleteLatch.countDown();
            }
        });
        final int assignmentsBeforeScaleOut = assignmentsCompleted.get();
        kafkaStreams1.start();
        TestUtils.waitForCondition(() -> {
            assignmentLock.lock();
            try {
                if (assignmentsCompleted.get() > assignmentsBeforeScaleOut) {
                    assertFalseNoRetry(assignmentsStable.get(assignmentsBeforeScaleOut + 1), "the first assignment after adding a node should be unstable while we warm up the state.");
                    return true;
                } else {
                    return false;
                }
            } finally {
                assignmentLock.unlock();
            }
        }, 120_000L, "Never saw a first assignment after scale out: " + assignmentsCompleted.get());
        TestUtils.waitForCondition(assignmentStable::get, 120_000L, "Assignment hasn't become stable: " + assignmentsCompleted.get() + " Note, if this does fail, check and see if the new instance just failed to catch up within" + " the probing rebalance interval. A full minute should be long enough to read ~500 records" + " in any test environment, but you never know...");
        restoreCompleteLatch.await();
        // We should finalize the restoration without having restored any records (because they're already in
        // the store. Otherwise, we failed to properly re-use the state from the standby.
        assertThat(instance1TotalRestored.get(), is(0L));
        // Belt-and-suspenders check that we never even attempt to restore any records.
        assertThat(instance1NumRestored.get(), is(-1L));
    }
}

Example 87 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableKTableForeignKeyJoinMaterializationIntegrationTest method shouldEmitTombstoneWhenDeletingNonJoiningRecords.

@Test
public void shouldEmitTombstoneWhenDeletingNonJoiningRecords() {
    final Topology topology = getTopology(streamsConfig, "store");
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
        final TestInputTopic<String, String> left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
        final TestOutputTopic<String, String> outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
        final KeyValueStore<String, String> store = driver.getKeyValueStore("store");
        left.pipeInput("lhs1", "lhsValue1|rhs1");
        assertThat(outputTopic.readKeyValuesToMap(), is(emptyMap()));
        if (materialized && queryable) {
            assertThat(asMap(store), is(emptyMap()));
        }
        // Deleting a non-joining record produces an unnecessary tombstone for inner joins, because
        // it's not possible to know whether a result was previously emitted.
        left.pipeInput("lhs1", (String) null);
        {
            if (materialized && queryable) {
                // in only this specific case, the record cache will actually be activated and
                // suppress the unnecessary tombstone. This is because the cache is able to determine
                // for sure that there has never been a previous result. (Because the "old" and "new" values
                // are both null, and the underlying store is also missing the record in question).
                assertThat(outputTopic.readKeyValuesToMap(), is(emptyMap()));
                assertThat(asMap(store), is(emptyMap()));
            } else {
                assertThat(outputTopic.readKeyValuesToMap(), is(mkMap(mkEntry("lhs1", null))));
            }
        }
        // Deleting a non-existing record is idempotent
        left.pipeInput("lhs1", (String) null);
        {
            assertThat(outputTopic.readKeyValuesToMap(), is(emptyMap()));
            if (materialized && queryable) {
                assertThat(asMap(store), is(emptyMap()));
            }
        }
    }
}

Example 88 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class AbstractResetIntegrationTest method setupTopologyWithIntermediateTopic.

@SuppressWarnings("deprecation")
private Topology setupTopologyWithIntermediateTopic(final boolean useRepartitioned, final String outputTopic2) {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Long, String> input = builder.stream(INPUT_TOPIC);
    // use map to trigger internal re-partitioning before groupByKey
    input.map(KeyValue::new).groupByKey().count().toStream().to(OUTPUT_TOPIC, Produced.with(Serdes.Long(), Serdes.Long()));
    final KStream<Long, String> stream;
    if (useRepartitioned) {
        stream = input.repartition();
    } else {
        input.to(INTERMEDIATE_USER_TOPIC);
        stream = builder.stream(INTERMEDIATE_USER_TOPIC);
    }
    stream.groupByKey().windowedBy(TimeWindows.of(ofMillis(35)).advanceBy(ofMillis(10))).count().toStream().map((key, value) -> new KeyValue<>(key.window().start() + key.window().end(), value)).to(outputTopic2, Produced.with(Serdes.Long(), Serdes.Long()));
    return builder.build();
}

Example 89 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class AbstractResetIntegrationTest method setupTopologyWithoutIntermediateUserTopic.

protected Topology setupTopologyWithoutIntermediateUserTopic() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Long, String> input = builder.stream(INPUT_TOPIC);
    // use map to trigger internal re-partitioning before groupByKey
    input.map((key, value) -> new KeyValue<>(key, key)).to(OUTPUT_TOPIC, Produced.with(Serdes.Long(), Serdes.Long()));
    return builder.build();
}

Example 90 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KStreamKTableJoinTest method shouldCreateRepartitionTopicsWithUserProvidedName.

@Test
public void shouldCreateRepartitionTopicsWithUserProvidedName() {
    final StreamsBuilder builder = new StreamsBuilder();
    final Properties props = new Properties();
    props.put(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, StreamsConfig.NO_OPTIMIZATION);
    final KStream<String, String> streamA = builder.stream("topic", Consumed.with(Serdes.String(), Serdes.String()));
    final KTable<String, String> tableB = builder.table("topic2", Consumed.with(Serdes.String(), Serdes.String()));
    final KTable<String, String> tableC = builder.table("topic3", Consumed.with(Serdes.String(), Serdes.String()));
    final KStream<String, String> rekeyedStream = streamA.map((k, v) -> new KeyValue<>(v, k));
    rekeyedStream.join(tableB, (value1, value2) -> value1 + value2, Joined.with(Serdes.String(), Serdes.String(), Serdes.String(), "first-join")).to("out-one");
    rekeyedStream.join(tableC, (value1, value2) -> value1 + value2, Joined.with(Serdes.String(), Serdes.String(), Serdes.String(), "second-join")).to("out-two");
    final Topology topology = builder.build(props);
    System.out.println(topology.describe().toString());
    assertEquals(expectedTopologyWithUserProvidedRepartitionTopicNames, topology.describe().toString());
}