Examples with UniqueTopicSerdeScope org.apache.kafka.streams.utils.UniqueTopicSerdeScope used on opensource projects

Example 1 with UniqueTopicSerdeScope

use of org.apache.kafka.streams.utils.UniqueTopicSerdeScope in project kafka by apache.

the class KTableKTableForeignKeyInnerJoinCustomPartitionerIntegrationTest method prepareTopology.

private static KafkaStreams prepareTopology(final String queryableName, final Properties streamsConfig) {
    final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<String, String> table1 = builder.stream(TABLE_1, Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))).repartition(repartitionA()).toTable(Named.as("table.a"));
    final KTable<String, String> table2 = builder.stream(TABLE_2, Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))).repartition(repartitionB()).toTable(Named.as("table.b"));
    final Materialized<String, String, KeyValueStore<Bytes, byte[]>> materialized;
    if (queryableName != null) {
        materialized = Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as(queryableName).withKeySerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true)).withValueSerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)).withCachingDisabled();
    } else {
        throw new RuntimeException("Current implementation of joinOnForeignKey requires a materialized store");
    }
    final ValueJoiner<String, String, String> joiner = (value1, value2) -> "value1=" + value1 + ",value2=" + value2;
    final TableJoined<String, String> tableJoined = TableJoined.with((topic, key, value, numPartitions) -> Math.abs(getKeyB(key).hashCode()) % numPartitions, (topic, key, value, numPartitions) -> Math.abs(key.hashCode()) % numPartitions);
    table1.join(table2, KTableKTableForeignKeyInnerJoinCustomPartitionerIntegrationTest::getKeyB, joiner, tableJoined, materialized).toStream().to(OUTPUT, Produced.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)));
    return new KafkaStreams(builder.build(streamsConfig), streamsConfig);
}

Example 2 with UniqueTopicSerdeScope

use of org.apache.kafka.streams.utils.UniqueTopicSerdeScope in project kafka by apache.

the class KTableKTableForeignKeyJoinScenarioTest method shouldUseExpectedTopicsWithSerde.

@Test
public void shouldUseExpectedTopicsWithSerde() {
    final String applicationId = "ktable-ktable-joinOnForeignKey";
    final Properties streamsConfig = mkProperties(mkMap(mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, applicationId), mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath())));
    final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<Integer, String> left = builder.table(LEFT_TABLE, Consumed.with(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)));
    final KTable<Integer, String> right = builder.table(RIGHT_TABLE, Consumed.with(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)));
    left.join(right, value -> Integer.parseInt(value.split("\\|")[1]), (value1, value2) -> "(" + value1 + "," + value2 + ")", Materialized.with(null, serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))).toStream().to(OUTPUT);
    final Topology topology = builder.build(streamsConfig);
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
        final TestInputTopic<Integer, String> leftInput = driver.createInputTopic(LEFT_TABLE, new IntegerSerializer(), new StringSerializer());
        final TestInputTopic<Integer, String> rightInput = driver.createInputTopic(RIGHT_TABLE, new IntegerSerializer(), new StringSerializer());
        leftInput.pipeInput(2, "lhsValue1|1");
        rightInput.pipeInput(1, "rhsValue1");
    }
    // verifying primarily that no extra pseudo-topics were used, but it's nice to also verify the rest of the
    // topics our serdes serialize data for
    assertThat(serdeScope.registeredTopics(), is(mkSet(// expected pseudo-topics
    applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-fk--key", applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-pk--key", applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-vh--value", // internal topics
    applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic--key", applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic--key", applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic--value", applicationId + "-left_table-STATE-STORE-0000000000-changelog--key", applicationId + "-left_table-STATE-STORE-0000000000-changelog--value", applicationId + "-right_table-STATE-STORE-0000000003-changelog--key", applicationId + "-right_table-STATE-STORE-0000000003-changelog--value", // output topics
    "output-topic--key", "output-topic--value")));
}

Example 3 with UniqueTopicSerdeScope

use of org.apache.kafka.streams.utils.UniqueTopicSerdeScope in project kafka by apache.

the class NamedTopologyIntegrationTest method shouldPrefixAllInternalTopicNamesWithNamedTopology.

@Test
public void shouldPrefixAllInternalTopicNamesWithNamedTopology() throws Exception {
    final String countTopologyName = "count-topology";
    final String fkjTopologyName = "FKJ-topology";
    final NamedTopologyBuilder countBuilder = streams.newNamedTopologyBuilder(countTopologyName);
    countBuilder.stream(INPUT_STREAM_1).groupBy((k, v) -> k).count();
    final NamedTopologyBuilder fkjBuilder = streams.newNamedTopologyBuilder(fkjTopologyName);
    final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
    final KTable<String, Long> left = fkjBuilder.table(INPUT_STREAM_2, Consumed.with(serdeScope.decorateSerde(Serdes.String(), props, true), serdeScope.decorateSerde(Serdes.Long(), props, false)));
    final KTable<String, Long> right = fkjBuilder.table(INPUT_STREAM_3, Consumed.with(serdeScope.decorateSerde(Serdes.String(), props, true), serdeScope.decorateSerde(Serdes.Long(), props, false)));
    left.join(right, Object::toString, (value1, value2) -> String.valueOf(value1 + value2), Materialized.with(null, serdeScope.decorateSerde(Serdes.String(), props, false)));
    streams.start(asList(fkjBuilder.build(), countBuilder.build()));
    waitForApplicationState(singletonList(streams), State.RUNNING, Duration.ofSeconds(60));
    final String countTopicPrefix = TOPIC_PREFIX + "-" + countTopologyName;
    final String fkjTopicPrefix = TOPIC_PREFIX + "-" + fkjTopologyName;
    final Set<String> internalTopics = CLUSTER.getAllTopicsInCluster().stream().filter(t -> t.contains(TOPIC_PREFIX)).filter(t -> t.endsWith("-repartition") || t.endsWith("-changelog") || t.endsWith("-topic")).collect(Collectors.toSet());
    assertThat(internalTopics, is(mkSet(countTopicPrefix + "-KSTREAM-AGGREGATE-STATE-STORE-0000000002-repartition", countTopicPrefix + "-KSTREAM-AGGREGATE-STATE-STORE-0000000002-changelog", fkjTopicPrefix + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic", fkjTopicPrefix + "-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic", fkjTopicPrefix + "-KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000010-changelog", fkjTopicPrefix + "-" + INPUT_STREAM_2 + "-STATE-STORE-0000000000-changelog", fkjTopicPrefix + "-" + INPUT_STREAM_3 + "-STATE-STORE-0000000003-changelog")));
}

Example 4 with UniqueTopicSerdeScope

use of org.apache.kafka.streams.utils.UniqueTopicSerdeScope in project kafka by apache.

the class KTableKTableForeignKeyInnerJoinMultiIntegrationTest method prepareTopology.

private static KafkaStreams prepareTopology(final String queryableName, final String queryableNameTwo, final Properties streamsConfig) {
    final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<Integer, Float> table1 = builder.table(TABLE_1, Consumed.with(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true), serdeScope.decorateSerde(Serdes.Float(), streamsConfig, false)));
    final KTable<String, Long> table2 = builder.table(TABLE_2, Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true), serdeScope.decorateSerde(Serdes.Long(), streamsConfig, false)));
    final KTable<Integer, String> table3 = builder.table(TABLE_3, Consumed.with(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)));
    final Materialized<Integer, String, KeyValueStore<Bytes, byte[]>> materialized;
    if (queryableName != null) {
        materialized = Materialized.<Integer, String, KeyValueStore<Bytes, byte[]>>as(queryableName).withKeySerde(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true)).withValueSerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)).withCachingDisabled();
    } else {
        throw new RuntimeException("Current implementation of joinOnForeignKey requires a materialized store");
    }
    final Materialized<Integer, String, KeyValueStore<Bytes, byte[]>> materializedTwo;
    if (queryableNameTwo != null) {
        materializedTwo = Materialized.<Integer, String, KeyValueStore<Bytes, byte[]>>as(queryableNameTwo).withKeySerde(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true)).withValueSerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)).withCachingDisabled();
    } else {
        throw new RuntimeException("Current implementation of joinOnForeignKey requires a materialized store");
    }
    final Function<Float, String> tableOneKeyExtractor = value -> Integer.toString((int) value.floatValue());
    final Function<String, Integer> joinedTableKeyExtractor = value -> {
        // Hardwired to return the desired foreign key as a test shortcut
        if (value.contains("value2=10"))
            return 10;
        else
            return 0;
    };
    final ValueJoiner<Float, Long, String> joiner = (value1, value2) -> "value1=" + value1 + ",value2=" + value2;
    final ValueJoiner<String, String, String> joinerTwo = (value1, value2) -> value1 + ",value3=" + value2;
    table1.join(table2, tableOneKeyExtractor, joiner, materialized).join(table3, joinedTableKeyExtractor, joinerTwo, materializedTwo).toStream().to(OUTPUT, Produced.with(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)));
    return new KafkaStreams(builder.build(streamsConfig), streamsConfig);
}

Example 5 with UniqueTopicSerdeScope

use of org.apache.kafka.streams.utils.UniqueTopicSerdeScope in project kafka by apache.

the class KTableKTableForeignKeyJoinIntegrationTest method getTopology.

private static Topology getTopology(final Properties streamsConfig, final String queryableStoreName, final boolean leftJoin, final boolean rejoin) {
    final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<String, String> left = builder.table(LEFT_TABLE, Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)));
    final KTable<String, String> right = builder.table(RIGHT_TABLE, Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)));
    final Function<String, String> extractor = value -> value.split("\\|")[1];
    final ValueJoiner<String, String, String> joiner = (value1, value2) -> "(" + value1 + "," + value2 + ")";
    final ValueJoiner<String, String, String> rejoiner = rejoin ? (value1, value2) -> "rejoin(" + value1 + "," + value2 + ")" : null;
    // the cache suppresses some of the unnecessary tombstones we want to make assertions about
    final Materialized<String, String, KeyValueStore<Bytes, byte[]>> mainMaterialized = queryableStoreName == null ? Materialized.<String, String, KeyValueStore<Bytes, byte[]>>with(null, serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)).withCachingDisabled() : Materialized.<String, String>as(Stores.inMemoryKeyValueStore(queryableStoreName)).withValueSerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)).withCachingDisabled();
    final Materialized<String, String, KeyValueStore<Bytes, byte[]>> rejoinMaterialized = !rejoin ? null : queryableStoreName == null ? Materialized.with(null, serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)) : // to really test this confuguration
    Materialized.<String, String>as(Stores.inMemoryKeyValueStore(queryableStoreName + "-rejoin")).withValueSerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)).withCachingDisabled();
    if (leftJoin) {
        final KTable<String, String> fkJoin = left.leftJoin(right, extractor, joiner, mainMaterialized);
        fkJoin.toStream().to(OUTPUT);
        // also make sure the FK join is set up right for downstream operations that require materialization
        if (rejoin) {
            fkJoin.leftJoin(left, rejoiner, rejoinMaterialized).toStream().to(REJOIN_OUTPUT);
        }
    } else {
        final KTable<String, String> fkJoin = left.join(right, extractor, joiner, mainMaterialized);
        fkJoin.toStream().to(OUTPUT);
        // also make sure the FK join is set up right for downstream operations that require materialization
        if (rejoin) {
            fkJoin.join(left, rejoiner, rejoinMaterialized).toStream().to(REJOIN_OUTPUT);
        }
    }
    return builder.build(streamsConfig);
}