Search in sources :

Example 66 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class StreamsMetadataStateTest method before.

@Before
public void before() throws Exception {
    builder = new StreamsBuilder();
    final KStream<Object, Object> one = builder.stream("topic-one");
    one.groupByKey().count("table-one");
    final KStream<Object, Object> two = builder.stream("topic-two");
    two.groupByKey().count("table-two");
    builder.stream("topic-three").groupByKey().count("table-three");
    one.merge(two).groupByKey().count("merged-table");
    builder.stream("topic-four").mapValues(new ValueMapper<Object, Object>() {

        @Override
        public Object apply(final Object value) {
            return value;
        }
    });
    builder.globalTable("global-topic", Consumed.with(null, null), Materialized.<Object, Object, KeyValueStore<Bytes, byte[]>>as(globalTable));
    StreamsBuilderTest.internalTopologyBuilder(builder).setApplicationId("appId");
    topic1P0 = new TopicPartition("topic-one", 0);
    topic1P1 = new TopicPartition("topic-one", 1);
    topic2P0 = new TopicPartition("topic-two", 0);
    topic2P1 = new TopicPartition("topic-two", 1);
    topic3P0 = new TopicPartition("topic-three", 0);
    topic4P0 = new TopicPartition("topic-four", 0);
    hostOne = new HostInfo("host-one", 8080);
    hostTwo = new HostInfo("host-two", 9090);
    hostThree = new HostInfo("host-three", 7070);
    hostToPartitions = new HashMap<>();
    hostToPartitions.put(hostOne, Utils.mkSet(topic1P0, topic2P1, topic4P0));
    hostToPartitions.put(hostTwo, Utils.mkSet(topic2P0, topic1P1));
    hostToPartitions.put(hostThree, Collections.singleton(topic3P0));
    partitionInfos = Arrays.asList(new PartitionInfo("topic-one", 0, null, null, null), new PartitionInfo("topic-one", 1, null, null, null), new PartitionInfo("topic-two", 0, null, null, null), new PartitionInfo("topic-two", 1, null, null, null), new PartitionInfo("topic-three", 0, null, null, null), new PartitionInfo("topic-four", 0, null, null, null));
    cluster = new Cluster(null, Collections.<Node>emptyList(), partitionInfos, Collections.<String>emptySet(), Collections.<String>emptySet());
    discovery = new StreamsMetadataState(StreamsBuilderTest.internalTopologyBuilder(builder), hostOne);
    discovery.onChange(hostToPartitions, cluster);
    partitioner = new StreamPartitioner<String, Object>() {

        @Override
        public Integer partition(final String key, final Object value, final int numPartitions) {
            return 1;
        }
    };
}
Also used : Node(org.apache.kafka.common.Node) Cluster(org.apache.kafka.common.Cluster) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) Bytes(org.apache.kafka.common.utils.Bytes) TopicPartition(org.apache.kafka.common.TopicPartition) PartitionInfo(org.apache.kafka.common.PartitionInfo) HostInfo(org.apache.kafka.streams.state.HostInfo) Before(org.junit.Before)

Example 67 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class StreamsPartitionAssignorTest method shouldNotAddStandbyTaskPartitionsToPartitionsForHost.

@Test
public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() throws Exception {
    final StreamsBuilder builder = new StreamsBuilder();
    final InternalTopologyBuilder internalTopologyBuilder = StreamsBuilderTest.internalTopologyBuilder(builder);
    internalTopologyBuilder.setApplicationId(applicationId);
    builder.stream("topic1").groupByKey().count();
    final UUID uuid = UUID.randomUUID();
    mockTaskManager(Collections.<TaskId>emptySet(), Collections.<TaskId>emptySet(), uuid, internalTopologyBuilder);
    Map<String, Object> props = new HashMap<>();
    props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
    props.put(StreamsConfig.APPLICATION_SERVER_CONFIG, userEndPoint);
    configurePartitionAssignor(props);
    partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
    final Map<String, PartitionAssignor.Subscription> subscriptions = new HashMap<>();
    final Set<TaskId> emptyTasks = Collections.emptySet();
    subscriptions.put("consumer1", new PartitionAssignor.Subscription(Collections.singletonList("topic1"), new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode()));
    subscriptions.put("consumer2", new PartitionAssignor.Subscription(Collections.singletonList("topic1"), new SubscriptionInfo(UUID.randomUUID(), emptyTasks, emptyTasks, "other:9090").encode()));
    final Set<TopicPartition> allPartitions = Utils.mkSet(t1p0, t1p1, t1p2);
    final Map<String, PartitionAssignor.Assignment> assign = partitionAssignor.assign(metadata, subscriptions);
    final PartitionAssignor.Assignment consumer1Assignment = assign.get("consumer1");
    final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumer1Assignment.userData());
    final Set<TopicPartition> consumer1partitions = assignmentInfo.partitionsByHost().get(new HostInfo("localhost", 8080));
    final Set<TopicPartition> consumer2Partitions = assignmentInfo.partitionsByHost().get(new HostInfo("other", 9090));
    final HashSet<TopicPartition> allAssignedPartitions = new HashSet<>(consumer1partitions);
    allAssignedPartitions.addAll(consumer2Partitions);
    assertThat(consumer1partitions, not(allPartitions));
    assertThat(consumer2Partitions, not(allPartitions));
    assertThat(allAssignedPartitions, equalTo(allPartitions));
}
Also used : TaskId(org.apache.kafka.streams.processor.TaskId) HashMap(java.util.HashMap) MockInternalTopicManager(org.apache.kafka.test.MockInternalTopicManager) SubscriptionInfo(org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) AssignmentInfo(org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo) TopicPartition(org.apache.kafka.common.TopicPartition) PartitionAssignor(org.apache.kafka.clients.consumer.internals.PartitionAssignor) UUID(java.util.UUID) HostInfo(org.apache.kafka.streams.state.HostInfo) HashSet(java.util.HashSet) StreamsBuilderTest(org.apache.kafka.streams.StreamsBuilderTest) Test(org.junit.Test)

Example 68 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class StreamsPartitionAssignorTest method shouldNotLoopInfinitelyOnMissingMetadataAndShouldNotCreateRelatedTasks.

@Test
public void shouldNotLoopInfinitelyOnMissingMetadataAndShouldNotCreateRelatedTasks() throws Exception {
    final StreamsBuilder builder = new StreamsBuilder();
    final InternalTopologyBuilder internalTopologyBuilder = StreamsBuilderTest.internalTopologyBuilder(builder);
    internalTopologyBuilder.setApplicationId(applicationId);
    KStream<Object, Object> stream1 = builder.stream("topic1").selectKey(new KeyValueMapper<Object, Object, Object>() {

        @Override
        public Object apply(final Object key, final Object value) {
            return null;
        }
    }).groupByKey().count(Materialized.<Object, Long, KeyValueStore<Bytes, byte[]>>as("count")).toStream().map(new KeyValueMapper<Object, Long, KeyValue<Object, Object>>() {

        @Override
        public KeyValue<Object, Object> apply(final Object key, final Long value) {
            return null;
        }
    });
    builder.stream("unknownTopic").selectKey(new KeyValueMapper<Object, Object, Object>() {

        @Override
        public Object apply(final Object key, final Object value) {
            return null;
        }
    }).join(stream1, new ValueJoiner() {

        @Override
        public Object apply(final Object value1, final Object value2) {
            return null;
        }
    }, JoinWindows.of(0));
    final UUID uuid = UUID.randomUUID();
    final String client = "client1";
    mockTaskManager(Collections.<TaskId>emptySet(), Collections.<TaskId>emptySet(), UUID.randomUUID(), internalTopologyBuilder);
    configurePartitionAssignor(Collections.<String, Object>emptyMap());
    final MockInternalTopicManager mockInternalTopicManager = new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer);
    partitionAssignor.setInternalTopicManager(mockInternalTopicManager);
    final Map<String, PartitionAssignor.Subscription> subscriptions = new HashMap<>();
    final Set<TaskId> emptyTasks = Collections.emptySet();
    subscriptions.put(client, new PartitionAssignor.Subscription(Collections.singletonList("unknownTopic"), new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode()));
    final Map<String, PartitionAssignor.Assignment> assignment = partitionAssignor.assign(metadata, subscriptions);
    final Map<String, Integer> expectedCreatedInternalTopics = new HashMap<>();
    expectedCreatedInternalTopics.put(applicationId + "-count-repartition", 3);
    expectedCreatedInternalTopics.put(applicationId + "-count-changelog", 3);
    assertThat(mockInternalTopicManager.readyTopics, equalTo(expectedCreatedInternalTopics));
    final List<TopicPartition> expectedAssignment = Arrays.asList(new TopicPartition("topic1", 0), new TopicPartition("topic1", 1), new TopicPartition("topic1", 2), new TopicPartition(applicationId + "-count-repartition", 0), new TopicPartition(applicationId + "-count-repartition", 1), new TopicPartition(applicationId + "-count-repartition", 2));
    assertThat(new HashSet<>(assignment.get(client).partitions()), equalTo(new HashSet<>(expectedAssignment)));
}
Also used : KeyValue(org.apache.kafka.streams.KeyValue) TaskId(org.apache.kafka.streams.processor.TaskId) HashMap(java.util.HashMap) MockInternalTopicManager(org.apache.kafka.test.MockInternalTopicManager) KeyValueMapper(org.apache.kafka.streams.kstream.KeyValueMapper) SubscriptionInfo(org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo) Bytes(org.apache.kafka.common.utils.Bytes) ValueJoiner(org.apache.kafka.streams.kstream.ValueJoiner) PartitionAssignor(org.apache.kafka.clients.consumer.internals.PartitionAssignor) UUID(java.util.UUID) HashSet(java.util.HashSet) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) TopicPartition(org.apache.kafka.common.TopicPartition) StreamsBuilderTest(org.apache.kafka.streams.StreamsBuilderTest) Test(org.junit.Test)

Example 69 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class StreamsPartitionAssignorTest method shouldGenerateTasksForAllCreatedPartitions.

@Test
public void shouldGenerateTasksForAllCreatedPartitions() throws Exception {
    final StreamsBuilder builder = new StreamsBuilder();
    final InternalTopologyBuilder internalTopologyBuilder = StreamsBuilderTest.internalTopologyBuilder(builder);
    internalTopologyBuilder.setApplicationId(applicationId);
    // KStream with 3 partitions
    KStream<Object, Object> stream1 = builder.stream("topic1").map(new KeyValueMapper<Object, Object, KeyValue<Object, Object>>() {

        @Override
        public KeyValue<Object, Object> apply(final Object key, final Object value) {
            return new KeyValue<>(key, value);
        }
    });
    // KTable with 4 partitions
    KTable<Object, Long> table1 = builder.table("topic3").groupBy(new KeyValueMapper<Object, Object, KeyValue<Object, Object>>() {

        @Override
        public KeyValue<Object, Object> apply(final Object key, final Object value) {
            return new KeyValue<>(key, value);
        }
    }).count();
    // joining the stream and the table
    // this triggers the enforceCopartitioning() routine in the StreamsPartitionAssignor,
    // forcing the stream.map to get repartitioned to a topic with four partitions.
    stream1.join(table1, new ValueJoiner() {

        @Override
        public Object apply(final Object value1, final Object value2) {
            return null;
        }
    });
    final UUID uuid = UUID.randomUUID();
    final String client = "client1";
    mockTaskManager(Collections.<TaskId>emptySet(), Collections.<TaskId>emptySet(), UUID.randomUUID(), internalTopologyBuilder);
    configurePartitionAssignor(Collections.<String, Object>emptyMap());
    final MockInternalTopicManager mockInternalTopicManager = new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer);
    partitionAssignor.setInternalTopicManager(mockInternalTopicManager);
    final Map<String, PartitionAssignor.Subscription> subscriptions = new HashMap<>();
    final Set<TaskId> emptyTasks = Collections.emptySet();
    subscriptions.put(client, new PartitionAssignor.Subscription(Utils.mkList("topic1", "topic3"), new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode()));
    final Map<String, PartitionAssignor.Assignment> assignment = partitionAssignor.assign(metadata, subscriptions);
    final Map<String, Integer> expectedCreatedInternalTopics = new HashMap<>();
    expectedCreatedInternalTopics.put(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 4);
    expectedCreatedInternalTopics.put(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-changelog", 4);
    expectedCreatedInternalTopics.put(applicationId + "-KSTREAM-MAP-0000000001-repartition", 4);
    expectedCreatedInternalTopics.put(applicationId + "-topic3-STATE-STORE-0000000002-changelog", 4);
    // check if all internal topics were created as expected
    assertThat(mockInternalTopicManager.readyTopics, equalTo(expectedCreatedInternalTopics));
    final List<TopicPartition> expectedAssignment = Arrays.asList(new TopicPartition("topic1", 0), new TopicPartition("topic1", 1), new TopicPartition("topic1", 2), new TopicPartition("topic3", 0), new TopicPartition("topic3", 1), new TopicPartition("topic3", 2), new TopicPartition("topic3", 3), new TopicPartition(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 0), new TopicPartition(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 1), new TopicPartition(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 2), new TopicPartition(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 3), new TopicPartition(applicationId + "-KSTREAM-MAP-0000000001-repartition", 0), new TopicPartition(applicationId + "-KSTREAM-MAP-0000000001-repartition", 1), new TopicPartition(applicationId + "-KSTREAM-MAP-0000000001-repartition", 2), new TopicPartition(applicationId + "-KSTREAM-MAP-0000000001-repartition", 3));
    // check if we created a task for all expected topicPartitions.
    assertThat(new HashSet<>(assignment.get(client).partitions()), equalTo(new HashSet<>(expectedAssignment)));
}
Also used : KeyValue(org.apache.kafka.streams.KeyValue) TaskId(org.apache.kafka.streams.processor.TaskId) HashMap(java.util.HashMap) MockInternalTopicManager(org.apache.kafka.test.MockInternalTopicManager) KeyValueMapper(org.apache.kafka.streams.kstream.KeyValueMapper) SubscriptionInfo(org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo) ValueJoiner(org.apache.kafka.streams.kstream.ValueJoiner) PartitionAssignor(org.apache.kafka.clients.consumer.internals.PartitionAssignor) UUID(java.util.UUID) HashSet(java.util.HashSet) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) TopicPartition(org.apache.kafka.common.TopicPartition) StreamsBuilderTest(org.apache.kafka.streams.StreamsBuilderTest) Test(org.junit.Test)

Example 70 with StreamsBuilder

use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class SimpleBenchmark method createKafkaStreamsKTableKTableJoin.

private KafkaStreams createKafkaStreamsKTableKTableJoin(Properties streamConfig, String kTableTopic1, String kTableTopic2, final CountDownLatch latch) {
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<Long, byte[]> input1 = builder.table(kTableTopic1);
    final KTable<Long, byte[]> input2 = builder.table(kTableTopic2);
    input1.leftJoin(input2, VALUE_JOINER).foreach(new CountDownAction(latch));
    return createKafkaStreamsWithExceptionHandler(builder, streamConfig);
}
Also used : StreamsBuilder(org.apache.kafka.streams.StreamsBuilder)

Aggregations

StreamsBuilder (org.apache.kafka.streams.StreamsBuilder)189 Test (org.junit.Test)121 KafkaStreams (org.apache.kafka.streams.KafkaStreams)72 Properties (java.util.Properties)61 KeyValue (org.apache.kafka.streams.KeyValue)42 MockProcessorSupplier (org.apache.kafka.test.MockProcessorSupplier)30 StreamsBuilderTest (org.apache.kafka.streams.StreamsBuilderTest)27 Serdes (org.apache.kafka.common.serialization.Serdes)21 KeyValueMapper (org.apache.kafka.streams.kstream.KeyValueMapper)21 Before (org.junit.Before)19 StreamsConfig (org.apache.kafka.streams.StreamsConfig)18 KStream (org.apache.kafka.streams.kstream.KStream)18 Predicate (org.apache.kafka.streams.kstream.Predicate)18 IntegrationTest (org.apache.kafka.test.IntegrationTest)18 Bytes (org.apache.kafka.common.utils.Bytes)16 HashSet (java.util.HashSet)15 ValueMapper (org.apache.kafka.streams.kstream.ValueMapper)14 HashMap (java.util.HashMap)13 KTable (org.apache.kafka.streams.kstream.KTable)13 Produced (org.apache.kafka.streams.kstream.Produced)13