Examples with PartitionInfo org.apache.kafka.common.PartitionInfo used on opensource projects

Example 86 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldAssignEvenlyAcrossConsumersOneClientMultipleThreads.

@Test
public void shouldAssignEvenlyAcrossConsumersOneClientMultipleThreads() {
    builder.addSource(null, "source1", null, null, null, "topic1");
    builder.addSource(null, "source2", null, null, null, "topic2");
    builder.addProcessor("processor", new MockApiProcessorSupplier<>(), "source1");
    builder.addProcessor("processorII", new MockApiProcessorSupplier<>(), "source2");
    final List<PartitionInfo> localInfos = asList(new PartitionInfo("topic1", 0, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic1", 1, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic1", 2, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic1", 3, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic2", 0, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic2", 1, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic2", 2, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic2", 3, Node.noNode(), new Node[0], new Node[0]));
    final Cluster localMetadata = new Cluster("cluster", Collections.singletonList(Node.noNode()), localInfos, emptySet(), emptySet());
    final List<String> topics = asList("topic1", "topic2");
    configureDefault();
    subscriptions.put("consumer10", new Subscription(topics, defaultSubscriptionInfo.encode()));
    subscriptions.put("consumer11", new Subscription(topics, defaultSubscriptionInfo.encode()));
    final Map<String, Assignment> assignments = partitionAssignor.assign(localMetadata, new GroupSubscription(subscriptions)).groupAssignment();
    // check assigned partitions
    assertEquals(mkSet(mkSet(t2p2, t1p0, t1p2, t2p0), mkSet(t1p1, t2p1, t1p3, t2p3)), mkSet(new HashSet<>(assignments.get("consumer10").partitions()), new HashSet<>(assignments.get("consumer11").partitions())));
    // the first consumer
    final AssignmentInfo info10 = AssignmentInfo.decode(assignments.get("consumer10").userData());
    final List<TaskId> expectedInfo10TaskIds = asList(TASK_0_0, TASK_0_2, TASK_1_0, TASK_1_2);
    assertEquals(expectedInfo10TaskIds, info10.activeTasks());
    // the second consumer
    final AssignmentInfo info11 = AssignmentInfo.decode(assignments.get("consumer11").userData());
    final List<TaskId> expectedInfo11TaskIds = asList(TASK_0_1, TASK_0_3, TASK_1_1, TASK_1_3);
    assertEquals(expectedInfo11TaskIds, info11.activeTasks());
}

Example 87 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldNotFailOnBranchedMultiLevelRepartitionConnectedTopology.

@Test
public void shouldNotFailOnBranchedMultiLevelRepartitionConnectedTopology() {
    // Test out a topology with 3 level of sub-topology as:
    // 0
    // /   \
    // 1    3
    // \  /
    // 2
    // where each pair of the sub topology is connected by repartition topic.
    // The purpose of this test is to verify the robustness of the stream partition assignor algorithm,
    // especially whether it could build the repartition topic counts (step zero) with a complex topology.
    // The traversal path 0 -> 1 -> 2 -> 3 hits the case where sub-topology 2 will be initialized while its
    // parent 3 hasn't been initialized yet.
    builder.addSource(null, "KSTREAM-SOURCE-0000000000", null, null, null, "input-stream");
    builder.addProcessor("KSTREAM-FLATMAPVALUES-0000000001", new MockApiProcessorSupplier<>(), "KSTREAM-SOURCE-0000000000");
    builder.addProcessor("KSTREAM-BRANCH-0000000002", new MockApiProcessorSupplier<>(), "KSTREAM-FLATMAPVALUES-0000000001");
    builder.addProcessor("KSTREAM-BRANCHCHILD-0000000003", new MockApiProcessorSupplier<>(), "KSTREAM-BRANCH-0000000002");
    builder.addProcessor("KSTREAM-BRANCHCHILD-0000000004", new MockApiProcessorSupplier<>(), "KSTREAM-BRANCH-0000000002");
    builder.addProcessor("KSTREAM-MAP-0000000005", new MockApiProcessorSupplier<>(), "KSTREAM-BRANCHCHILD-0000000003");
    builder.addInternalTopic("odd_store-repartition", InternalTopicProperties.empty());
    builder.addProcessor("odd_store-repartition-filter", new MockApiProcessorSupplier<>(), "KSTREAM-MAP-0000000005");
    builder.addSink("odd_store-repartition-sink", "odd_store-repartition", null, null, null, "odd_store-repartition-filter");
    builder.addSource(null, "odd_store-repartition-source", null, null, null, "odd_store-repartition");
    builder.addProcessor("KSTREAM-REDUCE-0000000006", new MockApiProcessorSupplier<>(), "odd_store-repartition-source");
    builder.addProcessor("KTABLE-TOSTREAM-0000000010", new MockApiProcessorSupplier<>(), "KSTREAM-REDUCE-0000000006");
    builder.addProcessor("KSTREAM-PEEK-0000000011", new MockApiProcessorSupplier<>(), "KTABLE-TOSTREAM-0000000010");
    builder.addProcessor("KSTREAM-MAP-0000000012", new MockApiProcessorSupplier<>(), "KSTREAM-PEEK-0000000011");
    builder.addInternalTopic("odd_store_2-repartition", InternalTopicProperties.empty());
    builder.addProcessor("odd_store_2-repartition-filter", new MockApiProcessorSupplier<>(), "KSTREAM-MAP-0000000012");
    builder.addSink("odd_store_2-repartition-sink", "odd_store_2-repartition", null, null, null, "odd_store_2-repartition-filter");
    builder.addSource(null, "odd_store_2-repartition-source", null, null, null, "odd_store_2-repartition");
    builder.addProcessor("KSTREAM-REDUCE-0000000013", new MockApiProcessorSupplier<>(), "odd_store_2-repartition-source");
    builder.addProcessor("KSTREAM-MAP-0000000017", new MockApiProcessorSupplier<>(), "KSTREAM-BRANCHCHILD-0000000004");
    builder.addInternalTopic("even_store-repartition", InternalTopicProperties.empty());
    builder.addProcessor("even_store-repartition-filter", new MockApiProcessorSupplier<>(), "KSTREAM-MAP-0000000017");
    builder.addSink("even_store-repartition-sink", "even_store-repartition", null, null, null, "even_store-repartition-filter");
    builder.addSource(null, "even_store-repartition-source", null, null, null, "even_store-repartition");
    builder.addProcessor("KSTREAM-REDUCE-0000000018", new MockApiProcessorSupplier<>(), "even_store-repartition-source");
    builder.addProcessor("KTABLE-TOSTREAM-0000000022", new MockApiProcessorSupplier<>(), "KSTREAM-REDUCE-0000000018");
    builder.addProcessor("KSTREAM-PEEK-0000000023", new MockApiProcessorSupplier<>(), "KTABLE-TOSTREAM-0000000022");
    builder.addProcessor("KSTREAM-MAP-0000000024", new MockApiProcessorSupplier<>(), "KSTREAM-PEEK-0000000023");
    builder.addInternalTopic("even_store_2-repartition", InternalTopicProperties.empty());
    builder.addProcessor("even_store_2-repartition-filter", new MockApiProcessorSupplier<>(), "KSTREAM-MAP-0000000024");
    builder.addSink("even_store_2-repartition-sink", "even_store_2-repartition", null, null, null, "even_store_2-repartition-filter");
    builder.addSource(null, "even_store_2-repartition-source", null, null, null, "even_store_2-repartition");
    builder.addProcessor("KSTREAM-REDUCE-0000000025", new MockApiProcessorSupplier<>(), "even_store_2-repartition-source");
    builder.addProcessor("KTABLE-JOINTHIS-0000000030", new MockApiProcessorSupplier<>(), "KSTREAM-REDUCE-0000000013");
    builder.addProcessor("KTABLE-JOINOTHER-0000000031", new MockApiProcessorSupplier<>(), "KSTREAM-REDUCE-0000000025");
    builder.addProcessor("KTABLE-MERGE-0000000029", new MockApiProcessorSupplier<>(), "KTABLE-JOINTHIS-0000000030", "KTABLE-JOINOTHER-0000000031");
    builder.addProcessor("KTABLE-TOSTREAM-0000000032", new MockApiProcessorSupplier<>(), "KTABLE-MERGE-0000000029");
    final List<String> topics = asList("input-stream", "test-even_store-repartition", "test-even_store_2-repartition", "test-odd_store-repartition", "test-odd_store_2-repartition");
    configureDefault();
    subscriptions.put("consumer10", new Subscription(topics, defaultSubscriptionInfo.encode()));
    final Cluster metadata = new Cluster("cluster", Collections.singletonList(Node.noNode()), Collections.singletonList(new PartitionInfo("input-stream", 0, Node.noNode(), new Node[0], new Node[0])), emptySet(), emptySet());
    // This shall fail if we have bugs in the repartition topic creation due to the inconsistent order of sub-topologies.
    partitionAssignor.assign(metadata, new GroupSubscription(subscriptions));
}

Example 88 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class StreamsMetadataStateTest method shouldGetInstanceWithKeyAndCustomPartitioner.

@Test
public void shouldGetInstanceWithKeyAndCustomPartitioner() {
    final TopicPartition tp4 = new TopicPartition("topic-three", 1);
    hostToActivePartitions.put(hostTwo, mkSet(topic2P0, tp4));
    metadataState.onChange(hostToActivePartitions, hostToStandbyPartitions, cluster.withPartitions(Collections.singletonMap(tp4, new PartitionInfo("topic-three", 1, null, null, null))));
    final KeyQueryMetadata expected = new KeyQueryMetadata(hostTwo, Collections.emptySet(), 1);
    final KeyQueryMetadata actual = metadataState.getKeyQueryMetadataForKey("table-three", "the-key", partitioner);
    assertEquals(expected, actual);
    assertEquals(1, actual.partition());
}

Example 89 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class StreamsMetadataStateTest method shouldGetAllStreamsInstancesWithNoStores.

@Test
public void shouldGetAllStreamsInstancesWithNoStores() {
    builder.stream("topic-five").filter((key, value) -> true).to("some-other-topic");
    final TopicPartition tp5 = new TopicPartition("topic-five", 1);
    final HostInfo hostFour = new HostInfo("host-four", 8080);
    hostToActivePartitions.put(hostFour, mkSet(tp5));
    metadataState.onChange(hostToActivePartitions, Collections.emptyMap(), cluster.withPartitions(Collections.singletonMap(tp5, new PartitionInfo("topic-five", 1, null, null, null))));
    final StreamsMetadata expected = new StreamsMetadataImpl(hostFour, Collections.singleton(globalTable), Collections.singleton(tp5), Collections.emptySet(), Collections.emptySet());
    final Collection<StreamsMetadata> actual = metadataState.getAllMetadata();
    assertTrue("expected " + actual + " to contain " + expected, actual.contains(expected));
}

Example 90 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class RecordCollectorTest method shouldThrowIfTopicIsUnknownOnSendWithPartitioner.

@Test
public void shouldThrowIfTopicIsUnknownOnSendWithPartitioner() {
    final RecordCollector collector = new RecordCollectorImpl(logContext, taskId, new StreamsProducer(config, processId + "-StreamThread-1", new MockClientSupplier() {

        @Override
        public Producer<byte[], byte[]> getProducer(final Map<String, Object> config) {
            return new MockProducer<byte[], byte[]>(cluster, true, new DefaultPartitioner(), byteArraySerializer, byteArraySerializer) {

                @Override
                public List<PartitionInfo> partitionsFor(final String topic) {
                    return Collections.emptyList();
                }
            };
        }
    }, null, null, logContext, Time.SYSTEM), productionExceptionHandler, streamsMetrics);
    collector.initialize();
    final StreamsException thrown = assertThrows(StreamsException.class, () -> collector.send(topic, "3", "0", null, null, stringSerializer, stringSerializer, streamPartitioner));
    assertThat(thrown.getMessage(), equalTo("Could not get partition information for topic topic for task 0_0." + " This can happen if the topic does not exist."));
}