Examples with PartitionInfo org.apache.kafka.common.PartitionInfo used on opensource projects

Example 41 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class FetcherTest method newMetadataResponse.

private MetadataResponse newMetadataResponse(String topic, Errors error) {
    List<MetadataResponse.PartitionMetadata> partitionsMetadata = new ArrayList<>();
    if (error == Errors.NONE) {
        for (PartitionInfo partitionInfo : cluster.partitionsForTopic(topic)) {
            partitionsMetadata.add(new MetadataResponse.PartitionMetadata(Errors.NONE, partitionInfo.partition(), partitionInfo.leader(), Arrays.asList(partitionInfo.replicas()), Arrays.asList(partitionInfo.inSyncReplicas())));
        }
    }
    MetadataResponse.TopicMetadata topicMetadata = new MetadataResponse.TopicMetadata(error, topic, false, partitionsMetadata);
    return new MetadataResponse(cluster.nodes(), null, MetadataResponse.NO_CONTROLLER_ID, Arrays.asList(topicMetadata));
}

Example 42 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class MetadataTest method testListenerCanUnregister.

@Test
public void testListenerCanUnregister() {
    long time = 0;
    final Set<String> topics = new HashSet<>();
    metadata.update(Cluster.empty(), Collections.<String>emptySet(), time);
    final Metadata.Listener listener = new Metadata.Listener() {

        @Override
        public void onMetadataUpdate(Cluster cluster, Set<String> unavailableTopics) {
            topics.clear();
            topics.addAll(cluster.topics());
        }
    };
    metadata.addListener(listener);
    metadata.update(new Cluster("cluster", Collections.singletonList(new Node(0, "host1", 1000)), Arrays.asList(new PartitionInfo("topic", 0, null, null, null), new PartitionInfo("topic1", 0, null, null, null)), Collections.<String>emptySet(), Collections.<String>emptySet()), Collections.<String>emptySet(), 100);
    metadata.removeListener(listener);
    metadata.update(new Cluster("cluster", Arrays.asList(new Node(0, "host1", 1000)), Arrays.asList(new PartitionInfo("topic2", 0, null, null, null), new PartitionInfo("topic3", 0, null, null, null)), Collections.<String>emptySet(), Collections.<String>emptySet()), Collections.<String>emptySet(), 100);
    assertEquals("Listener did not update topics list correctly", new HashSet<>(Arrays.asList("topic", "topic1")), topics);
}

Example 43 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class MetadataTest method testClusterListenerGetsNotifiedOfUpdate.

@Test
public void testClusterListenerGetsNotifiedOfUpdate() {
    long time = 0;
    MockClusterResourceListener mockClusterListener = new MockClusterResourceListener();
    ClusterResourceListeners listeners = new ClusterResourceListeners();
    listeners.maybeAdd(mockClusterListener);
    metadata = new Metadata(refreshBackoffMs, metadataExpireMs, false, listeners);
    String hostName = "www.example.com";
    Cluster cluster = Cluster.bootstrap(Arrays.asList(new InetSocketAddress(hostName, 9002)));
    metadata.update(cluster, Collections.<String>emptySet(), time);
    assertFalse("ClusterResourceListener should not called when metadata is updated with bootstrap Cluster", MockClusterResourceListener.IS_ON_UPDATE_CALLED.get());
    metadata.update(new Cluster("dummy", Arrays.asList(new Node(0, "host1", 1000)), Arrays.asList(new PartitionInfo("topic", 0, null, null, null), new PartitionInfo("topic1", 0, null, null, null)), Collections.<String>emptySet(), Collections.<String>emptySet()), Collections.<String>emptySet(), 100);
    assertEquals("MockClusterResourceListener did not get cluster metadata correctly", "dummy", mockClusterListener.clusterResource().clusterId());
    assertTrue("MockClusterResourceListener should be called when metadata is updated with non-bootstrap Cluster", MockClusterResourceListener.IS_ON_UPDATE_CALLED.get());
}

Example 44 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project flink by apache.

the class FlinkKafkaConsumer09 method convertToFlinkKafkaTopicPartition.

// ------------------------------------------------------------------------
//  Utilities 
// ------------------------------------------------------------------------
/**
	 * Converts a list of Kafka PartitionInfo's to Flink's KafkaTopicPartition (which are serializable)
	 * 
	 * @param partitions A list of Kafka PartitionInfos.
	 * @return A list of KafkaTopicPartitions
	 */
private static List<KafkaTopicPartition> convertToFlinkKafkaTopicPartition(List<PartitionInfo> partitions) {
    checkNotNull(partitions);
    List<KafkaTopicPartition> ret = new ArrayList<>(partitions.size());
    for (PartitionInfo pi : partitions) {
        ret.add(new KafkaTopicPartition(pi.topic(), pi.partition()));
    }
    return ret;
}

Example 45 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project flink by apache.

the class FlinkKafkaProducerBaseTest method testPartitionerOpenedWithDeterminatePartitionList.

/**
	 * Tests that partitions list is determinate and correctly provided to custom partitioner
	 */
@Test
public void testPartitionerOpenedWithDeterminatePartitionList() throws Exception {
    KafkaPartitioner mockPartitioner = mock(KafkaPartitioner.class);
    RuntimeContext mockRuntimeContext = mock(RuntimeContext.class);
    when(mockRuntimeContext.getIndexOfThisSubtask()).thenReturn(0);
    when(mockRuntimeContext.getNumberOfParallelSubtasks()).thenReturn(1);
    // out-of-order list of 4 partitions
    List<PartitionInfo> mockPartitionsList = new ArrayList<>(4);
    mockPartitionsList.add(new PartitionInfo(DummyFlinkKafkaProducer.DUMMY_TOPIC, 3, null, null, null));
    mockPartitionsList.add(new PartitionInfo(DummyFlinkKafkaProducer.DUMMY_TOPIC, 1, null, null, null));
    mockPartitionsList.add(new PartitionInfo(DummyFlinkKafkaProducer.DUMMY_TOPIC, 0, null, null, null));
    mockPartitionsList.add(new PartitionInfo(DummyFlinkKafkaProducer.DUMMY_TOPIC, 2, null, null, null));
    final DummyFlinkKafkaProducer producer = new DummyFlinkKafkaProducer(FakeStandardProducerConfig.get(), mockPartitioner);
    producer.setRuntimeContext(mockRuntimeContext);
    final KafkaProducer mockProducer = producer.getMockKafkaProducer();
    when(mockProducer.partitionsFor(anyString())).thenReturn(mockPartitionsList);
    when(mockProducer.metrics()).thenReturn(null);
    producer.open(new Configuration());
    // the out-of-order partitions list should be sorted before provided to the custom partitioner's open() method
    int[] correctPartitionList = { 0, 1, 2, 3 };
    verify(mockPartitioner).open(0, 1, correctPartitionList);
}