Examples with PartitionInfo org.apache.kafka.common.PartitionInfo used on opensource projects

Example 26 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project flink by apache.

the class FlinkKafkaProducerBaseTest method testPartitionerOpenedWithDeterminatePartitionList.

/**
	 * Tests that partitions list is determinate and correctly provided to custom partitioner
	 */
@Test
public void testPartitionerOpenedWithDeterminatePartitionList() throws Exception {
    KafkaPartitioner mockPartitioner = mock(KafkaPartitioner.class);
    RuntimeContext mockRuntimeContext = mock(RuntimeContext.class);
    when(mockRuntimeContext.getIndexOfThisSubtask()).thenReturn(0);
    when(mockRuntimeContext.getNumberOfParallelSubtasks()).thenReturn(1);
    // out-of-order list of 4 partitions
    List<PartitionInfo> mockPartitionsList = new ArrayList<>(4);
    mockPartitionsList.add(new PartitionInfo(DummyFlinkKafkaProducer.DUMMY_TOPIC, 3, null, null, null));
    mockPartitionsList.add(new PartitionInfo(DummyFlinkKafkaProducer.DUMMY_TOPIC, 1, null, null, null));
    mockPartitionsList.add(new PartitionInfo(DummyFlinkKafkaProducer.DUMMY_TOPIC, 0, null, null, null));
    mockPartitionsList.add(new PartitionInfo(DummyFlinkKafkaProducer.DUMMY_TOPIC, 2, null, null, null));
    final DummyFlinkKafkaProducer producer = new DummyFlinkKafkaProducer(FakeStandardProducerConfig.get(), mockPartitioner);
    producer.setRuntimeContext(mockRuntimeContext);
    final KafkaProducer mockProducer = producer.getMockKafkaProducer();
    when(mockProducer.partitionsFor(anyString())).thenReturn(mockPartitionsList);
    when(mockProducer.metrics()).thenReturn(null);
    producer.open(new Configuration());
    // the out-of-order partitions list should be sorted before provided to the custom partitioner's open() method
    int[] correctPartitionList = { 0, 1, 2, 3 };
    verify(mockPartitioner).open(0, 1, correctPartitionList);
}

Example 27 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class KafkaProducerTest method testMetadataFetchOnStaleMetadata.

@PrepareOnlyThisForTest(Metadata.class)
@Test
public void testMetadataFetchOnStaleMetadata() throws Exception {
    Properties props = new Properties();
    props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
    KafkaProducer<String, String> producer = new KafkaProducer<>(props, new StringSerializer(), new StringSerializer());
    Metadata metadata = PowerMock.createNiceMock(Metadata.class);
    MemberModifier.field(KafkaProducer.class, "metadata").set(producer, metadata);
    String topic = "topic";
    ProducerRecord<String, String> initialRecord = new ProducerRecord<>(topic, "value");
    // Create a record with a partition higher than the initial (outdated) partition range
    ProducerRecord<String, String> extendedRecord = new ProducerRecord<>(topic, 2, null, "value");
    Collection<Node> nodes = Collections.singletonList(new Node(0, "host1", 1000));
    final Cluster emptyCluster = new Cluster(null, nodes, Collections.<PartitionInfo>emptySet(), Collections.<String>emptySet(), Collections.<String>emptySet());
    final Cluster initialCluster = new Cluster("dummy", Collections.singletonList(new Node(0, "host1", 1000)), Arrays.asList(new PartitionInfo(topic, 0, null, null, null)), Collections.<String>emptySet(), Collections.<String>emptySet());
    final Cluster extendedCluster = new Cluster("dummy", Collections.singletonList(new Node(0, "host1", 1000)), Arrays.asList(new PartitionInfo(topic, 0, null, null, null), new PartitionInfo(topic, 1, null, null, null), new PartitionInfo(topic, 2, null, null, null)), Collections.<String>emptySet(), Collections.<String>emptySet());
    // Expect exactly one fetch for each attempt to refresh while topic metadata is not available
    final int refreshAttempts = 5;
    EasyMock.expect(metadata.fetch()).andReturn(emptyCluster).times(refreshAttempts - 1);
    EasyMock.expect(metadata.fetch()).andReturn(initialCluster).once();
    EasyMock.expect(metadata.fetch()).andThrow(new IllegalStateException("Unexpected call to metadata.fetch()")).anyTimes();
    PowerMock.replay(metadata);
    producer.send(initialRecord);
    PowerMock.verify(metadata);
    // Expect exactly one fetch if topic metadata is available and records are still within range
    PowerMock.reset(metadata);
    EasyMock.expect(metadata.fetch()).andReturn(initialCluster).once();
    EasyMock.expect(metadata.fetch()).andThrow(new IllegalStateException("Unexpected call to metadata.fetch()")).anyTimes();
    PowerMock.replay(metadata);
    producer.send(initialRecord, null);
    PowerMock.verify(metadata);
    // Expect exactly two fetches if topic metadata is available but metadata response still returns
    // the same partition size (either because metadata are still stale at the broker too or because
    // there weren't any partitions added in the first place).
    PowerMock.reset(metadata);
    EasyMock.expect(metadata.fetch()).andReturn(initialCluster).once();
    EasyMock.expect(metadata.fetch()).andReturn(initialCluster).once();
    EasyMock.expect(metadata.fetch()).andThrow(new IllegalStateException("Unexpected call to metadata.fetch()")).anyTimes();
    PowerMock.replay(metadata);
    try {
        producer.send(extendedRecord, null);
        fail("Expected KafkaException to be raised");
    } catch (KafkaException e) {
    // expected
    }
    PowerMock.verify(metadata);
    // Expect exactly two fetches if topic metadata is available but outdated for the given record
    PowerMock.reset(metadata);
    EasyMock.expect(metadata.fetch()).andReturn(initialCluster).once();
    EasyMock.expect(metadata.fetch()).andReturn(extendedCluster).once();
    EasyMock.expect(metadata.fetch()).andThrow(new IllegalStateException("Unexpected call to metadata.fetch()")).anyTimes();
    PowerMock.replay(metadata);
    producer.send(extendedRecord, null);
    PowerMock.verify(metadata);
}

Example 28 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class FetcherTest method newMetadataResponse.

private MetadataResponse newMetadataResponse(String topic, Errors error) {
    List<MetadataResponse.PartitionMetadata> partitionsMetadata = new ArrayList<>();
    if (error == Errors.NONE) {
        for (PartitionInfo partitionInfo : cluster.partitionsForTopic(topic)) {
            partitionsMetadata.add(new MetadataResponse.PartitionMetadata(Errors.NONE, partitionInfo.partition(), partitionInfo.leader(), Arrays.asList(partitionInfo.replicas()), Arrays.asList(partitionInfo.inSyncReplicas())));
        }
    }
    MetadataResponse.TopicMetadata topicMetadata = new MetadataResponse.TopicMetadata(error, topic, false, partitionsMetadata);
    return new MetadataResponse(cluster.nodes(), null, MetadataResponse.NO_CONTROLLER_ID, Arrays.asList(topicMetadata));
}

Example 29 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class MetadataTest method testListenerCanUnregister.

@Test
public void testListenerCanUnregister() {
    long time = 0;
    final Set<String> topics = new HashSet<>();
    metadata.update(Cluster.empty(), Collections.<String>emptySet(), time);
    final Metadata.Listener listener = new Metadata.Listener() {

        @Override
        public void onMetadataUpdate(Cluster cluster, Set<String> unavailableTopics) {
            topics.clear();
            topics.addAll(cluster.topics());
        }
    };
    metadata.addListener(listener);
    metadata.update(new Cluster("cluster", Collections.singletonList(new Node(0, "host1", 1000)), Arrays.asList(new PartitionInfo("topic", 0, null, null, null), new PartitionInfo("topic1", 0, null, null, null)), Collections.<String>emptySet(), Collections.<String>emptySet()), Collections.<String>emptySet(), 100);
    metadata.removeListener(listener);
    metadata.update(new Cluster("cluster", Arrays.asList(new Node(0, "host1", 1000)), Arrays.asList(new PartitionInfo("topic2", 0, null, null, null), new PartitionInfo("topic3", 0, null, null, null)), Collections.<String>emptySet(), Collections.<String>emptySet()), Collections.<String>emptySet(), 100);
    assertEquals("Listener did not update topics list correctly", new HashSet<>(Arrays.asList("topic", "topic1")), topics);
}

Example 30 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class MetadataTest method testClusterListenerGetsNotifiedOfUpdate.

@Test
public void testClusterListenerGetsNotifiedOfUpdate() {
    long time = 0;
    MockClusterResourceListener mockClusterListener = new MockClusterResourceListener();
    ClusterResourceListeners listeners = new ClusterResourceListeners();
    listeners.maybeAdd(mockClusterListener);
    metadata = new Metadata(refreshBackoffMs, metadataExpireMs, false, listeners);
    String hostName = "www.example.com";
    Cluster cluster = Cluster.bootstrap(Arrays.asList(new InetSocketAddress(hostName, 9002)));
    metadata.update(cluster, Collections.<String>emptySet(), time);
    assertFalse("ClusterResourceListener should not called when metadata is updated with bootstrap Cluster", MockClusterResourceListener.IS_ON_UPDATE_CALLED.get());
    metadata.update(new Cluster("dummy", Arrays.asList(new Node(0, "host1", 1000)), Arrays.asList(new PartitionInfo("topic", 0, null, null, null), new PartitionInfo("topic1", 0, null, null, null)), Collections.<String>emptySet(), Collections.<String>emptySet()), Collections.<String>emptySet(), 100);
    assertEquals("MockClusterResourceListener did not get cluster metadata correctly", "dummy", mockClusterListener.clusterResource().clusterId());
    assertTrue("MockClusterResourceListener should be called when metadata is updated with non-bootstrap Cluster", MockClusterResourceListener.IS_ON_UPDATE_CALLED.get());
}