Examples with GlobalPartitionInformation org.apache.storm.kafka.trident.GlobalPartitionInformation used on opensource projects

Example 6 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project storm by apache.

the class KafkaUtilsTest method moreTasksThanPartitions.

@Test
public void moreTasksThanPartitions() {
    GlobalPartitionInformation globalPartitionInformation = TestUtils.buildPartitionInfo(1);
    List<GlobalPartitionInformation> partitions = new ArrayList<GlobalPartitionInformation>();
    partitions.add(globalPartitionInformation);
    int numTasks = 2;
    assertEquals(1, KafkaUtils.calculatePartitionsForTask(partitions, numTasks, 0).size());
    assertEquals(0, KafkaUtils.calculatePartitionsForTask(partitions, numTasks, 1).size());
}

Example 7 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project storm by apache.

the class KafkaUtilsTest method runPartitionToTaskMappingTest.

public void runPartitionToTaskMappingTest(int numPartitions, int partitionsPerTask) {
    GlobalPartitionInformation globalPartitionInformation = TestUtils.buildPartitionInfo(numPartitions);
    List<GlobalPartitionInformation> partitions = new ArrayList<GlobalPartitionInformation>();
    partitions.add(globalPartitionInformation);
    int numTasks = numPartitions / partitionsPerTask;
    for (int i = 0; i < numTasks; i++) {
        assertEquals(partitionsPerTask, KafkaUtils.calculatePartitionsForTask(partitions, numTasks, i).size());
    }
}

Example 8 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project heron by twitter.

the class KafkaUtilsTest method runPartitionToTaskMappingTest.

public void runPartitionToTaskMappingTest(int numPartitions, int partitionsPerTask) {
    GlobalPartitionInformation globalPartitionInformation = TestUtils.buildPartitionInfo(numPartitions);
    List<GlobalPartitionInformation> partitions = new ArrayList<GlobalPartitionInformation>();
    partitions.add(globalPartitionInformation);
    int numTasks = numPartitions / partitionsPerTask;
    for (int i = 0; i < numTasks; i++) {
        assertEquals(partitionsPerTask, KafkaUtils.calculatePartitionsForTask(partitions, numTasks, i).size());
    }
}

Example 9 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project heron by twitter.

the class DynamicBrokersReader method getBrokerInfo.

/**
   * Get all partitions with their current leaders
   */
public List<GlobalPartitionInformation> getBrokerInfo() throws SocketTimeoutException {
    List<String> topics = getTopics();
    List<GlobalPartitionInformation> partitions = new ArrayList<GlobalPartitionInformation>();
    for (String topic : topics) {
        GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(topic, this.isWildcardTopic);
        try {
            int numPartitionsForTopic = getNumPartitions(topic);
            String brokerInfoPath = brokerPath();
            for (int partition = 0; partition < numPartitionsForTopic; partition++) {
                int leader = getLeaderFor(topic, partition);
                String path = brokerInfoPath + "/" + leader;
                try {
                    byte[] brokerData = curator.getData().forPath(path);
                    Broker hp = getBrokerHost(brokerData);
                    globalPartitionInformation.addPartition(partition, hp);
                } catch (org.apache.zookeeper.KeeperException.NoNodeException e) {
                    LOG.error("Node {} does not exist ", path);
                }
            }
        } catch (SocketTimeoutException e) {
            throw e;
        // SUPPRESS CHECKSTYLE IllegalCatch
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        LOG.info("Read partition info from zookeeper: " + globalPartitionInformation);
        partitions.add(globalPartitionInformation);
    }
    return partitions;
}

Example 10 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project heron by twitter.

the class DynamicBrokersReaderTest method testGetBrokerInfoWildcardMatch.

@Test
public void testGetBrokerInfoWildcardMatch() throws Exception {
    String host = "localhost";
    int port = 9092;
    int partition = 0;
    addPartition(partition, host, port, topic);
    addPartition(partition, host, port, secondTopic);
    List<GlobalPartitionInformation> partitions = wildCardBrokerReader.getBrokerInfo();
    GlobalPartitionInformation brokerInfo = getByTopic(partitions, topic);
    assertNotNull(brokerInfo);
    assertEquals(1, brokerInfo.getOrderedPartitions().size());
    assertEquals(port, brokerInfo.getBrokerFor(partition).port);
    assertEquals(host, brokerInfo.getBrokerFor(partition).host);
    brokerInfo = getByTopic(partitions, secondTopic);
    assertNotNull(brokerInfo);
    assertEquals(1, brokerInfo.getOrderedPartitions().size());
    assertEquals(port, brokerInfo.getBrokerFor(partition).port);
    assertEquals(host, brokerInfo.getBrokerFor(partition).host);
    addPartition(partition, host, port, thirdTopic);
    //Discover newly added topic
    partitions = wildCardBrokerReader.getBrokerInfo();
    assertNotNull(getByTopic(partitions, topic));
    assertNotNull(getByTopic(partitions, secondTopic));
    assertNotNull(getByTopic(partitions, secondTopic));
}