Examples with GlobalPartitionInformation org.apache.storm.kafka.trident.GlobalPartitionInformation used on opensource projects

Example 16 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project storm by apache.

the class DynamicBrokersReader method getBrokerInfo.

/**
     * Get all partitions with their current leaders
     */
public List<GlobalPartitionInformation> getBrokerInfo() throws SocketTimeoutException {
    List<String> topics = getTopics();
    List<GlobalPartitionInformation> partitions = new ArrayList<GlobalPartitionInformation>();
    for (String topic : topics) {
        GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(topic, this._isWildcardTopic);
        try {
            int numPartitionsForTopic = getNumPartitions(topic);
            String brokerInfoPath = brokerPath();
            for (int partition = 0; partition < numPartitionsForTopic; partition++) {
                int leader = getLeaderFor(topic, partition);
                String path = brokerInfoPath + "/" + leader;
                try {
                    byte[] brokerData = _curator.getData().forPath(path);
                    Broker hp = getBrokerHost(brokerData);
                    globalPartitionInformation.addPartition(partition, hp);
                } catch (org.apache.zookeeper.KeeperException.NoNodeException e) {
                    LOG.error("Node {} does not exist ", path);
                }
            }
        } catch (SocketTimeoutException e) {
            throw e;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        LOG.info("Read partition info from zookeeper: " + globalPartitionInformation);
        partitions.add(globalPartitionInformation);
    }
    return partitions;
}

Example 17 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project storm by apache.

the class DynamicBrokersReaderTest method testMultiplePartitionsOnSameHost.

@Test
public void testMultiplePartitionsOnSameHost() throws Exception {
    String host = "localhost";
    int port = 9092;
    int partition = 0;
    int secondPartition = partition + 1;
    addPartition(partition, 0, host, port, topic);
    addPartition(secondPartition, 0, host, port, topic);
    List<GlobalPartitionInformation> partitions = dynamicBrokersReader.getBrokerInfo();
    GlobalPartitionInformation brokerInfo = getByTopic(partitions, topic);
    assertNotNull(brokerInfo);
    assertEquals(2, brokerInfo.getOrderedPartitions().size());
    assertEquals(port, brokerInfo.getBrokerFor(partition).port);
    assertEquals(host, brokerInfo.getBrokerFor(partition).host);
    assertEquals(port, brokerInfo.getBrokerFor(secondPartition).port);
    assertEquals(host, brokerInfo.getBrokerFor(secondPartition).host);
}

Example 18 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project storm by apache.

the class DynamicBrokersReaderTest method testMultiplePartitionsOnDifferentHosts.

@Test
public void testMultiplePartitionsOnDifferentHosts() throws Exception {
    String host = "localhost";
    int port = 9092;
    int secondPort = 9093;
    int partition = 0;
    int secondPartition = partition + 1;
    addPartition(partition, 0, host, port, topic);
    addPartition(secondPartition, 1, host, secondPort, topic);
    List<GlobalPartitionInformation> partitions = dynamicBrokersReader.getBrokerInfo();
    GlobalPartitionInformation brokerInfo = getByTopic(partitions, topic);
    assertNotNull(brokerInfo);
    assertEquals(2, brokerInfo.getOrderedPartitions().size());
    assertEquals(port, brokerInfo.getBrokerFor(partition).port);
    assertEquals(host, brokerInfo.getBrokerFor(partition).host);
    assertEquals(secondPort, brokerInfo.getBrokerFor(secondPartition).port);
    assertEquals(host, brokerInfo.getBrokerFor(secondPartition).host);
}

Example 19 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project storm by apache.

the class KafkaUtilsTest method setup.

@Before
public void setup() {
    broker = new KafkaTestBroker();
    GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(TEST_TOPIC);
    globalPartitionInformation.addPartition(0, Broker.fromString(broker.getBrokerConnectionString()));
    brokerHosts = new StaticHosts(globalPartitionInformation);
    config = new KafkaConfig(brokerHosts, TEST_TOPIC);
    simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient");
}

Example 20 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project storm by apache.

the class KafkaUtilsTest method assignInvalidTask.

@Test(expected = IllegalArgumentException.class)
public void assignInvalidTask() {
    GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(TEST_TOPIC);
    List<GlobalPartitionInformation> partitions = new ArrayList<GlobalPartitionInformation>();
    partitions.add(globalPartitionInformation);
    KafkaUtils.calculatePartitionsForTask(partitions, 1, 1);
}