Examples with GlobalPartitionInformation org.apache.storm.kafka.trident.GlobalPartitionInformation used on opensource projects

Example 1 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project storm by apache.

the class KafkaUtils method calculatePartitionsForTask.

public static List<Partition> calculatePartitionsForTask(List<GlobalPartitionInformation> partitons, int totalTasks, int taskIndex) {
    Preconditions.checkArgument(taskIndex < totalTasks, "task index must be less that total tasks");
    List<Partition> taskPartitions = new ArrayList<Partition>();
    List<Partition> partitions = new ArrayList<Partition>();
    for (GlobalPartitionInformation partitionInformation : partitons) {
        partitions.addAll(partitionInformation.getOrderedPartitions());
    }
    int numPartitions = partitions.size();
    if (numPartitions < totalTasks) {
        LOG.warn("there are more tasks than partitions (tasks: " + totalTasks + "; partitions: " + numPartitions + "), some tasks will be idle");
    }
    for (int i = taskIndex; i < numPartitions; i += totalTasks) {
        Partition taskPartition = partitions.get(i);
        taskPartitions.add(taskPartition);
    }
    logPartitionMapping(totalTasks, taskIndex, taskPartitions);
    return taskPartitions;
}

Example 2 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project storm by apache.

the class KafkaBoltTest method setupKafkaConsumer.

private void setupKafkaConsumer() {
    GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(TEST_TOPIC);
    globalPartitionInformation.addPartition(0, Broker.fromString(broker.getBrokerConnectionString()));
    BrokerHosts brokerHosts = new StaticHosts(globalPartitionInformation);
    kafkaConfig = new KafkaConfig(brokerHosts, TEST_TOPIC);
    simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient");
}

Example 3 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project storm by apache.

the class DynamicBrokersReaderTest method testGetBrokerInfo.

@Test
public void testGetBrokerInfo() throws Exception {
    String host = "localhost";
    int port = 9092;
    int partition = 0;
    addPartition(partition, host, port, topic);
    List<GlobalPartitionInformation> partitions = dynamicBrokersReader.getBrokerInfo();
    GlobalPartitionInformation brokerInfo = getByTopic(partitions, topic);
    assertNotNull(brokerInfo);
    assertEquals(1, brokerInfo.getOrderedPartitions().size());
    assertEquals(port, brokerInfo.getBrokerFor(partition).port);
    assertEquals(host, brokerInfo.getBrokerFor(partition).host);
}

Example 4 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project storm by apache.

the class DynamicBrokersReaderTest method testSwitchHostForPartition.

@Test
public void testSwitchHostForPartition() throws Exception {
    String host = "localhost";
    int port = 9092;
    int partition = 0;
    addPartition(partition, host, port, topic);
    List<GlobalPartitionInformation> partitions = dynamicBrokersReader.getBrokerInfo();
    GlobalPartitionInformation brokerInfo = getByTopic(partitions, topic);
    assertNotNull(brokerInfo);
    assertEquals(port, brokerInfo.getBrokerFor(partition).port);
    assertEquals(host, brokerInfo.getBrokerFor(partition).host);
    String newHost = host + "switch";
    int newPort = port + 1;
    addPartition(partition, newHost, newPort, topic);
    partitions = dynamicBrokersReader.getBrokerInfo();
    brokerInfo = getByTopic(partitions, topic);
    assertNotNull(brokerInfo);
    assertEquals(newPort, brokerInfo.getBrokerFor(partition).port);
    assertEquals(newHost, brokerInfo.getBrokerFor(partition).host);
}

Example 5 with GlobalPartitionInformation

use of org.apache.storm.kafka.trident.GlobalPartitionInformation in project storm by apache.

the class DynamicBrokersReaderTest method testGetBrokerInfoWildcardMatch.

@Test
public void testGetBrokerInfoWildcardMatch() throws Exception {
    String host = "localhost";
    int port = 9092;
    int partition = 0;
    addPartition(partition, host, port, topic);
    addPartition(partition, host, port, secondTopic);
    List<GlobalPartitionInformation> partitions = wildCardBrokerReader.getBrokerInfo();
    GlobalPartitionInformation brokerInfo = getByTopic(partitions, topic);
    assertNotNull(brokerInfo);
    assertEquals(1, brokerInfo.getOrderedPartitions().size());
    assertEquals(port, brokerInfo.getBrokerFor(partition).port);
    assertEquals(host, brokerInfo.getBrokerFor(partition).host);
    brokerInfo = getByTopic(partitions, secondTopic);
    assertNotNull(brokerInfo);
    assertEquals(1, brokerInfo.getOrderedPartitions().size());
    assertEquals(port, brokerInfo.getBrokerFor(partition).port);
    assertEquals(host, brokerInfo.getBrokerFor(partition).host);
    addPartition(partition, host, port, thirdTopic);
    //Discover newly added topic
    partitions = wildCardBrokerReader.getBrokerInfo();
    assertNotNull(getByTopic(partitions, topic));
    assertNotNull(getByTopic(partitions, secondTopic));
    assertNotNull(getByTopic(partitions, secondTopic));
}