Search in sources :

Example 6 with TopicMetadataRequest

use of kafka.javaapi.TopicMetadataRequest in project jstorm by alibaba.

the class KafkaConsumer method findLeader.

protected PartitionMetadata findLeader(int partition) {
    PartitionMetadata returnMetaData = null;
    int errors = 0;
    int size = brokerList.size();
    Host brokerHost = brokerList.get(brokerIndex);
    try {
        if (consumer == null) {
            consumer = new SimpleConsumer(brokerHost.getHost(), brokerHost.getPort(), config.socketTimeoutMs, config.socketReceiveBufferBytes, config.clientId);
        }
    } catch (Exception e) {
        LOG.warn(e.getMessage(), e);
        consumer = null;
    }
    int i = brokerIndex;
    loop: while (i < size && errors < size + 1) {
        Host host = brokerList.get(i);
        i = (i + 1) % size;
        // next index
        brokerIndex = i;
        try {
            if (consumer == null) {
                consumer = new SimpleConsumer(host.getHost(), host.getPort(), config.socketTimeoutMs, config.socketReceiveBufferBytes, config.clientId);
            }
            List<String> topics = Collections.singletonList(config.topic);
            TopicMetadataRequest req = new TopicMetadataRequest(topics);
            kafka.javaapi.TopicMetadataResponse resp = null;
            try {
                resp = consumer.send(req);
            } catch (Exception e) {
                errors += 1;
                LOG.error("findLeader error, broker:" + host.toString() + ", will change to next broker index:" + (i + 1) % size);
                if (consumer != null) {
                    consumer.close();
                    consumer = null;
                }
                continue;
            }
            List<TopicMetadata> metaData = resp.topicsMetadata();
            for (TopicMetadata item : metaData) {
                for (PartitionMetadata part : item.partitionsMetadata()) {
                    if (part.partitionId() == partition) {
                        returnMetaData = part;
                        break loop;
                    }
                }
            }
        } catch (Exception e) {
            LOG.error("Error communicating with Broker:" + host.toString() + ", find Leader for partition:" + partition);
        } finally {
            if (consumer != null) {
                consumer.close();
                consumer = null;
            }
        }
    }
    return returnMetaData;
}
Also used : TopicMetadataRequest(kafka.javaapi.TopicMetadataRequest) PartitionMetadata(kafka.javaapi.PartitionMetadata) List(java.util.List) LinkedList(java.util.LinkedList) SimpleConsumer(kafka.javaapi.consumer.SimpleConsumer) KafkaException(kafka.common.KafkaException) IOException(java.io.IOException) UnresolvedAddressException(java.nio.channels.UnresolvedAddressException) SocketTimeoutException(java.net.SocketTimeoutException) ConnectException(java.net.ConnectException) TopicMetadata(kafka.javaapi.TopicMetadata)

Aggregations

TopicMetadata (kafka.javaapi.TopicMetadata)6 TopicMetadataRequest (kafka.javaapi.TopicMetadataRequest)6 PartitionMetadata (kafka.javaapi.PartitionMetadata)5 SimpleConsumer (kafka.javaapi.consumer.SimpleConsumer)5 TopicMetadataResponse (kafka.javaapi.TopicMetadataResponse)3 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 Broker (kafka.cluster.Broker)2 ConnectorSplit (com.facebook.presto.spi.ConnectorSplit)1 FixedSplitSource (com.facebook.presto.spi.FixedSplitSource)1 HostAddress (com.facebook.presto.spi.HostAddress)1 ImmutableList (com.google.common.collect.ImmutableList)1 HostAndPort (com.google.common.net.HostAndPort)1 ConnectException (java.net.ConnectException)1 SocketTimeoutException (java.net.SocketTimeoutException)1 URI (java.net.URI)1 URISyntaxException (java.net.URISyntaxException)1 URL (java.net.URL)1 UnknownHostException (java.net.UnknownHostException)1 ClosedChannelException (java.nio.channels.ClosedChannelException)1