Examples with TopicMetadata kafka.javaapi.TopicMetadata used on opensource projects

Example 1 with TopicMetadata

use of kafka.javaapi.TopicMetadata in project pinot by linkedin.

the class SimpleConsumerWrapper method getPartitionCount.

public synchronized int getPartitionCount(String topic, long timeoutMillis) {
    int unknownTopicReplyCount = 0;
    final int MAX_UNKNOWN_TOPIC_REPLY_COUNT = 10;
    int kafkaErrorCount = 0;
    final int MAX_KAFKA_ERROR_COUNT = 10;
    final long endTime = System.currentTimeMillis() + timeoutMillis;
    while (System.currentTimeMillis() < endTime) {
        // Try to get into a state where we're connected to Kafka
        while (!_currentState.isConnectedToKafkaBroker() && System.currentTimeMillis() < endTime) {
            _currentState.process();
        }
        if (endTime <= System.currentTimeMillis() && !_currentState.isConnectedToKafkaBroker()) {
            throw new TimeoutException("Failed to get the partition count for topic " + topic + " within " + timeoutMillis + " ms");
        }
        // Send the metadata request to Kafka
        TopicMetadataResponse topicMetadataResponse = null;
        try {
            topicMetadataResponse = _simpleConsumer.send(new TopicMetadataRequest(Collections.singletonList(topic)));
        } catch (Exception e) {
            _currentState.handleConsumerException(e);
            continue;
        }
        final TopicMetadata topicMetadata = topicMetadataResponse.topicsMetadata().get(0);
        final short errorCode = topicMetadata.errorCode();
        if (errorCode == Errors.NONE.code()) {
            return topicMetadata.partitionsMetadata().size();
        } else if (errorCode == Errors.LEADER_NOT_AVAILABLE.code()) {
            // If there is no leader, it'll take some time for a new leader to be elected, wait 100 ms before retrying
            Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        } else if (errorCode == Errors.INVALID_TOPIC_EXCEPTION.code()) {
            throw new RuntimeException("Invalid topic name " + topic);
        } else if (errorCode == Errors.UNKNOWN_TOPIC_OR_PARTITION.code()) {
            if (MAX_UNKNOWN_TOPIC_REPLY_COUNT < unknownTopicReplyCount) {
                throw new RuntimeException("Topic " + topic + " does not exist");
            } else {
                // Kafka topic creation can sometimes take some time, so we'll retry after a little bit
                unknownTopicReplyCount++;
                Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            }
        } else {
            // Retry after a short delay
            kafkaErrorCount++;
            if (MAX_KAFKA_ERROR_COUNT < kafkaErrorCount) {
                throw exceptionForKafkaErrorCode(errorCode);
            }
            Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        }
    }
    throw new TimeoutException();
}

Example 2 with TopicMetadata

use of kafka.javaapi.TopicMetadata in project voltdb by VoltDB.

the class KafkaStreamImporterConfig method getConfigsForPartitions.

private static Map<URI, KafkaStreamImporterConfig> getConfigsForPartitions(String key, List<HostAndPort> brokerList, final String topic, String groupId, String procedure, int soTimeout, int fetchSize, String commitPolicy, FormatterBuilder formatterBuilder) {
    SimpleConsumer consumer = null;
    Map<URI, KafkaStreamImporterConfig> configs = new HashMap<>();
    List<FailedMetaDataAttempt> attempts = new ArrayList<>();
    Iterator<HostAndPort> hpitr = brokerList.iterator();
    while (configs.isEmpty() && hpitr.hasNext()) {
        HostAndPort hp = hpitr.next();
        try {
            consumer = new SimpleConsumer(hp.getHost(), hp.getPort(), soTimeout, fetchSize, CLIENT_ID);
            TopicMetadataRequest req = new TopicMetadataRequest(singletonList(topic));
            kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
            List<TopicMetadata> metaData = resp.topicsMetadata();
            if (metaData == null) {
                attempts.add(new FailedMetaDataAttempt("Failed to get topic metadata for topic " + topic + " from host " + hp.getHost(), null));
                closeConsumer(consumer);
                consumer = null;
                continue;
            }
            int partitionCount = 0;
            for (TopicMetadata item : metaData) {
                for (PartitionMetadata part : item.partitionsMetadata()) {
                    ++partitionCount;
                    URI uri;
                    try {
                        uri = new URI("kafka", key, topic + "/partition/" + part.partitionId());
                    } catch (URISyntaxException ex) {
                        // Should not happen
                        throw new KafkaConfigurationException("unable to create topic resource URI", ex);
                    }
                    Broker leader = part.leader();
                    if (leader == null) {
                        attempts.add(new FailedMetaDataAttempt("Failed to get leader broker for topic " + topic + " partition " + part.partitionId() + " from host " + hp.getHost(), null));
                        continue;
                    }
                    KafkaStreamImporterConfig config = new KafkaStreamImporterConfig(uri, brokerList, topic, part.partitionId(), new HostAndPort(leader.host(), leader.port()), groupId, fetchSize, soTimeout, procedure, commitPolicy, formatterBuilder);
                    configs.put(uri, config);
                }
            }
            if (configs.size() != partitionCount) {
                configs.clear();
                closeConsumer(consumer);
                consumer = null;
            }
        } catch (Exception e) {
            attempts.add(new FailedMetaDataAttempt("Failed to send topic metadata request for topic " + topic + " from host " + hp.getHost(), e));
        } finally {
            closeConsumer(consumer);
        }
    }
    if (!attempts.isEmpty()) {
        attempts.forEach((attempt) -> {
            attempt.log();
        });
        attempts.clear();
        if (configs.isEmpty()) {
            throw new KafkaConfigurationException("Failed to get topic metadata for %s", topic);
        }
    }
    return configs;
}

Example 3 with TopicMetadata

use of kafka.javaapi.TopicMetadata in project flink by apache.

the class FlinkKafkaConsumer08 method getPartitionsForTopic.

// ------------------------------------------------------------------------
//  Kafka / ZooKeeper communication utilities
// ------------------------------------------------------------------------
/**
	 * Send request to Kafka to get partitions for topic.
	 * 
	 * @param topics The name of the topics.
	 * @param properties The properties for the Kafka Consumer that is used to query the partitions for the topic. 
	 */
public static List<KafkaTopicPartitionLeader> getPartitionsForTopic(List<String> topics, Properties properties) {
    String seedBrokersConfString = properties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG);
    final int numRetries = getInt(properties, GET_PARTITIONS_RETRIES_KEY, DEFAULT_GET_PARTITIONS_RETRIES);
    checkNotNull(seedBrokersConfString, "Configuration property %s not set", ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG);
    String[] seedBrokers = seedBrokersConfString.split(",");
    List<KafkaTopicPartitionLeader> partitions = new ArrayList<>();
    final String clientId = "flink-kafka-consumer-partition-lookup";
    final int soTimeout = getInt(properties, "socket.timeout.ms", 30000);
    final int bufferSize = getInt(properties, "socket.receive.buffer.bytes", 65536);
    Random rnd = new Random();
    retryLoop: for (int retry = 0; retry < numRetries; retry++) {
        // we pick a seed broker randomly to avoid overloading the first broker with all the requests when the
        // parallel source instances start. Still, we try all available brokers.
        int index = rnd.nextInt(seedBrokers.length);
        brokersLoop: for (int arrIdx = 0; arrIdx < seedBrokers.length; arrIdx++) {
            String seedBroker = seedBrokers[index];
            LOG.info("Trying to get topic metadata from broker {} in try {}/{}", seedBroker, retry, numRetries);
            if (++index == seedBrokers.length) {
                index = 0;
            }
            URL brokerUrl = NetUtils.getCorrectHostnamePort(seedBroker);
            SimpleConsumer consumer = null;
            try {
                consumer = new SimpleConsumer(brokerUrl.getHost(), brokerUrl.getPort(), soTimeout, bufferSize, clientId);
                TopicMetadataRequest req = new TopicMetadataRequest(topics);
                kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
                List<TopicMetadata> metaData = resp.topicsMetadata();
                // clear in case we have an incomplete list from previous tries
                partitions.clear();
                for (TopicMetadata item : metaData) {
                    if (item.errorCode() != ErrorMapping.NoError()) {
                        // warn and try more brokers
                        LOG.warn("Error while getting metadata from broker " + seedBroker + " to find partitions " + "for " + topics.toString() + ". Error: " + ErrorMapping.exceptionFor(item.errorCode()).getMessage());
                        continue brokersLoop;
                    }
                    if (!topics.contains(item.topic())) {
                        LOG.warn("Received metadata from topic " + item.topic() + " even though it was not requested. Skipping ...");
                        continue brokersLoop;
                    }
                    for (PartitionMetadata part : item.partitionsMetadata()) {
                        Node leader = brokerToNode(part.leader());
                        KafkaTopicPartition ktp = new KafkaTopicPartition(item.topic(), part.partitionId());
                        KafkaTopicPartitionLeader pInfo = new KafkaTopicPartitionLeader(ktp, leader);
                        partitions.add(pInfo);
                    }
                }
                // leave the loop through the brokers
                break retryLoop;
            } catch (Exception e) {
                //validates seed brokers in case of a ClosedChannelException
                validateSeedBrokers(seedBrokers, e);
                LOG.warn("Error communicating with broker {} to find partitions for {}. {} Message: {}", seedBroker, topics, e.getClass().getName(), e.getMessage());
                LOG.debug("Detailed trace", e);
                // we sleep a bit. Retrying immediately doesn't make sense in cases where Kafka is reorganizing the leader metadata
                try {
                    Thread.sleep(500);
                } catch (InterruptedException e1) {
                // sleep shorter.
                }
            } finally {
                if (consumer != null) {
                    consumer.close();
                }
            }
        }
    // brokers loop
    }
    // retries loop
    return partitions;
}

Example 4 with TopicMetadata

use of kafka.javaapi.TopicMetadata in project incubator-gobblin by apache.

the class Kafka08ConsumerClient method getTopics.

@Override
public List<KafkaTopic> getTopics() {
    List<TopicMetadata> topicMetadataList = getFilteredMetadataList();
    List<KafkaTopic> filteredTopics = Lists.newArrayList();
    for (TopicMetadata topicMetadata : topicMetadataList) {
        List<KafkaPartition> partitions = getPartitionsForTopic(topicMetadata);
        filteredTopics.add(new KafkaTopic(topicMetadata.topic(), partitions));
    }
    return filteredTopics;
}

Example 5 with TopicMetadata

use of kafka.javaapi.TopicMetadata in project druid by druid-io.

the class KafkaSimpleConsumer method findLeader.

private PartitionMetadata findLeader() throws InterruptedException {
    for (HostAndPort broker : replicaBrokers) {
        SimpleConsumer consumer = null;
        try {
            log.info("Finding new leader from Kafka brokers, try broker [%s]", broker.toString());
            consumer = new SimpleConsumer(broker.getHostText(), broker.getPort(), SO_TIMEOUT, BUFFER_SIZE, leaderLookupClientId);
            TopicMetadataResponse resp = consumer.send(new TopicMetadataRequest(Collections.singletonList(topic)));
            List<TopicMetadata> metaData = resp.topicsMetadata();
            for (TopicMetadata item : metaData) {
                if (topic.equals(item.topic())) {
                    for (PartitionMetadata part : item.partitionsMetadata()) {
                        if (part.partitionId() == partitionId) {
                            return part;
                        }
                    }
                }
            }
        } catch (Exception e) {
            ensureNotInterrupted(e);
            log.warn(e, "error communicating with Kafka Broker [%s] to find leader for [%s] - [%s]", broker, topic, partitionId);
        } finally {
            if (consumer != null) {
                consumer.close();
            }
        }
    }
    return null;
}