Examples with TopicMetadataResponse kafka.javaapi.TopicMetadataResponse used on opensource projects

Example 1 with TopicMetadataResponse

use of kafka.javaapi.TopicMetadataResponse in project pinot by linkedin.

the class SimpleConsumerWrapper method getPartitionCount.

public synchronized int getPartitionCount(String topic, long timeoutMillis) {
    int unknownTopicReplyCount = 0;
    final int MAX_UNKNOWN_TOPIC_REPLY_COUNT = 10;
    int kafkaErrorCount = 0;
    final int MAX_KAFKA_ERROR_COUNT = 10;
    final long endTime = System.currentTimeMillis() + timeoutMillis;
    while (System.currentTimeMillis() < endTime) {
        // Try to get into a state where we're connected to Kafka
        while (!_currentState.isConnectedToKafkaBroker() && System.currentTimeMillis() < endTime) {
            _currentState.process();
        }
        if (endTime <= System.currentTimeMillis() && !_currentState.isConnectedToKafkaBroker()) {
            throw new TimeoutException("Failed to get the partition count for topic " + topic + " within " + timeoutMillis + " ms");
        }
        // Send the metadata request to Kafka
        TopicMetadataResponse topicMetadataResponse = null;
        try {
            topicMetadataResponse = _simpleConsumer.send(new TopicMetadataRequest(Collections.singletonList(topic)));
        } catch (Exception e) {
            _currentState.handleConsumerException(e);
            continue;
        }
        final TopicMetadata topicMetadata = topicMetadataResponse.topicsMetadata().get(0);
        final short errorCode = topicMetadata.errorCode();
        if (errorCode == Errors.NONE.code()) {
            return topicMetadata.partitionsMetadata().size();
        } else if (errorCode == Errors.LEADER_NOT_AVAILABLE.code()) {
            // If there is no leader, it'll take some time for a new leader to be elected, wait 100 ms before retrying
            Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        } else if (errorCode == Errors.INVALID_TOPIC_EXCEPTION.code()) {
            throw new RuntimeException("Invalid topic name " + topic);
        } else if (errorCode == Errors.UNKNOWN_TOPIC_OR_PARTITION.code()) {
            if (MAX_UNKNOWN_TOPIC_REPLY_COUNT < unknownTopicReplyCount) {
                throw new RuntimeException("Topic " + topic + " does not exist");
            } else {
                // Kafka topic creation can sometimes take some time, so we'll retry after a little bit
                unknownTopicReplyCount++;
                Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            }
        } else {
            // Retry after a short delay
            kafkaErrorCount++;
            if (MAX_KAFKA_ERROR_COUNT < kafkaErrorCount) {
                throw exceptionForKafkaErrorCode(errorCode);
            }
            Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        }
    }
    throw new TimeoutException();
}

Example 2 with TopicMetadataResponse

use of kafka.javaapi.TopicMetadataResponse in project druid by druid-io.

the class KafkaSimpleConsumer method findLeader.

private PartitionMetadata findLeader() throws InterruptedException {
    for (HostAndPort broker : replicaBrokers) {
        SimpleConsumer consumer = null;
        try {
            log.info("Finding new leader from Kafka brokers, try broker [%s]", broker.toString());
            consumer = new SimpleConsumer(broker.getHostText(), broker.getPort(), SO_TIMEOUT, BUFFER_SIZE, leaderLookupClientId);
            TopicMetadataResponse resp = consumer.send(new TopicMetadataRequest(Collections.singletonList(topic)));
            List<TopicMetadata> metaData = resp.topicsMetadata();
            for (TopicMetadata item : metaData) {
                if (topic.equals(item.topic())) {
                    for (PartitionMetadata part : item.partitionsMetadata()) {
                        if (part.partitionId() == partitionId) {
                            return part;
                        }
                    }
                }
            }
        } catch (Exception e) {
            ensureNotInterrupted(e);
            log.warn(e, "error communicating with Kafka Broker [%s] to find leader for [%s] - [%s]", broker, topic, partitionId);
        } finally {
            if (consumer != null) {
                consumer.close();
            }
        }
    }
    return null;
}

Example 3 with TopicMetadataResponse

use of kafka.javaapi.TopicMetadataResponse in project presto by prestodb.

the class KafkaSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    KafkaTableHandle kafkaTableHandle = convertLayout(layout).getTable();
    SimpleConsumer simpleConsumer = consumerManager.getConsumer(selectRandom(nodes));
    TopicMetadataRequest topicMetadataRequest = new TopicMetadataRequest(ImmutableList.of(kafkaTableHandle.getTopicName()));
    TopicMetadataResponse topicMetadataResponse = simpleConsumer.send(topicMetadataRequest);
    ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder();
    for (TopicMetadata metadata : topicMetadataResponse.topicsMetadata()) {
        for (PartitionMetadata part : metadata.partitionsMetadata()) {
            log.debug("Adding Partition %s/%s", metadata.topic(), part.partitionId());
            Broker leader = part.leader();
            if (leader == null) {
                // Leader election going on...
                log.warn("No leader for partition %s/%s found!", metadata.topic(), part.partitionId());
                continue;
            }
            HostAddress partitionLeader = HostAddress.fromParts(leader.host(), leader.port());
            SimpleConsumer leaderConsumer = consumerManager.getConsumer(partitionLeader);
            // Kafka contains a reverse list of "end - start" pairs for the splits
            long[] offsets = findAllOffsets(leaderConsumer, metadata.topic(), part.partitionId());
            for (int i = offsets.length - 1; i > 0; i--) {
                KafkaSplit split = new KafkaSplit(connectorId, metadata.topic(), kafkaTableHandle.getKeyDataFormat(), kafkaTableHandle.getMessageDataFormat(), part.partitionId(), offsets[i], offsets[i - 1], partitionLeader);
                splits.add(split);
            }
        }
    }
    return new FixedSplitSource(splits.build());
}

Example 4 with TopicMetadataResponse

use of kafka.javaapi.TopicMetadataResponse in project apex-malhar by apache.

the class KafkaMetadataUtil method getTopicMetadata.

/**
 * @param brokerSet
 * @param topic
 * @return TopicMetadata for this specific topic via the brokerList<br>
 * null if topic is not found
 */
public static TopicMetadata getTopicMetadata(Set<String> brokerSet, String topic) {
    SimpleConsumer mdConsumer = null;
    if (brokerSet == null || brokerSet == null || brokerSet.size() == 0) {
        return null;
    }
    try {
        for (Iterator<String> iterator = brokerSet.iterator(); iterator.hasNext(); ) {
            String broker = iterator.next();
            logger.debug("Try to get Metadata for topic {} broker {}", topic, broker);
            try {
                mdConsumer = new SimpleConsumer(broker.split(":")[0], Integer.parseInt(broker.split(":")[1]), timeout, bufferSize, mdClientId);
                List<String> topics = new ArrayList<String>(1);
                topics.add(topic);
                kafka.javaapi.TopicMetadataRequest req = new kafka.javaapi.TopicMetadataRequest(topics);
                TopicMetadataResponse resp = mdConsumer.send(req);
                List<TopicMetadata> metaData = resp.topicsMetadata();
                for (TopicMetadata item : metaData) {
                    // There is at most 1 topic for this method
                    return item;
                }
            } catch (NumberFormatException e) {
                throw new IllegalArgumentException("Wrong format for broker url, should be \"broker1:port1\"");
            } catch (Exception e) {
                logger.warn("Broker {} is unavailable or in bad state!", broker);
            // skip and try next broker
            }
        }
        return null;
    } finally {
        if (mdConsumer != null) {
            mdConsumer.close();
        }
    }
}