Examples with PartitionMetadata kafka.javaapi.PartitionMetadata used on opensource projects

Example 1 with PartitionMetadata

use of kafka.javaapi.PartitionMetadata in project voltdb by VoltDB.

the class KafkaStreamImporterConfig method getConfigsForPartitions.

private static Map<URI, KafkaStreamImporterConfig> getConfigsForPartitions(String key, List<HostAndPort> brokerList, final String topic, String groupId, String procedure, int soTimeout, int fetchSize, String commitPolicy, FormatterBuilder formatterBuilder) {
    SimpleConsumer consumer = null;
    Map<URI, KafkaStreamImporterConfig> configs = new HashMap<>();
    List<FailedMetaDataAttempt> attempts = new ArrayList<>();
    Iterator<HostAndPort> hpitr = brokerList.iterator();
    while (configs.isEmpty() && hpitr.hasNext()) {
        HostAndPort hp = hpitr.next();
        try {
            consumer = new SimpleConsumer(hp.getHost(), hp.getPort(), soTimeout, fetchSize, CLIENT_ID);
            TopicMetadataRequest req = new TopicMetadataRequest(singletonList(topic));
            kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
            List<TopicMetadata> metaData = resp.topicsMetadata();
            if (metaData == null) {
                attempts.add(new FailedMetaDataAttempt("Failed to get topic metadata for topic " + topic + " from host " + hp.getHost(), null));
                closeConsumer(consumer);
                consumer = null;
                continue;
            }
            int partitionCount = 0;
            for (TopicMetadata item : metaData) {
                for (PartitionMetadata part : item.partitionsMetadata()) {
                    ++partitionCount;
                    URI uri;
                    try {
                        uri = new URI("kafka", key, topic + "/partition/" + part.partitionId());
                    } catch (URISyntaxException ex) {
                        // Should not happen
                        throw new KafkaConfigurationException("unable to create topic resource URI", ex);
                    }
                    Broker leader = part.leader();
                    if (leader == null) {
                        attempts.add(new FailedMetaDataAttempt("Failed to get leader broker for topic " + topic + " partition " + part.partitionId() + " from host " + hp.getHost(), null));
                        continue;
                    }
                    KafkaStreamImporterConfig config = new KafkaStreamImporterConfig(uri, brokerList, topic, part.partitionId(), new HostAndPort(leader.host(), leader.port()), groupId, fetchSize, soTimeout, procedure, commitPolicy, formatterBuilder);
                    configs.put(uri, config);
                }
            }
            if (configs.size() != partitionCount) {
                configs.clear();
                closeConsumer(consumer);
                consumer = null;
            }
        } catch (Exception e) {
            attempts.add(new FailedMetaDataAttempt("Failed to send topic metadata request for topic " + topic + " from host " + hp.getHost(), e));
        } finally {
            closeConsumer(consumer);
        }
    }
    if (!attempts.isEmpty()) {
        attempts.forEach((attempt) -> {
            attempt.log();
        });
        attempts.clear();
        if (configs.isEmpty()) {
            throw new KafkaConfigurationException("Failed to get topic metadata for %s", topic);
        }
    }
    return configs;
}

Example 2 with PartitionMetadata

use of kafka.javaapi.PartitionMetadata in project flink by apache.

the class FlinkKafkaConsumer08 method getPartitionsForTopic.

// ------------------------------------------------------------------------
//  Kafka / ZooKeeper communication utilities
// ------------------------------------------------------------------------
/**
	 * Send request to Kafka to get partitions for topic.
	 * 
	 * @param topics The name of the topics.
	 * @param properties The properties for the Kafka Consumer that is used to query the partitions for the topic. 
	 */
public static List<KafkaTopicPartitionLeader> getPartitionsForTopic(List<String> topics, Properties properties) {
    String seedBrokersConfString = properties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG);
    final int numRetries = getInt(properties, GET_PARTITIONS_RETRIES_KEY, DEFAULT_GET_PARTITIONS_RETRIES);
    checkNotNull(seedBrokersConfString, "Configuration property %s not set", ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG);
    String[] seedBrokers = seedBrokersConfString.split(",");
    List<KafkaTopicPartitionLeader> partitions = new ArrayList<>();
    final String clientId = "flink-kafka-consumer-partition-lookup";
    final int soTimeout = getInt(properties, "socket.timeout.ms", 30000);
    final int bufferSize = getInt(properties, "socket.receive.buffer.bytes", 65536);
    Random rnd = new Random();
    retryLoop: for (int retry = 0; retry < numRetries; retry++) {
        // we pick a seed broker randomly to avoid overloading the first broker with all the requests when the
        // parallel source instances start. Still, we try all available brokers.
        int index = rnd.nextInt(seedBrokers.length);
        brokersLoop: for (int arrIdx = 0; arrIdx < seedBrokers.length; arrIdx++) {
            String seedBroker = seedBrokers[index];
            LOG.info("Trying to get topic metadata from broker {} in try {}/{}", seedBroker, retry, numRetries);
            if (++index == seedBrokers.length) {
                index = 0;
            }
            URL brokerUrl = NetUtils.getCorrectHostnamePort(seedBroker);
            SimpleConsumer consumer = null;
            try {
                consumer = new SimpleConsumer(brokerUrl.getHost(), brokerUrl.getPort(), soTimeout, bufferSize, clientId);
                TopicMetadataRequest req = new TopicMetadataRequest(topics);
                kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
                List<TopicMetadata> metaData = resp.topicsMetadata();
                // clear in case we have an incomplete list from previous tries
                partitions.clear();
                for (TopicMetadata item : metaData) {
                    if (item.errorCode() != ErrorMapping.NoError()) {
                        // warn and try more brokers
                        LOG.warn("Error while getting metadata from broker " + seedBroker + " to find partitions " + "for " + topics.toString() + ". Error: " + ErrorMapping.exceptionFor(item.errorCode()).getMessage());
                        continue brokersLoop;
                    }
                    if (!topics.contains(item.topic())) {
                        LOG.warn("Received metadata from topic " + item.topic() + " even though it was not requested. Skipping ...");
                        continue brokersLoop;
                    }
                    for (PartitionMetadata part : item.partitionsMetadata()) {
                        Node leader = brokerToNode(part.leader());
                        KafkaTopicPartition ktp = new KafkaTopicPartition(item.topic(), part.partitionId());
                        KafkaTopicPartitionLeader pInfo = new KafkaTopicPartitionLeader(ktp, leader);
                        partitions.add(pInfo);
                    }
                }
                // leave the loop through the brokers
                break retryLoop;
            } catch (Exception e) {
                //validates seed brokers in case of a ClosedChannelException
                validateSeedBrokers(seedBrokers, e);
                LOG.warn("Error communicating with broker {} to find partitions for {}. {} Message: {}", seedBroker, topics, e.getClass().getName(), e.getMessage());
                LOG.debug("Detailed trace", e);
                // we sleep a bit. Retrying immediately doesn't make sense in cases where Kafka is reorganizing the leader metadata
                try {
                    Thread.sleep(500);
                } catch (InterruptedException e1) {
                // sleep shorter.
                }
            } finally {
                if (consumer != null) {
                    consumer.close();
                }
            }
        }
    // brokers loop
    }
    // retries loop
    return partitions;
}

Example 3 with PartitionMetadata

use of kafka.javaapi.PartitionMetadata in project apex-malhar by apache.

the class AbstractKafkaInputOperator method definePartitions.

@Override
public Collection<Partitioner.Partition<AbstractKafkaInputOperator<K>>> definePartitions(Collection<Partitioner.Partition<AbstractKafkaInputOperator<K>>> partitions, Partitioner.PartitioningContext context) {
    // Initialize brokers from zookeepers
    getConsumer().initBrokers();
    boolean isInitialParitition = true;
    // check if it's the initial partition
    if (partitions.iterator().hasNext()) {
        isInitialParitition = partitions.iterator().next().getStats() == null;
    }
    // Operator partitions
    List<Partitioner.Partition<AbstractKafkaInputOperator<K>>> newPartitions = null;
    // initialize the offset
    Map<KafkaPartition, Long> initOffset = null;
    if (isInitialParitition && offsetManager != null) {
        initOffset = offsetManager.loadInitialOffsets();
        logger.info("Initial offsets: {} ", "{ " + Joiner.on(", ").useForNull("").withKeyValueSeparator(": ").join(initOffset) + " }");
    }
    Set<Integer> deletedOperators = Sets.newHashSet();
    Collection<Partition<AbstractKafkaInputOperator<K>>> resultPartitions = partitions;
    boolean numPartitionsChanged = false;
    switch(strategy) {
        // Each operator partition will consume from only one kafka partition
        case ONE_TO_ONE:
            if (isInitialParitition) {
                lastRepartitionTime = System.currentTimeMillis();
                logger.info("[ONE_TO_ONE]: Initializing partition(s)");
                // get partition metadata for topics.
                // Whatever operator is using high-level or simple kafka consumer, the operator always create a temporary simple kafka consumer to get the metadata of the topic
                // The initial value of brokerList of the KafkaConsumer is used to retrieve the topic metadata
                Map<String, List<PartitionMetadata>> kafkaPartitions = KafkaMetadataUtil.getPartitionsForTopic(getConsumer().brokers, getConsumer().getTopic());
                // initialize the number of operator partitions according to number of kafka partitions
                newPartitions = new LinkedList<Partitioner.Partition<AbstractKafkaInputOperator<K>>>();
                for (Map.Entry<String, List<PartitionMetadata>> kp : kafkaPartitions.entrySet()) {
                    String clusterId = kp.getKey();
                    for (PartitionMetadata pm : kp.getValue()) {
                        logger.info("[ONE_TO_ONE]: Create operator partition for cluster {}, topic {}, kafka partition {} ", clusterId, getConsumer().topic, pm.partitionId());
                        newPartitions.add(createPartition(Sets.newHashSet(new KafkaPartition(clusterId, consumer.topic, pm.partitionId())), initOffset));
                    }
                }
                resultPartitions = newPartitions;
                numPartitionsChanged = true;
            } else if (newWaitingPartition.size() != 0) {
                // add partition for new kafka partition
                for (KafkaPartition newPartition : newWaitingPartition) {
                    logger.info("[ONE_TO_ONE]: Add operator partition for cluster {}, topic {}, partition {}", newPartition.getClusterId(), getConsumer().topic, newPartition.getPartitionId());
                    partitions.add(createPartition(Sets.newHashSet(newPartition), null));
                }
                newWaitingPartition.clear();
                resultPartitions = partitions;
                numPartitionsChanged = true;
            }
            break;
        // and guarantee the total intake rate for each operator partition is below some threshold
        case ONE_TO_MANY:
            if (getConsumer() instanceof HighlevelKafkaConsumer) {
                throw new UnsupportedOperationException("[ONE_TO_MANY]: The high-level consumer is not supported for ONE_TO_MANY partition strategy.");
            }
            if (isInitialParitition || newWaitingPartition.size() != 0) {
                lastRepartitionTime = System.currentTimeMillis();
                logger.info("[ONE_TO_MANY]: Initializing partition(s)");
                // get partition metadata for topics.
                // Whatever operator is using high-level or simple kafka consumer, the operator always create a temporary simple kafka consumer to get the metadata of the topic
                // The initial value of brokerList of the KafkaConsumer is used to retrieve the topic metadata
                Map<String, List<PartitionMetadata>> kafkaPartitions = KafkaMetadataUtil.getPartitionsForTopic(getConsumer().brokers, getConsumer().getTopic());
                int size = initialPartitionCount;
                @SuppressWarnings("unchecked") Set<KafkaPartition>[] kps = (Set<KafkaPartition>[]) Array.newInstance((new HashSet<KafkaPartition>()).getClass(), size);
                int i = 0;
                for (Map.Entry<String, List<PartitionMetadata>> en : kafkaPartitions.entrySet()) {
                    String clusterId = en.getKey();
                    for (PartitionMetadata pm : en.getValue()) {
                        if (kps[i % size] == null) {
                            kps[i % size] = new HashSet<KafkaPartition>();
                        }
                        kps[i % size].add(new KafkaPartition(clusterId, consumer.topic, pm.partitionId()));
                        i++;
                    }
                }
                size = i > size ? size : i;
                newPartitions = new ArrayList<Partitioner.Partition<AbstractKafkaInputOperator<K>>>(size);
                for (i = 0; i < size; i++) {
                    logger.info("[ONE_TO_MANY]: Create operator partition for kafka partition(s): {} ", StringUtils.join(kps[i], ", "));
                    newPartitions.add(createPartition(kps[i], initOffset));
                }
                // Add the existing partition Ids to the deleted operators
                for (Partition<AbstractKafkaInputOperator<K>> op : partitions) {
                    deletedOperators.add(op.getPartitionedInstance().operatorId);
                }
                newWaitingPartition.clear();
                resultPartitions = newPartitions;
                numPartitionsChanged = true;
            }
            break;
        case ONE_TO_MANY_HEURISTIC:
            throw new UnsupportedOperationException("[ONE_TO_MANY_HEURISTIC]: Not implemented yet");
        default:
            break;
    }
    if (numPartitionsChanged) {
        List<WindowDataManager> managers = windowDataManager.partition(resultPartitions.size(), deletedOperators);
        int i = 0;
        for (Partition<AbstractKafkaInputOperator<K>> partition : resultPartitions) {
            partition.getPartitionedInstance().setWindowDataManager(managers.get(i++));
        }
    }
    return resultPartitions;
}

Example 4 with PartitionMetadata

use of kafka.javaapi.PartitionMetadata in project jstorm by alibaba.

the class KafkaConsumer method findLeaderConsumer.

private SimpleConsumer findLeaderConsumer(int partition) {
    try {
        if (consumer != null) {
            return consumer;
        }
        PartitionMetadata metadata = findLeader(partition);
        if (metadata == null) {
            leaderBroker = null;
            consumer = null;
            return null;
        }
        leaderBroker = metadata.leader();
        consumer = new SimpleConsumer(leaderBroker.host(), leaderBroker.port(), config.socketTimeoutMs, config.socketReceiveBufferBytes, config.clientId);
        return consumer;
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
    }
    return null;
}

Example 5 with PartitionMetadata

use of kafka.javaapi.PartitionMetadata in project druid by druid-io.

the class KafkaSimpleConsumer method findLeader.

private PartitionMetadata findLeader() throws InterruptedException {
    for (HostAndPort broker : replicaBrokers) {
        SimpleConsumer consumer = null;
        try {
            log.info("Finding new leader from Kafka brokers, try broker [%s]", broker.toString());
            consumer = new SimpleConsumer(broker.getHostText(), broker.getPort(), SO_TIMEOUT, BUFFER_SIZE, leaderLookupClientId);
            TopicMetadataResponse resp = consumer.send(new TopicMetadataRequest(Collections.singletonList(topic)));
            List<TopicMetadata> metaData = resp.topicsMetadata();
            for (TopicMetadata item : metaData) {
                if (topic.equals(item.topic())) {
                    for (PartitionMetadata part : item.partitionsMetadata()) {
                        if (part.partitionId() == partitionId) {
                            return part;
                        }
                    }
                }
            }
        } catch (Exception e) {
            ensureNotInterrupted(e);
            log.warn(e, "error communicating with Kafka Broker [%s] to find leader for [%s] - [%s]", broker, topic, partitionId);
        } finally {
            if (consumer != null) {
                consumer.close();
            }
        }
    }
    return null;
}