Examples with SimpleConsumer kafka.javaapi.consumer.SimpleConsumer used on opensource projects

Example 16 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project cdap by caskdata.

the class KafkaConsumer method getConsumer.

private SimpleConsumer getConsumer() {
    if (consumer != null) {
        return consumer;
    }
    BrokerInfo leader = brokerService.getLeader(topic, partition);
    consumer = new SimpleConsumer(leader.getHost(), leader.getPort(), TIMEOUT_MS, BUFFER_SIZE_BYTES, clientName);
    return consumer;
}

Example 17 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project cdap by caskdata.

the class KafkaConsumer method fetchOffsetBefore.

/**
   * Fetch offset before given time.
   * @param timeMillis offset to fetch before timeMillis.
   * @return Kafka message offset
   */
public long fetchOffsetBefore(long timeMillis) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = Maps.newHashMap();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(timeMillis, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, CurrentVersion(), clientName);
    SimpleConsumer consumer = getConsumer();
    OffsetResponse response = consumer.getOffsetsBefore(request);
    if (response.hasError()) {
        // Try once more
        closeConsumer();
        consumer = getConsumer();
        response = consumer.getOffsetsBefore(request);
        if (response.hasError()) {
            closeConsumer();
            throw new RuntimeException(String.format("Error fetching offset data from broker %s:%d for topic %s, partition %d. Error code: %d", consumer.host(), consumer.port(), topic, partition, response.errorCode(topic, partition)));
        }
    }
    long[] offsets = response.offsets(topic, partition);
    if (offsets.length > 0) {
        return offsets[0];
    }
    // Otherwise throw exception.
    if (timeMillis != kafka.api.OffsetRequest.EarliestTime()) {
        return fetchOffsetBefore(kafka.api.OffsetRequest.EarliestTime());
    }
    closeConsumer();
    throw new RuntimeException(String.format("Got zero offsets in offset response for time %s from broker %s:%d for topic %s, partition %d", timeMillis, consumer.host(), consumer.port(), topic, partition));
}

Example 18 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project storm by apache.

the class KafkaOffsetLagUtil method getLogHeadOffsets.

private static Map<String, Map<Integer, Long>> getLogHeadOffsets(Map<String, List<TopicPartition>> leadersAndTopicPartitions) {
    Map<String, Map<Integer, Long>> result = new HashMap<>();
    if (leadersAndTopicPartitions != null) {
        PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(OffsetRequest.LatestTime(), 1);
        SimpleConsumer simpleConsumer = null;
        for (Map.Entry<String, List<TopicPartition>> leader : leadersAndTopicPartitions.entrySet()) {
            try {
                simpleConsumer = new SimpleConsumer(leader.getKey().split(":")[0], Integer.parseInt(leader.getKey().split(":")[1]), 10000, 64 * 1024, "LogHeadOffsetRequest");
                Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
                for (TopicPartition topicPartition : leader.getValue()) {
                    requestInfo.put(new TopicAndPartition(topicPartition.topic(), topicPartition.partition()), partitionOffsetRequestInfo);
                    if (!result.containsKey(topicPartition.topic())) {
                        result.put(topicPartition.topic(), new HashMap<Integer, Long>());
                    }
                }
                kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), "LogHeadOffsetRequest");
                OffsetResponse response = simpleConsumer.getOffsetsBefore(request);
                for (TopicPartition topicPartition : leader.getValue()) {
                    result.get(topicPartition.topic()).put(topicPartition.partition(), response.offsets(topicPartition.topic(), topicPartition.partition())[0]);
                }
            } finally {
                if (simpleConsumer != null) {
                    simpleConsumer.close();
                }
            }
        }
    }
    return result;
}

Example 19 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project storm by apache.

the class TridentKafkaEmitter method reEmitPartitionBatch.

/**
     * re-emit the batch described by the meta data provided
     *
     * @param attempt
     * @param collector
     * @param partition
     * @param meta
     */
private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map meta) {
    LOG.info("re-emitting batch, attempt " + attempt);
    String instanceId = (String) meta.get("instanceId");
    if (!_config.ignoreZkOffsets || instanceId.equals(_topologyInstanceId)) {
        SimpleConsumer consumer = _connections.register(partition);
        long offset = (Long) meta.get("offset");
        long nextOffset = (Long) meta.get("nextOffset");
        ByteBufferMessageSet msgs = null;
        msgs = fetchMessages(consumer, partition, offset);
        if (msgs != null) {
            for (MessageAndOffset msg : msgs) {
                if (offset == nextOffset) {
                    break;
                }
                if (offset > nextOffset) {
                    throw new RuntimeException("Error when re-emitting batch. overshot the end offset");
                }
                emit(collector, msg.message(), partition, msg.offset(), attempt);
                offset = msg.nextOffset();
            }
        }
    }
}

Example 20 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project storm by apache.

the class TridentKafkaEmitter method failFastEmitNewPartitionBatch.

private Map failFastEmitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMeta) {
    SimpleConsumer consumer = _connections.register(partition);
    Map ret = doEmitNewPartitionBatch(consumer, partition, collector, lastMeta, attempt);
    Long offset = (Long) ret.get("offset");
    Long endOffset = (Long) ret.get("nextOffset");
    _kafkaOffsetMetric.setOffsetData(partition, new PartitionManager.OffsetData(endOffset, offset));
    return ret;
}