Examples with OffsetResponse kafka.javaapi.OffsetResponse used on opensource projects

Example 11 with OffsetResponse

use of kafka.javaapi.OffsetResponse in project voltdb by VoltDB.

the class KafkaTopicPartitionImporter method getLastOffset.

public long getLastOffset() {
    final int partition = m_topicAndPartition.partition();
    final String topic = m_topicAndPartition.topic();
    OffsetResponse response = getTopicOffset(EARLIEST_OFFSET);
    if (response == null)
        return -1L;
    long earliest = response.offsets(topic, partition)[0];
    response = getTopicOffset(LATEST_OFFSET);
    if (response == null)
        return -1L;
    long latest = response.offsets(topic, partition)[0];
    if (latest == earliest)
        return latest;
    OffsetFetchResponse ofr = getClientTopicOffset();
    if (ofr == null)
        return earliest;
    long current = ofr.offsets().get(m_topicAndPartition).offset();
    if (current < earliest)
        return earliest;
    if (current < latest)
        return current;
    return latest;
}

Example 12 with OffsetResponse

use of kafka.javaapi.OffsetResponse in project opennms by OpenNMS.

the class KafkaOffsetProvider method getLastOffset.

public long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) {
    long lastOffset = 0;
    try {
        List<String> topics = Collections.singletonList(topic);
        TopicMetadataRequest req = new TopicMetadataRequest(topics);
        kafka.javaapi.TopicMetadataResponse topicMetadataResponse = consumer.send(req);
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
        for (TopicMetadata topicMetadata : topicMetadataResponse.topicsMetadata()) {
            for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) {
                if (partitionMetadata.partitionId() == partition) {
                    String partitionHost = partitionMetadata.leader().host();
                    consumer = getConsumer(partitionHost, partitionMetadata.leader().port());
                    break;
                }
            }
        }
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), KafkaOffsetConstants.CLIENT_NAME);
        OffsetResponse response = consumer.getOffsetsBefore(request);
        if (response.hasError()) {
            LOGGER.error("Error fetching Offset Data from the Broker. Reason: {}", response.errorCode(topic, partition));
            lastOffset = 0;
        }
        long[] offsets = response.offsets(topic, partition);
        lastOffset = offsets[0];
    } catch (Exception e) {
        LOGGER.error("Error while collecting the log Size for topic: {}:{} ", topic, partition, e);
    }
    return lastOffset;
}

Example 13 with OffsetResponse

use of kafka.javaapi.OffsetResponse in project cdap by caskdata.

the class KafkaUtil method getOffsetByTimestamp.

/**
 * Fetch the starting offset of the last segment whose latest message is published before the given timestamp.
 * The timestamp can also be special value {@link OffsetRequest$#EarliestTime()}
 * or {@link OffsetRequest$#LatestTime()}.
 *
 * @param consumer the consumer to send request to and receive response from
 * @param topic the topic for fetching the offset from
 * @param partition the partition for fetching the offset from
 * @param timestamp the timestamp to use for fetching last offset before it
 * @return the latest offset
 *
 * @throws NotLeaderForPartitionException if the broker that the consumer is talking to is not the leader
 *                                        for the given topic and partition.
 * @throws UnknownTopicOrPartitionException if the topic or partition is not known by the Kafka server
 * @throws UnknownServerException if the Kafka server responded with error.
 */
public static long getOffsetByTimestamp(SimpleConsumer consumer, String topic, int partition, long timestamp) throws KafkaException {
    // Fire offset request
    OffsetRequest request = new OffsetRequest(ImmutableMap.of(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(timestamp, 1)), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
    OffsetResponse response = consumer.getOffsetsBefore(request);
    if (response.hasError()) {
        throw Errors.forCode(response.errorCode(topic, partition)).exception();
    }
    // Retrieve offsets from response
    long[] offsets = response.offsets(topic, partition);
    if (offsets.length == 0) {
        if (timestamp != kafka.api.OffsetRequest.EarliestTime()) {
            // Hence, use the earliest time to find out the offset
            return getOffsetByTimestamp(consumer, topic, partition, kafka.api.OffsetRequest.EarliestTime());
        }
        // This shouldn't happen. The find earliest offset response should return at least one offset.
        throw new UnknownServerException("Empty offsets received from offsets request on " + topic + ":" + partition + " from broker " + consumer.host() + ":" + consumer.port());
    }
    LOG.debug("Offset {} fetched for {}:{} with timestamp {}.", offsets[0], topic, partition, timestamp);
    return offsets[0];
}