Examples with OffsetRequest kafka.javaapi.OffsetRequest used on opensource projects

Example 6 with OffsetRequest

use of kafka.javaapi.OffsetRequest in project jstorm by alibaba.

the class KafkaConsumer method getOffset.

public long getOffset(String topic, int partition, long startOffsetTime) {
    SimpleConsumer simpleConsumer = findLeaderConsumer(partition);
    if (simpleConsumer == null) {
        LOG.error("Error consumer is null get offset from partition:" + partition);
        return -1;
    }
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId());
    long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition);
    if (offsets.length > 0) {
        return offsets[0];
    } else {
        return NO_OFFSET;
    }
}

Example 7 with OffsetRequest

use of kafka.javaapi.OffsetRequest in project heron by twitter.

the class KafkaUtils method getOffset.

public static long getOffset(SimpleConsumer consumer, String topic, int partition, long startOffsetTime) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
    long[] offsets = consumer.getOffsetsBefore(request).offsets(topic, partition);
    if (offsets.length > 0) {
        return offsets[0];
    } else {
        return NO_OFFSET;
    }
}

Example 8 with OffsetRequest

use of kafka.javaapi.OffsetRequest in project cdap by caskdata.

the class KafkaUtil method getOffsetByTimestamp.

/**
   * Fetch the starting offset of the last segment whose latest message is published before the given timestamp.
   * The timestamp can also be special value {@link OffsetRequest$#EarliestTime()}
   * or {@link OffsetRequest$#LatestTime()}.
   *
   * @param consumer the consumer to send request to and receive response from
   * @param topic the topic for fetching the offset from
   * @param partition the partition for fetching the offset from
   * @param timestamp the timestamp to use for fetching last offset before it
   * @return the latest offset
   *
   * @throws NotLeaderForPartitionException if the broker that the consumer is talking to is not the leader
   *                                        for the given topic and partition.
   * @throws UnknownTopicOrPartitionException if the topic or partition is not known by the Kafka server
   * @throws UnknownServerException if the Kafka server responded with error.
   */
public static long getOffsetByTimestamp(SimpleConsumer consumer, String topic, int partition, long timestamp) throws KafkaException {
    // Fire offset request
    OffsetRequest request = new OffsetRequest(ImmutableMap.of(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(timestamp, 1)), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
    OffsetResponse response = consumer.getOffsetsBefore(request);
    if (response.hasError()) {
        throw Errors.forCode(response.errorCode(topic, partition)).exception();
    }
    // Retrieve offsets from response
    long[] offsets = response.offsets(topic, partition);
    if (offsets.length == 0) {
        if (timestamp != kafka.api.OffsetRequest.EarliestTime()) {
            // Hence, use the earliest time to find out the offset
            return getOffsetByTimestamp(consumer, topic, partition, kafka.api.OffsetRequest.EarliestTime());
        }
        // This shouldn't happen. The find earliest offset response should return at least one offset.
        throw new UnknownServerException("Empty offsets received from offsets request on " + topic + ":" + partition + " from broker " + consumer.host() + ":" + consumer.port());
    }
    LOG.debug("Offset {} fetched for {}:{} with timestamp {}.", offsets[0], topic, partition, timestamp);
    return offsets[0];
}