Examples with FetchRequest kafka.api.FetchRequest used on opensource projects

Example 1 with FetchRequest

use of kafka.api.FetchRequest in project kafka by apache.

the class SimpleConsumerDemo method main.

public static void main(String[] args) throws Exception {
    generateData();
    SimpleConsumer simpleConsumer = new SimpleConsumer(KafkaProperties.KAFKA_SERVER_URL, KafkaProperties.KAFKA_SERVER_PORT, KafkaProperties.CONNECTION_TIMEOUT, KafkaProperties.KAFKA_PRODUCER_BUFFER_SIZE, KafkaProperties.CLIENT_ID);
    System.out.println("Testing single fetch");
    FetchRequest req = new FetchRequestBuilder().clientId(KafkaProperties.CLIENT_ID).addFetch(KafkaProperties.TOPIC2, 0, 0L, 100).build();
    FetchResponse fetchResponse = simpleConsumer.fetch(req);
    printMessages(fetchResponse.messageSet(KafkaProperties.TOPIC2, 0));
    System.out.println("Testing single multi-fetch");
    Map<String, List<Integer>> topicMap = new HashMap<>();
    topicMap.put(KafkaProperties.TOPIC2, Collections.singletonList(0));
    topicMap.put(KafkaProperties.TOPIC3, Collections.singletonList(0));
    req = new FetchRequestBuilder().clientId(KafkaProperties.CLIENT_ID).addFetch(KafkaProperties.TOPIC2, 0, 0L, 100).addFetch(KafkaProperties.TOPIC3, 0, 0L, 100).build();
    fetchResponse = simpleConsumer.fetch(req);
    int fetchReq = 0;
    for (Map.Entry<String, List<Integer>> entry : topicMap.entrySet()) {
        String topic = entry.getKey();
        for (Integer offset : entry.getValue()) {
            System.out.println("Response from fetch request no: " + ++fetchReq);
            printMessages(fetchResponse.messageSet(topic, offset));
        }
    }
}

Example 2 with FetchRequest

use of kafka.api.FetchRequest in project druid by druid-io.

the class KafkaSimpleConsumer method fetch.

public Iterable<BytesMessageWithOffset> fetch(long offset, int timeoutMs) throws InterruptedException {
    FetchResponse response = null;
    Broker previousLeader = leaderBroker;
    while (true) {
        ensureConsumer(previousLeader);
        FetchRequest request = new FetchRequestBuilder().clientId(clientId).addFetch(topic, partitionId, offset, FETCH_SIZE).maxWait(timeoutMs).minBytes(1).build();
        log.debug("fetch offset %s", offset);
        try {
            response = consumer.fetch(request);
        } catch (Exception e) {
            ensureNotInterrupted(e);
            log.warn(e, "caught exception in fetch {} - {}", topic, partitionId);
            response = null;
        }
        if (response == null || response.hasError()) {
            short errorCode = response != null ? response.errorCode(topic, partitionId) : ErrorMapping.UnknownCode();
            log.warn("fetch %s - %s with offset %s encounters error: [%s]", topic, partitionId, offset, errorCode);
            boolean needNewLeader = false;
            if (errorCode == ErrorMapping.RequestTimedOutCode()) {
                log.info("kafka request timed out, response[%s]", response);
            } else if (errorCode == ErrorMapping.OffsetOutOfRangeCode()) {
                long newOffset = getOffset(earliest);
                log.info("got [%s] offset[%s] for [%s][%s]", earliest ? "earliest" : "latest", newOffset, topic, partitionId);
                if (newOffset < 0) {
                    needNewLeader = true;
                } else {
                    offset = newOffset;
                    continue;
                }
            } else {
                needNewLeader = true;
            }
            if (needNewLeader) {
                stopConsumer();
                previousLeader = leaderBroker;
                leaderBroker = null;
                continue;
            }
        } else {
            break;
        }
    }
    return response != null ? filterAndDecode(response.messageSet(topic, partitionId), offset) : EMPTY_MSGS;
}

Example 3 with FetchRequest

use of kafka.api.FetchRequest in project jstorm by alibaba.

the class KafkaConsumer method fetchMessages.

public ByteBufferMessageSet fetchMessages(int partition, long offset) throws IOException {
    String topic = config.topic;
    FetchRequest req = new FetchRequestBuilder().clientId(config.clientId).addFetch(topic, partition, offset, config.fetchMaxBytes).maxWait(config.fetchWaitMaxMs).build();
    FetchResponse fetchResponse = null;
    SimpleConsumer simpleConsumer = null;
    try {
        simpleConsumer = findLeaderConsumer(partition);
        if (simpleConsumer == null) {
            // LOG.error(message);
            return null;
        }
        fetchResponse = simpleConsumer.fetch(req);
    } catch (Exception e) {
        if (e instanceof ConnectException || e instanceof SocketTimeoutException || e instanceof IOException || e instanceof UnresolvedAddressException) {
            LOG.warn("Network error when fetching messages:", e);
            if (simpleConsumer != null) {
                String host = simpleConsumer.host();
                int port = simpleConsumer.port();
                simpleConsumer = null;
                throw new KafkaException("Network error when fetching messages: " + host + ":" + port + " , " + e.getMessage(), e);
            }
        } else {
            throw new RuntimeException(e);
        }
    }
    if (fetchResponse.hasError()) {
        short code = fetchResponse.errorCode(topic, partition);
        if (code == ErrorMapping.OffsetOutOfRangeCode() && config.resetOffsetIfOutOfRange) {
            long startOffset = getOffset(topic, partition, config.startOffsetTime);
            offset = startOffset;
        }
        if (leaderBroker != null) {
            LOG.error("fetch data from kafka topic[" + config.topic + "] host[" + leaderBroker.host() + ":" + leaderBroker.port() + "] partition[" + partition + "] error:" + code);
        } else {
        }
        return null;
    } else {
        ByteBufferMessageSet msgs = fetchResponse.messageSet(topic, partition);
        return msgs;
    }
}

Example 4 with FetchRequest

use of kafka.api.FetchRequest in project heron by twitter.

the class KafkaUtils method fetchMessages.

/***
   * Fetch messages from kafka
   * @param config
   * @param consumer
   * @param partition
   * @param offset
   * @return
   * @throws TopicOffsetOutOfRangeException
   * @throws FailedFetchException
   * @throws RuntimeException
   */
public static ByteBufferMessageSet fetchMessages(KafkaConfig config, SimpleConsumer consumer, Partition partition, long offset) throws TopicOffsetOutOfRangeException, FailedFetchException, RuntimeException {
    ByteBufferMessageSet msgs = null;
    String topic = partition.topic;
    int partitionId = partition.partition;
    FetchRequestBuilder builder = new FetchRequestBuilder();
    FetchRequest fetchRequest = builder.addFetch(topic, partitionId, offset, config.fetchSizeBytes).clientId(config.clientId).maxWait(config.fetchMaxWait).minBytes(config.minFetchByte).build();
    FetchResponse fetchResponse;
    try {
        fetchResponse = consumer.fetch(fetchRequest);
    // SUPPRESS CHECKSTYLE IllegalCatch
    } catch (Exception e) {
        if (e instanceof ConnectException || e instanceof SocketTimeoutException || e instanceof IOException || e instanceof UnresolvedAddressException) {
            LOG.warn("Network error when fetching messages:", e);
            throw new FailedFetchException(e);
        } else {
            throw new RuntimeException(e);
        }
    }
    if (fetchResponse.hasError()) {
        KafkaError error = KafkaError.getError(fetchResponse.errorCode(topic, partitionId));
        if (error.equals(KafkaError.OFFSET_OUT_OF_RANGE) && config.useStartOffsetTimeIfOffsetOutOfRange) {
            String msg = partition + " Got fetch request with offset out of range: [" + offset + "]";
            LOG.warn(msg);
            throw new TopicOffsetOutOfRangeException(msg);
        } else {
            String message = "Error fetching data from [" + partition + "] for topic [" + topic + "]: [" + error + "]";
            LOG.error(message);
            throw new FailedFetchException(message);
        }
    } else {
        msgs = fetchResponse.messageSet(topic, partitionId);
    }
    return msgs;
}

Example 5 with FetchRequest

use of kafka.api.FetchRequest in project storm by apache.

the class KafkaUtils method fetchMessages.

public static ByteBufferMessageSet fetchMessages(KafkaConfig config, SimpleConsumer consumer, Partition partition, long offset) throws TopicOffsetOutOfRangeException, FailedFetchException, RuntimeException {
    ByteBufferMessageSet msgs = null;
    String topic = partition.topic;
    int partitionId = partition.partition;
    FetchRequestBuilder builder = new FetchRequestBuilder();
    FetchRequest fetchRequest = builder.addFetch(topic, partitionId, offset, config.fetchSizeBytes).clientId(config.clientId).maxWait(config.fetchMaxWait).minBytes(config.minFetchByte).build();
    FetchResponse fetchResponse;
    try {
        fetchResponse = consumer.fetch(fetchRequest);
    } catch (Exception e) {
        if (e instanceof ConnectException || e instanceof SocketTimeoutException || e instanceof IOException || e instanceof UnresolvedAddressException) {
            LOG.warn("Network error when fetching messages:", e);
            throw new FailedFetchException(e);
        } else {
            throw new RuntimeException(e);
        }
    }
    if (fetchResponse.hasError()) {
        KafkaError error = KafkaError.getError(fetchResponse.errorCode(topic, partitionId));
        if (error.equals(KafkaError.OFFSET_OUT_OF_RANGE) && config.useStartOffsetTimeIfOffsetOutOfRange) {
            String msg = partition + " Got fetch request with offset out of range: [" + offset + "]";
            LOG.warn(msg);
            throw new TopicOffsetOutOfRangeException(msg);
        } else {
            String message = "Error fetching data from [" + partition + "] for topic [" + topic + "]: [" + error + "]";
            LOG.error(message);
            throw new FailedFetchException(message);
        }
    } else {
        msgs = fetchResponse.messageSet(topic, partitionId);
    }
    LOG.debug("Messages fetched. [config = {}], [consumer = {}], [partition = {}], [offset = {}], [msgs = {}]", config, consumer, partition, offset, msgs);
    return msgs;
}