Examples with FetchRequestBuilder kafka.api.FetchRequestBuilder used on opensource projects

Example 6 with FetchRequestBuilder

use of kafka.api.FetchRequestBuilder in project storm by apache.

the class KafkaUtils method fetchMessages.

public static ByteBufferMessageSet fetchMessages(KafkaConfig config, SimpleConsumer consumer, Partition partition, long offset) throws TopicOffsetOutOfRangeException, FailedFetchException, RuntimeException {
    ByteBufferMessageSet msgs = null;
    String topic = partition.topic;
    int partitionId = partition.partition;
    FetchRequestBuilder builder = new FetchRequestBuilder();
    FetchRequest fetchRequest = builder.addFetch(topic, partitionId, offset, config.fetchSizeBytes).clientId(config.clientId).maxWait(config.fetchMaxWait).minBytes(config.minFetchByte).build();
    FetchResponse fetchResponse;
    try {
        fetchResponse = consumer.fetch(fetchRequest);
    } catch (Exception e) {
        if (e instanceof ConnectException || e instanceof SocketTimeoutException || e instanceof IOException || e instanceof UnresolvedAddressException) {
            LOG.warn("Network error when fetching messages:", e);
            throw new FailedFetchException(e);
        } else {
            throw new RuntimeException(e);
        }
    }
    if (fetchResponse.hasError()) {
        KafkaError error = KafkaError.getError(fetchResponse.errorCode(topic, partitionId));
        if (error.equals(KafkaError.OFFSET_OUT_OF_RANGE) && config.useStartOffsetTimeIfOffsetOutOfRange) {
            String msg = partition + " Got fetch request with offset out of range: [" + offset + "]";
            LOG.warn(msg);
            throw new TopicOffsetOutOfRangeException(msg);
        } else {
            String message = "Error fetching data from [" + partition + "] for topic [" + topic + "]: [" + error + "]";
            LOG.error(message);
            throw new FailedFetchException(message);
        }
    } else {
        msgs = fetchResponse.messageSet(topic, partitionId);
    }
    LOG.debug("Messages fetched. [config = {}], [consumer = {}], [partition = {}], [offset = {}], [msgs = {}]", config, consumer, partition, offset, msgs);
    return msgs;
}

Example 7 with FetchRequestBuilder

use of kafka.api.FetchRequestBuilder in project pinot by linkedin.

the class SimpleConsumerWrapper method fetchMessagesAndHighWatermark.

/**
   * Fetch messages and the per-partition high watermark from Kafka between the specified offsets.
   *
   * @param startOffset The offset of the first message desired, inclusive
   * @param endOffset The offset of the last message desired, exclusive, or {@link Long#MAX_VALUE} for no end offset.
   * @param timeoutMillis Timeout in milliseconds
   * @throws java.util.concurrent.TimeoutException If the operation could not be completed within {@code timeoutMillis}
   * milliseconds
   * @return An iterable containing messages fetched from Kafka and their offsets, as well as the high watermark for
   * this partition.
   */
public synchronized Pair<Iterable<MessageAndOffset>, Long> fetchMessagesAndHighWatermark(long startOffset, long endOffset, int timeoutMillis) throws java.util.concurrent.TimeoutException {
    Preconditions.checkState(!_metadataOnlyConsumer, "Cannot fetch messages from a metadata-only SimpleConsumerWrapper");
    // Ensure that we're connected to the leader
    // TODO Improve error handling
    final long connectEndTime = System.currentTimeMillis() + _connectTimeoutMillis;
    while (_currentState.getStateValue() != ConsumerState.CONNECTED_TO_PARTITION_LEADER && System.currentTimeMillis() < connectEndTime) {
        _currentState.process();
    }
    if (_currentState.getStateValue() != ConsumerState.CONNECTED_TO_PARTITION_LEADER && connectEndTime <= System.currentTimeMillis()) {
        throw new java.util.concurrent.TimeoutException();
    }
    FetchResponse fetchResponse = _simpleConsumer.fetch(new FetchRequestBuilder().minBytes(100000).maxWait(timeoutMillis).addFetch(_topic, _partition, startOffset, 500000).build());
    if (!fetchResponse.hasError()) {
        final Iterable<MessageAndOffset> messageAndOffsetIterable = buildOffsetFilteringIterable(fetchResponse.messageSet(_topic, _partition), startOffset, endOffset);
        return Pair.of(messageAndOffsetIterable, fetchResponse.highWatermark(_topic, _partition));
    } else {
        throw exceptionForKafkaErrorCode(fetchResponse.errorCode(_topic, _partition));
    }
}

Example 8 with FetchRequestBuilder

use of kafka.api.FetchRequestBuilder in project cdap by caskdata.

the class KafkaConsumer method fetchMessageSet.

private ByteBufferMessageSet fetchMessageSet(long fetchOffset) throws OffsetOutOfRangeException {
    Preconditions.checkArgument(fetchOffset >= 0, String.format("Illegal fetch offset %d", fetchOffset));
    int failureCount = 0;
    while (true) {
        SimpleConsumer consumer = getConsumer();
        FetchRequest req = new FetchRequestBuilder().clientId(clientName).addFetch(topic, partition, fetchOffset, BUFFER_SIZE_BYTES).maxWait(fetchTimeoutMs).build();
        FetchResponse fetchResponse = consumer.fetch(req);
        if (!fetchResponse.hasError()) {
            return fetchResponse.messageSet(topic, partition);
        }
        short errorCode = fetchResponse.errorCode(topic, partition);
        if (++failureCount >= MAX_KAFKA_FETCH_RETRIES) {
            throw new RuntimeException(String.format("Error fetching data from broker %s:%d for topic %s, partition %d. Error code: %d", consumer.host(), consumer.port(), topic, partition, errorCode));
        }
        LOG.warn("Error fetching data from broker {}:{} for topic {}, partition {}. Error code: {}", consumer.host(), consumer.port(), topic, partition, errorCode);
        if (errorCode == ErrorMapping.OffsetOutOfRangeCode()) {
            throw new OffsetOutOfRangeException(String.format("Requested offset %d is out of range for topic %s partition %d", fetchOffset, topic, partition));
        }
        closeConsumer();
    }
}

Example 9 with FetchRequestBuilder

use of kafka.api.FetchRequestBuilder in project cdap by caskdata.

the class KafkaUtil method fetchMessages.

/**
   * Fetches messages from the given topic, partition and offset using the provided {@link SimpleConsumer}.
   *
   * @return A {@link ByteBufferMessageSet} of the given topic, partition for messages fetched
   *
   * @throws OffsetOutOfRangeException if the given offset is out of range.
   * @throws NotLeaderForPartitionException if the broker that the consumer is talking to is not the leader
   *                                        for the given topic and partition.
   * @throws UnknownTopicOrPartitionException if the topic or partition is not known by the Kafka server
   * @throws UnknownServerException if the Kafka server responded with error.
   */
public static ByteBufferMessageSet fetchMessages(SimpleConsumer consumer, String topic, int partition, int fetchSize, long requestOffset) throws KafkaException {
    FetchRequest req = new FetchRequestBuilder().clientId(consumer.clientId()).addFetch(topic, partition, requestOffset, fetchSize).build();
    FetchResponse fetchResponse = consumer.fetch(req);
    if (fetchResponse.hasError()) {
        throw Errors.forCode(fetchResponse.errorCode(topic, partition)).exception();
    }
    return fetchResponse.messageSet(topic, partition);
}