Examples with ByteBufferMessageSet kafka.javaapi.message.ByteBufferMessageSet used on opensource projects

Example 11 with ByteBufferMessageSet

use of kafka.javaapi.message.ByteBufferMessageSet in project heron by twitter.

the class KafkaUtils method fetchMessages.

/***
   * Fetch messages from kafka
   * @param config
   * @param consumer
   * @param partition
   * @param offset
   * @return
   * @throws TopicOffsetOutOfRangeException
   * @throws FailedFetchException
   * @throws RuntimeException
   */
public static ByteBufferMessageSet fetchMessages(KafkaConfig config, SimpleConsumer consumer, Partition partition, long offset) throws TopicOffsetOutOfRangeException, FailedFetchException, RuntimeException {
    ByteBufferMessageSet msgs = null;
    String topic = partition.topic;
    int partitionId = partition.partition;
    FetchRequestBuilder builder = new FetchRequestBuilder();
    FetchRequest fetchRequest = builder.addFetch(topic, partitionId, offset, config.fetchSizeBytes).clientId(config.clientId).maxWait(config.fetchMaxWait).minBytes(config.minFetchByte).build();
    FetchResponse fetchResponse;
    try {
        fetchResponse = consumer.fetch(fetchRequest);
    // SUPPRESS CHECKSTYLE IllegalCatch
    } catch (Exception e) {
        if (e instanceof ConnectException || e instanceof SocketTimeoutException || e instanceof IOException || e instanceof UnresolvedAddressException) {
            LOG.warn("Network error when fetching messages:", e);
            throw new FailedFetchException(e);
        } else {
            throw new RuntimeException(e);
        }
    }
    if (fetchResponse.hasError()) {
        KafkaError error = KafkaError.getError(fetchResponse.errorCode(topic, partitionId));
        if (error.equals(KafkaError.OFFSET_OUT_OF_RANGE) && config.useStartOffsetTimeIfOffsetOutOfRange) {
            String msg = partition + " Got fetch request with offset out of range: [" + offset + "]";
            LOG.warn(msg);
            throw new TopicOffsetOutOfRangeException(msg);
        } else {
            String message = "Error fetching data from [" + partition + "] for topic [" + topic + "]: [" + error + "]";
            LOG.error(message);
            throw new FailedFetchException(message);
        }
    } else {
        msgs = fetchResponse.messageSet(topic, partitionId);
    }
    return msgs;
}

Example 12 with ByteBufferMessageSet

use of kafka.javaapi.message.ByteBufferMessageSet in project heron by twitter.

the class PartitionManager method fill.

private void fill() {
    long start = System.currentTimeMillis();
    Long offset;
    // Are there failed tuples? If so, fetch those first.
    offset = this.failedMsgRetryManager.nextFailedMessageToRetry();
    final boolean processingNewTuples = offset == null;
    if (processingNewTuples) {
        offset = emittedToOffset;
    }
    ByteBufferMessageSet msgs = null;
    try {
        msgs = KafkaUtils.fetchMessages(spoutConfig, consumer, partition, offset);
    } catch (TopicOffsetOutOfRangeException e) {
        offset = KafkaUtils.getOffset(consumer, partition.topic, partition.partition, kafka.api.OffsetRequest.EarliestTime());
        //fix bug [STORM-643] : remove outdated failed offsets
        if (!processingNewTuples) {
            // For the case of EarliestTime it would be better to discard
            // all the failed offsets, that are earlier than actual EarliestTime
            // offset, since they are anyway not there.
            // These calls to broker API will be then saved.
            Set<Long> omitted = this.failedMsgRetryManager.clearOffsetsBefore(offset);
            // Omitted messages have not been acked and may be lost
            if (null != omitted) {
                lostMessageCount.incrBy(omitted.size());
            }
            LOG.warn("Removing the failed offsets for {} that are out of range: {}", partition, omitted);
        }
        if (offset > emittedToOffset) {
            lostMessageCount.incrBy(offset - emittedToOffset);
            emittedToOffset = offset;
            LOG.warn("{} Using new offset: {}", partition, emittedToOffset);
        }
        return;
    }
    long millis = System.currentTimeMillis() - start;
    fetchAPILatencyMax.update(millis);
    fetchAPILatencyMean.update(millis);
    fetchAPICallCount.incr();
    if (msgs != null) {
        int numMessages = 0;
        for (MessageAndOffset msg : msgs) {
            final Long curOffset = msg.offset();
            if (curOffset < offset) {
                // Skip any old offsets.
                continue;
            }
            if (processingNewTuples || this.failedMsgRetryManager.shouldReEmitMsg(curOffset)) {
                numMessages += 1;
                if (!pending.containsKey(curOffset)) {
                    pending.put(curOffset, System.currentTimeMillis());
                }
                waitingToEmit.add(msg);
                emittedToOffset = Math.max(msg.nextOffset(), emittedToOffset);
                if (failedMsgRetryManager.shouldReEmitMsg(curOffset)) {
                    this.failedMsgRetryManager.retryStarted(curOffset);
                }
            }
        }
        fetchAPIMessageCount.incrBy(numMessages);
    }
}

Example 13 with ByteBufferMessageSet

use of kafka.javaapi.message.ByteBufferMessageSet in project heron by twitter.

the class KafkaUtilsTest method runGetValueOnlyTuplesTest.

private void runGetValueOnlyTuplesTest() {
    String value = "value";
    createTopicAndSendMessage(null, value);
    ByteBufferMessageSet messageAndOffsets = getLastMessage();
    for (MessageAndOffset msg : messageAndOffsets) {
        Iterable<List<Object>> lists = KafkaUtils.generateTuples(config, msg.message(), config.topic);
        assertEquals(value, lists.iterator().next().get(0));
    }
}

Example 14 with ByteBufferMessageSet

use of kafka.javaapi.message.ByteBufferMessageSet in project cdap by caskdata.

the class KafkaLogProcessorPipeline method fetchMessages.

/**
   * Fetch messages from Kafka.
   *
   * @param partition the partition to fetch from
   * @param offset the Kafka offset to fetch from
   * @return An {@link Iterable} of {@link MessageAndOffset}.
   *
   * @throws LeaderNotAvailableException if there is no Kafka broker to talk to.
   * @throws OffsetOutOfRangeException if the given offset is out of range.
   * @throws NotLeaderForPartitionException if the broker that the consumer is talking to is not the leader
   *                                        for the given topic and partition.
   * @throws UnknownTopicOrPartitionException if the topic or partition is not known by the Kafka server
   * @throws UnknownServerException if the Kafka server responded with error.
   */
private Iterable<MessageAndOffset> fetchMessages(int partition, long offset) throws KafkaException {
    String topic = config.getTopic();
    KafkaSimpleConsumer consumer = getKafkaConsumer(topic, partition);
    if (consumer == null) {
        throw new LeaderNotAvailableException("No broker to fetch messages for " + topic + ":" + partition);
    }
    LOG.trace("Fetching messages from Kafka on {}:{} for pipeline {} with offset {}", topic, partition, name, offset);
    try {
        ByteBufferMessageSet result = KafkaUtil.fetchMessages(consumer, topic, partition, config.getKafkaFetchBufferSize(), offset);
        LOG.trace("Fetched {} bytes from Kafka on {}:{} for pipeline {}", result.sizeInBytes(), topic, partition, name);
        return result;
    } catch (OffsetOutOfRangeException e) {
        // If the error is not offset out of range, clear the consumer cache
        kafkaConsumers.remove(consumer.getBrokerInfo());
        throw e;
    }
}

Example 15 with ByteBufferMessageSet

use of kafka.javaapi.message.ByteBufferMessageSet in project cdap by caskdata.

the class KafkaOffsetResolver method getEventTimeByOffset.

/**
   * Fetch a log event with {@code requestOffset} and deserialize it to get the log event time.
   *
   * @return the log event time of the message with {@code requestOffset}
   * @throws NotFoundException If cannot find a valid log event message at the given offset
   * @throws OffsetOutOfRangeException if the given offset is out of range.
   * @throws NotLeaderForPartitionException if the broker that the consumer is talking to is not the leader
   *                                        for the given topic and partition.
   * @throws UnknownTopicOrPartitionException if the topic or partition is not known by the Kafka server
   * @throws UnknownServerException if the Kafka server responded with error.
   */
private long getEventTimeByOffset(SimpleConsumer consumer, int partition, long requestOffset) throws NotFoundException {
    String topic = config.getTopic();
    ByteBufferMessageSet messageSet = KafkaUtil.fetchMessages(consumer, topic, partition, SINGLE_MESSAGE_MAX_SIZE, requestOffset);
    Iterator<MessageAndOffset> iterator = messageSet.iterator();
    if (!iterator.hasNext()) {
        throw new NotFoundException("No message found in " + topic + ":" + partition + " at offset " + requestOffset);
    }
    try {
        return serializer.decodeEventTimestamp(iterator.next().message().payload());
    } catch (IOException e) {
        // Fail to deserialize is the same as not found because in anywhere this is not the event we are looking for
        throw new NotFoundException("Invalid log event found in " + topic + ":" + partition + " at offset " + requestOffset);
    }
}