use of kafka.api.FetchRequest in project bagheera by mozilla-metrics.
the class ProducerTest method countMessages.
private int countMessages() throws InvalidProtocolBufferException {
SimpleConsumer consumer = new SimpleConsumer("localhost", KAFKA_BROKER_PORT, 100, 1024);
long offset = 0l;
int messageCount = 0;
for (int i = 0; i < BATCH_SIZE; i++) {
ByteBufferMessageSet messageSet = consumer.fetch(new FetchRequest(KAFKA_TOPIC, 0, offset, 1024));
Iterator<MessageAndOffset> iterator = messageSet.iterator();
MessageAndOffset msgAndOff;
while (iterator.hasNext()) {
messageCount++;
msgAndOff = iterator.next();
offset = msgAndOff.offset();
Message message2 = msgAndOff.message();
BagheeraMessage bmsg = BagheeraMessage.parseFrom(ByteString.copyFrom(message2.payload()));
String payload = new String(bmsg.getPayload().toByteArray());
System.out.println(String.format("Message %d @%d: %s", messageCount, offset, payload));
}
}
consumer.close();
return messageCount;
}
use of kafka.api.FetchRequest in project voltdb by VoltDB.
the class KafkaTopicPartitionImporter method accept.
@Override
protected void accept() {
info(null, "Starting partition fetcher for " + m_topicAndPartition);
long submitCount = 0;
PendingWorkTracker callbackTracker = new PendingWorkTracker();
Formatter formatter = m_config.getFormatterBuilder().create();
try {
//Start with the starting leader.
resetLeader();
int sleepCounter = 1;
while (shouldRun()) {
if (m_currentOffset.get() < 0) {
getOffsetCoordinator();
if (m_offsetManager.get() == null) {
sleepCounter = backoffSleep(sleepCounter);
continue;
}
long lastOffset = getLastOffset();
if (lastOffset == -1) {
sleepCounter = backoffSleep(sleepCounter);
continue;
}
m_gapTracker.resetTo(lastOffset);
m_lastCommittedOffset = lastOffset;
m_currentOffset.set(lastOffset);
if (m_currentOffset.get() < 0) {
//If we dont know the offset get it backoff if we fail.
sleepCounter = backoffSleep(sleepCounter);
info(null, "No valid offset found for " + m_topicAndPartition);
continue;
}
info(null, "Starting offset for " + m_topicAndPartition + " is " + m_currentOffset.get());
}
long currentFetchCount = 0;
//Build fetch request of we have a valid offset and not too many are pending.
FetchRequest req = m_fetchRequestBuilder.addFetch(m_topicAndPartition.topic(), m_topicAndPartition.partition(), m_currentOffset.get(), m_config.getFetchSize()).build();
FetchResponse fetchResponse = null;
try {
fetchResponse = m_consumer.fetch(req);
if (fetchResponse == null) {
sleepCounter = backoffSleep(sleepCounter);
continue;
}
} catch (Exception ex) {
rateLimitedLog(Level.WARN, ex, "Failed to fetch from " + m_topicAndPartition);
//See if its network error and find new leader for this partition.
if (ex instanceof IOException) {
resetLeader();
//find leader in resetLeader would sleep and backoff
continue;
}
sleepCounter = backoffSleep(sleepCounter);
continue;
}
if (fetchResponse.hasError()) {
// Something went wrong!
short code = fetchResponse.errorCode(m_topicAndPartition.topic(), m_topicAndPartition.partition());
warn(ErrorMapping.exceptionFor(code), "Failed to fetch messages for %s", m_topicAndPartition);
sleepCounter = backoffSleep(sleepCounter);
if (code == ErrorMapping.OffsetOutOfRangeCode()) {
// We asked for an invalid offset. For simple case ask for the last element to reset
info(null, "Invalid offset requested for " + m_topicAndPartition);
getOffsetCoordinator();
m_currentOffset.set(-1L);
continue;
}
resetLeader();
continue;
}
sleepCounter = 1;
for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(m_topicAndPartition.topic(), m_topicAndPartition.partition())) {
//You may be catchin up so dont sleep.
currentFetchCount++;
long currentOffset = messageAndOffset.offset();
//if currentOffset is less means we have already pushed it and also check pending queue.
if (currentOffset < m_currentOffset.get()) {
continue;
}
ByteBuffer payload = messageAndOffset.message().payload();
Object[] params = null;
try {
m_gapTracker.submit(messageAndOffset.nextOffset());
params = formatter.transform(payload);
Invocation invocation = new Invocation(m_config.getProcedure(), params);
TopicPartitionInvocationCallback cb = new TopicPartitionInvocationCallback(messageAndOffset.offset(), messageAndOffset.nextOffset(), callbackTracker, m_gapTracker, m_dead, m_pauseOffset);
if (!noTransaction) {
if (callProcedure(invocation, cb)) {
callbackTracker.produceWork();
} else {
if (isDebugEnabled()) {
debug(null, "Failed to process Invocation possibly bad data: " + Arrays.toString(params));
}
m_gapTracker.commit(messageAndOffset.nextOffset());
}
}
} catch (FormatException e) {
rateLimitedLog(Level.WARN, e, "Failed to tranform data: %s", Arrays.toString(params));
m_gapTracker.commit(messageAndOffset.nextOffset());
}
submitCount++;
m_currentOffset.set(messageAndOffset.nextOffset());
if (!shouldRun()) {
break;
}
}
if (!shouldRun()) {
break;
}
//wait to fetch more if we read nothing last time.
if (currentFetchCount == 0) {
try {
Thread.sleep(m_waitSleepMs);
} catch (InterruptedException ie) {
}
}
if (shouldCommit()) {
commitOffset(false);
}
}
} catch (Exception ex) {
error(ex, "Failed to start topic partition fetcher for " + m_topicAndPartition);
} finally {
final boolean usePausedOffset = m_pauseOffset.get() != -1;
boolean skipCommit = false;
if (usePausedOffset) {
// Paused offset is not guaranteed reliable until all callbacks have been called.
if (callbackTracker.waitForWorkToFinish() == false) {
if (m_pauseOffset.get() < m_lastCommittedOffset) {
warn(null, "Committing paused offset even though a timeout occurred waiting for pending stored procedures to finish.");
} else {
warn(null, "Refusing to commit paused offset because a timeout occurred waiting for pending stored procedures to finish.");
skipCommit = true;
}
}
}
if (skipCommit == false) {
// Force a commit. Paused offset will be re-acquired if needed.
commitOffset(usePausedOffset);
}
KafkaStreamImporterConfig.closeConsumer(m_consumer);
m_consumer = null;
BlockingChannel channel = m_offsetManager.getAndSet(null);
if (channel != null) {
try {
channel.disconnect();
} catch (Exception ignoreIt) {
}
}
}
m_dead.compareAndSet(false, true);
info(null, "Partition fetcher stopped for " + m_topicAndPartition + " Last commit point is: " + m_lastCommittedOffset + " Callback Rcvd: " + callbackTracker.getCallbackCount() + " Submitted: " + submitCount);
}
use of kafka.api.FetchRequest in project cdap by caskdata.
the class KafkaConsumer method fetchMessageSet.
private ByteBufferMessageSet fetchMessageSet(long fetchOffset) throws OffsetOutOfRangeException {
Preconditions.checkArgument(fetchOffset >= 0, String.format("Illegal fetch offset %d", fetchOffset));
int failureCount = 0;
while (true) {
SimpleConsumer consumer = getConsumer();
FetchRequest req = new FetchRequestBuilder().clientId(clientName).addFetch(topic, partition, fetchOffset, BUFFER_SIZE_BYTES).maxWait(fetchTimeoutMs).build();
FetchResponse fetchResponse = consumer.fetch(req);
if (!fetchResponse.hasError()) {
return fetchResponse.messageSet(topic, partition);
}
short errorCode = fetchResponse.errorCode(topic, partition);
if (++failureCount >= MAX_KAFKA_FETCH_RETRIES) {
throw new RuntimeException(String.format("Error fetching data from broker %s:%d for topic %s, partition %d. Error code: %d", consumer.host(), consumer.port(), topic, partition, errorCode));
}
LOG.warn("Error fetching data from broker {}:{} for topic {}, partition {}. Error code: {}", consumer.host(), consumer.port(), topic, partition, errorCode);
if (errorCode == ErrorMapping.OffsetOutOfRangeCode()) {
throw new OffsetOutOfRangeException(String.format("Requested offset %d is out of range for topic %s partition %d", fetchOffset, topic, partition));
}
closeConsumer();
}
}
use of kafka.api.FetchRequest in project cdap by caskdata.
the class KafkaUtil method fetchMessages.
/**
* Fetches messages from the given topic, partition and offset using the provided {@link SimpleConsumer}.
*
* @return A {@link ByteBufferMessageSet} of the given topic, partition for messages fetched
*
* @throws OffsetOutOfRangeException if the given offset is out of range.
* @throws NotLeaderForPartitionException if the broker that the consumer is talking to is not the leader
* for the given topic and partition.
* @throws UnknownTopicOrPartitionException if the topic or partition is not known by the Kafka server
* @throws UnknownServerException if the Kafka server responded with error.
*/
public static ByteBufferMessageSet fetchMessages(SimpleConsumer consumer, String topic, int partition, int fetchSize, long requestOffset) throws KafkaException {
FetchRequest req = new FetchRequestBuilder().clientId(consumer.clientId()).addFetch(topic, partition, requestOffset, fetchSize).build();
FetchResponse fetchResponse = consumer.fetch(req);
if (fetchResponse.hasError()) {
throw Errors.forCode(fetchResponse.errorCode(topic, partition)).exception();
}
return fetchResponse.messageSet(topic, partition);
}
Aggregations