use of kafka.javaapi.consumer.SimpleConsumer in project presto by prestodb.
the class KafkaSplitManager method getSplits.
@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout) {
KafkaTableHandle kafkaTableHandle = convertLayout(layout).getTable();
SimpleConsumer simpleConsumer = consumerManager.getConsumer(selectRandom(nodes));
TopicMetadataRequest topicMetadataRequest = new TopicMetadataRequest(ImmutableList.of(kafkaTableHandle.getTopicName()));
TopicMetadataResponse topicMetadataResponse = simpleConsumer.send(topicMetadataRequest);
ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder();
for (TopicMetadata metadata : topicMetadataResponse.topicsMetadata()) {
for (PartitionMetadata part : metadata.partitionsMetadata()) {
log.debug("Adding Partition %s/%s", metadata.topic(), part.partitionId());
Broker leader = part.leader();
if (leader == null) {
// Leader election going on...
log.warn("No leader for partition %s/%s found!", metadata.topic(), part.partitionId());
continue;
}
HostAddress partitionLeader = HostAddress.fromParts(leader.host(), leader.port());
SimpleConsumer leaderConsumer = consumerManager.getConsumer(partitionLeader);
// Kafka contains a reverse list of "end - start" pairs for the splits
long[] offsets = findAllOffsets(leaderConsumer, metadata.topic(), part.partitionId());
for (int i = offsets.length - 1; i > 0; i--) {
KafkaSplit split = new KafkaSplit(connectorId, metadata.topic(), kafkaTableHandle.getKeyDataFormat(), kafkaTableHandle.getMessageDataFormat(), part.partitionId(), offsets[i], offsets[i - 1], partitionLeader);
splits.add(split);
}
}
}
return new FixedSplitSource(splits.build());
}
use of kafka.javaapi.consumer.SimpleConsumer in project opennms by OpenNMS.
the class KafkaOffsetProvider method getConsumer.
public SimpleConsumer getConsumer(String host, int port) {
SimpleConsumer consumer = consumerMap.get(host);
if (consumer == null) {
consumer = new SimpleConsumer(host, port, KafkaOffsetConstants.TIMEOUT, KafkaOffsetConstants.BUFFERSIZE, KafkaOffsetConstants.CLIENT_NAME);
LOGGER.info("Created a new Kafka Consumer for host: " + host);
consumerMap.put(host, consumer);
}
return consumer;
}
use of kafka.javaapi.consumer.SimpleConsumer in project opennms by OpenNMS.
the class KafkaOffsetProvider method closeConnection.
public void closeConnection() throws InterruptedException {
for (SimpleConsumer consumer : consumerMap.values()) {
LOGGER.info("Closing connection for: " + consumer.host());
consumer.close();
}
}
use of kafka.javaapi.consumer.SimpleConsumer in project cdap by caskdata.
the class KafkaLogProcessorPipeline method shutDown.
@Override
protected void shutDown() throws Exception {
LOG.debug("Shutting down log processor pipeline for {}", name);
fetchExecutor.shutdownNow();
try {
context.stop();
// Persist the checkpoints. It can only be done after successfully stopping the appenders.
// Since persistCheckpoint never throw, putting it inside try is ok.
persistCheckpoints();
} catch (Exception e) {
// Just log, not to fail the shutdown
LOG.warn("Exception raised when stopping pipeline {}", name, e);
}
for (SimpleConsumer consumer : kafkaConsumers.values()) {
try {
consumer.close();
} catch (Exception e) {
// Just log, not to fail the shutdown
LOG.warn("Exception raised when closing Kafka consumer.", e);
}
}
LOG.info("Log processor pipeline for {} stopped with latest checkpoints {}", name, checkpoints);
}
use of kafka.javaapi.consumer.SimpleConsumer in project cdap by caskdata.
the class KafkaConsumer method fetchMessageSet.
private ByteBufferMessageSet fetchMessageSet(long fetchOffset) throws OffsetOutOfRangeException {
Preconditions.checkArgument(fetchOffset >= 0, String.format("Illegal fetch offset %d", fetchOffset));
int failureCount = 0;
while (true) {
SimpleConsumer consumer = getConsumer();
FetchRequest req = new FetchRequestBuilder().clientId(clientName).addFetch(topic, partition, fetchOffset, BUFFER_SIZE_BYTES).maxWait(fetchTimeoutMs).build();
FetchResponse fetchResponse = consumer.fetch(req);
if (!fetchResponse.hasError()) {
return fetchResponse.messageSet(topic, partition);
}
short errorCode = fetchResponse.errorCode(topic, partition);
if (++failureCount >= MAX_KAFKA_FETCH_RETRIES) {
throw new RuntimeException(String.format("Error fetching data from broker %s:%d for topic %s, partition %d. Error code: %d", consumer.host(), consumer.port(), topic, partition, errorCode));
}
LOG.warn("Error fetching data from broker {}:{} for topic {}, partition {}. Error code: {}", consumer.host(), consumer.port(), topic, partition, errorCode);
if (errorCode == ErrorMapping.OffsetOutOfRangeCode()) {
throw new OffsetOutOfRangeException(String.format("Requested offset %d is out of range for topic %s partition %d", fetchOffset, topic, partition));
}
closeConsumer();
}
}
Aggregations