Examples with SimpleConsumer kafka.javaapi.consumer.SimpleConsumer used on opensource projects

Example 36 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project presto by prestodb.

the class KafkaSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    KafkaTableHandle kafkaTableHandle = convertLayout(layout).getTable();
    SimpleConsumer simpleConsumer = consumerManager.getConsumer(selectRandom(nodes));
    TopicMetadataRequest topicMetadataRequest = new TopicMetadataRequest(ImmutableList.of(kafkaTableHandle.getTopicName()));
    TopicMetadataResponse topicMetadataResponse = simpleConsumer.send(topicMetadataRequest);
    ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder();
    for (TopicMetadata metadata : topicMetadataResponse.topicsMetadata()) {
        for (PartitionMetadata part : metadata.partitionsMetadata()) {
            log.debug("Adding Partition %s/%s", metadata.topic(), part.partitionId());
            Broker leader = part.leader();
            if (leader == null) {
                // Leader election going on...
                log.warn("No leader for partition %s/%s found!", metadata.topic(), part.partitionId());
                continue;
            }
            HostAddress partitionLeader = HostAddress.fromParts(leader.host(), leader.port());
            SimpleConsumer leaderConsumer = consumerManager.getConsumer(partitionLeader);
            // Kafka contains a reverse list of "end - start" pairs for the splits
            long[] offsets = findAllOffsets(leaderConsumer, metadata.topic(), part.partitionId());
            for (int i = offsets.length - 1; i > 0; i--) {
                KafkaSplit split = new KafkaSplit(connectorId, metadata.topic(), kafkaTableHandle.getKeyDataFormat(), kafkaTableHandle.getMessageDataFormat(), part.partitionId(), offsets[i], offsets[i - 1], partitionLeader);
                splits.add(split);
            }
        }
    }
    return new FixedSplitSource(splits.build());
}

Example 37 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project opennms by OpenNMS.

the class KafkaOffsetProvider method getConsumer.

public SimpleConsumer getConsumer(String host, int port) {
    SimpleConsumer consumer = consumerMap.get(host);
    if (consumer == null) {
        consumer = new SimpleConsumer(host, port, KafkaOffsetConstants.TIMEOUT, KafkaOffsetConstants.BUFFERSIZE, KafkaOffsetConstants.CLIENT_NAME);
        LOGGER.info("Created a new Kafka Consumer for host: " + host);
        consumerMap.put(host, consumer);
    }
    return consumer;
}

Example 38 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project opennms by OpenNMS.

the class KafkaOffsetProvider method closeConnection.

public void closeConnection() throws InterruptedException {
    for (SimpleConsumer consumer : consumerMap.values()) {
        LOGGER.info("Closing connection for: " + consumer.host());
        consumer.close();
    }
}

Example 39 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project cdap by caskdata.

the class KafkaLogProcessorPipeline method shutDown.

@Override
protected void shutDown() throws Exception {
    LOG.debug("Shutting down log processor pipeline for {}", name);
    fetchExecutor.shutdownNow();
    try {
        context.stop();
        // Persist the checkpoints. It can only be done after successfully stopping the appenders.
        // Since persistCheckpoint never throw, putting it inside try is ok.
        persistCheckpoints();
    } catch (Exception e) {
        // Just log, not to fail the shutdown
        LOG.warn("Exception raised when stopping pipeline {}", name, e);
    }
    for (SimpleConsumer consumer : kafkaConsumers.values()) {
        try {
            consumer.close();
        } catch (Exception e) {
            // Just log, not to fail the shutdown
            LOG.warn("Exception raised when closing Kafka consumer.", e);
        }
    }
    LOG.info("Log processor pipeline for {} stopped with latest checkpoints {}", name, checkpoints);
}

Example 40 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project cdap by caskdata.

the class KafkaConsumer method fetchMessageSet.

private ByteBufferMessageSet fetchMessageSet(long fetchOffset) throws OffsetOutOfRangeException {
    Preconditions.checkArgument(fetchOffset >= 0, String.format("Illegal fetch offset %d", fetchOffset));
    int failureCount = 0;
    while (true) {
        SimpleConsumer consumer = getConsumer();
        FetchRequest req = new FetchRequestBuilder().clientId(clientName).addFetch(topic, partition, fetchOffset, BUFFER_SIZE_BYTES).maxWait(fetchTimeoutMs).build();
        FetchResponse fetchResponse = consumer.fetch(req);
        if (!fetchResponse.hasError()) {
            return fetchResponse.messageSet(topic, partition);
        }
        short errorCode = fetchResponse.errorCode(topic, partition);
        if (++failureCount >= MAX_KAFKA_FETCH_RETRIES) {
            throw new RuntimeException(String.format("Error fetching data from broker %s:%d for topic %s, partition %d. Error code: %d", consumer.host(), consumer.port(), topic, partition, errorCode));
        }
        LOG.warn("Error fetching data from broker {}:{} for topic {}, partition {}. Error code: {}", consumer.host(), consumer.port(), topic, partition, errorCode);
        if (errorCode == ErrorMapping.OffsetOutOfRangeCode()) {
            throw new OffsetOutOfRangeException(String.format("Requested offset %d is out of range for topic %s partition %d", fetchOffset, topic, partition));
        }
        closeConsumer();
    }
}