Search in sources :

Example 46 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project apache-kafka-on-k8s by banzaicloud.

the class SimpleConsumerDemo method main.

public static void main(String[] args) throws Exception {
    generateData();
    SimpleConsumer simpleConsumer = new SimpleConsumer(KafkaProperties.KAFKA_SERVER_URL, KafkaProperties.KAFKA_SERVER_PORT, KafkaProperties.CONNECTION_TIMEOUT, KafkaProperties.KAFKA_PRODUCER_BUFFER_SIZE, KafkaProperties.CLIENT_ID);
    System.out.println("Testing single fetch");
    FetchRequest req = new FetchRequestBuilder().clientId(KafkaProperties.CLIENT_ID).addFetch(KafkaProperties.TOPIC2, 0, 0L, 100).build();
    FetchResponse fetchResponse = simpleConsumer.fetch(req);
    printMessages(fetchResponse.messageSet(KafkaProperties.TOPIC2, 0));
    System.out.println("Testing single multi-fetch");
    Map<String, List<Integer>> topicMap = new HashMap<>();
    topicMap.put(KafkaProperties.TOPIC2, Collections.singletonList(0));
    topicMap.put(KafkaProperties.TOPIC3, Collections.singletonList(0));
    req = new FetchRequestBuilder().clientId(KafkaProperties.CLIENT_ID).addFetch(KafkaProperties.TOPIC2, 0, 0L, 100).addFetch(KafkaProperties.TOPIC3, 0, 0L, 100).build();
    fetchResponse = simpleConsumer.fetch(req);
    int fetchReq = 0;
    for (Map.Entry<String, List<Integer>> entry : topicMap.entrySet()) {
        String topic = entry.getKey();
        for (Integer offset : entry.getValue()) {
            System.out.println("Response from fetch request no: " + ++fetchReq);
            printMessages(fetchResponse.messageSet(topic, offset));
        }
    }
}
Also used : HashMap(java.util.HashMap) FetchRequestBuilder(kafka.api.FetchRequestBuilder) FetchRequest(kafka.api.FetchRequest) List(java.util.List) FetchResponse(kafka.javaapi.FetchResponse) Map(java.util.Map) HashMap(java.util.HashMap) SimpleConsumer(kafka.javaapi.consumer.SimpleConsumer)

Example 47 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project jstorm by alibaba.

the class KafkaConsumer method getOffset.

public long getOffset(String topic, int partition, long startOffsetTime) {
    SimpleConsumer simpleConsumer = findLeaderConsumer(partition);
    if (simpleConsumer == null) {
        LOG.error("Error consumer is null get offset from partition:" + partition);
        return -1;
    }
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId());
    long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition);
    if (offsets.length > 0) {
        return offsets[0];
    } else {
        return NO_OFFSET;
    }
}
Also used : PartitionOffsetRequestInfo(kafka.api.PartitionOffsetRequestInfo) HashMap(java.util.HashMap) TopicAndPartition(kafka.common.TopicAndPartition) SimpleConsumer(kafka.javaapi.consumer.SimpleConsumer) OffsetRequest(kafka.javaapi.OffsetRequest)

Example 48 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project jstorm by alibaba.

the class KafkaConsumer method findLeader.

protected PartitionMetadata findLeader(int partition) {
    PartitionMetadata returnMetaData = null;
    int errors = 0;
    int size = brokerList.size();
    Host brokerHost = brokerList.get(brokerIndex);
    try {
        if (consumer == null) {
            consumer = new SimpleConsumer(brokerHost.getHost(), brokerHost.getPort(), config.socketTimeoutMs, config.socketReceiveBufferBytes, config.clientId);
        }
    } catch (Exception e) {
        LOG.warn(e.getMessage(), e);
        consumer = null;
    }
    int i = brokerIndex;
    loop: while (i < size && errors < size + 1) {
        Host host = brokerList.get(i);
        i = (i + 1) % size;
        // next index
        brokerIndex = i;
        try {
            if (consumer == null) {
                consumer = new SimpleConsumer(host.getHost(), host.getPort(), config.socketTimeoutMs, config.socketReceiveBufferBytes, config.clientId);
            }
            List<String> topics = Collections.singletonList(config.topic);
            TopicMetadataRequest req = new TopicMetadataRequest(topics);
            kafka.javaapi.TopicMetadataResponse resp = null;
            try {
                resp = consumer.send(req);
            } catch (Exception e) {
                errors += 1;
                LOG.error("findLeader error, broker:" + host.toString() + ", will change to next broker index:" + (i + 1) % size);
                if (consumer != null) {
                    consumer.close();
                    consumer = null;
                }
                continue;
            }
            List<TopicMetadata> metaData = resp.topicsMetadata();
            for (TopicMetadata item : metaData) {
                for (PartitionMetadata part : item.partitionsMetadata()) {
                    if (part.partitionId() == partition) {
                        returnMetaData = part;
                        break loop;
                    }
                }
            }
        } catch (Exception e) {
            LOG.error("Error communicating with Broker:" + host.toString() + ", find Leader for partition:" + partition);
        } finally {
            if (consumer != null) {
                consumer.close();
                consumer = null;
            }
        }
    }
    return returnMetaData;
}
Also used : TopicMetadataRequest(kafka.javaapi.TopicMetadataRequest) PartitionMetadata(kafka.javaapi.PartitionMetadata) List(java.util.List) LinkedList(java.util.LinkedList) SimpleConsumer(kafka.javaapi.consumer.SimpleConsumer) KafkaException(kafka.common.KafkaException) IOException(java.io.IOException) UnresolvedAddressException(java.nio.channels.UnresolvedAddressException) SocketTimeoutException(java.net.SocketTimeoutException) ConnectException(java.net.ConnectException) TopicMetadata(kafka.javaapi.TopicMetadata)

Aggregations

SimpleConsumer (kafka.javaapi.consumer.SimpleConsumer)48 FetchResponse (kafka.javaapi.FetchResponse)10 FetchRequest (kafka.api.FetchRequest)9 IOException (java.io.IOException)8 FetchRequestBuilder (kafka.api.FetchRequestBuilder)8 PartitionMetadata (kafka.javaapi.PartitionMetadata)8 ArrayList (java.util.ArrayList)7 HashMap (java.util.HashMap)7 TopicMetadata (kafka.javaapi.TopicMetadata)7 TopicMetadataRequest (kafka.javaapi.TopicMetadataRequest)6 MessageAndOffset (kafka.message.MessageAndOffset)6 List (java.util.List)5 Map (java.util.Map)5 TopicAndPartition (kafka.common.TopicAndPartition)4 ByteBufferMessageSet (kafka.javaapi.message.ByteBufferMessageSet)4 ConnectException (java.net.ConnectException)3 SocketTimeoutException (java.net.SocketTimeoutException)3 ByteBuffer (java.nio.ByteBuffer)3 UnresolvedAddressException (java.nio.channels.UnresolvedAddressException)3 PartitionOffsetRequestInfo (kafka.api.PartitionOffsetRequestInfo)3