Search in sources :

Example 1 with ConsumerIterator

use of kafka.consumer.ConsumerIterator in project graylog2-server by Graylog2.

the class KafkaTransport method doLaunch.

public void doLaunch(final MessageInput input) throws MisfireException {
    serverStatus.awaitRunning(new Runnable() {

        public void run() {
    // listen for lifecycle changes
    final Properties props = new Properties();
    props.put("", GROUP_ID);
    props.put("", "gl2-" + nodeId + "-" + input.getId());
    props.put("fetch.min.bytes", String.valueOf(configuration.getInt(CK_FETCH_MIN_BYTES)));
    props.put("", String.valueOf(configuration.getInt(CK_FETCH_WAIT_MAX)));
    props.put("zookeeper.connect", configuration.getString(CK_ZOOKEEPER));
    // Default auto commit interval is 60 seconds. Reduce to 1 second to minimize message duplication
    // if something breaks.
    props.put("", "1000");
    // Set a consumer timeout to avoid blocking on the consumer iterator.
    props.put("", "1000");
    final int numThreads = configuration.getInt(CK_THREADS);
    final ConsumerConfig consumerConfig = new ConsumerConfig(props);
    cc = Consumer.createJavaConsumerConnector(consumerConfig);
    final TopicFilter filter = new Whitelist(configuration.getString(CK_TOPIC_FILTER));
    final List<KafkaStream<byte[], byte[]>> streams = cc.createMessageStreamsByFilter(filter, numThreads);
    final ExecutorService executor = executorService(numThreads);
    // this is being used during shutdown to first stop all submitted jobs before committing the offsets back to zookeeper
    // and then shutting down the connection.
    // this is to avoid yanking away the connection from the consumer runnables
    stopLatch = new CountDownLatch(streams.size());
    for (final KafkaStream<byte[], byte[]> stream : streams) {
        executor.submit(new Runnable() {

            public void run() {
                final ConsumerIterator<byte[], byte[]> consumerIterator = stream.iterator();
                boolean retry;
                do {
                    retry = false;
                    try {
                        // noinspection WhileLoopReplaceableByForEach
                        while (consumerIterator.hasNext()) {
                            if (paused) {
                                // we try not to spin here, so we wait until the lifecycle goes back to running.
                                LOG.debug("Message processing is paused, blocking until message processing is turned back on.");
                            // check for being stopped before actually getting the message, otherwise we could end up losing that message
                            if (stopped) {
                            if (isThrottled()) {
                            // process the message, this will immediately mark the message as having been processed. this gets tricky
                            // if we get an exception about processing it down below.
                            final MessageAndMetadata<byte[], byte[]> message =;
                            final byte[] bytes = message.message();
                            // it is possible that the message is null
                            if (bytes == null) {
                            final RawMessage rawMessage = new RawMessage(bytes);
                            // TODO implement throttling
                    } catch (ConsumerTimeoutException e) {
                        // Happens when there is nothing to consume, retry to check again.
                        retry = true;
                    } catch (Exception e) {
                        LOG.error("Kafka consumer error, stopping consumer thread.", e);
                } while (retry && !stopped);
                // explicitly commit our offsets when stopping.
                // this might trigger a couple of times, but it won't hurt
    scheduler.scheduleAtFixedRate(new Runnable() {

        public void run() {
    }, 1, 1, TimeUnit.SECONDS);
Also used : TopicFilter(kafka.consumer.TopicFilter) MessageAndMetadata(kafka.message.MessageAndMetadata) KafkaStream(kafka.consumer.KafkaStream) Properties(java.util.Properties) CountDownLatch(java.util.concurrent.CountDownLatch) ConsumerTimeoutException(kafka.consumer.ConsumerTimeoutException) MisfireException(org.graylog2.plugin.inputs.MisfireException) ConsumerIterator(kafka.consumer.ConsumerIterator) InstrumentedExecutorService(com.codahale.metrics.InstrumentedExecutorService) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ExecutorService(java.util.concurrent.ExecutorService) Whitelist(kafka.consumer.Whitelist) ConsumerTimeoutException(kafka.consumer.ConsumerTimeoutException) ConsumerConfig(kafka.consumer.ConsumerConfig) RawMessage(org.graylog2.plugin.journal.RawMessage)


InstrumentedExecutorService (com.codahale.metrics.InstrumentedExecutorService)1 Properties (java.util.Properties)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 ExecutorService (java.util.concurrent.ExecutorService)1 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)1 ConsumerConfig (kafka.consumer.ConsumerConfig)1 ConsumerIterator (kafka.consumer.ConsumerIterator)1 ConsumerTimeoutException (kafka.consumer.ConsumerTimeoutException)1 KafkaStream (kafka.consumer.KafkaStream)1 TopicFilter (kafka.consumer.TopicFilter)1 Whitelist (kafka.consumer.Whitelist)1 MessageAndMetadata (kafka.message.MessageAndMetadata)1 MisfireException (org.graylog2.plugin.inputs.MisfireException)1 RawMessage (org.graylog2.plugin.journal.RawMessage)1