Examples with ConsumerConnector kafka.javaapi.consumer.ConsumerConnector used on opensource projects

Example 6 with ConsumerConnector

use of kafka.javaapi.consumer.ConsumerConnector in project incubator-atlas by apache.

the class KafkaNotification method close.

@Override
public void close() {
    if (producer != null) {
        producer.close();
        producer = null;
    }
    for (ConsumerConnector consumerConnector : consumerConnectors) {
        consumerConnector.shutdown();
    }
    consumerConnectors.clear();
}

Example 7 with ConsumerConnector

use of kafka.javaapi.consumer.ConsumerConnector in project druid by druid-io.

the class KafkaEightFirehoseFactory method connect.

@Override
public Firehose connect(final ByteBufferInputRowParser firehoseParser) throws IOException {
    Set<String> newDimExclus = Sets.union(firehoseParser.getParseSpec().getDimensionsSpec().getDimensionExclusions(), Sets.newHashSet("feed"));
    final ByteBufferInputRowParser theParser = firehoseParser.withParseSpec(firehoseParser.getParseSpec().withDimensionsSpec(firehoseParser.getParseSpec().getDimensionsSpec().withDimensionExclusions(newDimExclus)));
    final ConsumerConnector connector = Consumer.createJavaConsumerConnector(new ConsumerConfig(consumerProps));
    final Map<String, List<KafkaStream<byte[], byte[]>>> streams = connector.createMessageStreams(ImmutableMap.of(feed, 1));
    final List<KafkaStream<byte[], byte[]>> streamList = streams.get(feed);
    if (streamList == null || streamList.size() != 1) {
        return null;
    }
    final KafkaStream<byte[], byte[]> stream = streamList.get(0);
    final ConsumerIterator<byte[], byte[]> iter = stream.iterator();
    return new Firehose() {

        @Override
        public boolean hasMore() {
            return iter.hasNext();
        }

        @Override
        public InputRow nextRow() {
            try {
                final byte[] message = iter.next().message();
                if (message == null) {
                    return null;
                }
                return theParser.parse(ByteBuffer.wrap(message));
            } catch (InvalidMessageException e) {
                /*
          IF the CRC is caused within the wire transfer, this is not the best way to handel CRC.
          Probably it is better to shutdown the fireHose without commit and start it again.
           */
                log.error(e, "Message failed its checksum and it is corrupt, will skip it");
                return null;
            }
        }

        @Override
        public Runnable commit() {
            return new Runnable() {

                @Override
                public void run() {
                    /*
                 This is actually not going to do exactly what we want, cause it will be called asynchronously
                 after the persist is complete.  So, it's going to commit that it's processed more than was actually
                 persisted.  This is unfortunate, but good enough for now.  Should revisit along with an upgrade
                 of our Kafka version.
               */
                    log.info("committing offsets");
                    connector.commitOffsets();
                }
            };
        }

        @Override
        public void close() throws IOException {
            connector.shutdown();
        }
    };
}

Example 8 with ConsumerConnector

use of kafka.javaapi.consumer.ConsumerConnector in project incubator-atlas by apache.

the class KafkaNotification method createConsumers.

@VisibleForTesting
public <T> List<NotificationConsumer<T>> createConsumers(NotificationType notificationType, int numConsumers, boolean autoCommitEnabled) {
    String topic = TOPIC_MAP.get(notificationType);
    Properties consumerProperties = getConsumerProperties(notificationType);
    List<NotificationConsumer<T>> consumers = new ArrayList<>(numConsumers);
    for (int i = 0; i < numConsumers; i++) {
        ConsumerConnector consumerConnector = createConsumerConnector(consumerProperties);
        Map<String, Integer> topicCountMap = new HashMap<>();
        topicCountMap.put(topic, 1);
        StringDecoder decoder = new StringDecoder(null);
        Map<String, List<KafkaStream<String, String>>> streamsMap = consumerConnector.createMessageStreams(topicCountMap, decoder, decoder);
        List<KafkaStream<String, String>> kafkaConsumers = streamsMap.get(topic);
        for (KafkaStream stream : kafkaConsumers) {
            KafkaConsumer<T> kafkaConsumer = createKafkaConsumer(notificationType.getClassType(), notificationType.getDeserializer(), stream, i, consumerConnector, autoCommitEnabled);
            consumers.add(kafkaConsumer);
        }
        consumerConnectors.add(consumerConnector);
    }
    return consumers;
}

Example 9 with ConsumerConnector

use of kafka.javaapi.consumer.ConsumerConnector in project incubator-atlas by apache.

the class KafkaNotificationMockTest method testCreateConsumers.

@Test
@SuppressWarnings("unchecked")
public void testCreateConsumers() throws Exception {
    Properties properties = mock(Properties.class);
    when(properties.getProperty("entities.group.id")).thenReturn("atlas");
    final ConsumerConnector consumerConnector = mock(ConsumerConnector.class);
    Map<String, Integer> topicCountMap = new HashMap<>();
    topicCountMap.put(KafkaNotification.ATLAS_ENTITIES_TOPIC, 1);
    Map<String, List<KafkaStream<String, String>>> kafkaStreamsMap = new HashMap<>();
    List<KafkaStream<String, String>> kafkaStreams = new ArrayList<>();
    KafkaStream kafkaStream = mock(KafkaStream.class);
    kafkaStreams.add(kafkaStream);
    kafkaStreamsMap.put(KafkaNotification.ATLAS_ENTITIES_TOPIC, kafkaStreams);
    when(consumerConnector.createMessageStreams(eq(topicCountMap), any(StringDecoder.class), any(StringDecoder.class))).thenReturn(kafkaStreamsMap);
    final KafkaConsumer consumer1 = mock(KafkaConsumer.class);
    final KafkaConsumer consumer2 = mock(KafkaConsumer.class);
    KafkaNotification kafkaNotification = new TestKafkaNotification(properties, consumerConnector, consumer1, consumer2);
    List<NotificationConsumer<String>> consumers = kafkaNotification.createConsumers(NotificationInterface.NotificationType.ENTITIES, 2);
    verify(consumerConnector, times(2)).createMessageStreams(eq(topicCountMap), any(StringDecoder.class), any(StringDecoder.class));
    assertEquals(consumers.size(), 2);
    assertTrue(consumers.contains(consumer1));
    assertTrue(consumers.contains(consumer2));
}