Examples with MessageAndMetadata kafka.message.MessageAndMetadata used on opensource projects

Example 6 with MessageAndMetadata

use of kafka.message.MessageAndMetadata in project ignite by apache.

the class KafkaIgniteStreamerSelfTest method consumerStream.

/**
 * Consumes Kafka stream via Ignite.
 *
 * @param topic Topic name.
 * @param keyValMap Expected key value map.
 * @throws TimeoutException If timed out.
 * @throws InterruptedException If interrupted.
 */
private void consumerStream(String topic, Map<String, String> keyValMap) throws TimeoutException, InterruptedException {
    KafkaStreamer<String, String> kafkaStmr = null;
    Ignite ignite = grid();
    try (IgniteDataStreamer<String, String> stmr = ignite.dataStreamer(DEFAULT_CACHE_NAME)) {
        stmr.allowOverwrite(true);
        stmr.autoFlushFrequency(10);
        // Configure Kafka streamer.
        kafkaStmr = new KafkaStreamer<>();
        // Get the cache.
        IgniteCache<String, String> cache = ignite.cache(DEFAULT_CACHE_NAME);
        // Set Ignite instance.
        kafkaStmr.setIgnite(ignite);
        // Set data streamer instance.
        kafkaStmr.setStreamer(stmr);
        // Set the topic.
        kafkaStmr.setTopic(topic);
        // Set the number of threads.
        kafkaStmr.setThreads(4);
        // Set the consumer configuration.
        kafkaStmr.setConsumerConfig(createDefaultConsumerConfig(embeddedBroker.getZookeeperAddress(), "groupX"));
        kafkaStmr.setMultipleTupleExtractor(new StreamMultipleTupleExtractor<MessageAndMetadata<byte[], byte[]>, String, String>() {

            @Override
            public Map<String, String> extract(MessageAndMetadata<byte[], byte[]> msg) {
                Map<String, String> entries = new HashMap<>();
                try {
                    String key = new String(msg.key());
                    String val = new String(msg.message());
                    // Convert the message into number of cache entries with same key or dynamic key from actual message.
                    // For now using key as cache entry key and value as cache entry value - for test purpose.
                    entries.put(key, val);
                } catch (Exception ex) {
                    fail("Unexpected error." + ex);
                }
                return entries;
            }
        });
        // Start kafka streamer.
        kafkaStmr.start();
        final CountDownLatch latch = new CountDownLatch(CNT);
        IgniteBiPredicate<UUID, CacheEvent> locLsnr = new IgniteBiPredicate<UUID, CacheEvent>() {

            @Override
            public boolean apply(UUID uuid, CacheEvent evt) {
                latch.countDown();
                return true;
            }
        };
        ignite.events(ignite.cluster().forCacheNodes(DEFAULT_CACHE_NAME)).remoteListen(locLsnr, null, EVT_CACHE_OBJECT_PUT);
        // Checks all events successfully processed in 10 seconds.
        assertTrue(latch.await(10, TimeUnit.SECONDS));
        for (Map.Entry<String, String> entry : keyValMap.entrySet()) assertEquals(entry.getValue(), cache.get(entry.getKey()));
    } finally {
        if (kafkaStmr != null)
            kafkaStmr.stop();
    }
}