Examples with KafkaProducer org.apache.kafka.clients.producer.KafkaProducer used on opensource projects

Example 21 with KafkaProducer

use of org.apache.kafka.clients.producer.KafkaProducer in project storm by apache.

the class TestKafkaDataSourcesProvider method testKafkaSink.

@SuppressWarnings("unchecked")
@Test
public void testKafkaSink() {
    ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(URI.create("kafka://mock?topic=foo"), null, null, TBL_PROPERTIES, FIELDS);
    Assert.assertNotNull(ds);
    ISqlTridentDataSource.SqlTridentConsumer consumer = ds.getConsumer();
    Assert.assertEquals(TridentKafkaStateFactory.class, consumer.getStateFactory().getClass());
    Assert.assertEquals(TridentKafkaUpdater.class, consumer.getStateUpdater().getClass());
    TridentKafkaState state = (TridentKafkaState) consumer.getStateFactory().makeState(Collections.emptyMap(), null, 0, 1);
    KafkaProducer producer = mock(KafkaProducer.class);
    doReturn(mock(Future.class)).when(producer).send(any(ProducerRecord.class));
    Whitebox.setInternalState(state, "producer", producer);
    List<TridentTuple> tupleList = mockTupleList();
    for (TridentTuple t : tupleList) {
        state.updateState(Collections.singletonList(t), null);
        verify(producer).send(argThat(new KafkaMessageMatcher(t)));
    }
    verifyNoMoreInteractions(producer);
}

Example 22 with KafkaProducer

use of org.apache.kafka.clients.producer.KafkaProducer in project heron by twitter.

the class KafkaUtilsTest method createTopicAndSendMessage.

private void createTopicAndSendMessage(String key, String value) {
    Properties p = new Properties();
    p.put("acks", "1");
    p.put("bootstrap.servers", broker.getBrokerConnectionString());
    p.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    p.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    p.put("metadata.fetch.timeout.ms", 1000);
    KafkaProducer<String, String> producer = new KafkaProducer<String, String>(p);
    try {
        producer.send(new ProducerRecord<String, String>(config.topic, key, value)).get();
    // SUPPRESS CHECKSTYLE IllegalCatch
    } catch (Exception e) {
        Assert.fail(e.getMessage());
        LOG.error("Failed to do synchronous sending due to " + e, e);
    } finally {
        producer.close();
    }
}

Example 23 with KafkaProducer

use of org.apache.kafka.clients.producer.KafkaProducer in project helios by spotify.

the class EventSenderFactory method build.

public static List<EventSender> build(final Environment environment, final CommonConfiguration<?> config, final MetricRegistry metricRegistry, final String pubsubHealthcheckTopic) {
    final List<EventSender> senders = new ArrayList<>();
    final KafkaClientProvider kafkaClientProvider = new KafkaClientProvider(config.getKafkaBrokers());
    final Optional<KafkaProducer<String, byte[]>> kafkaProducer = kafkaClientProvider.getDefaultProducer();
    kafkaProducer.ifPresent(producer -> senders.add(new KafkaSender(producer)));
    final LifecycleEnvironment lifecycle = environment.lifecycle();
    if (!config.getPubsubPrefixes().isEmpty()) {
        final PubSub pubsub = PubSubOptions.getDefaultInstance().getService();
        final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setDaemon(true).setNameFormat("pubsub-healthchecker-%d").build());
        // choose an arbitrary prefix to use in the healthcheck. we assume if we can connect to
        // one we can connect to all
        final String topicToHealthcheck = config.getPubsubPrefixes().iterator().next() + pubsubHealthcheckTopic;
        final GooglePubSubSender.DefaultHealthChecker healthchecker = new GooglePubSubSender.DefaultHealthChecker(pubsub, topicToHealthcheck, executor, Duration.ofMinutes(5));
        metricRegistry.register("pubsub-health", (Gauge<Boolean>) healthchecker::isHealthy);
        for (final String prefix : config.getPubsubPrefixes()) {
            final GooglePubSubSender sender = GooglePubSubSender.create(pubsub, prefix, healthchecker);
            senders.add(sender);
        }
        lifecycle.manage(new ManagedPubSub(pubsub));
    }
    senders.forEach(lifecycle::manage);
    return senders;
}