Examples with ProducerRecord org.apache.kafka.clients.producer.ProducerRecord used on opensource projects

Example 31 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project kafka by apache.

the class RecordCollectorTest method shouldRetryWhenTimeoutExceptionOccursOnSend.

@SuppressWarnings("unchecked")
@Test
public void shouldRetryWhenTimeoutExceptionOccursOnSend() throws Exception {
    final AtomicInteger attempt = new AtomicInteger(0);
    RecordCollectorImpl collector = new RecordCollectorImpl(new MockProducer(cluster, true, new DefaultPartitioner(), byteArraySerializer, byteArraySerializer) {

        @Override
        public synchronized Future<RecordMetadata> send(final ProducerRecord record, final Callback callback) {
            if (attempt.getAndIncrement() == 0) {
                throw new TimeoutException();
            }
            return super.send(record, callback);
        }
    }, "test");
    collector.send("topic1", "3", "0", null, null, stringSerializer, stringSerializer, streamPartitioner);
    final Long offset = collector.offsets().get(new TopicPartition("topic1", 0));
    assertEquals(Long.valueOf(0L), offset);
}

Example 32 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project kafka by apache.

the class RecordCollectorImpl method send.

@Override
public <K, V> void send(final String topic, K key, V value, Integer partition, Long timestamp, Serializer<K> keySerializer, Serializer<V> valueSerializer, StreamPartitioner<? super K, ? super V> partitioner) {
    checkForException();
    byte[] keyBytes = keySerializer.serialize(topic, key);
    byte[] valBytes = valueSerializer.serialize(topic, value);
    if (partition == null && partitioner != null) {
        List<PartitionInfo> partitions = this.producer.partitionsFor(topic);
        if (partitions != null && partitions.size() > 0)
            partition = partitioner.partition(key, value, partitions.size());
    }
    ProducerRecord<byte[], byte[]> serializedRecord = new ProducerRecord<>(topic, partition, timestamp, keyBytes, valBytes);
    for (int attempt = 1; attempt <= MAX_SEND_ATTEMPTS; attempt++) {
        try {
            this.producer.send(serializedRecord, new Callback() {

                @Override
                public void onCompletion(RecordMetadata metadata, Exception exception) {
                    if (exception == null) {
                        if (sendException != null) {
                            return;
                        }
                        TopicPartition tp = new TopicPartition(metadata.topic(), metadata.partition());
                        offsets.put(tp, metadata.offset());
                    } else {
                        sendException = exception;
                        log.error("{} Error sending record to topic {}. No more offsets will be recorded for this task and the exception will eventually be thrown", logPrefix, topic, exception);
                    }
                }
            });
            return;
        } catch (TimeoutException e) {
            if (attempt == MAX_SEND_ATTEMPTS) {
                throw new StreamsException(String.format("%s Failed to send record to topic %s after %d attempts", logPrefix, topic, attempt));
            }
            log.warn("{} Timeout exception caught when sending record to topic {} attempt {}", logPrefix, topic, attempt);
            Utils.sleep(SEND_RETRY_BACKOFF);
        }
    }
}

Example 33 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project storm by apache.

the class KafkaBoltTest method testSimple.

@SuppressWarnings({ "unchecked", "serial" })
@Test
public void testSimple() {
    final KafkaProducer<String, String> producer = mock(KafkaProducer.class);
    when(producer.send(any(), any())).thenAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            Callback c = (Callback) invocation.getArguments()[1];
            c.onCompletion(null, null);
            return null;
        }
    });
    KafkaBolt<String, String> bolt = new KafkaBolt<String, String>() {

        @Override
        protected KafkaProducer<String, String> mkProducer(Properties props) {
            return producer;
        }
    };
    bolt.withTopicSelector("MY_TOPIC");
    OutputCollector collector = mock(OutputCollector.class);
    TopologyContext context = mock(TopologyContext.class);
    Map<String, Object> conf = new HashMap<>();
    bolt.prepare(conf, context, collector);
    MkTupleParam param = new MkTupleParam();
    param.setFields("key", "message");
    Tuple testTuple = Testing.testTuple(Arrays.asList("KEY", "VALUE"), param);
    bolt.execute(testTuple);
    verify(producer).send(argThat(new ArgumentMatcher<ProducerRecord<String, String>>() {

        @Override
        public boolean matches(Object argument) {
            LOG.info("GOT {} ->", argument);
            ProducerRecord<String, String> arg = (ProducerRecord<String, String>) argument;
            LOG.info("  {} {} {}", arg.topic(), arg.key(), arg.value());
            return "MY_TOPIC".equals(arg.topic()) && "KEY".equals(arg.key()) && "VALUE".equals(arg.value());
        }
    }), any(Callback.class));
    verify(collector).ack(testTuple);
}

Example 34 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project druid by druid-io.

the class ITKafkaIndexingServiceTest method testKafka.

@Test
public void testKafka() {
    LOG.info("Starting test: ITKafkaIndexingServiceTest");
    // create topic
    try {
        int sessionTimeoutMs = 10000;
        int connectionTimeoutMs = 10000;
        String zkHosts = config.getZookeeperHosts();
        zkClient = new ZkClient(zkHosts, sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);
        int numPartitions = 4;
        int replicationFactor = 1;
        Properties topicConfig = new Properties();
        AdminUtils.createTopic(zkClient, TOPIC_NAME, numPartitions, replicationFactor, topicConfig);
    } catch (TopicExistsException e) {
    // it's ok if the topic already exists
    } catch (Exception e) {
        throw new ISE(e, "could not create kafka topic");
    }
    String spec;
    try {
        LOG.info("supervisorSpec name: [%s]", INDEXER_FILE);
        spec = getTaskAsString(INDEXER_FILE).replaceAll("%%DATASOURCE%%", DATASOURCE).replaceAll("%%TOPIC%%", TOPIC_NAME).replaceAll("%%KAFKA_BROKER%%", config.getKafkaHost());
        LOG.info("supervisorSpec: [%s]\n", spec);
    } catch (Exception e) {
        LOG.error("could not read file [%s]", INDEXER_FILE);
        throw new ISE(e, "could not read file [%s]", INDEXER_FILE);
    }
    // start supervisor
    supervisorId = indexer.submitSupervisor(spec);
    LOG.info("Submitted supervisor");
    // set up kafka producer
    Properties properties = new Properties();
    properties.put("bootstrap.servers", config.getKafkaHost());
    LOG.info("Kafka bootstrap.servers: [%s]", config.getKafkaHost());
    properties.put("acks", "all");
    properties.put("retries", "3");
    KafkaProducer<String, String> producer = new KafkaProducer<>(properties, new StringSerializer(), new StringSerializer());
    DateTimeZone zone = DateTimeZone.forID("UTC");
    // format for putting into events
    DateTimeFormatter event_fmt = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z'");
    // timestamp to put on events
    DateTime dt = new DateTime(zone);
    // timestamp of 1st event
    dtFirst = dt;
    // timestamp of last event
    dtLast = dt;
    // these are used to compute the expected aggregations
    int added = 0;
    int num_events = 0;
    // send data to kafka
    while (num_events < NUM_EVENTS_TO_SEND) {
        num_events++;
        added += num_events;
        // construct the event to send
        String event = String.format(event_template, event_fmt.print(dt), num_events, 0, num_events);
        LOG.info("sending event: [%s]", event);
        try {
            producer.send(new ProducerRecord<String, String>(TOPIC_NAME, event)).get();
        } catch (Exception ioe) {
            throw Throwables.propagate(ioe);
        }
        dtLast = dt;
        dt = new DateTime(zone);
    }
    producer.close();
    LOG.info("Waiting for [%s] millis for Kafka indexing tasks to consume events", WAIT_TIME_MILLIS);
    try {
        Thread.sleep(WAIT_TIME_MILLIS);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new RuntimeException(e);
    }
    InputStream is = ITKafkaIndexingServiceTest.class.getResourceAsStream(QUERIES_FILE);
    if (null == is) {
        throw new ISE("could not open query file: %s", QUERIES_FILE);
    }
    // put the timestamps into the query structure
    String query_response_template;
    try {
        query_response_template = IOUtils.toString(is, "UTF-8");
    } catch (IOException e) {
        throw new ISE(e, "could not read query file: %s", QUERIES_FILE);
    }
    String queryStr = query_response_template.replaceAll("%%DATASOURCE%%", DATASOURCE).replace("%%TIMEBOUNDARY_RESPONSE_TIMESTAMP%%", TIMESTAMP_FMT.print(dtFirst)).replace("%%TIMEBOUNDARY_RESPONSE_MAXTIME%%", TIMESTAMP_FMT.print(dtLast)).replace("%%TIMEBOUNDARY_RESPONSE_MINTIME%%", TIMESTAMP_FMT.print(dtFirst)).replace("%%TIMESERIES_QUERY_START%%", INTERVAL_FMT.print(dtFirst)).replace("%%TIMESERIES_QUERY_END%%", INTERVAL_FMT.print(dtLast.plusMinutes(2))).replace("%%TIMESERIES_RESPONSE_TIMESTAMP%%", TIMESTAMP_FMT.print(dtFirst)).replace("%%TIMESERIES_ADDED%%", Integer.toString(added)).replace("%%TIMESERIES_NUMEVENTS%%", Integer.toString(num_events));
    // this query will probably be answered from the indexing tasks but possibly from 2 historical segments / 2 indexing
    try {
        this.queryHelper.testQueriesFromString(queryStr, 2);
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
    LOG.info("Shutting down Kafka Supervisor");
    indexer.shutdownSupervisor(supervisorId);
    // wait for all kafka indexing tasks to finish
    LOG.info("Waiting for all kafka indexing tasks to finish");
    RetryUtil.retryUntilTrue(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return (indexer.getPendingTasks().size() + indexer.getRunningTasks().size() + indexer.getWaitingTasks().size()) == 0;
        }
    }, "Waiting for Tasks Completion");
    // wait for segments to be handed off
    try {
        RetryUtil.retryUntil(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                return coordinator.areSegmentsLoaded(DATASOURCE);
            }
        }, true, 30000, 10, "Real-time generated segments loaded");
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
    LOG.info("segments are present");
    segmentsExist = true;
    // this query will be answered by at least 1 historical segment, most likely 2, and possibly up to all 4
    try {
        this.queryHelper.testQueriesFromString(queryStr, 2);
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 35 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project druid by druid-io.

the class ITKafkaTest method testKafka.

@Test
public void testKafka() {
    LOG.info("Starting test: ITKafkaTest");
    // create topic
    try {
        int sessionTimeoutMs = 10000;
        int connectionTimeoutMs = 10000;
        String zkHosts = config.getZookeeperHosts();
        zkClient = new ZkClient(zkHosts, sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);
        int numPartitions = 1;
        int replicationFactor = 1;
        Properties topicConfig = new Properties();
        AdminUtils.createTopic(zkClient, TOPIC_NAME, numPartitions, replicationFactor, topicConfig);
    } catch (TopicExistsException e) {
    // it's ok if the topic already exists
    } catch (Exception e) {
        throw new ISE(e, "could not create kafka topic");
    }
    // set up kafka producer
    Properties properties = new Properties();
    properties.put("bootstrap.servers", config.getKafkaHost());
    LOG.info("Kafka bootstrap.servers: [%s]", config.getKafkaHost());
    properties.put("acks", "all");
    properties.put("retries", "3");
    KafkaProducer<String, String> producer = new KafkaProducer<>(properties, new StringSerializer(), new StringSerializer());
    DateTimeZone zone = DateTimeZone.forID("UTC");
    // format for putting into events
    DateTimeFormatter event_fmt = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z'");
    // timestamp to put on events
    DateTime dt = new DateTime(zone);
    // timestamp of 1st event
    dtFirst = dt;
    // timestamp of last event
    dtLast = dt;
    // these are used to compute the expected aggregations
    int added = 0;
    int num_events = 10;
    // send data to kafka
    for (int i = 0; i < num_events; i++) {
        added += i;
        // construct the event to send
        String event = String.format(event_template, event_fmt.print(dt), i, 0, i);
        LOG.info("sending event: [%s]", event);
        try {
            // Send event to kafka
            producer.send(new ProducerRecord<String, String>(TOPIC_NAME, event)).get();
        } catch (Exception ioe) {
            throw Throwables.propagate(ioe);
        }
        dtLast = dt;
        dt = new DateTime(zone);
    }
    producer.close();
    String indexerSpec;
    // replace temp strings in indexer file
    try {
        LOG.info("indexerFile name: [%s]", INDEXER_FILE);
        indexerSpec = getTaskAsString(INDEXER_FILE).replaceAll("%%DATASOURCE%%", DATASOURCE).replaceAll("%%TOPIC%%", TOPIC_NAME).replaceAll("%%ZOOKEEPER_SERVER%%", config.getZookeeperHosts()).replaceAll("%%GROUP_ID%%", Long.toString(System.currentTimeMillis())).replaceAll("%%COUNT%%", Integer.toString(num_events));
        LOG.info("indexerFile: [%s]\n", indexerSpec);
    } catch (Exception e) {
        // log here so the message will appear in the console output
        LOG.error("could not read indexer file [%s]", INDEXER_FILE);
        throw new ISE(e, "could not read indexer file [%s]", INDEXER_FILE);
    }
    // start indexing task
    taskID = indexer.submitTask(indexerSpec);
    LOG.info("-------------SUBMITTED TASK");
    // wait for the task to finish
    indexer.waitUntilTaskCompletes(taskID, 20000, 30);
    // wait for segments to be handed off
    try {
        RetryUtil.retryUntil(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                return coordinator.areSegmentsLoaded(DATASOURCE);
            }
        }, true, 30000, 10, "Real-time generated segments loaded");
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
    LOG.info("segments are present");
    segmentsExist = true;
    // put the timestamps into the query structure
    String query_response_template = null;
    InputStream is = ITKafkaTest.class.getResourceAsStream(QUERIES_FILE);
    if (null == is) {
        throw new ISE("could not open query file: %s", QUERIES_FILE);
    }
    try {
        query_response_template = IOUtils.toString(is, "UTF-8");
    } catch (IOException e) {
        throw new ISE(e, "could not read query file: %s", QUERIES_FILE);
    }
    String queryStr = query_response_template.replaceAll("%%DATASOURCE%%", DATASOURCE).replace("%%TIMEBOUNDARY_RESPONSE_TIMESTAMP%%", TIMESTAMP_FMT.print(dtFirst)).replace("%%TIMEBOUNDARY_RESPONSE_MAXTIME%%", TIMESTAMP_FMT.print(dtLast)).replace("%%TIMEBOUNDARY_RESPONSE_MINTIME%%", TIMESTAMP_FMT.print(dtFirst)).replace("%%TIMESERIES_QUERY_START%%", INTERVAL_FMT.print(dtFirst)).replace("%%TIMESERIES_QUERY_END%%", INTERVAL_FMT.print(dtFirst.plusMinutes(MINUTES_TO_SEND + 2))).replace("%%TIMESERIES_RESPONSE_TIMESTAMP%%", TIMESTAMP_FMT.print(dtFirst)).replace("%%TIMESERIES_ADDED%%", Integer.toString(added)).replace("%%TIMESERIES_NUMEVENTS%%", Integer.toString(num_events));
    // this query will probably be answered from the realtime task
    try {
        this.queryHelper.testQueriesFromString(queryStr, 2);
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}