Examples with ProducerRecord org.apache.kafka.clients.producer.ProducerRecord used on opensource projects

Example 16 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project kafka by apache.

the class WorkerSourceTaskTest method expectSendRecord.

private Capture<ProducerRecord<byte[], byte[]>> expectSendRecord(boolean anyTimes, boolean isRetry, boolean succeed) throws InterruptedException {
    expectConvertKeyValue(anyTimes);
    expectApplyTransformationChain(anyTimes);
    Capture<ProducerRecord<byte[], byte[]>> sent = EasyMock.newCapture();
    // 1. Offset data is passed to the offset storage.
    if (!isRetry) {
        offsetWriter.offset(PARTITION, OFFSET);
        if (anyTimes)
            PowerMock.expectLastCall().anyTimes();
        else
            PowerMock.expectLastCall();
    }
    // 2. Converted data passed to the producer, which will need callbacks invoked for flush to work
    IExpectationSetters<Future<RecordMetadata>> expect = EasyMock.expect(producer.send(EasyMock.capture(sent), EasyMock.capture(producerCallbacks)));
    IAnswer<Future<RecordMetadata>> expectResponse = new IAnswer<Future<RecordMetadata>>() {

        @Override
        public Future<RecordMetadata> answer() throws Throwable {
            synchronized (producerCallbacks) {
                for (org.apache.kafka.clients.producer.Callback cb : producerCallbacks.getValues()) {
                    cb.onCompletion(new RecordMetadata(new TopicPartition("foo", 0), 0, 0, 0L, 0L, 0, 0), null);
                }
                producerCallbacks.reset();
            }
            return sendFuture;
        }
    };
    if (anyTimes)
        expect.andStubAnswer(expectResponse);
    else
        expect.andAnswer(expectResponse);
    // 3. As a result of a successful producer send callback, we'll notify the source task of the record commit
    expectTaskCommitRecord(anyTimes, succeed);
    return sent;
}

Example 17 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project kafka by apache.

the class WorkerSourceTaskTest method testSendRecordsPropagatesTimestamp.

@Test
public void testSendRecordsPropagatesTimestamp() throws Exception {
    final Long timestamp = System.currentTimeMillis();
    createWorkerTask();
    List<SourceRecord> records = Collections.singletonList(new SourceRecord(PARTITION, OFFSET, "topic", null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD, timestamp));
    Capture<ProducerRecord<byte[], byte[]>> sent = expectSendRecordAnyTimes();
    PowerMock.replayAll();
    Whitebox.setInternalState(workerTask, "toSend", records);
    Whitebox.invokeMethod(workerTask, "sendRecords");
    assertEquals(timestamp, sent.getValue().timestamp());
    PowerMock.verifyAll();
}

Example 18 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project kafka by apache.

the class WorkerSourceTaskTest method testSendRecordsCorruptTimestamp.

@Test(expected = InvalidRecordException.class)
public void testSendRecordsCorruptTimestamp() throws Exception {
    final Long timestamp = -3L;
    createWorkerTask();
    List<SourceRecord> records = Collections.singletonList(new SourceRecord(PARTITION, OFFSET, "topic", null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD, timestamp));
    Capture<ProducerRecord<byte[], byte[]>> sent = expectSendRecordAnyTimes();
    PowerMock.replayAll();
    Whitebox.setInternalState(workerTask, "toSend", records);
    Whitebox.invokeMethod(workerTask, "sendRecords");
    assertEquals(null, sent.getValue().timestamp());
    PowerMock.verifyAll();
}

Example 19 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project incubator-atlas by apache.

the class KafkaNotification method sendInternalToProducer.

@VisibleForTesting
void sendInternalToProducer(Producer p, NotificationType type, String[] messages) throws NotificationException {
    String topic = TOPIC_MAP.get(type);
    List<MessageContext> messageContexts = new ArrayList<>();
    for (String message : messages) {
        ProducerRecord record = new ProducerRecord(topic, message);
        LOG.debug("Sending message for topic {}: {}", topic, message);
        Future future = p.send(record);
        messageContexts.add(new MessageContext(future, message));
    }
    List<String> failedMessages = new ArrayList<>();
    Exception lastFailureException = null;
    for (MessageContext context : messageContexts) {
        try {
            RecordMetadata response = context.getFuture().get();
            LOG.debug("Sent message for topic - {}, partition - {}, offset - {}", response.topic(), response.partition(), response.offset());
        } catch (Exception e) {
            lastFailureException = e;
            failedMessages.add(context.getMessage());
        }
    }
    if (lastFailureException != null) {
        throw new NotificationException(lastFailureException, failedMessages);
    }
}

Example 20 with ProducerRecord

use of org.apache.kafka.clients.producer.ProducerRecord in project incubator-atlas by apache.

the class KafkaNotificationMockTest method shouldSendMessagesSuccessfully.

@Test
@SuppressWarnings("unchecked")
public void shouldSendMessagesSuccessfully() throws NotificationException, ExecutionException, InterruptedException {
    Properties configProperties = mock(Properties.class);
    KafkaNotification kafkaNotification = new KafkaNotification(configProperties);
    Producer producer = mock(Producer.class);
    String topicName = kafkaNotification.getTopicName(NotificationInterface.NotificationType.HOOK);
    String message = "This is a test message";
    Future returnValue = mock(Future.class);
    when(returnValue.get()).thenReturn(new RecordMetadata(new TopicPartition(topicName, 0), 0, 0));
    ProducerRecord expectedRecord = new ProducerRecord(topicName, message);
    when(producer.send(expectedRecord)).thenReturn(returnValue);
    kafkaNotification.sendInternalToProducer(producer, NotificationInterface.NotificationType.HOOK, new String[] { message });
    verify(producer).send(expectedRecord);
}