Examples with RecordMetadata org.apache.kafka.clients.producer.RecordMetadata used on opensource projects

Example 16 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project ksql by confluentinc.

the class TopicProducer method produceInputData.

/**
 * Topic topicName will be automatically created if it doesn't exist.
 * @param topicName
 * @param recordsToPublish
 * @param schema
 * @return
 * @throws InterruptedException
 * @throws TimeoutException
 * @throws ExecutionException
 */
public Map<String, RecordMetadata> produceInputData(String topicName, Map<String, GenericRow> recordsToPublish, Schema schema) throws InterruptedException, TimeoutException, ExecutionException {
    KafkaProducer<String, GenericRow> producer = new KafkaProducer<>(producerConfig, new StringSerializer(), new KsqlJsonSerializer(schema));
    Map<String, RecordMetadata> result = new HashMap<>();
    for (Map.Entry<String, GenericRow> recordEntry : recordsToPublish.entrySet()) {
        String key = recordEntry.getKey();
        ProducerRecord<String, GenericRow> producerRecord = new ProducerRecord<>(topicName, key, recordEntry.getValue());
        Future<RecordMetadata> recordMetadataFuture = producer.send(producerRecord);
        result.put(key, recordMetadataFuture.get(TEST_RECORD_FUTURE_TIMEOUT_MS, TimeUnit.MILLISECONDS));
    }
    producer.close();
    return result;
}

Example 17 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project apache-kafka-on-k8s by banzaicloud.

the class WorkerSourceTaskTest method expectSendRecord.

@SuppressWarnings("unchecked")
private Capture<ProducerRecord<byte[], byte[]>> expectSendRecord(boolean anyTimes, boolean isRetry, boolean succeed) throws InterruptedException {
    expectConvertKeyValue(anyTimes);
    expectApplyTransformationChain(anyTimes);
    Capture<ProducerRecord<byte[], byte[]>> sent = EasyMock.newCapture();
    // 1. Offset data is passed to the offset storage.
    if (!isRetry) {
        offsetWriter.offset(PARTITION, OFFSET);
        if (anyTimes)
            PowerMock.expectLastCall().anyTimes();
        else
            PowerMock.expectLastCall();
    }
    // 2. Converted data passed to the producer, which will need callbacks invoked for flush to work
    IExpectationSetters<Future<RecordMetadata>> expect = EasyMock.expect(producer.send(EasyMock.capture(sent), EasyMock.capture(producerCallbacks)));
    IAnswer<Future<RecordMetadata>> expectResponse = new IAnswer<Future<RecordMetadata>>() {

        @Override
        public Future<RecordMetadata> answer() throws Throwable {
            synchronized (producerCallbacks) {
                for (org.apache.kafka.clients.producer.Callback cb : producerCallbacks.getValues()) {
                    cb.onCompletion(new RecordMetadata(new TopicPartition("foo", 0), 0, 0, 0L, 0L, 0, 0), null);
                }
                producerCallbacks.reset();
            }
            return sendFuture;
        }
    };
    if (anyTimes)
        expect.andStubAnswer(expectResponse);
    else
        expect.andAnswer(expectResponse);
    // 3. As a result of a successful producer send callback, we'll notify the source task of the record commit
    expectTaskCommitRecord(anyTimes, succeed);
    return sent;
}

Example 18 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project apache-kafka-on-k8s by banzaicloud.

the class RecordAccumulatorTest method testAbortUnsentBatches.

@Test
public void testAbortUnsentBatches() throws Exception {
    long lingerMs = Long.MAX_VALUE;
    int numRecords = 100;
    final AtomicInteger numExceptionReceivedInCallback = new AtomicInteger(0);
    final RecordAccumulator accum = createTestRecordAccumulator(128 + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 64 * 1024, CompressionType.NONE, lingerMs);
    final KafkaException cause = new KafkaException();
    class TestCallback implements Callback {

        @Override
        public void onCompletion(RecordMetadata metadata, Exception exception) {
            assertEquals(cause, exception);
            numExceptionReceivedInCallback.incrementAndGet();
        }
    }
    for (int i = 0; i < numRecords; i++) accum.append(new TopicPartition(topic, i % 3), 0L, key, value, null, new TestCallback(), maxBlockTimeMs);
    RecordAccumulator.ReadyCheckResult result = accum.ready(cluster, time.milliseconds());
    assertFalse(result.readyNodes.isEmpty());
    Map<Integer, List<ProducerBatch>> drained = accum.drain(cluster, result.readyNodes, Integer.MAX_VALUE, time.milliseconds());
    assertTrue(accum.hasUndrained());
    assertTrue(accum.hasIncomplete());
    accum.abortUndrainedBatches(cause);
    int numDrainedRecords = 0;
    for (Map.Entry<Integer, List<ProducerBatch>> drainedEntry : drained.entrySet()) {
        for (ProducerBatch batch : drainedEntry.getValue()) {
            assertTrue(batch.isClosed());
            assertFalse(batch.produceFuture.completed());
            numDrainedRecords += batch.recordCount;
        }
    }
    assertTrue(numDrainedRecords > 0);
    assertTrue(numExceptionReceivedInCallback.get() > 0);
    assertEquals(numRecords, numExceptionReceivedInCallback.get() + numDrainedRecords);
    assertFalse(accum.hasUndrained());
    assertTrue(accum.hasIncomplete());
}

Example 19 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project apache-kafka-on-k8s by banzaicloud.

the class SenderTest method testSequenceNumberIncrement.

@Test
public void testSequenceNumberIncrement() throws InterruptedException {
    final long producerId = 343434L;
    TransactionManager transactionManager = new TransactionManager();
    transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId, (short) 0));
    setupWithTransactionState(transactionManager);
    client.setNode(new Node(1, "localhost", 33343));
    int maxRetries = 10;
    Metrics m = new Metrics();
    SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
    Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, 50, transactionManager, apiVersions);
    Future<RecordMetadata> responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    client.prepareResponse(new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            if (body instanceof ProduceRequest) {
                ProduceRequest request = (ProduceRequest) body;
                MemoryRecords records = request.partitionRecordsOrFail().get(tp0);
                Iterator<MutableRecordBatch> batchIterator = records.batches().iterator();
                assertTrue(batchIterator.hasNext());
                RecordBatch batch = batchIterator.next();
                assertFalse(batchIterator.hasNext());
                assertEquals(0, batch.baseSequence());
                assertEquals(producerId, batch.producerId());
                assertEquals(0, batch.producerEpoch());
                return true;
            }
            return false;
        }
    }, produceResponse(tp0, 0, Errors.NONE, 0));
    // connect.
    sender.run(time.milliseconds());
    // send.
    sender.run(time.milliseconds());
    // receive response
    sender.run(time.milliseconds());
    assertTrue(responseFuture.isDone());
    assertEquals(0L, (long) transactionManager.lastAckedSequence(tp0));
    assertEquals(1L, (long) transactionManager.sequenceNumber(tp0));
}

Example 20 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project apache-kafka-on-k8s by banzaicloud.

the class SenderTest method testIdempotenceWithMultipleInflightsRetriedInOrder.

@Test
public void testIdempotenceWithMultipleInflightsRetriedInOrder() throws Exception {
    // Send multiple in flight requests, retry them all one at a time, in the correct order.
    final long producerId = 343434L;
    TransactionManager transactionManager = new TransactionManager();
    setupWithTransactionState(transactionManager);
    prepareAndReceiveInitProducerId(producerId, Errors.NONE);
    assertTrue(transactionManager.hasProducerId());
    assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
    // Send first ProduceRequest
    Future<RecordMetadata> request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    sender.run(time.milliseconds());
    String nodeId = client.requests().peek().destination();
    Node node = new Node(Integer.valueOf(nodeId), "localhost", 0);
    assertEquals(1, client.inFlightRequestCount());
    assertEquals(1, transactionManager.sequenceNumber(tp0).longValue());
    assertEquals(-1, transactionManager.lastAckedSequence(tp0));
    // Send second ProduceRequest
    Future<RecordMetadata> request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    sender.run(time.milliseconds());
    // Send third ProduceRequest
    Future<RecordMetadata> request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    sender.run(time.milliseconds());
    assertEquals(3, client.inFlightRequestCount());
    assertEquals(3, transactionManager.sequenceNumber(tp0).longValue());
    assertEquals(-1, transactionManager.lastAckedSequence(tp0));
    assertFalse(request1.isDone());
    assertFalse(request2.isDone());
    assertFalse(request3.isDone());
    assertTrue(client.isReady(node, time.milliseconds()));
    sendIdempotentProducerResponse(0, tp0, Errors.LEADER_NOT_AVAILABLE, -1L);
    // receive response 0
    sender.run(time.milliseconds());
    // Queue the fourth request, it shouldn't be sent until the first 3 complete.
    Future<RecordMetadata> request4 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    assertEquals(2, client.inFlightRequestCount());
    assertEquals(-1, transactionManager.lastAckedSequence(tp0));
    sendIdempotentProducerResponse(1, tp0, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, -1L);
    // re send request 1, receive response 2
    sender.run(time.milliseconds());
    sendIdempotentProducerResponse(2, tp0, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, -1L);
    // receive response 3
    sender.run(time.milliseconds());
    assertEquals(-1, transactionManager.lastAckedSequence(tp0));
    assertEquals(1, client.inFlightRequestCount());
    // Do nothing, we are reduced to one in flight request during retries.
    sender.run(time.milliseconds());
    // the batch for request 4 shouldn't have been drained, and hence the sequence should not have been incremented.
    assertEquals(3, transactionManager.sequenceNumber(tp0).longValue());
    assertEquals(1, client.inFlightRequestCount());
    assertEquals(-1, transactionManager.lastAckedSequence(tp0));
    sendIdempotentProducerResponse(0, tp0, Errors.NONE, 0L);
    // receive response 1
    sender.run(time.milliseconds());
    assertEquals(0, transactionManager.lastAckedSequence(tp0));
    assertTrue(request1.isDone());
    assertEquals(0, request1.get().offset());
    assertFalse(client.hasInFlightRequests());
    // send request 2;
    sender.run(time.milliseconds());
    assertEquals(1, client.inFlightRequestCount());
    sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L);
    // receive response 2
    sender.run(time.milliseconds());
    assertEquals(1, transactionManager.lastAckedSequence(tp0));
    assertTrue(request2.isDone());
    assertEquals(1, request2.get().offset());
    assertFalse(client.hasInFlightRequests());
    // send request 3
    sender.run(time.milliseconds());
    assertEquals(1, client.inFlightRequestCount());
    sendIdempotentProducerResponse(2, tp0, Errors.NONE, 2L);
    // receive response 3, send request 4 since we are out of 'retry' mode.
    sender.run(time.milliseconds());
    assertEquals(2, transactionManager.lastAckedSequence(tp0));
    assertTrue(request3.isDone());
    assertEquals(2, request3.get().offset());
    assertEquals(1, client.inFlightRequestCount());
    sendIdempotentProducerResponse(3, tp0, Errors.NONE, 3L);
    // receive response 4
    sender.run(time.milliseconds());
    assertEquals(3, transactionManager.lastAckedSequence(tp0));
    assertTrue(request4.isDone());
    assertEquals(3, request4.get().offset());
}