Examples with MemoryRecords org.apache.kafka.common.record.MemoryRecords used on opensource projects

Example 1 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class RequestResponseTest method createFetchResponse.

private FetchResponse createFetchResponse() {
    LinkedHashMap<TopicPartition, FetchResponse.PartitionData> responseData = new LinkedHashMap<>();
    MemoryRecords records = MemoryRecords.readableRecords(ByteBuffer.allocate(10));
    responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData(Errors.NONE, 1000000, records));
    return new FetchResponse(responseData, 25);
}

Example 2 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class RequestResponseTest method createProduceRequest.

private ProduceRequest createProduceRequest() {
    Map<TopicPartition, MemoryRecords> produceData = new HashMap<>();
    produceData.put(new TopicPartition("test", 0), MemoryRecords.readableRecords(ByteBuffer.allocate(10)));
    return new ProduceRequest.Builder((short) 1, 5000, produceData).build();
}

Example 3 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class ProduceRequest method toStruct.

/**
     * Visible for testing.
     */
@Override
public Struct toStruct() {
    // Store it in a local variable to protect against concurrent updates
    Map<TopicPartition, MemoryRecords> partitionRecords = partitionRecordsOrFail();
    Struct struct = new Struct(ApiKeys.PRODUCE.requestSchema(version()));
    Map<String, Map<Integer, MemoryRecords>> recordsByTopic = CollectionUtils.groupDataByTopic(partitionRecords);
    struct.set(ACKS_KEY_NAME, acks);
    struct.set(TIMEOUT_KEY_NAME, timeout);
    List<Struct> topicDatas = new ArrayList<>(recordsByTopic.size());
    for (Map.Entry<String, Map<Integer, MemoryRecords>> entry : recordsByTopic.entrySet()) {
        Struct topicData = struct.instance(TOPIC_DATA_KEY_NAME);
        topicData.set(TOPIC_KEY_NAME, entry.getKey());
        List<Struct> partitionArray = new ArrayList<>();
        for (Map.Entry<Integer, MemoryRecords> partitionEntry : entry.getValue().entrySet()) {
            MemoryRecords records = partitionEntry.getValue();
            Struct part = topicData.instance(PARTITION_DATA_KEY_NAME).set(PARTITION_KEY_NAME, partitionEntry.getKey()).set(RECORD_SET_KEY_NAME, records);
            partitionArray.add(part);
        }
        topicData.set(PARTITION_DATA_KEY_NAME, partitionArray.toArray());
        topicDatas.add(topicData);
    }
    struct.set(TOPIC_DATA_KEY_NAME, topicDatas.toArray());
    return struct;
}

Example 4 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project apache-kafka-on-k8s by banzaicloud.

the class SenderTest method testSequenceNumberIncrement.

@Test
public void testSequenceNumberIncrement() throws InterruptedException {
    final long producerId = 343434L;
    TransactionManager transactionManager = new TransactionManager();
    transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId, (short) 0));
    setupWithTransactionState(transactionManager);
    client.setNode(new Node(1, "localhost", 33343));
    int maxRetries = 10;
    Metrics m = new Metrics();
    SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
    Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, 50, transactionManager, apiVersions);
    Future<RecordMetadata> responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    client.prepareResponse(new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            if (body instanceof ProduceRequest) {
                ProduceRequest request = (ProduceRequest) body;
                MemoryRecords records = request.partitionRecordsOrFail().get(tp0);
                Iterator<MutableRecordBatch> batchIterator = records.batches().iterator();
                assertTrue(batchIterator.hasNext());
                RecordBatch batch = batchIterator.next();
                assertFalse(batchIterator.hasNext());
                assertEquals(0, batch.baseSequence());
                assertEquals(producerId, batch.producerId());
                assertEquals(0, batch.producerEpoch());
                return true;
            }
            return false;
        }
    }, produceResponse(tp0, 0, Errors.NONE, 0));
    // connect.
    sender.run(time.milliseconds());
    // send.
    sender.run(time.milliseconds());
    // receive response
    sender.run(time.milliseconds());
    assertTrue(responseFuture.isDone());
    assertEquals(0L, (long) transactionManager.lastAckedSequence(tp0));
    assertEquals(1L, (long) transactionManager.sequenceNumber(tp0));
}

Example 5 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project apache-kafka-on-k8s by banzaicloud.

the class SenderTest method produceRequestMatcher.

private MockClient.RequestMatcher produceRequestMatcher(final TopicPartition tp, final ProducerIdAndEpoch producerIdAndEpoch, final int sequence, final boolean isTransactional) {
    return new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            if (!(body instanceof ProduceRequest))
                return false;
            ProduceRequest request = (ProduceRequest) body;
            Map<TopicPartition, MemoryRecords> recordsMap = request.partitionRecordsOrFail();
            MemoryRecords records = recordsMap.get(tp);
            if (records == null)
                return false;
            List<MutableRecordBatch> batches = TestUtils.toList(records.batches());
            if (batches.isEmpty() || batches.size() > 1)
                return false;
            MutableRecordBatch batch = batches.get(0);
            return batch.baseOffset() == 0L && batch.baseSequence() == sequence && batch.producerId() == producerIdAndEpoch.producerId && batch.producerEpoch() == producerIdAndEpoch.epoch && batch.isTransactional() == isTransactional;
        }
    };
}