Examples with MemoryRecordsBuilder org.apache.kafka.common.record.MemoryRecordsBuilder used on opensource projects

Example 16 with MemoryRecordsBuilder

use of org.apache.kafka.common.record.MemoryRecordsBuilder in project kafka by apache.

the class FetcherTest method testFetcherLeadMetric.

@Test
public void testFetcherLeadMetric() {
    buildFetcher();
    assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 0);
    MetricName minLeadMetric = metrics.metricInstance(metricsRegistry.recordsLeadMin);
    Map<String, String> tags = new HashMap<>(2);
    tags.put("topic", tp0.topic());
    tags.put("partition", String.valueOf(tp0.partition()));
    MetricName partitionLeadMetric = metrics.metricName("records-lead", metricGroup, "", tags);
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric recordsFetchLeadMin = allMetrics.get(minLeadMetric);
    // recordsFetchLeadMin should be initialized to NaN
    assertEquals(Double.NaN, (Double) recordsFetchLeadMin.metricValue(), EPSILON);
    // recordsFetchLeadMin should be position - logStartOffset after receiving an empty FetchResponse
    fetchRecords(tidp0, MemoryRecords.EMPTY, Errors.NONE, 100L, -1L, 0L, 0);
    assertEquals(0L, (Double) recordsFetchLeadMin.metricValue(), EPSILON);
    KafkaMetric partitionLead = allMetrics.get(partitionLeadMetric);
    assertEquals(0L, (Double) partitionLead.metricValue(), EPSILON);
    // recordsFetchLeadMin should be position - logStartOffset after receiving a non-empty FetchResponse
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
    for (int v = 0; v < 3; v++) {
        builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
    }
    fetchRecords(tidp0, builder.build(), Errors.NONE, 200L, -1L, 0L, 0);
    assertEquals(0L, (Double) recordsFetchLeadMin.metricValue(), EPSILON);
    assertEquals(3L, (Double) partitionLead.metricValue(), EPSILON);
    // verify de-registration of partition lag
    subscriptions.unsubscribe();
    fetcher.sendFetches();
    assertFalse(allMetrics.containsKey(partitionLeadMetric));
}

Example 17 with MemoryRecordsBuilder

use of org.apache.kafka.common.record.MemoryRecordsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class ProducerBatchTest method testAppendedChecksumMagicV0AndV1.

@Test
public void testAppendedChecksumMagicV0AndV1() {
    for (byte magic : Arrays.asList(MAGIC_VALUE_V0, MAGIC_VALUE_V1)) {
        MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(128), magic, CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
        ProducerBatch batch = new ProducerBatch(new TopicPartition("topic", 1), builder, now);
        byte[] key = "hi".getBytes();
        byte[] value = "there".getBytes();
        FutureRecordMetadata future = batch.tryAppend(now, key, value, Record.EMPTY_HEADERS, null, now);
        assertNotNull(future);
        byte attributes = LegacyRecord.computeAttributes(magic, CompressionType.NONE, TimestampType.CREATE_TIME);
        long expectedChecksum = LegacyRecord.computeChecksum(magic, attributes, now, key, value);
        assertEquals(expectedChecksum, future.checksumOrNull().longValue());
    }
}

Example 18 with MemoryRecordsBuilder

use of org.apache.kafka.common.record.MemoryRecordsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class RecordBatchIterationBenchmark method createBatch.

private ByteBuffer createBatch(int batchSize) {
    byte[] value = new byte[messageSize];
    final ByteBuffer buf = ByteBuffer.allocate(AbstractRecords.estimateSizeInBytesUpperBound(messageVersion, compressionType, new byte[0], value, Record.EMPTY_HEADERS) * batchSize);
    final MemoryRecordsBuilder builder = MemoryRecords.builder(buf, messageVersion, compressionType, TimestampType.CREATE_TIME, startingOffset);
    for (int i = 0; i < batchSize; ++i) {
        switch(bytes) {
            case ONES:
                Arrays.fill(value, (byte) 1);
                break;
            case RANDOM:
                random.nextBytes(value);
                break;
        }
        builder.append(0, null, value);
    }
    return builder.build().buffer();
}

Example 19 with MemoryRecordsBuilder

use of org.apache.kafka.common.record.MemoryRecordsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class KafkaConsumerTest method fetchResponse.

private FetchResponse fetchResponse(Map<TopicPartition, FetchInfo> fetches) {
    LinkedHashMap<TopicPartition, PartitionData> tpResponses = new LinkedHashMap<>();
    for (Map.Entry<TopicPartition, FetchInfo> fetchEntry : fetches.entrySet()) {
        TopicPartition partition = fetchEntry.getKey();
        long fetchOffset = fetchEntry.getValue().offset;
        int fetchCount = fetchEntry.getValue().count;
        final MemoryRecords records;
        if (fetchCount == 0) {
            records = MemoryRecords.EMPTY;
        } else {
            MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, fetchOffset);
            for (int i = 0; i < fetchCount; i++) builder.append(0L, ("key-" + i).getBytes(), ("value-" + i).getBytes());
            records = builder.build();
        }
        tpResponses.put(partition, new FetchResponse.PartitionData(Errors.NONE, 0, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records));
    }
    return new FetchResponse(Errors.NONE, tpResponses, 0, INVALID_SESSION_ID);
}

Example 20 with MemoryRecordsBuilder

use of org.apache.kafka.common.record.MemoryRecordsBuilder in project apache-kafka-on-k8s by banzaicloud.

the class ProduceRequestTest method testV3AndAboveShouldContainOnlyOneRecordBatch.

@Test
public void testV3AndAboveShouldContainOnlyOneRecordBatch() {
    ByteBuffer buffer = ByteBuffer.allocate(256);
    MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
    builder.append(10L, null, "a".getBytes());
    builder.close();
    builder = MemoryRecords.builder(buffer, CompressionType.NONE, TimestampType.CREATE_TIME, 1L);
    builder.append(11L, "1".getBytes(), "b".getBytes());
    builder.append(12L, null, "c".getBytes());
    builder.close();
    buffer.flip();
    Map<TopicPartition, MemoryRecords> produceData = new HashMap<>();
    produceData.put(new TopicPartition("test", 0), MemoryRecords.readableRecords(buffer));
    ProduceRequest.Builder requestBuilder = ProduceRequest.Builder.forCurrentMagic((short) 1, 5000, produceData);
    assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder);
}