Examples with MemoryRecordsBuilder org.apache.kafka.common.record.MemoryRecordsBuilder used on opensource projects

Example 51 with MemoryRecordsBuilder

use of org.apache.kafka.common.record.MemoryRecordsBuilder in project kafka by apache.

the class FetcherTest method testFetchResponseMetricsPartialResponse.

@Test
public void testFetchResponseMetricsPartialResponse() {
    buildFetcher();
    assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 1);
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg));
    KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg));
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
    for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
    MemoryRecords records = builder.build();
    int expectedBytes = 0;
    for (Record record : records.records()) {
        if (record.offset() >= 1)
            expectedBytes += record.sizeInBytes();
    }
    fetchRecords(tidp0, records, Errors.NONE, 100L, 0);
    assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON);
    assertEquals(2, (Double) recordsCountAverage.metricValue(), EPSILON);
}

Example 52 with MemoryRecordsBuilder

use of org.apache.kafka.common.record.MemoryRecordsBuilder in project kafka by apache.

the class FetcherTest method testFetchResponseMetricsWithOnePartitionError.

@Test
public void testFetchResponseMetricsWithOnePartitionError() {
    buildFetcher();
    assignFromUser(mkSet(tp0, tp1));
    subscriptions.seek(tp0, 0);
    subscriptions.seek(tp1, 0);
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg));
    KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg));
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
    for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
    MemoryRecords records = builder.build();
    Map<TopicIdPartition, FetchResponseData.PartitionData> partitions = new HashMap<>();
    partitions.put(tidp0, new FetchResponseData.PartitionData().setPartitionIndex(tp0.partition()).setHighWatermark(100).setLogStartOffset(0).setRecords(records));
    partitions.put(tidp1, new FetchResponseData.PartitionData().setPartitionIndex(tp1.partition()).setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()).setHighWatermark(100).setLogStartOffset(0));
    assertEquals(1, fetcher.sendFetches());
    client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)));
    consumerClient.poll(time.timer(0));
    fetcher.collectFetch();
    int expectedBytes = 0;
    for (Record record : records.records()) expectedBytes += record.sizeInBytes();
    assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON);
    assertEquals(3, (Double) recordsCountAverage.metricValue(), EPSILON);
}

Example 53 with MemoryRecordsBuilder

use of org.apache.kafka.common.record.MemoryRecordsBuilder in project kafka by apache.

the class FetcherTest method testSubscriptionPositionUpdatedWithEpoch.

@Test
public void testSubscriptionPositionUpdatedWithEpoch() {
    // Create some records that include a leader epoch (1)
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, TimestampType.CREATE_TIME, 0L, RecordBatch.NO_TIMESTAMP, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, 1);
    builder.appendWithOffset(0L, 0L, "key".getBytes(), "value-1".getBytes());
    builder.appendWithOffset(1L, 0L, "key".getBytes(), "value-2".getBytes());
    builder.appendWithOffset(2L, 0L, "key".getBytes(), "value-3".getBytes());
    MemoryRecords records = builder.build();
    buildFetcher();
    assignFromUser(singleton(tp0));
    // Initialize the epoch=1
    Map<String, Integer> partitionCounts = new HashMap<>();
    partitionCounts.put(tp0.topic(), 4);
    MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWithIds("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> 1, topicIds);
    metadata.updateWithCurrentRequestVersion(metadataResponse, false, 0L);
    // Seek
    subscriptions.seek(tp0, 0);
    // Do a normal fetch
    assertEquals(1, fetcher.sendFetches());
    assertFalse(fetcher.hasCompletedFetches());
    client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
    consumerClient.pollNoWakeup();
    assertTrue(fetcher.hasCompletedFetches());
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
    assertTrue(partitionRecords.containsKey(tp0));
    assertEquals(subscriptions.position(tp0).offset, 3L);
    assertOptional(subscriptions.position(tp0).offsetEpoch, value -> assertEquals(value.intValue(), 1));
}

Example 54 with MemoryRecordsBuilder

use of org.apache.kafka.common.record.MemoryRecordsBuilder in project kafka by apache.

the class FetcherTest method testLeaderEpochInConsumerRecord.

@Test
public void testLeaderEpochInConsumerRecord() {
    buildFetcher();
    assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 0);
    Integer partitionLeaderEpoch = 1;
    ByteBuffer buffer = ByteBuffer.allocate(1024);
    MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, TimestampType.CREATE_TIME, 0L, System.currentTimeMillis(), partitionLeaderEpoch);
    builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes());
    builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes());
    builder.close();
    partitionLeaderEpoch += 7;
    builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, TimestampType.CREATE_TIME, 2L, System.currentTimeMillis(), partitionLeaderEpoch);
    builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes());
    builder.close();
    partitionLeaderEpoch += 5;
    builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, TimestampType.CREATE_TIME, 3L, System.currentTimeMillis(), partitionLeaderEpoch);
    builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes());
    builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes());
    builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes());
    builder.close();
    buffer.flip();
    MemoryRecords records = MemoryRecords.readableRecords(buffer);
    assertEquals(1, fetcher.sendFetches());
    assertFalse(fetcher.hasCompletedFetches());
    client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
    consumerClient.poll(time.timer(0));
    assertTrue(fetcher.hasCompletedFetches());
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
    assertTrue(partitionRecords.containsKey(tp0));
    assertEquals(6, partitionRecords.get(tp0).size());
    for (ConsumerRecord<byte[], byte[]> record : partitionRecords.get(tp0)) {
        int expectedLeaderEpoch = Integer.parseInt(Utils.utf8(record.value()));
        assertEquals(Optional.of(expectedLeaderEpoch), record.leaderEpoch());
    }
}

Example 55 with MemoryRecordsBuilder

use of org.apache.kafka.common.record.MemoryRecordsBuilder in project kafka by apache.

the class FetcherTest method testFetcherMetrics.

/*
     * Send multiple requests. Verify that the client side quota metrics have the right values
     */
@Test
public void testFetcherMetrics() {
    buildFetcher();
    assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 0);
    MetricName maxLagMetric = metrics.metricInstance(metricsRegistry.recordsLagMax);
    Map<String, String> tags = new HashMap<>();
    tags.put("topic", tp0.topic());
    tags.put("partition", String.valueOf(tp0.partition()));
    MetricName partitionLagMetric = metrics.metricName("records-lag", metricGroup, tags);
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric recordsFetchLagMax = allMetrics.get(maxLagMetric);
    // recordsFetchLagMax should be initialized to NaN
    assertEquals(Double.NaN, (Double) recordsFetchLagMax.metricValue(), EPSILON);
    // recordsFetchLagMax should be hw - fetchOffset after receiving an empty FetchResponse
    fetchRecords(tidp0, MemoryRecords.EMPTY, Errors.NONE, 100L, 0);
    assertEquals(100, (Double) recordsFetchLagMax.metricValue(), EPSILON);
    KafkaMetric partitionLag = allMetrics.get(partitionLagMetric);
    assertEquals(100, (Double) partitionLag.metricValue(), EPSILON);
    // recordsFetchLagMax should be hw - offset of the last message after receiving a non-empty FetchResponse
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
    for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
    fetchRecords(tidp0, builder.build(), Errors.NONE, 200L, 0);
    assertEquals(197, (Double) recordsFetchLagMax.metricValue(), EPSILON);
    assertEquals(197, (Double) partitionLag.metricValue(), EPSILON);
    // verify de-registration of partition lag
    subscriptions.unsubscribe();
    fetcher.sendFetches();
    assertFalse(allMetrics.containsKey(partitionLagMetric));
}