Examples with GrowableBufferSupplier org.apache.kafka.common.utils.BufferSupplier.GrowableBufferSupplier used on opensource projects

Example 1 with GrowableBufferSupplier

use of org.apache.kafka.common.utils.BufferSupplier.GrowableBufferSupplier in project kafka by apache.

the class SnapshotWriterReaderTest method validateDelimiters.

private int validateDelimiters(RawSnapshotReader snapshot, long lastContainedLogTime) {
    assertNotEquals(0, snapshot.sizeInBytes());
    int countRecords = 0;
    Iterator<RecordBatch> recordBatches = Utils.covariantCast(snapshot.records().batchIterator());
    assertTrue(recordBatches.hasNext());
    RecordBatch batch = recordBatches.next();
    Iterator<Record> records = batch.streamingIterator(new GrowableBufferSupplier());
    // Verify existence of the header record
    assertTrue(batch.isControlBatch());
    assertTrue(records.hasNext());
    Record record = records.next();
    countRecords += 1;
    SnapshotHeaderRecord headerRecord = ControlRecordUtils.deserializedSnapshotHeaderRecord(record);
    assertEquals(headerRecord.version(), ControlRecordUtils.SNAPSHOT_HEADER_HIGHEST_VERSION);
    assertEquals(headerRecord.lastContainedLogTimestamp(), lastContainedLogTime);
    assertFalse(records.hasNext());
    // Loop over remaining records
    while (recordBatches.hasNext()) {
        batch = recordBatches.next();
        records = batch.streamingIterator(new GrowableBufferSupplier());
        while (records.hasNext()) {
            countRecords += 1;
            record = records.next();
        }
    }
    // Verify existence of the footer record in the end
    assertTrue(batch.isControlBatch());
    SnapshotFooterRecord footerRecord = ControlRecordUtils.deserializedSnapshotFooterRecord(record);
    assertEquals(footerRecord.version(), ControlRecordUtils.SNAPSHOT_HEADER_HIGHEST_VERSION);
    return countRecords;
}

Example 2 with GrowableBufferSupplier

use of org.apache.kafka.common.utils.BufferSupplier.GrowableBufferSupplier in project kafka by apache.

the class FileRawSnapshotTest method testBatchWriteReadSnapshot.

@Test
public void testBatchWriteReadSnapshot() throws IOException {
    OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3);
    int bufferSize = 256;
    int batchSize = 3;
    int numberOfBatches = 10;
    try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
        for (int i = 0; i < numberOfBatches; i++) {
            ByteBuffer[] buffers = IntStream.range(0, batchSize).mapToObj(ignore -> ByteBuffer.wrap(randomBytes(bufferSize))).toArray(ByteBuffer[]::new);
            snapshot.append(buildRecords(buffers));
        }
        snapshot.freeze();
    }
    try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) {
        int countBatches = 0;
        int countRecords = 0;
        Iterator<RecordBatch> batches = Utils.covariantCast(snapshot.records().batchIterator());
        while (batches.hasNext()) {
            RecordBatch batch = batches.next();
            countBatches += 1;
            Iterator<Record> records = batch.streamingIterator(new GrowableBufferSupplier());
            while (records.hasNext()) {
                Record record = records.next();
                countRecords += 1;
                assertFalse(record.hasKey());
                assertTrue(record.hasValue());
                assertEquals(bufferSize, record.value().remaining());
            }
        }
        assertEquals(numberOfBatches, countBatches);
        assertEquals(numberOfBatches * batchSize, countRecords);
    }
}

Example 3 with GrowableBufferSupplier

use of org.apache.kafka.common.utils.BufferSupplier.GrowableBufferSupplier in project kafka by apache.

the class FileRawSnapshotTest method testBufferWriteReadSnapshot.

@Test
public void testBufferWriteReadSnapshot() throws IOException {
    OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3);
    int bufferSize = 256;
    int batchSize = 3;
    int numberOfBatches = 10;
    int expectedSize = 0;
    try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
        for (int i = 0; i < numberOfBatches; i++) {
            ByteBuffer[] buffers = IntStream.range(0, batchSize).mapToObj(ignore -> ByteBuffer.wrap(randomBytes(bufferSize))).toArray(ByteBuffer[]::new);
            UnalignedMemoryRecords records = buildRecords(buffers);
            snapshot.append(records);
            expectedSize += records.sizeInBytes();
        }
        assertEquals(expectedSize, snapshot.sizeInBytes());
        snapshot.freeze();
    }
    // File should exist and the size should be the sum of all the buffers
    assertTrue(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch)));
    assertEquals(expectedSize, Files.size(Snapshots.snapshotPath(tempDir, offsetAndEpoch)));
    try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) {
        int countBatches = 0;
        int countRecords = 0;
        Iterator<RecordBatch> batches = Utils.covariantCast(snapshot.records().batchIterator());
        while (batches.hasNext()) {
            RecordBatch batch = batches.next();
            countBatches += 1;
            Iterator<Record> records = batch.streamingIterator(new GrowableBufferSupplier());
            while (records.hasNext()) {
                Record record = records.next();
                countRecords += 1;
                assertFalse(record.hasKey());
                assertTrue(record.hasValue());
                assertEquals(bufferSize, record.value().remaining());
            }
        }
        assertEquals(numberOfBatches, countBatches);
        assertEquals(numberOfBatches * batchSize, countRecords);
    }
}

Example 4 with GrowableBufferSupplier

use of org.apache.kafka.common.utils.BufferSupplier.GrowableBufferSupplier in project kafka by apache.

the class FileRawSnapshotTest method testWriteReadSnapshot.

@Test
public void testWriteReadSnapshot() throws IOException {
    OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3);
    int bufferSize = 256;
    int numberOfBatches = 10;
    ByteBuffer expectedBuffer = ByteBuffer.wrap(randomBytes(bufferSize));
    try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
        UnalignedMemoryRecords records = buildRecords(expectedBuffer);
        for (int i = 0; i < numberOfBatches; i++) {
            snapshot.append(records);
        }
        snapshot.freeze();
    }
    try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) {
        int countBatches = 0;
        int countRecords = 0;
        Iterator<RecordBatch> batches = Utils.covariantCast(snapshot.records().batchIterator());
        while (batches.hasNext()) {
            RecordBatch batch = batches.next();
            countBatches += 1;
            Iterator<Record> records = batch.streamingIterator(new GrowableBufferSupplier());
            while (records.hasNext()) {
                Record record = records.next();
                countRecords += 1;
                assertFalse(record.hasKey());
                assertTrue(record.hasValue());
                assertEquals(bufferSize, record.value().remaining());
                assertEquals(expectedBuffer, record.value());
            }
        }
        assertEquals(numberOfBatches, countBatches);
        assertEquals(numberOfBatches, countRecords);
    }
}