Examples with UnalignedMemoryRecords org.apache.kafka.common.record.UnalignedMemoryRecords used on opensource projects

Example 6 with UnalignedMemoryRecords

use of org.apache.kafka.common.record.UnalignedMemoryRecords in project kafka by apache.

the class FileRawSnapshotTest method testCreateSnapshotWithSameId.

@Test
public void testCreateSnapshotWithSameId() {
    OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(20L, 2);
    int bufferSize = 256;
    int numberOfBatches = 1;
    try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
        UnalignedMemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize)));
        for (int i = 0; i < numberOfBatches; i++) {
            snapshot.append(records);
        }
        snapshot.freeze();
    }
    // Create another snapshot with the same id
    try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
        UnalignedMemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize)));
        for (int i = 0; i < numberOfBatches; i++) {
            snapshot.append(records);
        }
        snapshot.freeze();
    }
}

Example 7 with UnalignedMemoryRecords

use of org.apache.kafka.common.record.UnalignedMemoryRecords in project kafka by apache.

the class FileRawSnapshotTest method testBufferWriteReadSnapshot.

@Test
public void testBufferWriteReadSnapshot() throws IOException {
    OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3);
    int bufferSize = 256;
    int batchSize = 3;
    int numberOfBatches = 10;
    int expectedSize = 0;
    try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
        for (int i = 0; i < numberOfBatches; i++) {
            ByteBuffer[] buffers = IntStream.range(0, batchSize).mapToObj(ignore -> ByteBuffer.wrap(randomBytes(bufferSize))).toArray(ByteBuffer[]::new);
            UnalignedMemoryRecords records = buildRecords(buffers);
            snapshot.append(records);
            expectedSize += records.sizeInBytes();
        }
        assertEquals(expectedSize, snapshot.sizeInBytes());
        snapshot.freeze();
    }
    // File should exist and the size should be the sum of all the buffers
    assertTrue(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch)));
    assertEquals(expectedSize, Files.size(Snapshots.snapshotPath(tempDir, offsetAndEpoch)));
    try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) {
        int countBatches = 0;
        int countRecords = 0;
        Iterator<RecordBatch> batches = Utils.covariantCast(snapshot.records().batchIterator());
        while (batches.hasNext()) {
            RecordBatch batch = batches.next();
            countBatches += 1;
            Iterator<Record> records = batch.streamingIterator(new GrowableBufferSupplier());
            while (records.hasNext()) {
                Record record = records.next();
                countRecords += 1;
                assertFalse(record.hasKey());
                assertTrue(record.hasValue());
                assertEquals(bufferSize, record.value().remaining());
            }
        }
        assertEquals(numberOfBatches, countBatches);
        assertEquals(numberOfBatches * batchSize, countRecords);
    }
}

Example 8 with UnalignedMemoryRecords

use of org.apache.kafka.common.record.UnalignedMemoryRecords in project kafka by apache.

the class FileRawSnapshotTest method testWriteReadSnapshot.

@Test
public void testWriteReadSnapshot() throws IOException {
    OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3);
    int bufferSize = 256;
    int numberOfBatches = 10;
    ByteBuffer expectedBuffer = ByteBuffer.wrap(randomBytes(bufferSize));
    try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
        UnalignedMemoryRecords records = buildRecords(expectedBuffer);
        for (int i = 0; i < numberOfBatches; i++) {
            snapshot.append(records);
        }
        snapshot.freeze();
    }
    try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) {
        int countBatches = 0;
        int countRecords = 0;
        Iterator<RecordBatch> batches = Utils.covariantCast(snapshot.records().batchIterator());
        while (batches.hasNext()) {
            RecordBatch batch = batches.next();
            countBatches += 1;
            Iterator<Record> records = batch.streamingIterator(new GrowableBufferSupplier());
            while (records.hasNext()) {
                Record record = records.next();
                countRecords += 1;
                assertFalse(record.hasKey());
                assertTrue(record.hasValue());
                assertEquals(bufferSize, record.value().remaining());
                assertEquals(expectedBuffer, record.value());
            }
        }
        assertEquals(numberOfBatches, countBatches);
        assertEquals(numberOfBatches, countRecords);
    }
}

Example 9 with UnalignedMemoryRecords

use of org.apache.kafka.common.record.UnalignedMemoryRecords in project kafka by apache.

the class FileRawSnapshotTest method testAbortedSnapshot.

@Test
public void testAbortedSnapshot() throws IOException {
    OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(20L, 2);
    int bufferSize = 256;
    int numberOfBatches = 10;
    try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
        UnalignedMemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize)));
        for (int i = 0; i < numberOfBatches; i++) {
            snapshot.append(records);
        }
    }
    // File should not exist since freeze was not called before
    assertFalse(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch)));
    assertEquals(0, Files.list(Snapshots.snapshotDir(tempDir)).count());
}

Example 10 with UnalignedMemoryRecords

use of org.apache.kafka.common.record.UnalignedMemoryRecords in project kafka by apache.

the class KafkaRaftClientSnapshotTest method testFetchSnapshotRequestAsLeader.

@Test
public void testFetchSnapshotRequestAsLeader() throws Exception {
    int localId = 0;
    Set<Integer> voters = Utils.mkSet(localId, localId + 1);
    OffsetAndEpoch snapshotId = new OffsetAndEpoch(1, 1);
    List<String> records = Arrays.asList("foo", "bar");
    RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).appendToLog(snapshotId.epoch, Arrays.asList("a")).build();
    context.becomeLeader();
    int epoch = context.currentEpoch();
    context.advanceLocalLeaderHighWatermarkToLogEndOffset();
    try (SnapshotWriter<String> snapshot = context.client.createSnapshot(snapshotId.offset - 1, snapshotId.epoch, 0).get()) {
        assertEquals(snapshotId, snapshot.snapshotId());
        snapshot.append(records);
        snapshot.freeze();
    }
    RawSnapshotReader snapshot = context.log.readSnapshot(snapshotId).get();
    context.deliverRequest(fetchSnapshotRequest(context.metadataPartition, epoch, snapshotId, Integer.MAX_VALUE, 0));
    context.client.poll();
    FetchSnapshotResponseData.PartitionSnapshot response = context.assertSentFetchSnapshotResponse(context.metadataPartition).get();
    assertEquals(Errors.NONE, Errors.forCode(response.errorCode()));
    assertEquals(snapshot.sizeInBytes(), response.size());
    assertEquals(0, response.position());
    assertEquals(snapshot.sizeInBytes(), response.unalignedRecords().sizeInBytes());
    UnalignedMemoryRecords memoryRecords = (UnalignedMemoryRecords) snapshot.slice(0, Math.toIntExact(snapshot.sizeInBytes()));
    assertEquals(memoryRecords.buffer(), ((UnalignedMemoryRecords) response.unalignedRecords()).buffer());
}