use of org.apache.kafka.raft.OffsetAndEpoch in project kafka by apache.
the class FileRawSnapshotTest method testBatchWriteReadSnapshot.
@Test
public void testBatchWriteReadSnapshot() throws IOException {
OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3);
int bufferSize = 256;
int batchSize = 3;
int numberOfBatches = 10;
try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
for (int i = 0; i < numberOfBatches; i++) {
ByteBuffer[] buffers = IntStream.range(0, batchSize).mapToObj(ignore -> ByteBuffer.wrap(randomBytes(bufferSize))).toArray(ByteBuffer[]::new);
snapshot.append(buildRecords(buffers));
}
snapshot.freeze();
}
try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) {
int countBatches = 0;
int countRecords = 0;
Iterator<RecordBatch> batches = Utils.covariantCast(snapshot.records().batchIterator());
while (batches.hasNext()) {
RecordBatch batch = batches.next();
countBatches += 1;
Iterator<Record> records = batch.streamingIterator(new GrowableBufferSupplier());
while (records.hasNext()) {
Record record = records.next();
countRecords += 1;
assertFalse(record.hasKey());
assertTrue(record.hasValue());
assertEquals(bufferSize, record.value().remaining());
}
}
assertEquals(numberOfBatches, countBatches);
assertEquals(numberOfBatches * batchSize, countRecords);
}
}
use of org.apache.kafka.raft.OffsetAndEpoch in project kafka by apache.
the class FileRawSnapshotTest method testCreateSnapshotWithSameId.
@Test
public void testCreateSnapshotWithSameId() {
OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(20L, 2);
int bufferSize = 256;
int numberOfBatches = 1;
try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
UnalignedMemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize)));
for (int i = 0; i < numberOfBatches; i++) {
snapshot.append(records);
}
snapshot.freeze();
}
// Create another snapshot with the same id
try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
UnalignedMemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize)));
for (int i = 0; i < numberOfBatches; i++) {
snapshot.append(records);
}
snapshot.freeze();
}
}
use of org.apache.kafka.raft.OffsetAndEpoch in project kafka by apache.
the class FileRawSnapshotTest method testBufferWriteReadSnapshot.
@Test
public void testBufferWriteReadSnapshot() throws IOException {
OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3);
int bufferSize = 256;
int batchSize = 3;
int numberOfBatches = 10;
int expectedSize = 0;
try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
for (int i = 0; i < numberOfBatches; i++) {
ByteBuffer[] buffers = IntStream.range(0, batchSize).mapToObj(ignore -> ByteBuffer.wrap(randomBytes(bufferSize))).toArray(ByteBuffer[]::new);
UnalignedMemoryRecords records = buildRecords(buffers);
snapshot.append(records);
expectedSize += records.sizeInBytes();
}
assertEquals(expectedSize, snapshot.sizeInBytes());
snapshot.freeze();
}
// File should exist and the size should be the sum of all the buffers
assertTrue(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch)));
assertEquals(expectedSize, Files.size(Snapshots.snapshotPath(tempDir, offsetAndEpoch)));
try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) {
int countBatches = 0;
int countRecords = 0;
Iterator<RecordBatch> batches = Utils.covariantCast(snapshot.records().batchIterator());
while (batches.hasNext()) {
RecordBatch batch = batches.next();
countBatches += 1;
Iterator<Record> records = batch.streamingIterator(new GrowableBufferSupplier());
while (records.hasNext()) {
Record record = records.next();
countRecords += 1;
assertFalse(record.hasKey());
assertTrue(record.hasValue());
assertEquals(bufferSize, record.value().remaining());
}
}
assertEquals(numberOfBatches, countBatches);
assertEquals(numberOfBatches * batchSize, countRecords);
}
}
use of org.apache.kafka.raft.OffsetAndEpoch in project kafka by apache.
the class FileRawSnapshotTest method testWriteReadSnapshot.
@Test
public void testWriteReadSnapshot() throws IOException {
OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3);
int bufferSize = 256;
int numberOfBatches = 10;
ByteBuffer expectedBuffer = ByteBuffer.wrap(randomBytes(bufferSize));
try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
UnalignedMemoryRecords records = buildRecords(expectedBuffer);
for (int i = 0; i < numberOfBatches; i++) {
snapshot.append(records);
}
snapshot.freeze();
}
try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) {
int countBatches = 0;
int countRecords = 0;
Iterator<RecordBatch> batches = Utils.covariantCast(snapshot.records().batchIterator());
while (batches.hasNext()) {
RecordBatch batch = batches.next();
countBatches += 1;
Iterator<Record> records = batch.streamingIterator(new GrowableBufferSupplier());
while (records.hasNext()) {
Record record = records.next();
countRecords += 1;
assertFalse(record.hasKey());
assertTrue(record.hasValue());
assertEquals(bufferSize, record.value().remaining());
assertEquals(expectedBuffer, record.value());
}
}
assertEquals(numberOfBatches, countBatches);
assertEquals(numberOfBatches, countRecords);
}
}
use of org.apache.kafka.raft.OffsetAndEpoch in project kafka by apache.
the class FileRawSnapshotTest method testAbortedSnapshot.
@Test
public void testAbortedSnapshot() throws IOException {
OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(20L, 2);
int bufferSize = 256;
int numberOfBatches = 10;
try (FileRawSnapshotWriter snapshot = createSnapshotWriter(tempDir, offsetAndEpoch)) {
UnalignedMemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize)));
for (int i = 0; i < numberOfBatches; i++) {
snapshot.append(records);
}
}
// File should not exist since freeze was not called before
assertFalse(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch)));
assertEquals(0, Files.list(Snapshots.snapshotDir(tempDir)).count());
}
Aggregations