Examples with RawSnapshotWriter org.apache.kafka.snapshot.RawSnapshotWriter used on opensource projects

Example 6 with RawSnapshotWriter

use of org.apache.kafka.snapshot.RawSnapshotWriter in project kafka by apache.

the class MockLogTest method testUpdateLogStartOffsetWillRemoveOlderSnapshot.

@Test
public void testUpdateLogStartOffsetWillRemoveOlderSnapshot() throws IOException {
    int numberOfRecords = 10;
    int epoch = 1;
    OffsetAndEpoch sameEpochSnapshotId = new OffsetAndEpoch(numberOfRecords, epoch);
    appendBatch(numberOfRecords, epoch);
    log.updateHighWatermark(new LogOffsetMetadata(sameEpochSnapshotId.offset));
    try (RawSnapshotWriter snapshot = log.createNewSnapshot(sameEpochSnapshotId).get()) {
        snapshot.freeze();
    }
    OffsetAndEpoch greaterEpochSnapshotId = new OffsetAndEpoch(2 * numberOfRecords, epoch + 1);
    appendBatch(numberOfRecords, greaterEpochSnapshotId.epoch);
    log.updateHighWatermark(new LogOffsetMetadata(greaterEpochSnapshotId.offset));
    try (RawSnapshotWriter snapshot = log.createNewSnapshot(greaterEpochSnapshotId).get()) {
        snapshot.freeze();
    }
    assertTrue(log.deleteBeforeSnapshot(greaterEpochSnapshotId));
    assertEquals(Optional.empty(), log.readSnapshot(sameEpochSnapshotId));
}

Example 7 with RawSnapshotWriter

use of org.apache.kafka.snapshot.RawSnapshotWriter in project kafka by apache.

the class MockLogTest method testUpdateLogStartOffset.

@Test
public void testUpdateLogStartOffset() throws IOException {
    int offset = 10;
    int epoch = 0;
    OffsetAndEpoch snapshotId = new OffsetAndEpoch(offset, epoch);
    appendBatch(offset, epoch);
    log.updateHighWatermark(new LogOffsetMetadata(offset));
    try (RawSnapshotWriter snapshot = log.createNewSnapshot(snapshotId).get()) {
        snapshot.freeze();
    }
    assertTrue(log.deleteBeforeSnapshot(snapshotId));
    assertEquals(offset, log.startOffset());
    assertEquals(epoch, log.lastFetchedEpoch());
    assertEquals(offset, log.endOffset().offset);
    int newRecords = 10;
    appendBatch(newRecords, epoch + 1);
    log.updateHighWatermark(new LogOffsetMetadata(offset + newRecords));
    // Start offset should not change since a new snapshot was not generated
    assertFalse(log.deleteBeforeSnapshot(new OffsetAndEpoch(offset + newRecords, epoch)));
    assertEquals(offset, log.startOffset());
    assertEquals(epoch + 1, log.lastFetchedEpoch());
    assertEquals(offset + newRecords, log.endOffset().offset);
    assertEquals(offset + newRecords, log.highWatermark().offset);
}

Example 8 with RawSnapshotWriter

use of org.apache.kafka.snapshot.RawSnapshotWriter in project kafka by apache.

the class MockLogTest method testCreateSnapshotMuchEalierEpoch.

@Test
public void testCreateSnapshotMuchEalierEpoch() {
    int numberOfRecords = 10;
    int epoch = 2;
    OffsetAndEpoch snapshotId = new OffsetAndEpoch(numberOfRecords, epoch);
    appendBatch(numberOfRecords, epoch);
    log.updateHighWatermark(new LogOffsetMetadata(numberOfRecords));
    try (RawSnapshotWriter snapshot = log.createNewSnapshot(snapshotId).get()) {
        snapshot.freeze();
    }
    assertTrue(log.deleteBeforeSnapshot(snapshotId));
    assertEquals(snapshotId.offset, log.startOffset());
    assertThrows(IllegalArgumentException.class, () -> log.createNewSnapshot(new OffsetAndEpoch(numberOfRecords, epoch - 1)));
}

Example 9 with RawSnapshotWriter

use of org.apache.kafka.snapshot.RawSnapshotWriter in project kafka by apache.

the class MockLogTest method testAppendAsFollower.

@Test
public void testAppendAsFollower() throws IOException {
    final long initialOffset = 5;
    final int epoch = 3;
    SimpleRecord recordFoo = new SimpleRecord("foo".getBytes());
    try (RawSnapshotWriter snapshot = log.storeSnapshot(new OffsetAndEpoch(initialOffset, 0)).get()) {
        snapshot.freeze();
    }
    log.truncateToLatestSnapshot();
    log.appendAsFollower(MemoryRecords.withRecords(initialOffset, CompressionType.NONE, epoch, recordFoo));
    assertEquals(initialOffset, log.startOffset());
    assertEquals(initialOffset + 1, log.endOffset().offset);
    assertEquals(3, log.lastFetchedEpoch());
    Records records = log.read(5L, Isolation.UNCOMMITTED).records;
    List<ByteBuffer> extractRecords = new ArrayList<>();
    for (Record record : records.records()) {
        extractRecords.add(record.value());
    }
    assertEquals(1, extractRecords.size());
    assertEquals(recordFoo.value(), extractRecords.get(0));
    assertEquals(new OffsetAndEpoch(5, 0), log.endOffsetForEpoch(0));
    assertEquals(new OffsetAndEpoch(log.endOffset().offset, epoch), log.endOffsetForEpoch(epoch));
}

Example 10 with RawSnapshotWriter

use of org.apache.kafka.snapshot.RawSnapshotWriter in project kafka by apache.

the class MockLogTest method testCreateExistingSnapshot.

@Test
public void testCreateExistingSnapshot() {
    int numberOfRecords = 10;
    int epoch = 1;
    OffsetAndEpoch snapshotId = new OffsetAndEpoch(numberOfRecords, epoch);
    appendBatch(numberOfRecords, epoch);
    log.updateHighWatermark(new LogOffsetMetadata(numberOfRecords));
    try (RawSnapshotWriter snapshot = log.createNewSnapshot(snapshotId).get()) {
        snapshot.freeze();
    }
    assertTrue(log.deleteBeforeSnapshot(snapshotId));
    assertEquals(snapshotId.offset, log.startOffset());
    assertEquals(Optional.empty(), log.createNewSnapshot(snapshotId));
}