Examples with LogEntryProto org.apache.ratis.proto.RaftProtos.LogEntryProto used on opensource projects

Example 46 with LogEntryProto

use of org.apache.ratis.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestCacheEviction method testEvictionInSegmentedLog.

@Test
public void testEvictionInSegmentedLog() throws Exception {
    final RaftProperties prop = new RaftProperties();
    prop.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, SimpleStateMachine4Testing.class, StateMachine.class);
    RaftServerConfigKeys.Log.setSegmentSizeMax(prop, SizeInBytes.valueOf("8KB"));
    RaftServerConfigKeys.Log.setPreallocatedSize(prop, SizeInBytes.valueOf("8KB"));
    final RaftPeerId peerId = RaftPeerId.valueOf("s0");
    final RaftGroupId groupId = RaftGroupId.randomId();
    final RaftGroupMemberId memberId = RaftGroupMemberId.valueOf(peerId, groupId);
    final int maxCachedNum = RaftServerConfigKeys.Log.segmentCacheNumMax(prop);
    File storageDir = getTestDir();
    RaftServerConfigKeys.setStorageDir(prop, Collections.singletonList(storageDir));
    RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir);
    final DivisionInfo info = Mockito.mock(DivisionInfo.class);
    Mockito.when(info.getLastAppliedIndex()).thenReturn(0L);
    Mockito.when(info.getFollowerNextIndices()).thenReturn(new long[] {});
    final SegmentedRaftLog raftLog = RaftServerTestUtil.newSegmentedRaftLog(memberId, info, storage, prop);
    raftLog.open(RaftLog.INVALID_LOG_INDEX, null);
    List<SegmentRange> slist = TestSegmentedRaftLog.prepareRanges(0, maxCachedNum, 7, 0);
    LogEntryProto[] entries = generateEntries(slist);
    raftLog.append(entries).forEach(CompletableFuture::join);
    // check the current cached segment number: the last segment is still open
    Assert.assertEquals(maxCachedNum - 1, raftLog.getRaftLogCache().getCachedSegmentNum());
    Mockito.when(info.getLastAppliedIndex()).thenReturn(35L);
    Mockito.when(info.getFollowerNextIndices()).thenReturn(new long[] { 21, 40, 40 });
    slist = TestSegmentedRaftLog.prepareRanges(maxCachedNum, maxCachedNum + 2, 7, 7 * maxCachedNum);
    entries = generateEntries(slist);
    raftLog.append(entries).forEach(CompletableFuture::join);
    // check the cached segment number again. since the slowest follower is on
    // index 21, the eviction should happen and evict 3 segments
    Assert.assertEquals(maxCachedNum + 1 - 3, raftLog.getRaftLogCache().getCachedSegmentNum());
}

Example 47 with LogEntryProto

use of org.apache.ratis.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestRaftLogReadWrite method testReadWithPadding.

/**
 * Simulate the scenario that the peer is shutdown without truncating
 * log segment file padding. Make sure the reader can correctly handle this.
 */
@Test
public void testReadWithPadding() throws IOException {
    final RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir);
    final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage);
    long size = SegmentedRaftLogFormat.getHeaderLength();
    LogEntryProto[] entries = new LogEntryProto[100];
    final SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, ByteBuffer.allocateDirect(bufferSize));
    size += writeMessages(entries, out);
    out.flush();
    // make sure the file contains padding
    Assert.assertEquals(RaftServerConfigKeys.Log.PREALLOCATED_SIZE_DEFAULT.getSize(), openSegment.length());
    // check if the reader can correctly read the log file
    final LogEntryProto[] readEntries = readLog(openSegment, 0, RaftLog.INVALID_LOG_INDEX, true);
    Assert.assertArrayEquals(entries, readEntries);
    out.close();
    Assert.assertEquals(size, openSegment.length());
}

Example 48 with LogEntryProto

use of org.apache.ratis.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestRaftLogReadWrite method testReadWithCorruptPadding.

/**
 * corrupt the padding by inserting non-zero bytes. Make sure the reader
 * throws exception.
 */
@Test
public void testReadWithCorruptPadding() throws IOException {
    final RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir);
    final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage);
    LogEntryProto[] entries = new LogEntryProto[10];
    final SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(openSegment, false, 16 * 1024 * 1024, 4 * 1024 * 1024, ByteBuffer.allocateDirect(bufferSize));
    for (int i = 0; i < 10; i++) {
        SimpleOperation m = new SimpleOperation("m" + i);
        entries[i] = LogProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, i);
        out.write(entries[i]);
    }
    out.flush();
    // make sure the file contains padding
    Assert.assertEquals(4 * 1024 * 1024, openSegment.length());
    try (FileOutputStream fout = new FileOutputStream(openSegment, true)) {
        ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[] { -1, 1 });
        fout.getChannel().write(byteBuffer, 16 * 1024 * 1024 - 10);
    }
    List<LogEntryProto> list = new ArrayList<>();
    try (SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream(openSegment, 0, RaftLog.INVALID_LOG_INDEX, true)) {
        LogEntryProto entry;
        while ((entry = in.nextEntry()) != null) {
            list.add(entry);
        }
        Assert.fail("should fail since we corrupt the padding");
    } catch (IOException e) {
        boolean findVerifyTerminator = false;
        for (StackTraceElement s : e.getStackTrace()) {
            if (s.getMethodName().equals("verifyTerminator")) {
                findVerifyTerminator = true;
                break;
            }
        }
        Assert.assertTrue(findVerifyTerminator);
    }
    Assert.assertArrayEquals(entries, list.toArray(new LogEntryProto[list.size()]));
}

Example 49 with LogEntryProto

use of org.apache.ratis.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestSegmentedRaftLogCache method prepareLogSegment.

private LogSegment prepareLogSegment(long start, long end, boolean isOpen) {
    LogSegment s = LogSegment.newOpenSegment(null, start, null);
    for (long i = start; i <= end; i++) {
        SimpleOperation m = new SimpleOperation("m" + i);
        LogEntryProto entry = LogProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, i);
        s.appendToOpenSegment(entry, LogSegment.Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE);
    }
    if (!isOpen) {
        s.close();
    }
    return s;
}

Example 50 with LogEntryProto

use of org.apache.ratis.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestSegmentedRaftLogCache method testAppendEntry.

@Test
public void testAppendEntry() throws Exception {
    LogSegment closedSegment = prepareLogSegment(0, 99, false);
    cache.addSegment(closedSegment);
    final SimpleOperation m = new SimpleOperation("m");
    try {
        LogEntryProto entry = LogProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, 0);
        cache.appendEntry(entry, LogSegment.Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE);
        Assert.fail("the open segment is null");
    } catch (IllegalStateException ignored) {
    }
    LogSegment openSegment = prepareLogSegment(100, 100, true);
    cache.addSegment(openSegment);
    for (long index = 101; index < 200; index++) {
        LogEntryProto entry = LogProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, index);
        cache.appendEntry(entry, LogSegment.Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE);
    }
    Assert.assertNotNull(cache.getOpenSegment());
    checkCache(0, 199, 100);
}