Examples with LogEntryProto org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto used on opensource projects

Example 31 with LogEntryProto

use of org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestRaftLogSegment method testPreallocationAndAppend.

/**
 * Keep appending and check if pre-allocation is correct
 */
@Test
public void testPreallocationAndAppend() throws Exception {
    final SizeInBytes max = SizeInBytes.valueOf(2, TraditionalBinaryPrefix.MEGA);
    RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
    final File file = storage.getStorageDir().getOpenLogFile(0);
    final byte[] content = new byte[1024];
    Arrays.fill(content, (byte) 1);
    SimpleOperation op = new SimpleOperation(new String(content));
    LogEntryProto entry = ProtoUtils.toLogEntryProto(op.getLogEntryContent(), 0, 0, clientId, callId);
    final long entrySize = LogSegment.getEntrySize(entry);
    long totalSize = SegmentedRaftLog.HEADER_BYTES.length;
    long preallocated = 16 * 1024;
    try (LogOutputStream out = new LogOutputStream(file, false, max.getSize(), 16 * 1024, 10 * 1024)) {
        Assert.assertEquals(preallocated, file.length());
        while (totalSize + entrySize < max.getSize()) {
            totalSize += entrySize;
            out.write(entry);
            if (totalSize > preallocated) {
                Assert.assertEquals("totalSize==" + totalSize, preallocated + 16 * 1024, file.length());
                preallocated += 16 * 1024;
            }
        }
    }
    Assert.assertEquals(totalSize, file.length());
}

Example 32 with LogEntryProto

use of org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestRaftLogSegment method testPreallocateSegment.

@Test
public void testPreallocateSegment() throws Exception {
    RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
    final File file = storage.getStorageDir().getOpenLogFile(0);
    final int[] maxSizes = new int[] { 1024, 1025, 1024 * 1024 - 1, 1024 * 1024, 1024 * 1024 + 1, 2 * 1024 * 1024 - 1, 2 * 1024 * 1024, 2 * 1024 * 1024 + 1, 8 * 1024 * 1024 };
    final int[] preallocated = new int[] { 512, 1024, 1025, 1024 * 1024, 1024 * 1024 + 1, 2 * 1024 * 1024 };
    // make sure preallocation is correct with different max/pre-allocated size
    for (int max : maxSizes) {
        for (int a : preallocated) {
            try (LogOutputStream ignored = new LogOutputStream(file, false, max, a, bufferSize)) {
                Assert.assertEquals("max=" + max + ", a=" + a, file.length(), Math.min(max, a));
            }
            try (LogInputStream in = new LogInputStream(file, 0, INVALID_LOG_INDEX, true)) {
                LogEntryProto entry = in.nextEntry();
                Assert.assertNull(entry);
            }
        }
    }
    // test the scenario where an entry's size is larger than the max size
    final byte[] content = new byte[1024 * 2];
    Arrays.fill(content, (byte) 1);
    final long size;
    try (LogOutputStream out = new LogOutputStream(file, false, 1024, 1024, bufferSize)) {
        SimpleOperation op = new SimpleOperation(new String(content));
        LogEntryProto entry = ProtoUtils.toLogEntryProto(op.getLogEntryContent(), 0, 0, clientId, callId);
        size = LogSegment.getEntrySize(entry);
        out.write(entry);
    }
    Assert.assertEquals(file.length(), size + SegmentedRaftLog.HEADER_BYTES.length);
    try (LogInputStream in = new LogInputStream(file, 0, INVALID_LOG_INDEX, true)) {
        LogEntryProto entry = in.nextEntry();
        Assert.assertArrayEquals(content, entry.getSmLogEntry().getData().toByteArray());
        Assert.assertNull(in.nextEntry());
    }
}

Example 33 with LogEntryProto

use of org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestSegmentedRaftLog method testAppendEntriesWithInconsistency.

/**
 * Test append with inconsistent entries
 */
@Test
public void testAppendEntriesWithInconsistency() throws Exception {
    // prepare the log for truncation
    List<SegmentRange> ranges = prepareRanges(5, 200, 0);
    List<LogEntryProto> entries = prepareLogEntries(ranges, null);
    RaftServerImpl server = mock(RaftServerImpl.class);
    RetryCache retryCache = RetryCacheTestUtil.createRetryCache();
    when(server.getRetryCache()).thenReturn(retryCache);
    doCallRealMethod().when(server).failClientRequest(any(LogEntryProto.class));
    try (SegmentedRaftLog raftLog = new SegmentedRaftLog(peerId, server, storage, -1, properties)) {
        raftLog.open(RaftServerConstants.INVALID_LOG_INDEX, null);
        entries.stream().forEach(entry -> RetryCacheTestUtil.createEntry(retryCache, entry));
        // append entries to the raftlog
        entries.stream().map(raftLog::appendEntry).forEach(CompletableFuture::join);
    }
    // append entries whose first 100 entries are the same with existing log,
    // and the next 100 are with different term
    SegmentRange r1 = new SegmentRange(550, 599, 2, false);
    SegmentRange r2 = new SegmentRange(600, 649, 3, false);
    SegmentRange r3 = new SegmentRange(650, 749, 10, false);
    List<LogEntryProto> newEntries = prepareLogEntries(Arrays.asList(r1, r2, r3), null);
    try (SegmentedRaftLog raftLog = new SegmentedRaftLog(peerId, server, storage, -1, properties)) {
        raftLog.open(RaftServerConstants.INVALID_LOG_INDEX, null);
        raftLog.append(newEntries.toArray(new LogEntryProto[newEntries.size()])).forEach(CompletableFuture::join);
        checkFailedEntries(entries, 650, retryCache);
        checkEntries(raftLog, entries, 0, 650);
        checkEntries(raftLog, newEntries, 100, 100);
        Assert.assertEquals(newEntries.get(newEntries.size() - 1), getLastEntry(raftLog));
        Assert.assertEquals(newEntries.get(newEntries.size() - 1).getIndex(), raftLog.getLatestFlushedIndex());
    }
    // load the raftlog again and check
    try (SegmentedRaftLog raftLog = new SegmentedRaftLog(peerId, server, storage, -1, properties)) {
        raftLog.open(RaftServerConstants.INVALID_LOG_INDEX, null);
        checkEntries(raftLog, entries, 0, 650);
        checkEntries(raftLog, newEntries, 100, 100);
        Assert.assertEquals(newEntries.get(newEntries.size() - 1), getLastEntry(raftLog));
        Assert.assertEquals(newEntries.get(newEntries.size() - 1).getIndex(), raftLog.getLatestFlushedIndex());
        RaftLogCache cache = raftLog.getRaftLogCache();
        Assert.assertEquals(5, cache.getNumOfSegments());
    }
}

Example 34 with LogEntryProto

use of org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestSegmentedRaftLog method checkEntries.

private void checkEntries(RaftLog raftLog, List<LogEntryProto> expected, int offset, int size) throws IOException {
    if (size > 0) {
        for (int i = offset; i < size + offset; i++) {
            LogEntryProto entry = raftLog.get(expected.get(i).getIndex());
            Assert.assertEquals(expected.get(i), entry);
        }
        TermIndex[] termIndices = raftLog.getEntries(expected.get(offset).getIndex(), expected.get(offset + size - 1).getIndex() + 1);
        LogEntryProto[] entriesFromLog = Arrays.stream(termIndices).map(ti -> {
            try {
                return raftLog.get(ti.getIndex());
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }).toArray(LogEntryProto[]::new);
        LogEntryProto[] expectedArray = expected.subList(offset, offset + size).stream().toArray(LogEntryProto[]::new);
        Assert.assertArrayEquals(expectedArray, entriesFromLog);
    }
}

Example 35 with LogEntryProto

use of org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestSegmentedRaftLog method prepareLog.

private LogEntryProto[] prepareLog(List<SegmentRange> list) throws IOException {
    List<LogEntryProto> entryList = new ArrayList<>();
    for (SegmentRange range : list) {
        File file = range.isOpen ? storage.getStorageDir().getOpenLogFile(range.start) : storage.getStorageDir().getClosedLogFile(range.start, range.end);
        final int size = (int) (range.end - range.start + 1);
        LogEntryProto[] entries = new LogEntryProto[size];
        try (LogOutputStream out = new LogOutputStream(file, false, segmentMaxSize, preallocatedSize, bufferSize)) {
            for (int i = 0; i < size; i++) {
                SimpleOperation m = new SimpleOperation("m" + (i + range.start));
                entries[i] = ProtoUtils.toLogEntryProto(m.getLogEntryContent(), range.term, i + range.start, clientId, callId);
                out.write(entries[i]);
            }
        }
        Collections.addAll(entryList, entries);
    }
    return entryList.toArray(new LogEntryProto[entryList.size()]);
}