Examples with LogEntryProto org.apache.ratis.proto.RaftProtos.LogEntryProto used on opensource projects

Example 51 with LogEntryProto

use of org.apache.ratis.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestLogSegment method prepareLog.

File prepareLog(boolean isOpen, long startIndex, int numEntries, long term, boolean isLastEntryPartiallyWritten) throws IOException {
    if (!isOpen) {
        Preconditions.assertTrue(!isLastEntryPartiallyWritten, "For closed log, the last entry cannot be partially written.");
    }
    RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir);
    final File file = LogSegmentStartEnd.valueOf(startIndex, startIndex + numEntries - 1, isOpen).getFile(storage);
    final LogEntryProto[] entries = new LogEntryProto[numEntries];
    try (SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(file, false, segmentMaxSize, preallocatedSize, ByteBuffer.allocateDirect(bufferSize))) {
        for (int i = 0; i < entries.length; i++) {
            SimpleOperation op = new SimpleOperation("m" + i);
            entries[i] = LogProtoUtils.toLogEntryProto(op.getLogEntryContent(), term, i + startIndex);
            out.write(entries[i]);
        }
    }
    if (isLastEntryPartiallyWritten) {
        final int entrySize = size(entries[entries.length - 1]);
        final int truncatedEntrySize = ThreadLocalRandom.current().nextInt(entrySize - 1) + 1;
        // 0 < truncatedEntrySize < entrySize
        final long fileLength = file.length();
        final long truncatedFileLength = fileLength - (entrySize - truncatedEntrySize);
        LOG.info("truncate last entry: entry(size={}, truncated={}), file(length={}, truncated={})", entrySize, truncatedEntrySize, fileLength, truncatedFileLength);
        FileUtils.truncateFile(file, truncatedFileLength);
    }
    storage.close();
    return file;
}

Example 52 with LogEntryProto

use of org.apache.ratis.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestLogSegment method testAppendWithGap.

@Test
public void testAppendWithGap() throws Exception {
    LogSegment segment = LogSegment.newOpenSegment(null, 1000, null);
    SimpleOperation op = new SimpleOperation("m");
    final StateMachineLogEntryProto m = op.getLogEntryContent();
    try {
        LogEntryProto entry = LogProtoUtils.toLogEntryProto(m, 0, 1001);
        segment.appendToOpenSegment(entry, LogSegment.Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE);
        Assert.fail("should fail since the entry's index needs to be 1000");
    } catch (IllegalStateException e) {
    // the exception is expected.
    }
    LogEntryProto entry = LogProtoUtils.toLogEntryProto(m, 0, 1000);
    segment.appendToOpenSegment(entry, LogSegment.Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE);
    try {
        entry = LogProtoUtils.toLogEntryProto(m, 0, 1002);
        segment.appendToOpenSegment(entry, LogSegment.Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE);
        Assert.fail("should fail since the entry's index needs to be 1001");
    } catch (IllegalStateException e) {
    // the exception is expected.
    }
}

Example 53 with LogEntryProto

use of org.apache.ratis.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestLogSegment method testAppendEntries.

@Test
public void testAppendEntries() throws Exception {
    final long start = 1000;
    LogSegment segment = LogSegment.newOpenSegment(null, start, null);
    long size = SegmentedRaftLogFormat.getHeaderLength();
    final long max = 8 * 1024 * 1024;
    checkLogSegment(segment, start, start - 1, true, size, 0);
    // append till full
    long term = 0;
    int i = 0;
    while (size < max) {
        SimpleOperation op = new SimpleOperation("m" + i);
        LogEntryProto entry = LogProtoUtils.toLogEntryProto(op.getLogEntryContent(), term, i++ + start);
        size += getEntrySize(entry, LogSegment.Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE);
        segment.appendToOpenSegment(entry, LogSegment.Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE);
    }
    Assert.assertTrue(segment.getTotalFileSize() >= max);
    checkLogSegment(segment, start, i - 1 + start, true, size, term);
}

Example 54 with LogEntryProto

use of org.apache.ratis.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestLogSegment method testPreallocationAndAppend.

/**
 * Keep appending and check if pre-allocation is correct
 */
@Test
public void testPreallocationAndAppend() throws Exception {
    final SizeInBytes max = SizeInBytes.valueOf(2, TraditionalBinaryPrefix.MEGA);
    RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir);
    final File file = LogSegmentStartEnd.valueOf(0).getFile(storage);
    final byte[] content = new byte[1024];
    Arrays.fill(content, (byte) 1);
    SimpleOperation op = new SimpleOperation(new String(content));
    LogEntryProto entry = LogProtoUtils.toLogEntryProto(op.getLogEntryContent(), 0, 0);
    final long entrySize = LogSegment.getEntrySize(entry, LogSegment.Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE);
    long totalSize = SegmentedRaftLogFormat.getHeaderLength();
    long preallocated = 16 * 1024;
    try (SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(file, false, max.getSize(), 16 * 1024, ByteBuffer.allocateDirect(10 * 1024))) {
        Assert.assertEquals(preallocated, file.length());
        while (totalSize + entrySize < max.getSize()) {
            totalSize += entrySize;
            out.write(entry);
            if (totalSize > preallocated) {
                Assert.assertEquals("totalSize==" + totalSize, preallocated + 16 * 1024, file.length());
                preallocated += 16 * 1024;
            }
        }
    }
    Assert.assertEquals(totalSize, file.length());
}

Example 55 with LogEntryProto

use of org.apache.ratis.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class TestLogSegment method testPreallocateSegment.

@Test
public void testPreallocateSegment() throws Exception {
    RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir);
    final File file = LogSegmentStartEnd.valueOf(0).getFile(storage);
    final int[] maxSizes = new int[] { 1024, 1025, 1024 * 1024 - 1, 1024 * 1024, 1024 * 1024 + 1, 2 * 1024 * 1024 - 1, 2 * 1024 * 1024, 2 * 1024 * 1024 + 1, 8 * 1024 * 1024 };
    final int[] preallocated = new int[] { 512, 1024, 1025, 1024 * 1024, 1024 * 1024 + 1, 2 * 1024 * 1024 };
    // make sure preallocation is correct with different max/pre-allocated size
    for (int max : maxSizes) {
        for (int a : preallocated) {
            try (SegmentedRaftLogOutputStream ignored = new SegmentedRaftLogOutputStream(file, false, max, a, ByteBuffer.allocateDirect(bufferSize))) {
                Assert.assertEquals("max=" + max + ", a=" + a, file.length(), Math.min(max, a));
            }
            try (SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream(file, 0, INVALID_LOG_INDEX, true)) {
                LogEntryProto entry = in.nextEntry();
                Assert.assertNull(entry);
            }
        }
    }
    // test the scenario where an entry's size is larger than the max size
    final byte[] content = new byte[1024 * 2];
    Arrays.fill(content, (byte) 1);
    final long size;
    try (SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(file, false, 1024, 1024, ByteBuffer.allocateDirect(bufferSize))) {
        SimpleOperation op = new SimpleOperation(new String(content));
        LogEntryProto entry = LogProtoUtils.toLogEntryProto(op.getLogEntryContent(), 0, 0);
        size = LogSegment.getEntrySize(entry, LogSegment.Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE);
        out.write(entry);
    }
    Assert.assertEquals(file.length(), size + SegmentedRaftLogFormat.getHeaderLength());
    try (SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream(file, 0, INVALID_LOG_INDEX, true)) {
        LogEntryProto entry = in.nextEntry();
        Assert.assertArrayEquals(content, entry.getStateMachineLogEntry().getLogData().toByteArray());
        Assert.assertNull(in.nextEntry());
    }
}