Examples with SimpleOperation org.apache.ratis.RaftTestUtil.SimpleOperation used on opensource projects

Example 16 with SimpleOperation

use of org.apache.ratis.RaftTestUtil.SimpleOperation in project incubator-ratis by apache.

the class TestRaftLogSegment method testPreallocationAndAppend.

/**
 * Keep appending and check if pre-allocation is correct
 */
@Test
public void testPreallocationAndAppend() throws Exception {
    final SizeInBytes max = SizeInBytes.valueOf(2, TraditionalBinaryPrefix.MEGA);
    RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
    final File file = storage.getStorageDir().getOpenLogFile(0);
    final byte[] content = new byte[1024];
    Arrays.fill(content, (byte) 1);
    SimpleOperation op = new SimpleOperation(new String(content));
    LogEntryProto entry = ProtoUtils.toLogEntryProto(op.getLogEntryContent(), 0, 0, clientId, callId);
    final long entrySize = LogSegment.getEntrySize(entry);
    long totalSize = SegmentedRaftLog.HEADER_BYTES.length;
    long preallocated = 16 * 1024;
    try (LogOutputStream out = new LogOutputStream(file, false, max.getSize(), 16 * 1024, 10 * 1024)) {
        Assert.assertEquals(preallocated, file.length());
        while (totalSize + entrySize < max.getSize()) {
            totalSize += entrySize;
            out.write(entry);
            if (totalSize > preallocated) {
                Assert.assertEquals("totalSize==" + totalSize, preallocated + 16 * 1024, file.length());
                preallocated += 16 * 1024;
            }
        }
    }
    Assert.assertEquals(totalSize, file.length());
}

Example 17 with SimpleOperation

use of org.apache.ratis.RaftTestUtil.SimpleOperation in project incubator-ratis by apache.

the class TestRaftLogSegment method testPreallocateSegment.

@Test
public void testPreallocateSegment() throws Exception {
    RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
    final File file = storage.getStorageDir().getOpenLogFile(0);
    final int[] maxSizes = new int[] { 1024, 1025, 1024 * 1024 - 1, 1024 * 1024, 1024 * 1024 + 1, 2 * 1024 * 1024 - 1, 2 * 1024 * 1024, 2 * 1024 * 1024 + 1, 8 * 1024 * 1024 };
    final int[] preallocated = new int[] { 512, 1024, 1025, 1024 * 1024, 1024 * 1024 + 1, 2 * 1024 * 1024 };
    // make sure preallocation is correct with different max/pre-allocated size
    for (int max : maxSizes) {
        for (int a : preallocated) {
            try (LogOutputStream ignored = new LogOutputStream(file, false, max, a, bufferSize)) {
                Assert.assertEquals("max=" + max + ", a=" + a, file.length(), Math.min(max, a));
            }
            try (LogInputStream in = new LogInputStream(file, 0, INVALID_LOG_INDEX, true)) {
                LogEntryProto entry = in.nextEntry();
                Assert.assertNull(entry);
            }
        }
    }
    // test the scenario where an entry's size is larger than the max size
    final byte[] content = new byte[1024 * 2];
    Arrays.fill(content, (byte) 1);
    final long size;
    try (LogOutputStream out = new LogOutputStream(file, false, 1024, 1024, bufferSize)) {
        SimpleOperation op = new SimpleOperation(new String(content));
        LogEntryProto entry = ProtoUtils.toLogEntryProto(op.getLogEntryContent(), 0, 0, clientId, callId);
        size = LogSegment.getEntrySize(entry);
        out.write(entry);
    }
    Assert.assertEquals(file.length(), size + SegmentedRaftLog.HEADER_BYTES.length);
    try (LogInputStream in = new LogInputStream(file, 0, INVALID_LOG_INDEX, true)) {
        LogEntryProto entry = in.nextEntry();
        Assert.assertArrayEquals(content, entry.getSmLogEntry().getData().toByteArray());
        Assert.assertNull(in.nextEntry());
    }
}

Example 18 with SimpleOperation

use of org.apache.ratis.RaftTestUtil.SimpleOperation in project incubator-ratis by apache.

the class TestSegmentedRaftLog method prepareLog.

private LogEntryProto[] prepareLog(List<SegmentRange> list) throws IOException {
    List<LogEntryProto> entryList = new ArrayList<>();
    for (SegmentRange range : list) {
        File file = range.isOpen ? storage.getStorageDir().getOpenLogFile(range.start) : storage.getStorageDir().getClosedLogFile(range.start, range.end);
        final int size = (int) (range.end - range.start + 1);
        LogEntryProto[] entries = new LogEntryProto[size];
        try (LogOutputStream out = new LogOutputStream(file, false, segmentMaxSize, preallocatedSize, bufferSize)) {
            for (int i = 0; i < size; i++) {
                SimpleOperation m = new SimpleOperation("m" + (i + range.start));
                entries[i] = ProtoUtils.toLogEntryProto(m.getLogEntryContent(), range.term, i + range.start, clientId, callId);
                out.write(entries[i]);
            }
        }
        Collections.addAll(entryList, entries);
    }
    return entryList.toArray(new LogEntryProto[entryList.size()]);
}

Example 19 with SimpleOperation

use of org.apache.ratis.RaftTestUtil.SimpleOperation in project incubator-ratis by apache.

the class TestRaftLogReadWrite method writeMessages.

private long writeMessages(LogEntryProto[] entries, SegmentedRaftLogOutputStream out) throws IOException {
    long size = 0;
    for (int i = 0; i < entries.length; i++) {
        SimpleOperation m = new SimpleOperation("m" + i);
        entries[i] = LogProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, i);
        final int s = entries[i].getSerializedSize();
        size += CodedOutputStream.computeUInt32SizeNoTag(s) + s + 4;
        out.write(entries[i]);
    }
    return size;
}

Example 20 with SimpleOperation

use of org.apache.ratis.RaftTestUtil.SimpleOperation in project incubator-ratis by apache.

the class TestRaftLogReadWrite method testReadWithCorruptPadding.

/**
 * corrupt the padding by inserting non-zero bytes. Make sure the reader
 * throws exception.
 */
@Test
public void testReadWithCorruptPadding() throws IOException {
    final RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir);
    final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage);
    LogEntryProto[] entries = new LogEntryProto[10];
    final SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(openSegment, false, 16 * 1024 * 1024, 4 * 1024 * 1024, ByteBuffer.allocateDirect(bufferSize));
    for (int i = 0; i < 10; i++) {
        SimpleOperation m = new SimpleOperation("m" + i);
        entries[i] = LogProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, i);
        out.write(entries[i]);
    }
    out.flush();
    // make sure the file contains padding
    Assert.assertEquals(4 * 1024 * 1024, openSegment.length());
    try (FileOutputStream fout = new FileOutputStream(openSegment, true)) {
        ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[] { -1, 1 });
        fout.getChannel().write(byteBuffer, 16 * 1024 * 1024 - 10);
    }
    List<LogEntryProto> list = new ArrayList<>();
    try (SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream(openSegment, 0, RaftLog.INVALID_LOG_INDEX, true)) {
        LogEntryProto entry;
        while ((entry = in.nextEntry()) != null) {
            list.add(entry);
        }
        Assert.fail("should fail since we corrupt the padding");
    } catch (IOException e) {
        boolean findVerifyTerminator = false;
        for (StackTraceElement s : e.getStackTrace()) {
            if (s.getMethodName().equals("verifyTerminator")) {
                findVerifyTerminator = true;
                break;
            }
        }
        Assert.assertTrue(findVerifyTerminator);
    }
    Assert.assertArrayEquals(entries, list.toArray(new LogEntryProto[list.size()]));
}