Examples with LogSequenceNumber herddb.log.LogSequenceNumber used on opensource projects

Example 66 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileCommitLogTest method testDiskFullLogBrokenEntry.

@Test
public void testDiskFullLogBrokenEntry() throws Exception {
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), ServerConfiguration.PROPERTY_MAX_LOG_FILE_SIZE_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_BYTES_DEFAULT, ServerConfiguration.PROPERTY_MAX_SYNC_TIME_DEFAULT, ServerConfiguration.PROPERTY_REQUIRE_FSYNC_DEFAULT, // do not use O_DIRECT, we are creating a broken file, O_DIRECT will add padding at unpredictable points
    false, ServerConfiguration.PROPERTY_DEFERRED_SYNC_PERIOD_DEFAULT, NullStatsLogger.INSTANCE)) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            for (int i = 0; i < 100; i++) {
                log.log(LogEntryFactory.beginTransaction(0), true).getLogSequenceNumber();
                writeCount++;
            }
            FileCommitLog fileCommitLog = (FileCommitLog) log;
            // simulate end of disk
            byte[] dummyEntry = LogEntryFactory.beginTransaction(0).serialize();
            // header
            fileCommitLog.getWriter().out.write(ENTRY_START);
            fileCommitLog.getWriter().out.writeLong(0);
            // just half entry
            fileCommitLog.getWriter().out.write(dummyEntry, 0, dummyEntry.length / 2);
            // missing entry footer
            fileCommitLog.getWriter().out.flush();
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
        // must be able to read twice
        AtomicInteger readCount2 = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount2.incrementAndGet();
                }
            }, true);
        }
        assertEquals(writeCount, readCount.get());
    }
}

Example 67 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileCommitLogTest method testDiskFullLogMissingFooter.

@Test
public void testDiskFullLogMissingFooter() throws Exception {
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), ServerConfiguration.PROPERTY_MAX_LOG_FILE_SIZE_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_BYTES_DEFAULT, ServerConfiguration.PROPERTY_MAX_SYNC_TIME_DEFAULT, ServerConfiguration.PROPERTY_REQUIRE_FSYNC_DEFAULT, // do not use O_DIRECT, we are creating a broken file, O_DIRECT will add padding at unpredictable points
    false, ServerConfiguration.PROPERTY_DEFERRED_SYNC_PERIOD_DEFAULT, NullStatsLogger.INSTANCE)) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            for (int i = 0; i < 100; i++) {
                log.log(LogEntryFactory.beginTransaction(0), true).getLogSequenceNumber();
                writeCount++;
            }
            FileCommitLog fileCommitLog = (FileCommitLog) log;
            // simulate end of disk
            byte[] dummyEntry = LogEntryFactory.beginTransaction(0).serialize();
            // header
            fileCommitLog.getWriter().out.write(ENTRY_START);
            fileCommitLog.getWriter().out.writeLong(0);
            // entry
            fileCommitLog.getWriter().out.write(dummyEntry);
            // missing entry footer
            fileCommitLog.getWriter().out.flush();
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
        // must be able to read twice
        AtomicInteger readCount2 = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount2.incrementAndGet();
                }
            }, true);
        }
        assertEquals(writeCount, readCount.get());
    }
}

Example 68 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileCommitLogTest method testDeferredSync.

@Test
public void testDeferredSync() throws Exception {
    TestStatsProvider testStatsProvider = new TestStatsProvider();
    TestStatsProvider.TestStatsLogger statsLogger = testStatsProvider.getStatsLogger("test");
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), ServerConfiguration.PROPERTY_MAX_LOG_FILE_SIZE_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_BYTES_DEFAULT, 1, false, /* require fsync */
    false, // each second
    1, statsLogger)) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (FileCommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            for (int i = 0; i < 10_000; i++) {
                log.log(LogEntryFactory.beginTransaction(0), false);
                writeCount++;
            }
            Counter deferredSyncs = statsLogger.scope("aa").getCounter("deferredSyncs");
            TestUtils.waitForCondition(() -> {
                int qsize = log.getQueueSize();
                Long _deferredSyncs = deferredSyncs.get();
                return qsize == 0 && _deferredSyncs != null && _deferredSyncs > 0;
            }, TestUtils.NOOP, 100);
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
    }
}

Example 69 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileCommitLogTest method testMaxSyncTime.

@Test
public void testMaxSyncTime() throws Exception {
    TestStatsProvider testStatsProvider = new TestStatsProvider();
    TestStatsProvider.TestStatsLogger statsLogger = testStatsProvider.getStatsLogger("test");
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), ServerConfiguration.PROPERTY_MAX_LOG_FILE_SIZE_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_BYTES_DEFAULT, // 1ms
    1, true, /* require fsync */
    false, /* O_DIRECT */
    ServerConfiguration.PROPERTY_DEFERRED_SYNC_PERIOD_DEFAULT, statsLogger)) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (FileCommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            log.log(LogEntryFactory.beginTransaction(0), true).getLogSequenceNumber();
            writeCount = 1;
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
    }
}

Example 70 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileCommitLogTest method testLogMultiFiles.

@Test
public void testLogMultiFiles() throws Exception {
    TestStatsProvider testStatsProvider = new TestStatsProvider();
    TestStatsProvider.TestStatsLogger statsLogger = testStatsProvider.getStatsLogger("test");
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), // 2K Bbyte files,
    1024 * 2, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_BYTES_DEFAULT, ServerConfiguration.PROPERTY_MAX_SYNC_TIME_DEFAULT, false, false, /* O_DIRECT */
    ServerConfiguration.PROPERTY_DEFERRED_SYNC_PERIOD_DEFAULT, statsLogger)) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (FileCommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            for (int i = 0; i < 10_000; i++) {
                log.log(LogEntryFactory.beginTransaction(0), false);
                writeCount++;
            }
            TestUtils.waitForCondition(() -> {
                int qsize = log.getQueueSize();
                return qsize == 0;
            }, TestUtils.NOOP, 100);
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
        // this number really depends on disk format
        // this test in the future will be updated when we change the format
        assertEquals(145L, statsLogger.scope("aa").getCounter("newfiles").get().longValue());
    }
}