Examples with CommitLog herddb.log.CommitLog used on opensource projects

Example 6 with CommitLog

use of herddb.log.CommitLog in project herddb by diennea.

the class MemoryCommitLogManager method createCommitLog.

@Override
public CommitLog createCommitLog(String tableSpace, String tablespaceName, String localNodeId) {
    return new CommitLog() {

        AtomicLong offset = new AtomicLong(-1);

        @Override
        public CommitLogResult log(LogEntry entry, boolean synch) throws LogNotAvailableException {
            if (isHasListeners()) {
                synch = true;
            }
            if (testSerialize) {
                // NOOP, but trigger serialization subsystem
                try {
                    entry.serialize(ExtendedDataOutputStream.NULL);
                } catch (IOException err) {
                    throw new LogNotAvailableException(err);
                }
            }
            LogSequenceNumber logPos = new LogSequenceNumber(1, offset.incrementAndGet());
            notifyListeners(logPos, entry);
            return new CommitLogResult(logPos, !synch, synch);
        }

        @Override
        public LogSequenceNumber getLastSequenceNumber() {
            return new LogSequenceNumber(1, offset.get());
        }

        private volatile boolean closed;

        @Override
        public void close() throws LogNotAvailableException {
            closed = true;
        }

        @Override
        public boolean isFailed() {
            return false;
        }

        @Override
        public boolean isClosed() {
            return closed;
        }

        @Override
        public void recovery(LogSequenceNumber snapshotSequenceNumber, BiConsumer<LogSequenceNumber, LogEntry> consumer, boolean fencing) throws LogNotAvailableException {
        }

        @Override
        public void dropOldLedgers(LogSequenceNumber lastCheckPointSequenceNumber) throws LogNotAvailableException {
        }

        @Override
        public void startWriting(int expectedReplicaCount) throws LogNotAvailableException {
        }

        @Override
        public void clear() throws LogNotAvailableException {
        }
    };
}

Example 7 with CommitLog

use of herddb.log.CommitLog in project herddb by diennea.

the class FileCommitLogTest method testMaxBatchSizeBytes.

@Test
public void testMaxBatchSizeBytes() throws Exception {
    TestStatsProvider testStatsProvider = new TestStatsProvider();
    TestStatsProvider.TestStatsLogger statsLogger = testStatsProvider.getStatsLogger("test");
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), ServerConfiguration.PROPERTY_MAX_LOG_FILE_SIZE_DEFAULT, // no flush by batch size
    Integer.MAX_VALUE, // flush after 2 writes
    LogEntryFactory.beginTransaction(0).serialize().length * 2 - 1, // no flush by time
    Integer.MAX_VALUE, true, /* require fsync */
    false, /* O_DIRECT */
    ServerConfiguration.PROPERTY_DEFERRED_SYNC_PERIOD_DEFAULT, statsLogger)) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (FileCommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            CopyOnWriteArrayList<LogSequenceNumber> completed = new CopyOnWriteArrayList<>();
            CommitLogResult future = log.log(LogEntryFactory.beginTransaction(0), true);
            future.logSequenceNumber.thenAccept(completed::add);
            assertFalse(future.logSequenceNumber.isDone());
            CommitLogResult future2 = log.log(LogEntryFactory.beginTransaction(0), true);
            future2.logSequenceNumber.thenAccept(completed::add);
            future.logSequenceNumber.get(10, TimeUnit.SECONDS);
            future2.logSequenceNumber.get(10, TimeUnit.SECONDS);
            TestUtils.waitForCondition(() -> {
                return completed.size() == 2;
            }, NOOP, 100);
            writeCount = completed.size();
            assertTrue(completed.get(1).after(completed.get(0)));
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
    }
}

Example 8 with CommitLog

use of herddb.log.CommitLog in project herddb by diennea.

the class FileCommitLogTest method testDiskFullLogBrokenEntry.

@Test
public void testDiskFullLogBrokenEntry() throws Exception {
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), ServerConfiguration.PROPERTY_MAX_LOG_FILE_SIZE_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_BYTES_DEFAULT, ServerConfiguration.PROPERTY_MAX_SYNC_TIME_DEFAULT, ServerConfiguration.PROPERTY_REQUIRE_FSYNC_DEFAULT, // do not use O_DIRECT, we are creating a broken file, O_DIRECT will add padding at unpredictable points
    false, ServerConfiguration.PROPERTY_DEFERRED_SYNC_PERIOD_DEFAULT, NullStatsLogger.INSTANCE)) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            for (int i = 0; i < 100; i++) {
                log.log(LogEntryFactory.beginTransaction(0), true).getLogSequenceNumber();
                writeCount++;
            }
            FileCommitLog fileCommitLog = (FileCommitLog) log;
            // simulate end of disk
            byte[] dummyEntry = LogEntryFactory.beginTransaction(0).serialize();
            // header
            fileCommitLog.getWriter().out.write(ENTRY_START);
            fileCommitLog.getWriter().out.writeLong(0);
            // just half entry
            fileCommitLog.getWriter().out.write(dummyEntry, 0, dummyEntry.length / 2);
            // missing entry footer
            fileCommitLog.getWriter().out.flush();
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
        // must be able to read twice
        AtomicInteger readCount2 = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount2.incrementAndGet();
                }
            }, true);
        }
        assertEquals(writeCount, readCount.get());
    }
}

Example 9 with CommitLog

use of herddb.log.CommitLog in project herddb by diennea.

the class FileCommitLogTest method testDiskFullLogMissingFooter.

@Test
public void testDiskFullLogMissingFooter() throws Exception {
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), ServerConfiguration.PROPERTY_MAX_LOG_FILE_SIZE_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_BYTES_DEFAULT, ServerConfiguration.PROPERTY_MAX_SYNC_TIME_DEFAULT, ServerConfiguration.PROPERTY_REQUIRE_FSYNC_DEFAULT, // do not use O_DIRECT, we are creating a broken file, O_DIRECT will add padding at unpredictable points
    false, ServerConfiguration.PROPERTY_DEFERRED_SYNC_PERIOD_DEFAULT, NullStatsLogger.INSTANCE)) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            for (int i = 0; i < 100; i++) {
                log.log(LogEntryFactory.beginTransaction(0), true).getLogSequenceNumber();
                writeCount++;
            }
            FileCommitLog fileCommitLog = (FileCommitLog) log;
            // simulate end of disk
            byte[] dummyEntry = LogEntryFactory.beginTransaction(0).serialize();
            // header
            fileCommitLog.getWriter().out.write(ENTRY_START);
            fileCommitLog.getWriter().out.writeLong(0);
            // entry
            fileCommitLog.getWriter().out.write(dummyEntry);
            // missing entry footer
            fileCommitLog.getWriter().out.flush();
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
        // must be able to read twice
        AtomicInteger readCount2 = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount2.incrementAndGet();
                }
            }, true);
        }
        assertEquals(writeCount, readCount.get());
    }
}

Example 10 with CommitLog

use of herddb.log.CommitLog in project herddb by diennea.

the class FileCommitLogTest method testDeferredSync.

@Test
public void testDeferredSync() throws Exception {
    TestStatsProvider testStatsProvider = new TestStatsProvider();
    TestStatsProvider.TestStatsLogger statsLogger = testStatsProvider.getStatsLogger("test");
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), ServerConfiguration.PROPERTY_MAX_LOG_FILE_SIZE_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_BYTES_DEFAULT, 1, false, /* require fsync */
    false, // each second
    1, statsLogger)) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (FileCommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            for (int i = 0; i < 10_000; i++) {
                log.log(LogEntryFactory.beginTransaction(0), false);
                writeCount++;
            }
            Counter deferredSyncs = statsLogger.scope("aa").getCounter("deferredSyncs");
            TestUtils.waitForCondition(() -> {
                int qsize = log.getQueueSize();
                Long _deferredSyncs = deferredSyncs.get();
                return qsize == 0 && _deferredSyncs != null && _deferredSyncs > 0;
            }, TestUtils.NOOP, 100);
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
    }
}