Examples with LogSequenceNumber herddb.log.LogSequenceNumber used on opensource projects

Example 31 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileCommitLogTest method testLogsynch.

@Test
public void testLogsynch() throws Exception {
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath())) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            for (int i = 0; i < 100; i++) {
                log.log(LogEntryFactory.beginTransaction(0), true);
                writeCount++;
            }
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
    }
}

Example 32 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileCommitLogTest method testMaxBatchSize.

@Test
public void testMaxBatchSize() throws Exception {
    TestStatsProvider testStatsProvider = new TestStatsProvider();
    TestStatsProvider.TestStatsLogger statsLogger = testStatsProvider.getStatsLogger("test");
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), ServerConfiguration.PROPERTY_MAX_LOG_FILE_SIZE_DEFAULT, // flush only when we have 2 entries in the queue
    2, // no flush by size
    Integer.MAX_VALUE, // no flush by time
    Integer.MAX_VALUE, true, /* require fsync */
    false, /* O_DIRECT */
    ServerConfiguration.PROPERTY_DEFERRED_SYNC_PERIOD_DEFAULT, statsLogger)) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (FileCommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            CopyOnWriteArrayList<LogSequenceNumber> completed = new CopyOnWriteArrayList<>();
            CommitLogResult future = log.log(LogEntryFactory.beginTransaction(0), true);
            future.logSequenceNumber.thenAccept(completed::add);
            assertFalse(future.logSequenceNumber.isDone());
            CommitLogResult future2 = log.log(LogEntryFactory.beginTransaction(0), true);
            future2.logSequenceNumber.thenAccept(completed::add);
            future.logSequenceNumber.get(10, TimeUnit.SECONDS);
            future2.logSequenceNumber.get(10, TimeUnit.SECONDS);
            TestUtils.waitForCondition(() -> {
                return completed.size() == 2;
            }, NOOP, 100);
            writeCount = completed.size();
            assertTrue(completed.get(1).after(completed.get(0)));
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
    }
}

Example 33 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileCommitLogTest method testLogMultiFiles_O_DIRECT.

@Test
public void testLogMultiFiles_O_DIRECT() throws Exception {
    TestStatsProvider testStatsProvider = new TestStatsProvider();
    TestStatsProvider.TestStatsLogger statsLogger = testStatsProvider.getStatsLogger("test");
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), // 2K Bbyte files,
    1024 * 2, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_DEFAULT, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_BYTES_DEFAULT, ServerConfiguration.PROPERTY_MAX_SYNC_TIME_DEFAULT, false, true, /* O_DIRECT */
    ServerConfiguration.PROPERTY_DEFERRED_SYNC_PERIOD_DEFAULT, statsLogger)) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (FileCommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            for (int i = 0; i < 10_000; i++) {
                log.log(LogEntryFactory.beginTransaction(0), false);
                writeCount++;
            }
            TestUtils.waitForCondition(() -> {
                int qsize = log.getQueueSize();
                return qsize == 0;
            }, TestUtils.NOOP, 100);
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
        // this number really depends on disk format
        // this test in the future will be updated when we change the format
        assertEquals(145L, statsLogger.scope("aa").getCounter("newfiles").get().longValue());
    }
}

Example 34 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileCommitLogTest method testLog.

@Test
public void testLog() throws Exception {
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath())) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            for (int i = 0; i < 10_000; i++) {
                log.log(LogEntryFactory.beginTransaction(0), false);
                writeCount++;
            }
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
    }
}

Example 35 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookkeeperCommitLog method followTheLeader.

@Override
public void followTheLeader(LogSequenceNumber skipPast, BiConsumer<LogSequenceNumber, LogEntry> consumer) throws LogNotAvailableException {
    List<Long> actualList = metadataManager.getActualLedgersList(tableSpaceUUID).getActiveLedgers();
    List<Long> toRead = actualList;
    if (skipPast.ledgerId != -1) {
        toRead = toRead.stream().filter(l -> l >= skipPast.ledgerId).collect(Collectors.toList());
    }
    try {
        long nextEntry = skipPast.offset + 1;
        // LOGGER.log(Level.SEVERE, "followTheLeader "+tableSpace+" skipPast:{0} toRead: {1} actualList:{2}, nextEntry:{3}", new Object[]{skipPast, toRead, actualList, nextEntry});
        for (Long previous : toRead) {
            // LOGGER.log(Level.SEVERE, "followTheLeader openLedger " + previous + " nextEntry:" + nextEntry);
            LedgerHandle lh;
            try {
                lh = bookKeeper.openLedgerNoRecovery(previous, BookKeeper.DigestType.CRC32, sharedSecret.getBytes(StandardCharsets.UTF_8));
            } catch (BKException.BKLedgerRecoveryException e) {
                LOGGER.log(Level.SEVERE, "error", e);
                return;
            }
            try {
                long lastAddConfirmed = lh.getLastAddConfirmed();
                LOGGER.log(Level.FINE, "followTheLeader " + tableSpaceUUID + " openLedger {0} -> lastAddConfirmed:{1}, nextEntry:{2}", new Object[] { previous, lastAddConfirmed, nextEntry });
                if (nextEntry > lastAddConfirmed) {
                    nextEntry = 0;
                    continue;
                }
                Enumeration<LedgerEntry> entries = lh.readEntries(nextEntry, lh.getLastAddConfirmed());
                while (entries.hasMoreElements()) {
                    LedgerEntry e = entries.nextElement();
                    long entryId = e.getEntryId();
                    byte[] entryData = e.getEntry();
                    LogEntry statusEdit = LogEntry.deserialize(entryData);
                    // LOGGER.log(Level.SEVERE, "" + tableSpaceUUID + " followentry {0},{1} -> {2}", new Object[]{previous, entryId, statusEdit});
                    LogSequenceNumber number = new LogSequenceNumber(previous, entryId);
                    lastSequenceNumber.accumulateAndGet(number.offset, EnsureLongIncrementAccumulator.INSTANCE);
                    lastLedgerId = number.ledgerId;
                    currentLedgerId = number.ledgerId;
                    consumer.accept(number, statusEdit);
                }
            } finally {
                try {
                    lh.close();
                } catch (BKException err) {
                    LOGGER.log(Level.SEVERE, "error while closing ledger", err);
                } catch (InterruptedException err) {
                    LOGGER.log(Level.SEVERE, "error while closing ledger", err);
                    Thread.currentThread().interrupt();
                }
            }
        }
    } catch (InterruptedException | EOFException | BKException err) {
        LOGGER.log(Level.SEVERE, "internal error", err);
        throw new LogNotAvailableException(err);
    }
}