Examples with LogSequenceNumber herddb.log.LogSequenceNumber used on opensource projects

Example 61 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookKeeperDataStorageManager method readLogSequenceNumberFromTablesMetadataFile.

private static LogSequenceNumber readLogSequenceNumberFromTablesMetadataFile(String tableSpace, byte[] data, String znode) throws DataStorageManagerException {
    try (InputStream input = new ByteArrayInputStream(data);
        ExtendedDataInputStream din = new ExtendedDataInputStream(input)) {
        // version
        long version = din.readVLong();
        // flags for future implementations
        long flags = din.readVLong();
        if (version != 1 || flags != 0) {
            throw new DataStorageManagerException("corrupted table list znode" + znode);
        }
        String readname = din.readUTF();
        if (!readname.equals(tableSpace)) {
            throw new DataStorageManagerException("znode " + znode + " is not for spablespace " + tableSpace);
        }
        long ledgerId = din.readZLong();
        long offset = din.readZLong();
        return new LogSequenceNumber(ledgerId, offset);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
}

Example 62 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookKeeperDataStorageManager method readLogSequenceNumberFromCheckpointInfoFile.

private static LogSequenceNumber readLogSequenceNumberFromCheckpointInfoFile(String tableSpace, byte[] data, String checkPointFile) throws DataStorageManagerException, IOException {
    try (InputStream input = new ByteArrayInputStream(data);
        ExtendedDataInputStream din = new ExtendedDataInputStream(input)) {
        // version
        long version = din.readVLong();
        // flags for future implementations
        long flags = din.readVLong();
        if (version != 1 || flags != 0) {
            throw new IOException("corrupted checkpoint file");
        }
        String readname = din.readUTF();
        if (!readname.equals(tableSpace)) {
            throw new DataStorageManagerException("zonde " + checkPointFile + " is not for spablespace " + tableSpace + " but for " + readname);
        }
        long ledgerId = din.readZLong();
        long offset = din.readZLong();
        return new LogSequenceNumber(ledgerId, offset);
    }
}

Example 63 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookkeeperCommitLog method recovery.

@Override
public void recovery(LogSequenceNumber snapshotSequenceNumber, BiConsumer<LogSequenceNumber, LogEntry> consumer, boolean fencing) throws LogNotAvailableException {
    String tableSpaceDescription = tableSpaceDescription();
    this.actualLedgersList = metadataManager.getActualLedgersList(tableSpaceUUID);
    LOGGER.log(Level.INFO, "Actual ledgers list:{0} tableSpace {1}", new Object[] { actualLedgersList, tableSpaceDescription });
    this.lastLedgerId = snapshotSequenceNumber.ledgerId;
    this.currentLedgerId = snapshotSequenceNumber.ledgerId;
    this.lastSequenceNumber.set(snapshotSequenceNumber.offset);
    LOGGER.log(Level.INFO, "recovery from latest snapshotSequenceNumber:{0} tableSpace {1}, node {2}, fencing {3}", new Object[] { snapshotSequenceNumber, tableSpaceDescription, localNodeId, fencing });
    if (!isRecoveryAvailable(snapshotSequenceNumber, actualLedgersList, tableSpaceDescription)) {
        throw new FullRecoveryNeededException("Cannot recover from BookKeeper, not enough data, plese check the logs");
    }
    for (long ledgerId : actualLedgersList.getActiveLedgers()) {
        try {
            Versioned<LedgerMetadata> result = FutureUtils.result(bookKeeper.getLedgerManager().readLedgerMetadata(ledgerId));
            LedgerMetadata metadata = result.getValue();
            String ledgerLeader = extractLeaderFromMetadata(metadata.getCustomMetadata());
            LOGGER.log(Level.INFO, "Ledger {0}: {1} {2} created by {3}, LastEntryId {4} Length {5}", new Object[] { String.valueOf(ledgerId), metadata.getState(), metadata.getAllEnsembles(), ledgerLeader, metadata.getLastEntryId(), metadata.getLength() });
        } catch (BKException.BKNoSuchLedgerExistsException | BKException.BKNoSuchLedgerExistsOnMetadataServerException e) {
            if (ledgerId < snapshotSequenceNumber.ledgerId) {
                LOGGER.log(Level.INFO, "Actual ledgers list includes a not existing ledgerid:" + ledgerId + " tablespace " + tableSpaceDescription + ", but this ledger is not useful for recovery (snapshotSequenceNumber.ledgerId is " + snapshotSequenceNumber.ledgerId);
            } else {
                throw new FullRecoveryNeededException(new Exception("Actual ledgers list includes a not existing ledgerid:" + ledgerId + " tablespace " + tableSpaceDescription));
            }
        } catch (LogNotAvailableException e) {
            throw e;
        } catch (Exception e) {
            throw new LogNotAvailableException(e);
        }
    }
    try {
        for (long ledgerId : actualLedgersList.getActiveLedgers()) {
            if (ledgerId < snapshotSequenceNumber.ledgerId) {
                LOGGER.log(Level.FINER, "Skipping ledger {0}", ledgerId);
                continue;
            }
            ReadHandle handle;
            try {
                if (fencing) {
                    handle = bookKeeper.openLedger(ledgerId, BookKeeper.DigestType.CRC32C, SHARED_SECRET.getBytes(StandardCharsets.UTF_8));
                } else {
                    handle = bookKeeper.openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32C, SHARED_SECRET.getBytes(StandardCharsets.UTF_8));
                }
            } catch (org.apache.bookkeeper.client.api.BKException errorDuringOpen) {
                throw new LogNotAvailableException("Cannot open ledger " + ledgerId + " (fencing " + fencing + "): " + errorDuringOpen, errorDuringOpen);
            }
            try {
                long first;
                if (ledgerId == snapshotSequenceNumber.ledgerId) {
                    first = snapshotSequenceNumber.offset;
                    if (first == -1) {
                        // this can happen if checkpoint  happened while starting to follow a new ledger but actually no entry was ever read
                        LOGGER.log(Level.INFO, "Tablespace " + tableSpaceDescription + ", recovering from latest snapshot ledger " + ledgerId + ", first entry " + first + " is not valid. Adjusting to 0");
                        first = 0;
                    }
                    if (LOGGER.isLoggable(Level.FINE)) {
                        LOGGER.log(Level.FINE, "Tablespace " + tableSpaceDescription + ", recovering from latest snapshot ledger " + ledgerId + ", starting from entry " + first);
                    }
                } else {
                    first = 0;
                    if (LOGGER.isLoggable(Level.FINE)) {
                        LOGGER.log(Level.FINE, "Tablespace " + tableSpaceDescription + ", recovering from ledger " + ledgerId + ", starting from entry " + first);
                    }
                }
                long lastAddConfirmed = handle.getLastAddConfirmed();
                String ledgerLeader = extractLeaderFromMetadata(handle.getLedgerMetadata().getCustomMetadata());
                LOGGER.log(Level.INFO, "Tablespace " + tableSpaceDescription + ", Recovering from ledger " + ledgerId + ", first=" + first + " lastAddConfirmed=" + lastAddConfirmed + " written by " + ledgerLeader);
                if (lastAddConfirmed >= 0) {
                    for (long b = first; b <= lastAddConfirmed; ) {
                        long start = b;
                        long end = b + RECOVERY_BATCH_SIZE;
                        if (end > lastAddConfirmed) {
                            end = lastAddConfirmed;
                        }
                        b = end + 1;
                        double percent = ((start - first) * 100.0 / (lastAddConfirmed + 1));
                        int entriesToRead = (int) (1 + end - start);
                        if (LOGGER.isLoggable(Level.FINE)) {
                            LOGGER.log(Level.FINE, "{3} From entry {0}, to entry {1} ({2} %)", new Object[] { start, end, percent, tableSpaceDescription });
                        }
                        long _start = System.currentTimeMillis();
                        int localEntryCount = 0;
                        try (LedgerEntries entries = handle.read(start, end)) {
                            for (org.apache.bookkeeper.client.api.LedgerEntry entry : entries) {
                                long entryId = entry.getEntryId();
                                LogSequenceNumber number = new LogSequenceNumber(ledgerId, entryId);
                                LogEntry statusEdit = readLogEntry(entry);
                                lastLedgerId = ledgerId;
                                currentLedgerId = ledgerId;
                                lastSequenceNumber.set(entryId);
                                if (number.after(snapshotSequenceNumber)) {
                                    if (LOGGER.isLoggable(Level.FINEST)) {
                                        LOGGER.log(Level.FINEST, "rec " + tableSpaceName + " #" + localEntryCount + " {0}, {1}", new Object[] { number, statusEdit });
                                    }
                                    consumer.accept(number, statusEdit);
                                } else {
                                    if (LOGGER.isLoggable(Level.FINEST)) {
                                        LOGGER.log(Level.FINEST, "skip " + tableSpaceName + " #" + localEntryCount + " {0}<{1}, {2}", new Object[] { number, snapshotSequenceNumber, statusEdit });
                                    }
                                }
                                localEntryCount++;
                            }
                        }
                        LOGGER.log(Level.FINER, tableSpaceDescription() + " read " + localEntryCount + " entries from ledger " + ledgerId + ", expected " + entriesToRead);
                        if (localEntryCount != entriesToRead) {
                            throw new LogNotAvailableException(tableSpaceDescription() + " Read " + localEntryCount + " entries, expected " + entriesToRead);
                        }
                        lastLedgerId = ledgerId;
                        lastSequenceNumber.set(end);
                        long _stop = System.currentTimeMillis();
                        LOGGER.log(Level.INFO, "{4} From entry {0}, to entry {1} ({2} %) read time {3}", new Object[] { start, end, percent, (_stop - _start) + " ms", tableSpaceDescription });
                    }
                }
            } catch (RuntimeException err) {
                LOGGER.log(Level.SEVERE, "Internal error while recovering tablespace " + tableSpaceDescription() + ": " + err, err);
                throw err;
            } finally {
                handle.close();
            }
        }
        LOGGER.log(Level.INFO, "After recovery of {0} lastSequenceNumber {1}", new Object[] { tableSpaceDescription, getLastSequenceNumber() });
    } catch (IOException | InterruptedException | org.apache.bookkeeper.client.api.BKException err) {
        LOGGER.log(Level.SEVERE, "Fatal error during recovery of " + tableSpaceDescription(), err);
        signalLogFailed();
        throw new LogNotAvailableException(err);
    } catch (LogNotAvailableException err) {
        LOGGER.log(Level.SEVERE, "Fatal error during recovery of " + tableSpaceDescription(), err);
        signalLogFailed();
        throw err;
    }
}

Example 64 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookkeeperCommitLog method acceptEntryForFollower.

private boolean acceptEntryForFollower(org.apache.bookkeeper.client.api.LedgerEntry e, EntryAcceptor consumer) throws Exception {
    long entryId = e.getEntryId();
    LogEntry statusEdit = readLogEntry(e);
    LogSequenceNumber number = new LogSequenceNumber(e.getLedgerId(), entryId);
    if (LOGGER.isLoggable(Level.FINER)) {
        LOGGER.log(Level.FINER, "{0} follow entry {1}", new Object[] { tableSpaceDescription(), number });
    }
    if (lastLedgerId == number.ledgerId) {
        lastSequenceNumber.accumulateAndGet(number.offset, EnsureLongIncrementAccumulator.INSTANCE);
    } else {
        lastSequenceNumber.set(number.offset);
    }
    lastLedgerId = number.ledgerId;
    currentLedgerId = number.ledgerId;
    return consumer.accept(number, statusEdit);
}

Example 65 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileCommitLogTest method testMaxBatchSizeBytes.

@Test
public void testMaxBatchSizeBytes() throws Exception {
    TestStatsProvider testStatsProvider = new TestStatsProvider();
    TestStatsProvider.TestStatsLogger statsLogger = testStatsProvider.getStatsLogger("test");
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), ServerConfiguration.PROPERTY_MAX_LOG_FILE_SIZE_DEFAULT, // no flush by batch size
    Integer.MAX_VALUE, // flush after 2 writes
    LogEntryFactory.beginTransaction(0).serialize().length * 2 - 1, // no flush by time
    Integer.MAX_VALUE, true, /* require fsync */
    false, /* O_DIRECT */
    ServerConfiguration.PROPERTY_DEFERRED_SYNC_PERIOD_DEFAULT, statsLogger)) {
        manager.start();
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (FileCommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.startWriting(1);
            CopyOnWriteArrayList<LogSequenceNumber> completed = new CopyOnWriteArrayList<>();
            CommitLogResult future = log.log(LogEntryFactory.beginTransaction(0), true);
            future.logSequenceNumber.thenAccept(completed::add);
            assertFalse(future.logSequenceNumber.isDone());
            CommitLogResult future2 = log.log(LogEntryFactory.beginTransaction(0), true);
            future2.logSequenceNumber.thenAccept(completed::add);
            future.logSequenceNumber.get(10, TimeUnit.SECONDS);
            future2.logSequenceNumber.get(10, TimeUnit.SECONDS);
            TestUtils.waitForCondition(() -> {
                return completed.size() == 2;
            }, NOOP, 100);
            writeCount = completed.size();
            assertTrue(completed.get(1).after(completed.get(0)));
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt", "aa", "nodeid")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
    }
}