Examples with LogSequenceNumber herddb.log.LogSequenceNumber used on opensource projects

Example 21 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookKeeperDataStorageManager method getLastcheckpointSequenceNumber.

@Override
public LogSequenceNumber getLastcheckpointSequenceNumber(String tableSpace) throws DataStorageManagerException {
    try {
        String tableSpaceDirectory = getTableSpaceZNode(tableSpace);
        LogSequenceNumber max = LogSequenceNumber.START_OF_TIME;
        List<String> stream = zkGetChildren(tableSpaceDirectory);
        for (String p : stream) {
            if (isTablespaceCheckPointInfoFile(p)) {
                try {
                    byte[] content = readZNode(p, new Stat());
                    if (content != null) {
                        LogSequenceNumber logPositionInFile = readLogSequenceNumberFromCheckpointInfoFile(tableSpace, content, p);
                        if (logPositionInFile.after(max)) {
                            max = logPositionInFile;
                        }
                    }
                } catch (DataStorageManagerException ignore) {
                    LOGGER.log(Level.SEVERE, "unparsable checkpoint info file " + p, ignore);
                }
            }
        }
        return max;
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
}

Example 22 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookKeeperDataStorageManager method indexCheckpoint.

@Override
public List<PostCheckpointAction> indexCheckpoint(String tableSpace, String indexName, IndexStatus indexStatus, boolean pin) throws DataStorageManagerException {
    String dir = getIndexDirectory(tableSpace, indexName);
    LogSequenceNumber logPosition = indexStatus.sequenceNumber;
    String checkpointFile = getCheckPointsFile(dir, logPosition);
    Stat stat = new Stat();
    byte[] exists = readZNode(checkpointFile, stat);
    if (exists != null) {
        IndexStatus actualStatus = readIndexStatusFromFile(exists, checkpointFile);
        if (actualStatus != null && actualStatus.equals(indexStatus)) {
            LOGGER.log(Level.INFO, "indexCheckpoint " + tableSpace + ", " + indexName + ": " + indexStatus + " already saved on" + checkpointFile);
            return Collections.emptyList();
        }
    }
    LOGGER.log(Level.FINE, "indexCheckpoint " + tableSpace + ", " + indexName + ": " + indexStatus + " to file " + checkpointFile);
    byte[] content;
    try (ByteArrayOutputStream buffer = new ByteArrayOutputStream();
        XXHash64Utils.HashingOutputStream oo = new XXHash64Utils.HashingOutputStream(buffer);
        ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
        // version
        dataOutputKeys.writeVLong(1);
        // flags for future implementations
        dataOutputKeys.writeVLong(0);
        indexStatus.serialize(dataOutputKeys);
        dataOutputKeys.writeLong(oo.hash());
        dataOutputKeys.flush();
        content = buffer.toByteArray();
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    writeZNodeEnforceOwnership(tableSpace, checkpointFile, content, stat);
    /* Checkpoint pinning */
    final Map<Long, Integer> pins = pinIndexAndGetPages(tableSpace, indexName, indexStatus, pin);
    final Set<LogSequenceNumber> checkpoints = pinIndexAndGetCheckpoints(tableSpace, indexName, indexStatus, pin);
    long maxPageId = indexStatus.activePages.stream().max(Comparator.naturalOrder()).orElse(Long.MAX_VALUE);
    List<PostCheckpointAction> result = new ArrayList<>();
    // we can drop old page files now
    PagesMapping tableSpacePagesMapping = getTableSpacePagesMapping(tableSpace).getIndexPagesMapping(indexName);
    // we can drop old page files now
    for (Map.Entry<Long, Long> pages : tableSpacePagesMapping.pages.entrySet()) {
        long pageId = pages.getKey();
        long ledgerId = pages.getValue();
        LOGGER.log(Level.FINEST, "checkpoint pageId {0} ledgerId {1}", new Object[] { pageId, ledgerId });
        if (pageId > 0 && !pins.containsKey(pageId) && !indexStatus.activePages.contains(pageId) && pageId < maxPageId) {
            LOGGER.log(Level.FINEST, "checkpoint ledger " + ledgerId + " pageId " + pageId + ". will be deleted after checkpoint end");
            result.add(new DropLedgerForIndexAction(tableSpace, indexName, "delete index page " + pageId + " ledgerId " + ledgerId, pageId, ledgerId));
        }
    }
    // we can drop orphan ledgers
    for (Long ledgerId : tableSpacePagesMapping.oldLedgers) {
        LOGGER.log(Level.FINEST, "checkpoint ledger " + ledgerId + " without page. will be deleted after checkpoint end");
        result.add(new DropLedgerForIndexAction(tableSpace, indexName, "delete unused ledgerId " + ledgerId, Long.MAX_VALUE, ledgerId));
    }
    List<String> children = zkGetChildren(dir);
    for (String p : children) {
        if (isTableOrIndexCheckpointsFile(p) && !p.equals(checkpointFile)) {
            IndexStatus status = readIndexStatusFromFile(p);
            if (logPosition.after(status.sequenceNumber) && !checkpoints.contains(status.sequenceNumber)) {
                LOGGER.log(Level.FINEST, "checkpoint metadata file " + p + ". will be deleted after checkpoint end");
                result.add(new DeleteZNodeAction(tableSpace, indexName, "delete checkpoint metadata file " + p, p));
            }
        }
    }
    return result;
}

Example 23 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookKeeperDataStorageManager method readLogSequenceNumberFromIndexMetadataFile.

private static LogSequenceNumber readLogSequenceNumberFromIndexMetadataFile(String tableSpace, byte[] data, String znode) throws DataStorageManagerException {
    try (InputStream input = new ByteArrayInputStream(data);
        ExtendedDataInputStream din = new ExtendedDataInputStream(input)) {
        // version
        long version = din.readVLong();
        // flags for future implementations
        long flags = din.readVLong();
        if (version != 1 || flags != 0) {
            throw new DataStorageManagerException("corrupted index list znode " + znode);
        }
        String readname = din.readUTF();
        if (!readname.equals(tableSpace)) {
            throw new DataStorageManagerException("znode " + znode + " is not for spablespace " + tableSpace);
        }
        long ledgerId = din.readZLong();
        long offset = din.readZLong();
        return new LogSequenceNumber(ledgerId, offset);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
}

Example 24 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class ChangeDataCapture method run.

/**
 * Execute one run
 * @return the last sequence number, to be used to configure CDC for the next execution
 * @throws Exception
 */
public LogSequenceNumber run() throws Exception {
    if (zookeeperMetadataStorageManager == null) {
        throw new IllegalStateException("not started");
    }
    try (BookkeeperCommitLog cdc = manager.createCommitLog(tableSpaceUUID, tableSpaceUUID, "cdc")) {
        running = true;
        CommitLog.FollowerContext context = cdc.startFollowing(lastPosition);
        cdc.followTheLeader(lastPosition, new CommitLog.EntryAcceptor() {

            @Override
            public boolean accept(LogSequenceNumber lsn, LogEntry entry) throws Exception {
                applyEntry(entry, lsn);
                lastPosition = lsn;
                return !closed;
            }
        }, context);
        return lastPosition;
    } finally {
        running = false;
    }
}

Example 25 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookkeeperCommitLog method log.

@Override
public CommitLogResult log(LogEntry edit, boolean sync) {
    CompletableFuture<LogSequenceNumber> res;
    CommitFileWriter _writer = null;
    try {
        _writer = getValidWriter();
    } catch (LogNotAvailableException errorWhileRollingLedger) {
        LOGGER.log(Level.SEVERE, "Cannot get a valid writer for " + tableSpaceDescription(), errorWhileRollingLedger);
    }
    if (failed) {
        res = FutureUtils.exception(new LogNotAvailableException(new Exception("this commitlog is failed, tablespace " + tableSpaceDescription() + ", node " + this.localNodeId)).fillInStackTrace());
    } else if (closed || _writer == null) {
        res = FutureUtils.exception(new LogNotAvailableException(new Exception("this commitlog has been closed, tablespace " + tableSpaceDescription() + ", node " + this.localNodeId)).fillInStackTrace());
    } else {
        res = _writer.writeEntry(edit);
        // publish the lastSequenceNumber
        // we must return a new CompletableFuture that completes only
        // AFTER lastSequenceNumber is updated
        // otherwise while doing a checkpoint we could observe
        // an old value for lastSequenceNumber
        // in case of a slow system
        res = res.thenApply((pos) -> {
            if (lastLedgerId == pos.ledgerId) {
                lastSequenceNumber.accumulateAndGet(pos.offset, EnsureLongIncrementAccumulator.INSTANCE);
            }
            notifyListeners(pos, edit);
            return pos;
        });
    }
    if (!sync) {
        // sending a fake completed result
        return new CommitLogResult(CompletableFuture.completedFuture(null), true, /*
                     * deferred
                     */
        false);
    } else {
        return new CommitLogResult(res, false, /*
                     * deferred
                     */
        true);
    }
}