Examples with LogSequenceNumber herddb.log.LogSequenceNumber used on opensource projects

Example 16 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookKeeperDataStorageManager method writeTables.

@Override
public Collection<PostCheckpointAction> writeTables(String tableSpace, LogSequenceNumber sequenceNumber, List<Table> tables, List<Index> indexlist, boolean prepareActions) throws DataStorageManagerException {
    if (sequenceNumber.isStartOfTime() && !tables.isEmpty()) {
        throw new DataStorageManagerException("impossible to write a non empty table list at start-of-time");
    }
    // we need to flush current mappings, because here we are flushing
    // the status of all of the tables and indexes
    persistTableSpaceMapping(tableSpace);
    String tableSpaceDirectory = getTableSpaceZNode(tableSpace);
    String fileTables = getTablespaceTablesMetadataFile(tableSpace, sequenceNumber);
    String fileIndexes = getTablespaceIndexesMetadataFile(tableSpace, sequenceNumber);
    LOGGER.log(Level.FINE, "writeTables for tableSpace " + tableSpace + " sequenceNumber " + sequenceNumber + " to " + fileTables);
    try (VisibleByteArrayOutputStream buffer = new VisibleByteArrayOutputStream();
        ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
        // version
        dout.writeVLong(1);
        // flags for future implementations
        dout.writeVLong(0);
        dout.writeUTF(tableSpace);
        dout.writeZLong(sequenceNumber.ledgerId);
        dout.writeZLong(sequenceNumber.offset);
        dout.writeInt(tables.size());
        for (Table t : tables) {
            byte[] tableSerialized = t.serialize();
            dout.writeArray(tableSerialized);
        }
        dout.flush();
        writeZNodeEnforceOwnership(tableSpace, fileTables, buffer.toByteArray(), null);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    try (VisibleByteArrayOutputStream buffer = new VisibleByteArrayOutputStream();
        ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
        // version
        dout.writeVLong(1);
        // flags for future implementations
        dout.writeVLong(0);
        dout.writeUTF(tableSpace);
        dout.writeZLong(sequenceNumber.ledgerId);
        dout.writeZLong(sequenceNumber.offset);
        if (indexlist != null) {
            dout.writeInt(indexlist.size());
            for (Index t : indexlist) {
                byte[] indexSerialized = t.serialize();
                dout.writeArray(indexSerialized);
            }
        } else {
            dout.writeInt(0);
        }
        dout.flush();
        writeZNodeEnforceOwnership(tableSpace, fileIndexes, buffer.toByteArray(), null);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    Collection<PostCheckpointAction> result = new ArrayList<>();
    if (prepareActions) {
        List<String> stream = zkGetChildren(tableSpaceDirectory);
        for (String p : stream) {
            if (isTablespaceIndexesMetadataFile(p)) {
                try {
                    byte[] content = readZNode(p, new Stat());
                    if (content != null) {
                        LogSequenceNumber logPositionInFile = readLogSequenceNumberFromIndexMetadataFile(tableSpace, content, p);
                        if (sequenceNumber.after(logPositionInFile)) {
                            LOGGER.log(Level.FINEST, "indexes metadata file " + p + ". will be deleted after checkpoint end");
                            result.add(new DeleteZNodeAction(tableSpace, "indexes", "delete indexesmetadata file " + p, p));
                        }
                    }
                } catch (DataStorageManagerException ignore) {
                    LOGGER.log(Level.SEVERE, "Unparsable indexesmetadata file " + p, ignore);
                    result.add(new DeleteZNodeAction(tableSpace, "indexes", "delete unparsable indexesmetadata file " + p, p));
                }
            } else if (isTablespaceTablesMetadataFile(p)) {
                try {
                    byte[] content = readZNode(p, new Stat());
                    if (content != null) {
                        LogSequenceNumber logPositionInFile = readLogSequenceNumberFromTablesMetadataFile(tableSpace, content, p);
                        if (sequenceNumber.after(logPositionInFile)) {
                            LOGGER.log(Level.FINEST, "tables metadata file " + p + ". will be deleted after checkpoint end");
                            result.add(new DeleteZNodeAction(tableSpace, "tables", "delete tablesmetadata file " + p, p));
                        }
                    }
                } catch (DataStorageManagerException ignore) {
                    LOGGER.log(Level.SEVERE, "Unparsable tablesmetadata file " + p, ignore);
                    result.add(new DeleteZNodeAction(tableSpace, "transactions", "delete unparsable tablesmetadata file " + p, p));
                }
            }
        }
    }
    return result;
}

Example 17 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookkeeperCommitLogManager method scanRawLedger.

public static void scanRawLedger(long ledgerId, long fromId, long toId, herddb.client.ClientConfiguration clientConfiguration, ZookeeperMetadataStorageManager metadataStorageManager, Consumer<LogEntryWithSequenceNumber> consumer) throws Exception {
    ClientConfiguration config = new ClientConfiguration();
    config.setZkServers(metadataStorageManager.getZkAddress());
    config.setZkTimeout(metadataStorageManager.getZkSessionTimeout());
    config.setZkLedgersRootPath(clientConfiguration.getString(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_PATH, ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_PATH_DEFAULT));
    config.setEnableParallelRecoveryRead(true);
    config.setEnableDigestTypeAutodetection(true);
    try (org.apache.bookkeeper.client.api.BookKeeper bookKeeper = org.apache.bookkeeper.client.api.BookKeeper.newBuilder(config).build()) {
        try (ReadHandle lh = bookKeeper.newOpenLedgerOp().withRecovery(false).withLedgerId(ledgerId).withPassword(BookkeeperCommitLog.SHARED_SECRET.getBytes(StandardCharsets.UTF_8)).execute().get()) {
            long lastAddConfirmed = lh.readLastAddConfirmed();
            if (toId < 0) {
                toId = lastAddConfirmed;
            }
            LOG.log(Level.INFO, "Scanning Ledger {0} from {1} to {2} LAC {3}", new Object[] { ledgerId, fromId, toId, lastAddConfirmed });
            for (long id = fromId; id <= toId; id++) {
                try (LedgerEntries entries = lh.readUnconfirmed(id, id)) {
                    LedgerEntry entry = entries.getEntry(id);
                    LogEntry lEntry = LogEntry.deserialize(entry.getEntryBytes());
                    LogEntryWithSequenceNumber e = new LogEntryWithSequenceNumber(new LogSequenceNumber(ledgerId, id), lEntry);
                    consumer.accept(e);
                }
            }
        }
    }
}

Example 18 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookKeeperDataStorageManager method tableCheckpoint.

@Override
public List<PostCheckpointAction> tableCheckpoint(String tableSpace, String tableName, TableStatus tableStatus, boolean pin) throws DataStorageManagerException {
    // ensure that current mapping has been persisted safely
    persistTableSpaceMapping(tableSpace);
    LogSequenceNumber logPosition = tableStatus.sequenceNumber;
    String dir = getTableDirectory(tableSpace, tableName);
    String checkpointFile = getCheckPointsFile(dir, logPosition);
    Stat stat = new Stat();
    try {
        byte[] exists = readZNode(checkpointFile, stat);
        if (exists != null) {
            TableStatus actualStatus = readTableStatusFromFile(checkpointFile);
            if (actualStatus != null && actualStatus.equals(tableStatus)) {
                LOGGER.log(Level.FINE, "tableCheckpoint " + tableSpace + ", " + tableName + ": " + tableStatus + " (pin:" + pin + ") already saved on file " + checkpointFile);
                return Collections.emptyList();
            }
        }
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    LOGGER.log(Level.FINE, "tableCheckpoint " + tableSpace + ", " + tableName + ": " + tableStatus + " (pin:" + pin + ") to file " + checkpointFile);
    byte[] content;
    try (ByteArrayOutputStream buffer = new ByteArrayOutputStream();
        XXHash64Utils.HashingOutputStream oo = new XXHash64Utils.HashingOutputStream(buffer);
        ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
        // version
        dataOutputKeys.writeVLong(1);
        // flags for future implementations
        dataOutputKeys.writeVLong(0);
        tableStatus.serialize(dataOutputKeys);
        dataOutputKeys.writeLong(oo.hash());
        dataOutputKeys.flush();
        content = buffer.toByteArray();
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    writeZNodeEnforceOwnership(tableSpace, checkpointFile, content, stat);
    /* Checkpoint pinning */
    final Map<Long, Integer> pins = pinTableAndGetPages(tableSpace, tableName, tableStatus, pin);
    final Set<LogSequenceNumber> checkpoints = pinTableAndGetCheckpoints(tableSpace, tableName, tableStatus, pin);
    long maxPageId = tableStatus.activePages.keySet().stream().max(Comparator.naturalOrder()).orElse(Long.MAX_VALUE);
    List<PostCheckpointAction> result = new ArrayList<>();
    PagesMapping tableSpacePagesMapping = getTableSpacePagesMapping(tableSpace).getTablePagesMapping(tableName);
    // we can drop old page files now
    for (Map.Entry<Long, Long> pages : tableSpacePagesMapping.pages.entrySet()) {
        long pageId = pages.getKey();
        long ledgerId = pages.getValue();
        LOGGER.log(Level.FINEST, "checkpoint pageId {0} ledgerId {1}", new Object[] { pageId, ledgerId });
        if (pageId > 0 && !pins.containsKey(pageId) && !tableStatus.activePages.containsKey(pageId) && pageId < maxPageId) {
            LOGGER.log(Level.FINEST, "checkpoint ledger " + ledgerId + " pageId " + pageId + ". will be deleted after checkpoint end");
            result.add(new DropLedgerForTableAction(tableSpace, tableName, "delete page " + pageId + " ledgerId " + ledgerId, pageId, ledgerId));
        }
    }
    // we can drop orphan ledgers
    for (Long ledgerId : tableSpacePagesMapping.oldLedgers) {
        LOGGER.log(Level.FINEST, "checkpoint ledger " + ledgerId + " without page. will be deleted after checkpoint end");
        result.add(new DropLedgerForTableAction(tableSpace, tableName, "delete unused ledgerId " + ledgerId, Long.MAX_VALUE, ledgerId));
    }
    List<String> children = zkGetChildren(dir);
    try {
        for (String p : children) {
            if (isTableOrIndexCheckpointsFile(p) && !p.equals(checkpointFile)) {
                TableStatus status = readTableStatusFromFile(p);
                if (logPosition.after(status.sequenceNumber) && !checkpoints.contains(status.sequenceNumber)) {
                    LOGGER.log(Level.FINEST, "checkpoint metadata znode " + p + ". will be deleted after checkpoint end");
                    result.add(new DeleteZNodeAction(tableSpace, tableName, "delete checkpoint metadata znode " + p, p));
                }
            }
        }
    } catch (IOException err) {
        LOGGER.log(Level.SEVERE, "Could not list table dir " + dir, err);
    }
    return result;
}

Example 19 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookKeeperDataStorageManager method writeCheckpointSequenceNumber.

@Override
public Collection<PostCheckpointAction> writeCheckpointSequenceNumber(String tableSpace, LogSequenceNumber sequenceNumber) throws DataStorageManagerException {
    // ensure that current page mappings are persisted on ZK
    persistTableSpaceMapping(tableSpace);
    String checkPointFile = getTablespaceCheckPointInfoFile(tableSpace, sequenceNumber);
    LOGGER.log(Level.INFO, "checkpoint for {0} at {1} to {2}", new Object[] { tableSpace, sequenceNumber, checkPointFile });
    try (VisibleByteArrayOutputStream buffer = new VisibleByteArrayOutputStream();
        ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
        // version
        dout.writeVLong(1);
        // flags for future implementations
        dout.writeVLong(0);
        dout.writeUTF(tableSpace);
        dout.writeZLong(sequenceNumber.ledgerId);
        dout.writeZLong(sequenceNumber.offset);
        dout.flush();
        writeZNodeEnforceOwnership(tableSpace, checkPointFile, buffer.toByteArray(), null);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    String tableSpaceDirectory = getTableSpaceZNode(tableSpace);
    List<String> stream = zkGetChildren(tableSpaceDirectory);
    Collection<PostCheckpointAction> result = new ArrayList<>();
    for (String p : stream) {
        if (isTablespaceCheckPointInfoFile(p)) {
            try {
                byte[] content = readZNode(p, new Stat());
                if (content != null) {
                    LogSequenceNumber logPositionInFile = readLogSequenceNumberFromCheckpointInfoFile(tableSpace, content, p);
                    if (sequenceNumber.after(logPositionInFile)) {
                        LOGGER.log(Level.FINEST, "checkpoint info file " + p + ". will be deleted after checkpoint end");
                        result.add(new DeleteZNodeAction(tableSpace, "checkpoint", "delete checkpoint info file " + p, p));
                    }
                }
            } catch (DataStorageManagerException | IOException ignore) {
                LOGGER.log(Level.SEVERE, "unparsable checkpoint info file " + p, ignore);
            // do not auto-delete checkpoint files
            }
        }
    }
    return result;
}

Example 20 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class BookKeeperDataStorageManager method readLogSequenceNumberFromTransactionsFile.

private static LogSequenceNumber readLogSequenceNumberFromTransactionsFile(String tableSpace, byte[] data, String file) throws DataStorageManagerException {
    try (InputStream input = new ByteArrayInputStream(data);
        ExtendedDataInputStream din = new ExtendedDataInputStream(input)) {
        // version
        long version = din.readVLong();
        // flags for future implementations
        long flags = din.readVLong();
        if (version != 1 || flags != 0) {
            throw new DataStorageManagerException("corrupted transaction list znode " + file);
        }
        String readname = din.readUTF();
        if (!readname.equals(tableSpace)) {
            throw new DataStorageManagerException("znode " + file + " is not for spablespace " + tableSpace);
        }
        long ledgerId = din.readZLong();
        long offset = din.readZLong();
        return new LogSequenceNumber(ledgerId, offset);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
}