Examples with ExtendedDataOutputStream herddb.utils.ExtendedDataOutputStream used on opensource projects

Example 11 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class MemoryDataStorageManager method tableCheckpoint.

@Override
public List<PostCheckpointAction> tableCheckpoint(String tableSpace, String tableName, TableStatus tableStatus, boolean pin) throws DataStorageManagerException {
    /* Checkpoint pinning */
    final Map<Long, Integer> pins = pinTableAndGetPages(tableSpace, tableName, tableStatus, pin);
    final Set<LogSequenceNumber> checkpoints = pinTableAndGetCheckpoints(tableSpace, tableName, tableStatus, pin);
    List<Long> pagesForTable = new ArrayList<>();
    String prefix = tableSpace + "." + tableName + "_";
    for (String key : pages.keySet()) {
        if (key.startsWith(prefix)) {
            long pageId = Long.parseLong(key.substring(prefix.length()));
            if (!pins.containsKey(pageId)) {
                pagesForTable.add(pageId);
            }
        }
    }
    pagesForTable.removeAll(tableStatus.activePages.keySet());
    List<PostCheckpointAction> result = new ArrayList<>();
    for (long pageId : pagesForTable) {
        result.add(new PostCheckpointAction(tableSpace, tableName, "drop page " + pageId) {

            @Override
            public void run() {
                // remove only after checkpoint completed
                pages.remove(prefix + pageId);
                LOGGER.log(Level.SEVERE, "removing " + (prefix + pageId));
            }
        });
    }
    for (String oldStatus : tableStatuses.keySet()) {
        if (oldStatus.startsWith(prefix)) {
            /* Check for checkpoint skip only if match expected structure */
            final LogSequenceNumber log = evaluateLogSequenceNumber(prefix);
            if (log != null) {
                /* If is pinned skip this status*/
                if (checkpoints.contains(log)) {
                    continue;
                }
            }
            result.add(new PostCheckpointAction(tableSpace, tableName, "drop table checkpoint " + oldStatus) {

                @Override
                public void run() {
                    // remove only after checkpoint completed
                    tableStatuses.remove(oldStatus);
                }
            });
        }
    }
    VisibleByteArrayOutputStream oo = new VisibleByteArrayOutputStream(1024);
    try (ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
        tableStatus.serialize(dataOutputKeys);
        dataOutputKeys.flush();
        oo.write(oo.xxhash64());
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    /* Uses a copy to limit byte[] size at the min needed */
    tableStatuses.put(checkpointName(tableSpace, tableName, tableStatus.sequenceNumber), oo.toByteArray());
    return result;
}

Example 12 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class BookKeeperDataStorageManager method writeTables.

@Override
public Collection<PostCheckpointAction> writeTables(String tableSpace, LogSequenceNumber sequenceNumber, List<Table> tables, List<Index> indexlist, boolean prepareActions) throws DataStorageManagerException {
    if (sequenceNumber.isStartOfTime() && !tables.isEmpty()) {
        throw new DataStorageManagerException("impossible to write a non empty table list at start-of-time");
    }
    // we need to flush current mappings, because here we are flushing
    // the status of all of the tables and indexes
    persistTableSpaceMapping(tableSpace);
    String tableSpaceDirectory = getTableSpaceZNode(tableSpace);
    String fileTables = getTablespaceTablesMetadataFile(tableSpace, sequenceNumber);
    String fileIndexes = getTablespaceIndexesMetadataFile(tableSpace, sequenceNumber);
    LOGGER.log(Level.FINE, "writeTables for tableSpace " + tableSpace + " sequenceNumber " + sequenceNumber + " to " + fileTables);
    try (VisibleByteArrayOutputStream buffer = new VisibleByteArrayOutputStream();
        ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
        // version
        dout.writeVLong(1);
        // flags for future implementations
        dout.writeVLong(0);
        dout.writeUTF(tableSpace);
        dout.writeZLong(sequenceNumber.ledgerId);
        dout.writeZLong(sequenceNumber.offset);
        dout.writeInt(tables.size());
        for (Table t : tables) {
            byte[] tableSerialized = t.serialize();
            dout.writeArray(tableSerialized);
        }
        dout.flush();
        writeZNodeEnforceOwnership(tableSpace, fileTables, buffer.toByteArray(), null);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    try (VisibleByteArrayOutputStream buffer = new VisibleByteArrayOutputStream();
        ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
        // version
        dout.writeVLong(1);
        // flags for future implementations
        dout.writeVLong(0);
        dout.writeUTF(tableSpace);
        dout.writeZLong(sequenceNumber.ledgerId);
        dout.writeZLong(sequenceNumber.offset);
        if (indexlist != null) {
            dout.writeInt(indexlist.size());
            for (Index t : indexlist) {
                byte[] indexSerialized = t.serialize();
                dout.writeArray(indexSerialized);
            }
        } else {
            dout.writeInt(0);
        }
        dout.flush();
        writeZNodeEnforceOwnership(tableSpace, fileIndexes, buffer.toByteArray(), null);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    Collection<PostCheckpointAction> result = new ArrayList<>();
    if (prepareActions) {
        List<String> stream = zkGetChildren(tableSpaceDirectory);
        for (String p : stream) {
            if (isTablespaceIndexesMetadataFile(p)) {
                try {
                    byte[] content = readZNode(p, new Stat());
                    if (content != null) {
                        LogSequenceNumber logPositionInFile = readLogSequenceNumberFromIndexMetadataFile(tableSpace, content, p);
                        if (sequenceNumber.after(logPositionInFile)) {
                            LOGGER.log(Level.FINEST, "indexes metadata file " + p + ". will be deleted after checkpoint end");
                            result.add(new DeleteZNodeAction(tableSpace, "indexes", "delete indexesmetadata file " + p, p));
                        }
                    }
                } catch (DataStorageManagerException ignore) {
                    LOGGER.log(Level.SEVERE, "Unparsable indexesmetadata file " + p, ignore);
                    result.add(new DeleteZNodeAction(tableSpace, "indexes", "delete unparsable indexesmetadata file " + p, p));
                }
            } else if (isTablespaceTablesMetadataFile(p)) {
                try {
                    byte[] content = readZNode(p, new Stat());
                    if (content != null) {
                        LogSequenceNumber logPositionInFile = readLogSequenceNumberFromTablesMetadataFile(tableSpace, content, p);
                        if (sequenceNumber.after(logPositionInFile)) {
                            LOGGER.log(Level.FINEST, "tables metadata file " + p + ". will be deleted after checkpoint end");
                            result.add(new DeleteZNodeAction(tableSpace, "tables", "delete tablesmetadata file " + p, p));
                        }
                    }
                } catch (DataStorageManagerException ignore) {
                    LOGGER.log(Level.SEVERE, "Unparsable tablesmetadata file " + p, ignore);
                    result.add(new DeleteZNodeAction(tableSpace, "transactions", "delete unparsable tablesmetadata file " + p, p));
                }
            }
        }
    }
    return result;
}

Example 13 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class BookKeeperDataStorageManager method tableCheckpoint.

@Override
public List<PostCheckpointAction> tableCheckpoint(String tableSpace, String tableName, TableStatus tableStatus, boolean pin) throws DataStorageManagerException {
    // ensure that current mapping has been persisted safely
    persistTableSpaceMapping(tableSpace);
    LogSequenceNumber logPosition = tableStatus.sequenceNumber;
    String dir = getTableDirectory(tableSpace, tableName);
    String checkpointFile = getCheckPointsFile(dir, logPosition);
    Stat stat = new Stat();
    try {
        byte[] exists = readZNode(checkpointFile, stat);
        if (exists != null) {
            TableStatus actualStatus = readTableStatusFromFile(checkpointFile);
            if (actualStatus != null && actualStatus.equals(tableStatus)) {
                LOGGER.log(Level.FINE, "tableCheckpoint " + tableSpace + ", " + tableName + ": " + tableStatus + " (pin:" + pin + ") already saved on file " + checkpointFile);
                return Collections.emptyList();
            }
        }
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    LOGGER.log(Level.FINE, "tableCheckpoint " + tableSpace + ", " + tableName + ": " + tableStatus + " (pin:" + pin + ") to file " + checkpointFile);
    byte[] content;
    try (ByteArrayOutputStream buffer = new ByteArrayOutputStream();
        XXHash64Utils.HashingOutputStream oo = new XXHash64Utils.HashingOutputStream(buffer);
        ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
        // version
        dataOutputKeys.writeVLong(1);
        // flags for future implementations
        dataOutputKeys.writeVLong(0);
        tableStatus.serialize(dataOutputKeys);
        dataOutputKeys.writeLong(oo.hash());
        dataOutputKeys.flush();
        content = buffer.toByteArray();
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    writeZNodeEnforceOwnership(tableSpace, checkpointFile, content, stat);
    /* Checkpoint pinning */
    final Map<Long, Integer> pins = pinTableAndGetPages(tableSpace, tableName, tableStatus, pin);
    final Set<LogSequenceNumber> checkpoints = pinTableAndGetCheckpoints(tableSpace, tableName, tableStatus, pin);
    long maxPageId = tableStatus.activePages.keySet().stream().max(Comparator.naturalOrder()).orElse(Long.MAX_VALUE);
    List<PostCheckpointAction> result = new ArrayList<>();
    PagesMapping tableSpacePagesMapping = getTableSpacePagesMapping(tableSpace).getTablePagesMapping(tableName);
    // we can drop old page files now
    for (Map.Entry<Long, Long> pages : tableSpacePagesMapping.pages.entrySet()) {
        long pageId = pages.getKey();
        long ledgerId = pages.getValue();
        LOGGER.log(Level.FINEST, "checkpoint pageId {0} ledgerId {1}", new Object[] { pageId, ledgerId });
        if (pageId > 0 && !pins.containsKey(pageId) && !tableStatus.activePages.containsKey(pageId) && pageId < maxPageId) {
            LOGGER.log(Level.FINEST, "checkpoint ledger " + ledgerId + " pageId " + pageId + ". will be deleted after checkpoint end");
            result.add(new DropLedgerForTableAction(tableSpace, tableName, "delete page " + pageId + " ledgerId " + ledgerId, pageId, ledgerId));
        }
    }
    // we can drop orphan ledgers
    for (Long ledgerId : tableSpacePagesMapping.oldLedgers) {
        LOGGER.log(Level.FINEST, "checkpoint ledger " + ledgerId + " without page. will be deleted after checkpoint end");
        result.add(new DropLedgerForTableAction(tableSpace, tableName, "delete unused ledgerId " + ledgerId, Long.MAX_VALUE, ledgerId));
    }
    List<String> children = zkGetChildren(dir);
    try {
        for (String p : children) {
            if (isTableOrIndexCheckpointsFile(p) && !p.equals(checkpointFile)) {
                TableStatus status = readTableStatusFromFile(p);
                if (logPosition.after(status.sequenceNumber) && !checkpoints.contains(status.sequenceNumber)) {
                    LOGGER.log(Level.FINEST, "checkpoint metadata znode " + p + ". will be deleted after checkpoint end");
                    result.add(new DeleteZNodeAction(tableSpace, tableName, "delete checkpoint metadata znode " + p, p));
                }
            }
        }
    } catch (IOException err) {
        LOGGER.log(Level.SEVERE, "Could not list table dir " + dir, err);
    }
    return result;
}

Example 14 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class BookKeeperDataStorageManager method writeCheckpointSequenceNumber.

@Override
public Collection<PostCheckpointAction> writeCheckpointSequenceNumber(String tableSpace, LogSequenceNumber sequenceNumber) throws DataStorageManagerException {
    // ensure that current page mappings are persisted on ZK
    persistTableSpaceMapping(tableSpace);
    String checkPointFile = getTablespaceCheckPointInfoFile(tableSpace, sequenceNumber);
    LOGGER.log(Level.INFO, "checkpoint for {0} at {1} to {2}", new Object[] { tableSpace, sequenceNumber, checkPointFile });
    try (VisibleByteArrayOutputStream buffer = new VisibleByteArrayOutputStream();
        ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
        // version
        dout.writeVLong(1);
        // flags for future implementations
        dout.writeVLong(0);
        dout.writeUTF(tableSpace);
        dout.writeZLong(sequenceNumber.ledgerId);
        dout.writeZLong(sequenceNumber.offset);
        dout.flush();
        writeZNodeEnforceOwnership(tableSpace, checkPointFile, buffer.toByteArray(), null);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    String tableSpaceDirectory = getTableSpaceZNode(tableSpace);
    List<String> stream = zkGetChildren(tableSpaceDirectory);
    Collection<PostCheckpointAction> result = new ArrayList<>();
    for (String p : stream) {
        if (isTablespaceCheckPointInfoFile(p)) {
            try {
                byte[] content = readZNode(p, new Stat());
                if (content != null) {
                    LogSequenceNumber logPositionInFile = readLogSequenceNumberFromCheckpointInfoFile(tableSpace, content, p);
                    if (sequenceNumber.after(logPositionInFile)) {
                        LOGGER.log(Level.FINEST, "checkpoint info file " + p + ". will be deleted after checkpoint end");
                        result.add(new DeleteZNodeAction(tableSpace, "checkpoint", "delete checkpoint info file " + p, p));
                    }
                }
            } catch (DataStorageManagerException | IOException ignore) {
                LOGGER.log(Level.SEVERE, "unparsable checkpoint info file " + p, ignore);
            // do not auto-delete checkpoint files
            }
        }
    }
    return result;
}

Example 15 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class BookKeeperDataStorageManager method writeIndexPage.

private static long writeIndexPage(DataWriter writer, VisibleByteArrayOutputStream stream) throws IOException {
    try (ExtendedDataOutputStream dataOutput = new ExtendedDataOutputStream(stream)) {
        // version
        dataOutput.writeVLong(1);
        // flags for future implementations
        dataOutput.writeVLong(0);
        writer.write(dataOutput);
        dataOutput.flush();
        long hash = XXHash64Utils.hash(stream.getBuffer(), 0, stream.size());
        dataOutput.writeLong(hash);
        dataOutput.flush();
        return stream.size();
    }
}