Examples with PostCheckpointAction herddb.core.PostCheckpointAction used on opensource projects

Example 11 with PostCheckpointAction

use of herddb.core.PostCheckpointAction in project herddb by diennea.

the class FileDataStorageManager method tableCheckpoint.

@Override
public List<PostCheckpointAction> tableCheckpoint(String tableSpace, String tableName, TableStatus tableStatus, boolean pin) throws DataStorageManagerException {
    LogSequenceNumber logPosition = tableStatus.sequenceNumber;
    Path dir = getTableDirectory(tableSpace, tableName);
    Path checkpointFile = getTableCheckPointsFile(dir, logPosition);
    try {
        if (Files.isRegularFile(checkpointFile)) {
            TableStatus actualStatus = readTableStatusFromFile(checkpointFile);
            if (actualStatus != null && actualStatus.equals(tableStatus)) {
                LOGGER.log(Level.FINE, "tableCheckpoint " + tableSpace + ", " + tableName + ": " + tableStatus + " (pin:" + pin + ") already saved on file " + checkpointFile);
                return Collections.emptyList();
            }
        }
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    Path parent = getParent(checkpointFile);
    Path checkpointFileTemp = parent.resolve(checkpointFile.getFileName() + ".tmp");
    LOGGER.log(Level.FINE, "tableCheckpoint " + tableSpace + ", " + tableName + ": " + tableStatus + " (pin:" + pin + ") to file " + checkpointFile);
    try (ManagedFile file = ManagedFile.open(checkpointFileTemp, requirefsync);
        SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
        XXHash64Utils.HashingOutputStream oo = new XXHash64Utils.HashingOutputStream(buffer);
        ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
        // version
        dataOutputKeys.writeVLong(1);
        // flags for future implementations
        dataOutputKeys.writeVLong(0);
        tableStatus.serialize(dataOutputKeys);
        dataOutputKeys.writeLong(oo.hash());
        dataOutputKeys.flush();
        file.sync();
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    try {
        Files.move(checkpointFileTemp, checkpointFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    /* Checkpoint pinning */
    final Map<Long, Integer> pins = pinTableAndGetPages(tableSpace, tableName, tableStatus, pin);
    final Set<LogSequenceNumber> checkpoints = pinTableAndGetCheckpoints(tableSpace, tableName, tableStatus, pin);
    long maxPageId = tableStatus.activePages.keySet().stream().max(Comparator.naturalOrder()).orElse(Long.MAX_VALUE);
    List<PostCheckpointAction> result = new ArrayList<>();
    // we can drop old page files now
    List<Path> pageFiles = getTablePageFiles(tableSpace, tableName);
    for (Path p : pageFiles) {
        long pageId = getPageId(p);
        LOGGER.log(Level.FINEST, "checkpoint file {0} pageId {1}", new Object[] { p.toAbsolutePath(), pageId });
        if (pageId > 0 && !pins.containsKey(pageId) && !tableStatus.activePages.containsKey(pageId) && pageId < maxPageId) {
            LOGGER.log(Level.FINEST, "checkpoint file " + p.toAbsolutePath() + " pageId " + pageId + ". will be deleted after checkpoint end");
            result.add(new DeleteFileAction(tableSpace, tableName, "delete page " + pageId + " file " + p.toAbsolutePath(), p));
        }
    }
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
        for (Path p : stream) {
            if (isTableOrIndexCheckpointsFile(p) && !p.equals(checkpointFile)) {
                TableStatus status = readTableStatusFromFile(p);
                if (logPosition.after(status.sequenceNumber) && !checkpoints.contains(status.sequenceNumber)) {
                    LOGGER.log(Level.FINEST, "checkpoint metadata file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
                    result.add(new DeleteFileAction(tableSpace, tableName, "delete checkpoint metadata file " + p.toAbsolutePath(), p));
                }
            }
        }
    } catch (IOException err) {
        LOGGER.log(Level.SEVERE, "Could not list table dir " + dir, err);
    }
    return result;
}

Example 12 with PostCheckpointAction

use of herddb.core.PostCheckpointAction in project herddb by diennea.

the class FileDataStorageManager method writeCheckpointSequenceNumber.

@Override
public Collection<PostCheckpointAction> writeCheckpointSequenceNumber(String tableSpace, LogSequenceNumber sequenceNumber) throws DataStorageManagerException {
    Path tableSpaceDirectory = getTablespaceDirectory(tableSpace);
    try {
        Files.createDirectories(tableSpaceDirectory);
        Path checkPointFile = getTablespaceCheckPointInfoFile(tableSpace, sequenceNumber);
        Path parent = getParent(checkPointFile);
        Files.createDirectories(parent);
        Path checkpointFileTemp = parent.resolve(checkPointFile.getFileName() + ".tmp");
        LOGGER.log(Level.INFO, "checkpoint for " + tableSpace + " at " + sequenceNumber + " to " + checkPointFile.toAbsolutePath().toString());
        try (ManagedFile file = ManagedFile.open(checkpointFileTemp, requirefsync);
            SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
            ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
            // version
            dout.writeVLong(1);
            // flags for future implementations
            dout.writeVLong(0);
            dout.writeUTF(tableSpace);
            dout.writeZLong(sequenceNumber.ledgerId);
            dout.writeZLong(sequenceNumber.offset);
            dout.flush();
            file.sync();
        } catch (IOException err) {
            throw new DataStorageManagerException(err);
        }
        // write file atomically
        Files.move(checkpointFileTemp, checkPointFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    Collection<PostCheckpointAction> result = new ArrayList<>();
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(tableSpaceDirectory)) {
        for (Path p : stream) {
            if (isTablespaceCheckPointInfoFile(p)) {
                try {
                    LogSequenceNumber logPositionInFile = readLogSequenceNumberFromCheckpointInfoFile(tableSpace, p);
                    if (sequenceNumber.after(logPositionInFile)) {
                        LOGGER.log(Level.FINEST, "checkpoint info file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
                        result.add(new DeleteFileAction(tableSpace, "checkpoint", "delete checkpoint info file " + p.toAbsolutePath(), p));
                    }
                } catch (DataStorageManagerException ignore) {
                    LOGGER.log(Level.SEVERE, "unparsable checkpoint info file " + p.toAbsolutePath(), ignore);
                // do not auto-delete checkpoint files
                }
            }
        }
    } catch (IOException err) {
        LOGGER.log(Level.SEVERE, "Could not list dir " + tableSpaceDirectory, err);
    }
    return result;
}

Example 13 with PostCheckpointAction

use of herddb.core.PostCheckpointAction in project herddb by diennea.

the class BLinkKeyToPageIndex method checkpoint.

@Override
public List<PostCheckpointAction> checkpoint(LogSequenceNumber sequenceNumber, boolean pin) throws DataStorageManagerException {
    try {
        /* Tree can be null if no data was inserted (tree creation deferred to check evaluate key size) */
        final BLink<Bytes, Long> tree = this.tree;
        if (tree == null) {
            return Collections.emptyList();
        }
        BLinkMetadata<Bytes> metadata = getTree().checkpoint();
        byte[] metaPage = MetadataSerializer.INSTANCE.write(metadata);
        Set<Long> activePages = new HashSet<>();
        metadata.nodes.forEach(node -> activePages.add(node.storeId));
        IndexStatus indexStatus = new IndexStatus(indexName, sequenceNumber, newPageId.get(), activePages, metaPage);
        List<PostCheckpointAction> result = new ArrayList<>();
        result.addAll(dataStorageManager.indexCheckpoint(tableSpace, indexName, indexStatus, pin));
        LOGGER.log(Level.INFO, "checkpoint index {0} finished: logpos {1}, {2} pages", new Object[] { indexName, sequenceNumber, Integer.toString(metadata.nodes.size()) });
        LOGGER.log(Level.FINE, "checkpoint index {0} finished: logpos {1}, pages {2}", new Object[] { indexName, sequenceNumber, activePages.toString() });
        return result;
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
}

Example 14 with PostCheckpointAction

use of herddb.core.PostCheckpointAction in project herddb by diennea.

the class BRINIndexManager method checkpoint.

@Override
public List<PostCheckpointAction> checkpoint(LogSequenceNumber sequenceNumber, boolean pin) throws DataStorageManagerException {
    try {
        BlockRangeIndexMetadata<Bytes> metadata = data.checkpoint();
        /* Checks metadata consistency with actual data structure */
        if (VALIDATE_CHECKPOINT_METADATA) {
            boolean invalid = false;
            /*
                 * Metadata are saved/recovered in reverse order so "next" block has been already created
                 */
            Long nextID = null;
            for (BlockRangeIndexMetadata.BlockMetadata<Bytes> blockData : metadata.getBlocksMetadata()) {
                /* Medatada safety check (do not trust blindly ordering) */
                if (blockData.nextBlockId != null) {
                    if (nextID == null) {
                        LOGGER.log(Level.WARNING, "Wrong next block on index {0}, expected notingh but {0} found", new Object[] { index.name, blockData.nextBlockId });
                        invalid = true;
                    } else if (nextID != blockData.nextBlockId.longValue()) {
                        LOGGER.log(Level.WARNING, "Wrong next block on index {0}, expected {1} but {2} found", new Object[] { index.name, nextID, blockData.nextBlockId });
                        invalid = true;
                    }
                } else {
                    if (nextID != null) {
                        LOGGER.log(Level.WARNING, "Wrong next block on index {0}, expected {1} but nothing found", new Object[] { index.name, nextID });
                        invalid = true;
                    }
                }
                nextID = blockData.blockId;
            }
            if (invalid) {
                LOGGER.log(Level.WARNING, data.generateDetailedInternalStatus());
            }
        }
        PageContents page = new PageContents();
        page.type = PageContents.TYPE_METADATA;
        page.metadata = metadata.getBlocksMetadata();
        byte[] contents = page.serialize();
        Set<Long> activePages = new HashSet<>();
        page.metadata.forEach(b -> {
            activePages.add(b.pageId);
        });
        IndexStatus indexStatus = new IndexStatus(index.name, sequenceNumber, newPageId.get(), activePages, contents);
        List<PostCheckpointAction> result = new ArrayList<>();
        result.addAll(dataStorageManager.indexCheckpoint(tableSpaceUUID, index.uuid, indexStatus, pin));
        LOGGER.log(Level.INFO, "checkpoint index {0} finished: logpos {1}, {2} blocks", new Object[] { index.name, sequenceNumber, Integer.toString(page.metadata.size()) });
        LOGGER.log(Level.FINE, "checkpoint index {0} finished: logpos {1}, pages {2}", new Object[] { index.name, sequenceNumber, activePages });
        return result;
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
}

Example 15 with PostCheckpointAction

use of herddb.core.PostCheckpointAction in project herddb by diennea.

the class MemoryHashIndexManager method checkpoint.

@Override
public List<PostCheckpointAction> checkpoint(LogSequenceNumber sequenceNumber, boolean pin) throws DataStorageManagerException {
    if (createdInTransaction > 0) {
        LOGGER.log(Level.INFO, "checkpoint for index " + index.name + " skipped, this index is created on transaction " + createdInTransaction + " which is not committed");
        return Collections.emptyList();
    }
    List<PostCheckpointAction> result = new ArrayList<>();
    LOGGER.log(Level.INFO, "flush index {0}", new Object[] { index.name });
    long pageId = newPageId.getAndIncrement();
    Holder<Long> count = new Holder<>();
    dataStorageManager.writeIndexPage(tableSpaceUUID, index.uuid, pageId, (out) -> {
        long entries = 0;
        // version
        out.writeVLong(1);
        // flags for future implementations
        out.writeVLong(0);
        out.writeVInt(data.size());
        for (Map.Entry<Bytes, List<Bytes>> entry : data.entrySet()) {
            out.writeArray(entry.getKey());
            List<Bytes> entrydata = entry.getValue();
            out.writeVInt(entrydata.size());
            for (Bytes v : entrydata) {
                out.writeArray(v);
                ++entries;
            }
        }
        count.value = entries;
    });
    IndexStatus indexStatus = new IndexStatus(index.name, sequenceNumber, newPageId.get(), Collections.singleton(pageId), null);
    result.addAll(dataStorageManager.indexCheckpoint(tableSpaceUUID, index.uuid, indexStatus, pin));
    LOGGER.log(Level.INFO, "checkpoint index {0} finished: logpos {1}, {2} entries, page {3}", new Object[] { index.name, sequenceNumber, Long.toString(count.value), Long.toString(pageId) });
    return result;
}