Examples with ExtendedDataOutputStream herddb.utils.ExtendedDataOutputStream used on opensource projects

Example 6 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class FileDataStorageManager method writeTransactionsAtCheckpoint.

@Override
public Collection<PostCheckpointAction> writeTransactionsAtCheckpoint(String tableSpace, LogSequenceNumber sequenceNumber, Collection<Transaction> transactions) throws DataStorageManagerException {
    if (sequenceNumber.isStartOfTime() && !transactions.isEmpty()) {
        throw new DataStorageManagerException("impossible to write a non empty transactions list at start-of-time");
    }
    Path tableSpaceDirectory = getTablespaceDirectory(tableSpace);
    try {
        Files.createDirectories(tableSpaceDirectory);
        Path checkPointFile = getTablespaceTransactionsFile(tableSpace, sequenceNumber);
        Path parent = getParent(checkPointFile);
        Files.createDirectories(parent);
        Path checkpointFileTemp = parent.resolve(checkPointFile.getFileName() + ".tmp");
        LOGGER.log(Level.FINE, "writeTransactionsAtCheckpoint for tableSpace {0} sequenceNumber {1} to {2}, active transactions {3}", new Object[] { tableSpace, sequenceNumber, checkPointFile.toAbsolutePath().toString(), transactions.size() });
        try (ManagedFile file = ManagedFile.open(checkpointFileTemp, requirefsync);
            SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
            ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
            // version
            dout.writeVLong(1);
            // flags for future implementations
            dout.writeVLong(0);
            dout.writeUTF(tableSpace);
            dout.writeZLong(sequenceNumber.ledgerId);
            dout.writeZLong(sequenceNumber.offset);
            dout.writeInt(transactions.size());
            for (Transaction t : transactions) {
                t.serialize(dout);
            }
            dout.flush();
            file.sync();
        } catch (IOException err) {
            throw new DataStorageManagerException(err);
        }
        try {
            Files.move(checkpointFileTemp, checkPointFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
        } catch (IOException err) {
            throw new DataStorageManagerException(err);
        }
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    Collection<PostCheckpointAction> result = new ArrayList<>();
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(tableSpaceDirectory)) {
        for (Path p : stream) {
            if (isTransactionsFile(p)) {
                try {
                    LogSequenceNumber logPositionInFile = readLogSequenceNumberFromTransactionsFile(tableSpace, p);
                    if (sequenceNumber.after(logPositionInFile)) {
                        LOGGER.log(Level.FINEST, "transactions metadata file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
                        result.add(new DeleteFileAction(tableSpace, "transactions", "delete transactions file " + p.toAbsolutePath(), p));
                    }
                } catch (DataStorageManagerException ignore) {
                    LOGGER.log(Level.SEVERE, "Unparsable transactions file " + p.toAbsolutePath(), ignore);
                    result.add(new DeleteFileAction(tableSpace, "transactions", "delete unparsable transactions file " + p.toAbsolutePath(), p));
                }
            }
        }
    } catch (IOException err) {
        LOGGER.log(Level.SEVERE, "Could not list dir " + tableSpaceDirectory, err);
    }
    return result;
}

Example 7 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class FileDataStorageManager method indexCheckpoint.

@Override
public List<PostCheckpointAction> indexCheckpoint(String tableSpace, String indexName, IndexStatus indexStatus, boolean pin) throws DataStorageManagerException {
    Path dir = getIndexDirectory(tableSpace, indexName);
    LogSequenceNumber logPosition = indexStatus.sequenceNumber;
    Path checkpointFile = getTableCheckPointsFile(dir, logPosition);
    Path parent = getParent(checkpointFile);
    Path checkpointFileTemp = parent.resolve(checkpointFile.getFileName() + ".tmp");
    if (Files.isRegularFile(checkpointFile)) {
        IndexStatus actualStatus = readIndexStatusFromFile(checkpointFile);
        if (actualStatus != null && actualStatus.equals(indexStatus)) {
            LOGGER.log(Level.INFO, "indexCheckpoint " + tableSpace + ", " + indexName + ": " + indexStatus + " already saved on" + checkpointFile);
            return Collections.emptyList();
        }
    }
    LOGGER.log(Level.FINE, "indexCheckpoint " + tableSpace + ", " + indexName + ": " + indexStatus + " to file " + checkpointFile);
    try (ManagedFile file = ManagedFile.open(checkpointFileTemp, requirefsync);
        SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
        XXHash64Utils.HashingOutputStream oo = new XXHash64Utils.HashingOutputStream(buffer);
        ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
        // version
        dataOutputKeys.writeVLong(1);
        // flags for future implementations
        dataOutputKeys.writeVLong(0);
        indexStatus.serialize(dataOutputKeys);
        dataOutputKeys.writeLong(oo.hash());
        dataOutputKeys.flush();
        file.sync();
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    try {
        Files.move(checkpointFileTemp, checkpointFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    /* Checkpoint pinning */
    final Map<Long, Integer> pins = pinIndexAndGetPages(tableSpace, indexName, indexStatus, pin);
    final Set<LogSequenceNumber> checkpoints = pinIndexAndGetCheckpoints(tableSpace, indexName, indexStatus, pin);
    long maxPageId = indexStatus.activePages.stream().max(Comparator.naturalOrder()).orElse(Long.MAX_VALUE);
    List<PostCheckpointAction> result = new ArrayList<>();
    // we can drop old page files now
    List<Path> pageFiles = getIndexPageFiles(tableSpace, indexName);
    for (Path p : pageFiles) {
        long pageId = getPageId(p);
        LOGGER.log(Level.FINEST, "checkpoint file {0} pageId {1}", new Object[] { p.toAbsolutePath(), pageId });
        if (pageId > 0 && !pins.containsKey(pageId) && !indexStatus.activePages.contains(pageId) && pageId < maxPageId) {
            LOGGER.log(Level.FINEST, "checkpoint file " + p.toAbsolutePath() + " pageId " + pageId + ". will be deleted after checkpoint end");
            result.add(new DeleteFileAction(tableSpace, indexName, "delete page " + pageId + " file " + p.toAbsolutePath(), p));
        }
    }
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
        for (Path p : stream) {
            if (isTableOrIndexCheckpointsFile(p) && !p.equals(checkpointFile)) {
                IndexStatus status = readIndexStatusFromFile(p);
                if (logPosition.after(status.sequenceNumber) && !checkpoints.contains(status.sequenceNumber)) {
                    LOGGER.log(Level.FINEST, "checkpoint metadata file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
                    result.add(new DeleteFileAction(tableSpace, indexName, "delete checkpoint metadata file " + p.toAbsolutePath(), p));
                }
            }
        }
    } catch (IOException err) {
        LOGGER.log(Level.SEVERE, "Could not list indexName dir " + dir, err);
    }
    return result;
}

Example 8 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class FileMetadataStorageManager method persistTableSpaceOnDisk.

private void persistTableSpaceOnDisk(TableSpace tableSpace) throws MetadataStorageManagerException {
    Path tablespaceMetaTmp = baseDirectory.resolve(tableSpace.name.toLowerCase() + "." + System.nanoTime() + ".tmpmetadata");
    Path tablespaceMeta = baseDirectory.resolve(tableSpace.name.toLowerCase() + ".metadata");
    try (ManagedFile file = ManagedFile.open(tablespaceMetaTmp, ServerConfiguration.PROPERTY_REQUIRE_FSYNC_DEFAULT);
        SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), FileDataStorageManager.COPY_BUFFERS_SIZE);
        XXHash64Utils.HashingOutputStream oo = new XXHash64Utils.HashingOutputStream(buffer);
        ExtendedDataOutputStream dout = new ExtendedDataOutputStream(oo)) {
        // version
        dout.writeVLong(1);
        // flags for future implementations
        dout.writeVLong(0);
        tableSpace.serialize(dout);
        // footer
        dout.writeLong(oo.hash());
        dout.flush();
        file.sync();
    } catch (IOException err) {
        throw new MetadataStorageManagerException(err);
    }
    try {
        Files.move(tablespaceMetaTmp, tablespaceMeta, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
    } catch (IOException err) {
        throw new MetadataStorageManagerException(err);
    }
}

Example 9 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class MemoryDataStorageManager method indexCheckpoint.

@Override
public List<PostCheckpointAction> indexCheckpoint(String tableSpace, String indexName, IndexStatus indexStatus, boolean pin) throws DataStorageManagerException {
    /* Checkpoint pinning */
    final Map<Long, Integer> pins = pinIndexAndGetPages(tableSpace, indexName, indexStatus, pin);
    final Set<LogSequenceNumber> checkpoints = pinIndexAndGetCheckpoints(tableSpace, indexName, indexStatus, pin);
    List<Long> pagesForIndex = new ArrayList<>();
    String prefix = tableSpace + "." + indexName + "_";
    for (String key : indexpages.keySet()) {
        if (key.startsWith(prefix)) {
            long pageId = Long.parseLong(key.substring(prefix.length()));
            if (!pins.containsKey(pageId)) {
                pagesForIndex.add(pageId);
            }
        }
    }
    pagesForIndex.removeAll(indexStatus.activePages);
    List<PostCheckpointAction> result = new ArrayList<>();
    for (long pageId : pagesForIndex) {
        result.add(new PostCheckpointAction(tableSpace, indexName, "drop page " + pageId) {

            @Override
            public void run() {
                // remove only after checkpoint completed
                indexpages.remove(prefix + pageId);
            }
        });
    }
    for (String oldStatus : indexStatuses.keySet()) {
        if (oldStatus.startsWith(prefix)) {
            /* Check for checkpoint skip only if match expected structure */
            final LogSequenceNumber log = evaluateLogSequenceNumber(prefix);
            if (log != null) {
                /* If is pinned skip this status*/
                if (checkpoints.contains(log)) {
                    continue;
                }
            }
            result.add(new PostCheckpointAction(tableSpace, indexName, "drop index checkpoint " + oldStatus) {

                @Override
                public void run() {
                    // remove only after checkpoint completed
                    indexStatuses.remove(oldStatus);
                }
            });
        }
    }
    VisibleByteArrayOutputStream oo = new VisibleByteArrayOutputStream(1024);
    try (ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
        indexStatus.serialize(dataOutputKeys);
        dataOutputKeys.flush();
        oo.write(oo.xxhash64());
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    /* Uses a copy to limit byte[] size at the min needed */
    indexStatuses.put(checkpointName(tableSpace, indexName, indexStatus.sequenceNumber), oo.toByteArray());
    return result;
}

Example 10 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class MemoryDataStorageManager method writeIndexPage.

@Override
public void writeIndexPage(String tableSpace, String indexName, long pageId, DataWriter writer) throws DataStorageManagerException {
    Bytes page_wrapper;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream(1024);
        ExtendedDataOutputStream eout = new ExtendedDataOutputStream(out)) {
        writer.write(eout);
        eout.flush();
        page_wrapper = Bytes.from_array(out.toByteArray());
    } catch (IOException ex) {
        throw new DataStorageManagerException(ex);
    }
    indexpages.put(tableSpace + "." + indexName + "_" + pageId, page_wrapper);
}