Examples with LogSequenceNumber herddb.log.LogSequenceNumber used on opensource projects

Example 6 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileDataStorageManager method readLogSequenceNumberFromIndexMetadataFile.

private static LogSequenceNumber readLogSequenceNumberFromIndexMetadataFile(String tableSpace, Path file) throws DataStorageManagerException {
    try (InputStream input = new BufferedInputStream(Files.newInputStream(file, StandardOpenOption.READ), 4 * 1024 * 1024);
        ExtendedDataInputStream din = new ExtendedDataInputStream(input)) {
        // version
        long version = din.readVLong();
        // flags for future implementations
        long flags = din.readVLong();
        if (version != 1 || flags != 0) {
            throw new DataStorageManagerException("corrupted index list file " + file.toAbsolutePath());
        }
        String readname = din.readUTF();
        if (!readname.equals(tableSpace)) {
            throw new DataStorageManagerException("file " + file.toAbsolutePath() + " is not for spablespace " + tableSpace);
        }
        long ledgerId = din.readZLong();
        long offset = din.readZLong();
        return new LogSequenceNumber(ledgerId, offset);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
}

Example 7 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileDataStorageManager method writeTables.

@Override
public Collection<PostCheckpointAction> writeTables(String tableSpace, LogSequenceNumber sequenceNumber, List<Table> tables, List<Index> indexlist, boolean prepareActions) throws DataStorageManagerException {
    if (sequenceNumber.isStartOfTime() && !tables.isEmpty()) {
        throw new DataStorageManagerException("impossible to write a non empty table list at start-of-time");
    }
    Path tableSpaceDirectory = getTablespaceDirectory(tableSpace);
    try {
        Files.createDirectories(tableSpaceDirectory);
        Path fileTables = getTablespaceTablesMetadataFile(tableSpace, sequenceNumber);
        Path fileIndexes = getTablespaceIndexesMetadataFile(tableSpace, sequenceNumber);
        Path parent = getParent(fileTables);
        Files.createDirectories(parent);
        LOGGER.log(Level.FINE, "writeTables for tableSpace " + tableSpace + " sequenceNumber " + sequenceNumber + " to " + fileTables.toAbsolutePath().toString());
        try (ManagedFile file = ManagedFile.open(fileTables, requirefsync);
            SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
            ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
            // version
            dout.writeVLong(1);
            // flags for future implementations
            dout.writeVLong(0);
            dout.writeUTF(tableSpace);
            dout.writeZLong(sequenceNumber.ledgerId);
            dout.writeZLong(sequenceNumber.offset);
            dout.writeInt(tables.size());
            for (Table t : tables) {
                byte[] tableSerialized = t.serialize();
                dout.writeArray(tableSerialized);
            }
            dout.flush();
            file.sync();
        } catch (IOException err) {
            throw new DataStorageManagerException(err);
        }
        try (ManagedFile file = ManagedFile.open(fileIndexes, requirefsync);
            SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
            ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
            // version
            dout.writeVLong(1);
            // flags for future implementations
            dout.writeVLong(0);
            dout.writeUTF(tableSpace);
            dout.writeZLong(sequenceNumber.ledgerId);
            dout.writeZLong(sequenceNumber.offset);
            if (indexlist != null) {
                dout.writeInt(indexlist.size());
                for (Index t : indexlist) {
                    byte[] indexSerialized = t.serialize();
                    dout.writeArray(indexSerialized);
                }
            } else {
                dout.writeInt(0);
            }
            dout.flush();
            file.sync();
        } catch (IOException err) {
            throw new DataStorageManagerException(err);
        }
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    Collection<PostCheckpointAction> result = new ArrayList<>();
    if (prepareActions) {
        try (DirectoryStream<Path> stream = Files.newDirectoryStream(tableSpaceDirectory)) {
            for (Path p : stream) {
                if (isTablespaceIndexesMetadataFile(p)) {
                    try {
                        LogSequenceNumber logPositionInFile = readLogSequenceNumberFromIndexMetadataFile(tableSpace, p);
                        if (sequenceNumber.after(logPositionInFile)) {
                            LOGGER.log(Level.FINEST, "indexes metadata file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
                            result.add(new DeleteFileAction(tableSpace, "indexes", "delete indexesmetadata file " + p.toAbsolutePath(), p));
                        }
                    } catch (DataStorageManagerException ignore) {
                        LOGGER.log(Level.SEVERE, "Unparsable indexesmetadata file " + p.toAbsolutePath(), ignore);
                        result.add(new DeleteFileAction(tableSpace, "indexes", "delete unparsable indexesmetadata file " + p.toAbsolutePath(), p));
                    }
                } else if (isTablespaceTablesMetadataFile(p)) {
                    try {
                        LogSequenceNumber logPositionInFile = readLogSequenceNumberFromTablesMetadataFile(tableSpace, p);
                        if (sequenceNumber.after(logPositionInFile)) {
                            LOGGER.log(Level.FINEST, "tables metadata file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
                            result.add(new DeleteFileAction(tableSpace, "tables", "delete tablesmetadata file " + p.toAbsolutePath(), p));
                        }
                    } catch (DataStorageManagerException ignore) {
                        LOGGER.log(Level.SEVERE, "Unparsable tablesmetadata file " + p.toAbsolutePath(), ignore);
                        result.add(new DeleteFileAction(tableSpace, "transactions", "delete unparsable tablesmetadata file " + p.toAbsolutePath(), p));
                    }
                }
            }
        } catch (IOException err) {
            LOGGER.log(Level.SEVERE, "Could not list dir " + tableSpaceDirectory, err);
        }
    }
    return result;
}

Example 8 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileDataStorageManager method writeTransactionsAtCheckpoint.

@Override
public Collection<PostCheckpointAction> writeTransactionsAtCheckpoint(String tableSpace, LogSequenceNumber sequenceNumber, Collection<Transaction> transactions) throws DataStorageManagerException {
    if (sequenceNumber.isStartOfTime() && !transactions.isEmpty()) {
        throw new DataStorageManagerException("impossible to write a non empty transactions list at start-of-time");
    }
    Path tableSpaceDirectory = getTablespaceDirectory(tableSpace);
    try {
        Files.createDirectories(tableSpaceDirectory);
        Path checkPointFile = getTablespaceTransactionsFile(tableSpace, sequenceNumber);
        Path parent = getParent(checkPointFile);
        Files.createDirectories(parent);
        Path checkpointFileTemp = parent.resolve(checkPointFile.getFileName() + ".tmp");
        LOGGER.log(Level.FINE, "writeTransactionsAtCheckpoint for tableSpace {0} sequenceNumber {1} to {2}, active transactions {3}", new Object[] { tableSpace, sequenceNumber, checkPointFile.toAbsolutePath().toString(), transactions.size() });
        try (ManagedFile file = ManagedFile.open(checkpointFileTemp, requirefsync);
            SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
            ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
            // version
            dout.writeVLong(1);
            // flags for future implementations
            dout.writeVLong(0);
            dout.writeUTF(tableSpace);
            dout.writeZLong(sequenceNumber.ledgerId);
            dout.writeZLong(sequenceNumber.offset);
            dout.writeInt(transactions.size());
            for (Transaction t : transactions) {
                t.serialize(dout);
            }
            dout.flush();
            file.sync();
        } catch (IOException err) {
            throw new DataStorageManagerException(err);
        }
        try {
            Files.move(checkpointFileTemp, checkPointFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
        } catch (IOException err) {
            throw new DataStorageManagerException(err);
        }
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    Collection<PostCheckpointAction> result = new ArrayList<>();
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(tableSpaceDirectory)) {
        for (Path p : stream) {
            if (isTransactionsFile(p)) {
                try {
                    LogSequenceNumber logPositionInFile = readLogSequenceNumberFromTransactionsFile(tableSpace, p);
                    if (sequenceNumber.after(logPositionInFile)) {
                        LOGGER.log(Level.FINEST, "transactions metadata file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
                        result.add(new DeleteFileAction(tableSpace, "transactions", "delete transactions file " + p.toAbsolutePath(), p));
                    }
                } catch (DataStorageManagerException ignore) {
                    LOGGER.log(Level.SEVERE, "Unparsable transactions file " + p.toAbsolutePath(), ignore);
                    result.add(new DeleteFileAction(tableSpace, "transactions", "delete unparsable transactions file " + p.toAbsolutePath(), p));
                }
            }
        }
    } catch (IOException err) {
        LOGGER.log(Level.SEVERE, "Could not list dir " + tableSpaceDirectory, err);
    }
    return result;
}

Example 9 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileCommitLog method recovery.

@Override
public void recovery(LogSequenceNumber snapshotSequenceNumber, BiConsumer<LogSequenceNumber, LogEntry> consumer, boolean fencing) throws LogNotAvailableException {
    LOGGER.log(Level.INFO, "recovery {1}, snapshotSequenceNumber: {0}", new Object[] { snapshotSequenceNumber, tableSpaceName });
    // no lock is needed, we are at boot time
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(logDirectory)) {
        List<Path> names = new ArrayList<>();
        for (Path path : stream) {
            if (Files.isRegularFile(path) && (path.getFileName() + "").endsWith(LOGFILEEXTENSION)) {
                names.add(path);
            }
        }
        names.sort(Comparator.comparing(Path::toString));
        long offset = -1;
        for (Path p : names) {
            LOGGER.log(Level.INFO, "tablespace {1}, logfile is {0}", new Object[] { p.toAbsolutePath(), tableSpaceName });
            String name = (p.getFileName() + "").replace(LOGFILEEXTENSION, "");
            long ledgerId = Long.parseLong(name, 16);
            currentLedgerId = Math.max(currentLedgerId, ledgerId);
            offset = -1;
            try (CommitFileReader reader = new CommitFileReader(logDirectory, ledgerId)) {
                LogEntryWithSequenceNumber n = reader.nextEntry();
                while (n != null) {
                    offset = n.logSequenceNumber.offset;
                    if (n.logSequenceNumber.after(snapshotSequenceNumber)) {
                        LOGGER.log(Level.FINE, "RECOVER ENTRY {0}, {1}", new Object[] { n.logSequenceNumber, n.entry });
                        consumer.accept(n.logSequenceNumber, n.entry);
                    } else {
                        LOGGER.log(Level.FINE, "SKIP ENTRY {0}, {1}", new Object[] { n.logSequenceNumber, n.entry });
                    }
                    n = reader.nextEntry();
                }
            }
        }
        recoveredLogSequence = new LogSequenceNumber(currentLedgerId, offset);
        LOGGER.log(Level.INFO, "Tablespace {1}, max ledgerId is {0}", new Object[] { currentLedgerId, tableSpaceName });
    } catch (IOException | RuntimeException err) {
        failed = true;
        throw new LogNotAvailableException(err);
    }
}

Example 10 with LogSequenceNumber

use of herddb.log.LogSequenceNumber in project herddb by diennea.

the class FileCommitLog method writeEntry.

private int writeEntry(LogEntryHolderFuture entry) {
    try {
        CommitFileWriter writer = this.writer;
        if (writer == null) {
            throw new IOException("not yet writable");
        }
        long newSequenceNumber = ++writer.sequenceNumber;
        int written = writer.writeEntry(newSequenceNumber, entry.entry);
        if (writtenBytes > maxLogFileSize) {
            openNewLedger();
        }
        entry.done(new LogSequenceNumber(writer.ledgerId, newSequenceNumber));
        statsEntryLatency.registerSuccessfulEvent(System.currentTimeMillis() - entry.timestamp, TimeUnit.MILLISECONDS);
        return written;
    } catch (IOException | LogNotAvailableException err) {
        entry.error(err);
        statsEntryLatency.registerFailedEvent(System.currentTimeMillis() - entry.timestamp, TimeUnit.MILLISECONDS);
        return 0;
    }
}