use of herddb.utils.ManagedFile in project herddb by diennea.
the class FileDataStorageManager method writeTransactionsAtCheckpoint.
@Override
public Collection<PostCheckpointAction> writeTransactionsAtCheckpoint(String tableSpace, LogSequenceNumber sequenceNumber, Collection<Transaction> transactions) throws DataStorageManagerException {
if (sequenceNumber.isStartOfTime() && !transactions.isEmpty()) {
throw new DataStorageManagerException("impossible to write a non empty transactions list at start-of-time");
}
Path tableSpaceDirectory = getTablespaceDirectory(tableSpace);
try {
Files.createDirectories(tableSpaceDirectory);
Path checkPointFile = getTablespaceTransactionsFile(tableSpace, sequenceNumber);
Path parent = getParent(checkPointFile);
Files.createDirectories(parent);
Path checkpointFileTemp = parent.resolve(checkPointFile.getFileName() + ".tmp");
LOGGER.log(Level.FINE, "writeTransactionsAtCheckpoint for tableSpace {0} sequenceNumber {1} to {2}, active transactions {3}", new Object[] { tableSpace, sequenceNumber, checkPointFile.toAbsolutePath().toString(), transactions.size() });
try (ManagedFile file = ManagedFile.open(checkpointFileTemp);
SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
// version
dout.writeVLong(1);
// flags for future implementations
dout.writeVLong(0);
dout.writeUTF(tableSpace);
dout.writeZLong(sequenceNumber.ledgerId);
dout.writeZLong(sequenceNumber.offset);
dout.writeInt(transactions.size());
for (Transaction t : transactions) {
t.serialize(dout);
}
dout.flush();
file.sync();
} catch (IOException err) {
throw new DataStorageManagerException(err);
}
try {
Files.move(checkpointFileTemp, checkPointFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException err) {
throw new DataStorageManagerException(err);
}
} catch (IOException err) {
throw new DataStorageManagerException(err);
}
Collection<PostCheckpointAction> result = new ArrayList<>();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(tableSpaceDirectory)) {
for (Path p : stream) {
if (isTransactionsFile(p)) {
try {
LogSequenceNumber logPositionInFile = readLogSequenceNumberFromTransactionsFile(tableSpace, p);
if (sequenceNumber.after(logPositionInFile)) {
LOGGER.log(Level.FINEST, "transactions metadata file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
result.add(new DeleteFileAction("transactions", "delete transactions file " + p.toAbsolutePath(), p));
}
} catch (DataStorageManagerException ignore) {
LOGGER.log(Level.SEVERE, "Unparsable transactions file " + p.toAbsolutePath(), ignore);
result.add(new DeleteFileAction("transactions", "delete unparsable transactions file " + p.toAbsolutePath(), p));
}
}
}
} catch (IOException err) {
LOGGER.log(Level.SEVERE, "Could not list dir " + tableSpaceDirectory, err);
}
return result;
}
use of herddb.utils.ManagedFile in project herddb by diennea.
the class FileDataStorageManager method indexCheckpoint.
@Override
public List<PostCheckpointAction> indexCheckpoint(String tableSpace, String indexName, IndexStatus indexStatus, boolean pin) throws DataStorageManagerException {
Path dir = getIndexDirectory(tableSpace, indexName);
LogSequenceNumber logPosition = indexStatus.sequenceNumber;
Path checkpointFile = getTableCheckPointsFile(dir, logPosition);
Path parent = getParent(checkpointFile);
Path checkpointFileTemp = parent.resolve(checkpointFile.getFileName() + ".tmp");
try {
Files.createDirectories(dir);
if (Files.isRegularFile(checkpointFile)) {
IndexStatus actualStatus = readIndexStatusFromFile(checkpointFile);
if (actualStatus != null && actualStatus.equals(indexStatus)) {
LOGGER.log(Level.SEVERE, "indexCheckpoint " + tableSpace + ", " + indexName + ": " + indexStatus + " already saved on" + checkpointFile);
return Collections.emptyList();
}
}
} catch (IOException err) {
throw new DataStorageManagerException(err);
}
LOGGER.log(Level.FINE, "indexCheckpoint " + tableSpace + ", " + indexName + ": " + indexStatus + " to file " + checkpointFile);
try (ManagedFile file = ManagedFile.open(checkpointFileTemp);
SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
XXHash64Utils.HashingOutputStream oo = new XXHash64Utils.HashingOutputStream(buffer);
ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
// version
dataOutputKeys.writeVLong(1);
// flags for future implementations
dataOutputKeys.writeVLong(0);
indexStatus.serialize(dataOutputKeys);
dataOutputKeys.writeLong(oo.hash());
dataOutputKeys.flush();
file.sync();
} catch (IOException err) {
throw new DataStorageManagerException(err);
}
try {
Files.move(checkpointFileTemp, checkpointFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
} catch (IOException err) {
throw new DataStorageManagerException(err);
}
/* Checkpoint pinning */
final Map<Long, Integer> pins = pinIndexAndGetPages(tableSpace, indexName, indexStatus, pin);
final Set<LogSequenceNumber> checkpoints = pinIndexAndGetCheckpoints(tableSpace, indexName, indexStatus, pin);
long maxPageId = indexStatus.activePages.stream().max(Comparator.naturalOrder()).orElse(Long.MAX_VALUE);
List<PostCheckpointAction> result = new ArrayList<>();
// we can drop old page files now
List<Path> pageFiles = getIndexPageFiles(tableSpace, indexName);
for (Path p : pageFiles) {
long pageId = getPageId(p);
LOGGER.log(Level.FINEST, "checkpoint file {0} pageId {1}", new Object[] { p.toAbsolutePath(), pageId });
if (pageId > 0 && !pins.containsKey(pageId) && !indexStatus.activePages.contains(pageId) && pageId < maxPageId) {
LOGGER.log(Level.FINEST, "checkpoint file " + p.toAbsolutePath() + " pageId " + pageId + ". will be deleted after checkpoint end");
result.add(new DeleteFileAction(indexName, "delete page " + pageId + " file " + p.toAbsolutePath(), p));
}
}
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
for (Path p : stream) {
if (isTableOrIndexCheckpointsFile(p) && !p.equals(checkpointFile)) {
IndexStatus status = readIndexStatusFromFile(p);
if (logPosition.after(status.sequenceNumber) && !checkpoints.contains(status.sequenceNumber)) {
LOGGER.log(Level.FINEST, "checkpoint metadata file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
result.add(new DeleteFileAction(indexName, "delete checkpoint metadata file " + p.toAbsolutePath(), p));
}
}
}
} catch (IOException err) {
LOGGER.log(Level.SEVERE, "Could not list indexName dir " + dir, err);
}
return result;
}
use of herddb.utils.ManagedFile in project herddb by diennea.
the class FileMetadataStorageManager method persistTableSpaceOnDisk.
private void persistTableSpaceOnDisk(TableSpace tableSpace) throws MetadataStorageManagerException {
Path tablespaceMetaTmp = baseDirectory.resolve(tableSpace.name + "." + System.nanoTime() + ".tmpmetadata");
Path tablespaceMeta = baseDirectory.resolve(tableSpace.name + ".metadata");
try (ManagedFile file = ManagedFile.open(tablespaceMetaTmp);
SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), FileDataStorageManager.COPY_BUFFERS_SIZE);
XXHash64Utils.HashingOutputStream oo = new XXHash64Utils.HashingOutputStream(buffer);
ExtendedDataOutputStream dout = new ExtendedDataOutputStream(oo)) {
// version
dout.writeVLong(1);
// flags for future implementations
dout.writeVLong(0);
tableSpace.serialize(dout);
// footer
dout.writeLong(oo.hash());
dout.flush();
file.sync();
} catch (IOException err) {
throw new MetadataStorageManagerException(err);
}
try {
Files.move(tablespaceMetaTmp, tablespaceMeta, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
} catch (IOException err) {
throw new MetadataStorageManagerException(err);
}
}
use of herddb.utils.ManagedFile in project herddb by diennea.
the class FileDataStorageManager method writeCheckpointSequenceNumber.
@Override
public Collection<PostCheckpointAction> writeCheckpointSequenceNumber(String tableSpace, LogSequenceNumber sequenceNumber) throws DataStorageManagerException {
Path tableSpaceDirectory = getTablespaceDirectory(tableSpace);
try {
Files.createDirectories(tableSpaceDirectory);
Path checkPointFile = getTablespaceCheckPointInfoFile(tableSpace, sequenceNumber);
Path parent = getParent(checkPointFile);
Files.createDirectories(parent);
Path checkpointFileTemp = parent.resolve(checkPointFile.getFileName() + ".tmp");
LOGGER.log(Level.INFO, "checkpoint for " + tableSpace + " at " + sequenceNumber + " to " + checkPointFile.toAbsolutePath().toString());
try (ManagedFile file = ManagedFile.open(checkpointFileTemp);
SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
// version
dout.writeVLong(1);
// flags for future implementations
dout.writeVLong(0);
dout.writeUTF(tableSpace);
dout.writeZLong(sequenceNumber.ledgerId);
dout.writeZLong(sequenceNumber.offset);
dout.flush();
file.sync();
} catch (IOException err) {
throw new DataStorageManagerException(err);
}
// write file atomically
Files.move(checkpointFileTemp, checkPointFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException err) {
throw new DataStorageManagerException(err);
}
Collection<PostCheckpointAction> result = new ArrayList<>();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(tableSpaceDirectory)) {
for (Path p : stream) {
if (isTablespaceCheckPointInfoFile(p)) {
try {
LogSequenceNumber logPositionInFile = readLogSequenceNumberFromCheckpointInfoFile(tableSpace, p);
if (sequenceNumber.after(logPositionInFile)) {
LOGGER.log(Level.FINEST, "checkpoint info file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
result.add(new DeleteFileAction("checkpoint", "delete checkpoint info file " + p.toAbsolutePath(), p));
}
} catch (DataStorageManagerException ignore) {
LOGGER.log(Level.SEVERE, "unparsable checkpoint info file " + p.toAbsolutePath(), ignore);
// do not auto-delete checkpoint files
}
}
}
} catch (IOException err) {
LOGGER.log(Level.SEVERE, "Could not list dir " + tableSpaceDirectory, err);
}
return result;
}
use of herddb.utils.ManagedFile in project herddb by diennea.
the class FileDataStorageManager method tableCheckpoint.
@Override
public List<PostCheckpointAction> tableCheckpoint(String tableSpace, String tableName, TableStatus tableStatus, boolean pin) throws DataStorageManagerException {
LogSequenceNumber logPosition = tableStatus.sequenceNumber;
Path dir = getTableDirectory(tableSpace, tableName);
Path checkpointFile = getTableCheckPointsFile(dir, logPosition);
try {
Files.createDirectories(dir);
if (Files.isRegularFile(checkpointFile)) {
TableStatus actualStatus = readTableStatusFromFile(checkpointFile);
if (actualStatus != null && actualStatus.equals(tableStatus)) {
LOGGER.log(Level.INFO, "tableCheckpoint " + tableSpace + ", " + tableName + ": " + tableStatus + " already saved on file " + checkpointFile);
return Collections.emptyList();
}
}
} catch (IOException err) {
throw new DataStorageManagerException(err);
}
Path parent = getParent(checkpointFile);
Path checkpointFileTemp = parent.resolve(checkpointFile.getFileName() + ".tmp");
LOGGER.log(Level.FINE, "tableCheckpoint " + tableSpace + ", " + tableName + ": " + tableStatus + " to file " + checkpointFile);
try (ManagedFile file = ManagedFile.open(checkpointFileTemp);
SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
XXHash64Utils.HashingOutputStream oo = new XXHash64Utils.HashingOutputStream(buffer);
ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
// version
dataOutputKeys.writeVLong(1);
// flags for future implementations
dataOutputKeys.writeVLong(0);
tableStatus.serialize(dataOutputKeys);
dataOutputKeys.writeLong(oo.hash());
dataOutputKeys.flush();
file.sync();
} catch (IOException err) {
throw new DataStorageManagerException(err);
}
try {
Files.move(checkpointFileTemp, checkpointFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
} catch (IOException err) {
throw new DataStorageManagerException(err);
}
/* Checkpoint pinning */
final Map<Long, Integer> pins = pinTableAndGetPages(tableSpace, tableName, tableStatus, pin);
final Set<LogSequenceNumber> checkpoints = pinTableAndGetCheckpoints(tableSpace, tableName, tableStatus, pin);
long maxPageId = tableStatus.activePages.keySet().stream().max(Comparator.naturalOrder()).orElse(Long.MAX_VALUE);
List<PostCheckpointAction> result = new ArrayList<>();
// we can drop old page files now
List<Path> pageFiles = getTablePageFiles(tableSpace, tableName);
for (Path p : pageFiles) {
long pageId = getPageId(p);
LOGGER.log(Level.FINEST, "checkpoint file {0} pageId {1}", new Object[] { p.toAbsolutePath(), pageId });
if (pageId > 0 && !pins.containsKey(pageId) && !tableStatus.activePages.containsKey(pageId) && pageId < maxPageId) {
LOGGER.log(Level.FINEST, "checkpoint file " + p.toAbsolutePath() + " pageId " + pageId + ". will be deleted after checkpoint end");
result.add(new DeleteFileAction(tableName, "delete page " + pageId + " file " + p.toAbsolutePath(), p));
}
}
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
for (Path p : stream) {
if (isTableOrIndexCheckpointsFile(p) && !p.equals(checkpointFile)) {
TableStatus status = readTableStatusFromFile(p);
if (logPosition.after(status.sequenceNumber) && !checkpoints.contains(status.sequenceNumber)) {
LOGGER.log(Level.FINEST, "checkpoint metadata file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
result.add(new DeleteFileAction(tableName, "delete checkpoint metadata file " + p.toAbsolutePath(), p));
}
}
}
} catch (IOException err) {
LOGGER.log(Level.SEVERE, "Could not list table dir " + dir, err);
}
return result;
}
Aggregations