use of io.dingodb.raft.entity.LocalFileMetaOutter.LocalFileMeta in project dingo by dingodb.
the class CheckpointFile method load.
public Checkpoint load() throws IOException {
final ProtoBufFile file = new ProtoBufFile(this.path);
final LocalFileMeta meta = file.load();
if (meta != null) {
final byte[] data = meta.getUserMeta().toByteArray();
Checkpoint checkpoint = new Checkpoint(null, -1);
if (checkpoint.decode(data)) {
return checkpoint;
}
}
return null;
}
use of io.dingodb.raft.entity.LocalFileMetaOutter.LocalFileMeta in project dingo by dingodb.
the class LocalSnapshotCopier method filterBeforeCopy.
boolean filterBeforeCopy(final LocalSnapshotWriter writer, final SnapshotReader lastSnapshot) throws IOException {
final Set<String> existingFiles = writer.listFiles();
final ArrayDeque<String> toRemove = new ArrayDeque<>();
for (final String file : existingFiles) {
if (this.remoteSnapshot.getFileMeta(file) == null) {
toRemove.add(file);
writer.removeFile(file);
}
}
final Set<String> remoteFiles = this.remoteSnapshot.listFiles();
for (final String fileName : remoteFiles) {
final LocalFileMeta remoteMeta = (LocalFileMeta) this.remoteSnapshot.getFileMeta(fileName);
Requires.requireNonNull(remoteMeta, "remoteMeta");
if (!remoteMeta.hasChecksum()) {
// Re-download file if this file doesn't have checksum
writer.removeFile(fileName);
toRemove.add(fileName);
continue;
}
LocalFileMeta localMeta = (LocalFileMeta) writer.getFileMeta(fileName);
if (localMeta != null) {
if (localMeta.hasChecksum() && localMeta.getChecksum().equals(remoteMeta.getChecksum())) {
LOG.info("Keep file={} checksum={} in {}", fileName, remoteMeta.getChecksum(), writer.getPath());
continue;
}
// Remove files from writer so that the file is to be copied from
// remote_snapshot or last_snapshot
writer.removeFile(fileName);
toRemove.add(fileName);
}
// Try find files in last_snapshot
if (lastSnapshot == null) {
continue;
}
if ((localMeta = (LocalFileMeta) lastSnapshot.getFileMeta(fileName)) == null) {
continue;
}
if (!localMeta.hasChecksum() || !localMeta.getChecksum().equals(remoteMeta.getChecksum())) {
continue;
}
LOG.info("Found the same file ={} checksum={} in lastSnapshot={}", fileName, remoteMeta.getChecksum(), lastSnapshot.getPath());
if (localMeta.getSource() == FileSource.FILE_SOURCE_LOCAL) {
final String sourcePath = lastSnapshot.getPath() + File.separator + fileName;
final String destPath = writer.getPath() + File.separator + fileName;
FileUtils.deleteQuietly(new File(destPath));
try {
Files.createLink(Paths.get(destPath), Paths.get(sourcePath));
} catch (final IOException e) {
LOG.error("Fail to link {} to {}", sourcePath, destPath, e);
continue;
}
// Don't delete linked file
if (!toRemove.isEmpty() && toRemove.peekLast().equals(fileName)) {
toRemove.pollLast();
}
}
// Copy file from last_snapshot
writer.addFile(fileName, localMeta);
}
if (!writer.sync()) {
LOG.error("Fail to sync writer on path={}", writer.getPath());
return false;
}
for (final String fileName : toRemove) {
final String removePath = writer.getPath() + File.separator + fileName;
FileUtils.deleteQuietly(new File(removePath));
LOG.info("Deleted file: {}", removePath);
}
return true;
}
use of io.dingodb.raft.entity.LocalFileMetaOutter.LocalFileMeta in project dingo by dingodb.
the class LocalSnapshotWriter method addFile.
@Override
public boolean addFile(final String fileName, final Message fileMeta) {
final Builder metaBuilder = LocalFileMeta.newBuilder();
if (fileMeta != null) {
metaBuilder.mergeFrom(fileMeta);
}
final LocalFileMeta meta = metaBuilder.build();
return this.metaTable.addFile(fileName, meta);
}
use of io.dingodb.raft.entity.LocalFileMetaOutter.LocalFileMeta in project dingo by dingodb.
the class SnapshotFileReader method readFile.
@Override
public int readFile(final ByteBufferCollector metaBufferCollector, final String fileName, final long offset, final long maxCount) throws IOException, RetryAgainException {
// read the whole meta file.
if (fileName.equals(Snapshot.JRAFT_SNAPSHOT_META_FILE)) {
final ByteBuffer metaBuf = this.metaTable.saveToByteBufferAsRemote();
// because bufRef will flip the buffer before using, so we must set the meta buffer position to it's limit.
metaBuf.position(metaBuf.limit());
metaBufferCollector.setBuffer(metaBuf);
return EOF;
}
final LocalFileMeta fileMeta = this.metaTable.getFileMeta(fileName);
if (fileMeta == null) {
throw new FileNotFoundException("LocalFileMeta not found for " + fileName);
}
// go through throttle
long newMaxCount = maxCount;
if (this.snapshotThrottle != null) {
newMaxCount = this.snapshotThrottle.throttledByThroughput(maxCount);
if (newMaxCount < maxCount) {
// throughput is throttled to 0, try again.
if (newMaxCount == 0) {
throw new RetryAgainException("readFile throttled by throughput");
}
}
}
return readFileWithMeta(metaBufferCollector, fileName, fileMeta, offset, newMaxCount);
}
Aggregations