use of org.apache.ratis.server.storage.FileInfo in project incubator-ratis by apache.
the class SimpleStateMachineStorage method findLatestSnapshot.
public SingleFileSnapshotInfo findLatestSnapshot() throws IOException {
SingleFileSnapshotInfo latest = null;
try (DirectoryStream<Path> stream = Files.newDirectoryStream(smDir.toPath())) {
for (Path path : stream) {
Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
if (matcher.matches()) {
final long endIndex = Long.parseLong(matcher.group(2));
if (latest == null || endIndex > latest.getIndex()) {
final long term = Long.parseLong(matcher.group(1));
MD5Hash fileDigest = MD5FileUtil.readStoredMd5ForFile(path.toFile());
final FileInfo fileInfo = new FileInfo(path, fileDigest);
latest = new SingleFileSnapshotInfo(fileInfo, term, endIndex);
}
}
}
}
return latest;
}
use of org.apache.ratis.server.storage.FileInfo in project alluxio by Alluxio.
the class SnapshotDownloader method onNextInternal.
private void onNextInternal(R response) throws IOException {
TermIndex termIndex = TermIndex.valueOf(mDataGetter.apply(response).getSnapshotTerm(), mDataGetter.apply(response).getSnapshotIndex());
if (mTermIndex == null) {
LOG.info("Downloading new snapshot {} from {}", termIndex, mSource);
mTermIndex = termIndex;
// start a new file
mTempFile = RaftJournalUtils.createTempSnapshotFile(mStorage);
mTempFile.deleteOnExit();
mStream.onNext(mMessageBuilder.apply(0L));
} else {
if (!termIndex.equals(mTermIndex)) {
throw new IOException(String.format("Mismatched term index when downloading the snapshot. expected: %s actual: %s", mTermIndex, termIndex));
}
if (!mDataGetter.apply(response).hasChunk()) {
throw new IOException(String.format("A chunk for file %s is missing from the response %s.", mTempFile, response));
}
// write the chunk
if (mOutputStream == null) {
LOG.info("Start writing to temporary file {}", mTempFile.getPath());
mOutputStream = new FileOutputStream(mTempFile);
}
long position = mOutputStream.getChannel().position();
if (position != mDataGetter.apply(response).getOffset()) {
throw new IOException(String.format("Mismatched offset in file %d, expect %d, bytes written %d", position, mDataGetter.apply(response).getOffset(), mBytesWritten));
}
mOutputStream.write(mDataGetter.apply(response).getChunk().toByteArray());
mBytesWritten += mDataGetter.apply(response).getChunk().size();
LOG.debug("Written {} bytes to snapshot file {}", mBytesWritten, mTempFile.getPath());
if (mDataGetter.apply(response).getEof()) {
LOG.debug("Completed writing to temporary file {} with size {}", mTempFile.getPath(), mOutputStream.getChannel().position());
mOutputStream.close();
mOutputStream = null;
final MD5Hash digest = MD5FileUtil.computeMd5ForFile(mTempFile);
mSnapshotToInstall = new SingleFileSnapshotInfo(new FileInfo(mTempFile.toPath(), digest), mTermIndex.getTerm(), mTermIndex.getIndex());
mFuture.complete(mTermIndex);
LOG.info("Finished copying snapshot to local file {}.", mTempFile);
mStream.onCompleted();
} else {
mStream.onNext(mMessageBuilder.apply(mBytesWritten));
}
}
}
use of org.apache.ratis.server.storage.FileInfo in project alluxio by Alluxio.
the class SnapshotReplicationManager method installDownloadedSnapshot.
/**
* Installs a downloaded snapshot in the journal snapshot directory.
*
* @return the index of the installed snapshot
*/
private long installDownloadedSnapshot() {
if (!transitionState(DownloadState.DOWNLOADED, DownloadState.INSTALLING)) {
return RaftLog.INVALID_LOG_INDEX;
}
File tempFile = null;
try (Timer.Context ctx = MetricsSystem.timer(MetricKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_INSTALL_TIMER.getName()).time()) {
SnapshotInfo snapshot = mDownloadedSnapshot;
if (snapshot == null) {
throw new IllegalStateException("Snapshot is not completed");
}
FileInfo fileInfo = snapshot.getFiles().get(0);
tempFile = fileInfo.getPath().toFile();
if (!tempFile.exists()) {
throw new FileNotFoundException(String.format("Snapshot file %s is not found", tempFile));
}
SnapshotInfo latestSnapshot = mStorage.getLatestSnapshot();
TermIndex lastInstalled = latestSnapshot == null ? null : latestSnapshot.getTermIndex();
TermIndex downloaded = snapshot.getTermIndex();
if (lastInstalled != null && downloaded.compareTo(lastInstalled) < 0) {
throw new AbortedException(String.format("Snapshot to be installed %s is older than current snapshot %s", downloaded, lastInstalled));
}
final File snapshotFile = mStorage.getSnapshotFile(downloaded.getTerm(), downloaded.getIndex());
LOG.debug("Moving temp snapshot {} to file {}", tempFile, snapshotFile);
MD5FileUtil.saveMD5File(snapshotFile, fileInfo.getFileDigest());
if (!tempFile.renameTo(snapshotFile)) {
throw new IOException(String.format("Failed to rename %s to %s", tempFile, snapshotFile));
}
mStorage.loadLatestSnapshot();
LOG.info("Completed storing snapshot at {} to file {}", downloaded, snapshotFile);
return downloaded.getIndex();
} catch (Exception e) {
LOG.error("Failed to install snapshot", e);
if (tempFile != null) {
tempFile.delete();
}
return RaftLog.INVALID_LOG_INDEX;
} finally {
transitionState(DownloadState.INSTALLING, DownloadState.IDLE);
}
}
Aggregations