Search in sources :

Example 1 with FileInfo

use of org.apache.ratis.server.storage.FileInfo in project incubator-ratis by apache.

the class SimpleStateMachineStorage method findLatestSnapshot.

public SingleFileSnapshotInfo findLatestSnapshot() throws IOException {
    SingleFileSnapshotInfo latest = null;
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(smDir.toPath())) {
        for (Path path : stream) {
            Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
            if (matcher.matches()) {
                final long endIndex = Long.parseLong(matcher.group(2));
                if (latest == null || endIndex > latest.getIndex()) {
                    final long term = Long.parseLong(matcher.group(1));
                    MD5Hash fileDigest = MD5FileUtil.readStoredMd5ForFile(path.toFile());
                    final FileInfo fileInfo = new FileInfo(path, fileDigest);
                    latest = new SingleFileSnapshotInfo(fileInfo, term, endIndex);
                }
            }
        }
    }
    return latest;
}
Also used : Path(java.nio.file.Path) FileInfo(org.apache.ratis.server.storage.FileInfo) Matcher(java.util.regex.Matcher) MD5Hash(org.apache.ratis.io.MD5Hash)

Example 2 with FileInfo

use of org.apache.ratis.server.storage.FileInfo in project alluxio by Alluxio.

the class SnapshotDownloader method onNextInternal.

private void onNextInternal(R response) throws IOException {
    TermIndex termIndex = TermIndex.valueOf(mDataGetter.apply(response).getSnapshotTerm(), mDataGetter.apply(response).getSnapshotIndex());
    if (mTermIndex == null) {
        LOG.info("Downloading new snapshot {} from {}", termIndex, mSource);
        mTermIndex = termIndex;
        // start a new file
        mTempFile = RaftJournalUtils.createTempSnapshotFile(mStorage);
        mTempFile.deleteOnExit();
        mStream.onNext(mMessageBuilder.apply(0L));
    } else {
        if (!termIndex.equals(mTermIndex)) {
            throw new IOException(String.format("Mismatched term index when downloading the snapshot. expected: %s actual: %s", mTermIndex, termIndex));
        }
        if (!mDataGetter.apply(response).hasChunk()) {
            throw new IOException(String.format("A chunk for file %s is missing from the response %s.", mTempFile, response));
        }
        // write the chunk
        if (mOutputStream == null) {
            LOG.info("Start writing to temporary file {}", mTempFile.getPath());
            mOutputStream = new FileOutputStream(mTempFile);
        }
        long position = mOutputStream.getChannel().position();
        if (position != mDataGetter.apply(response).getOffset()) {
            throw new IOException(String.format("Mismatched offset in file %d, expect %d, bytes written %d", position, mDataGetter.apply(response).getOffset(), mBytesWritten));
        }
        mOutputStream.write(mDataGetter.apply(response).getChunk().toByteArray());
        mBytesWritten += mDataGetter.apply(response).getChunk().size();
        LOG.debug("Written {} bytes to snapshot file {}", mBytesWritten, mTempFile.getPath());
        if (mDataGetter.apply(response).getEof()) {
            LOG.debug("Completed writing to temporary file {} with size {}", mTempFile.getPath(), mOutputStream.getChannel().position());
            mOutputStream.close();
            mOutputStream = null;
            final MD5Hash digest = MD5FileUtil.computeMd5ForFile(mTempFile);
            mSnapshotToInstall = new SingleFileSnapshotInfo(new FileInfo(mTempFile.toPath(), digest), mTermIndex.getTerm(), mTermIndex.getIndex());
            mFuture.complete(mTermIndex);
            LOG.info("Finished copying snapshot to local file {}.", mTempFile);
            mStream.onCompleted();
        } else {
            mStream.onNext(mMessageBuilder.apply(mBytesWritten));
        }
    }
}
Also used : SingleFileSnapshotInfo(org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo) FileInfo(org.apache.ratis.server.storage.FileInfo) FileOutputStream(java.io.FileOutputStream) MD5Hash(org.apache.ratis.io.MD5Hash) IOException(java.io.IOException) TermIndex(org.apache.ratis.server.protocol.TermIndex)

Example 3 with FileInfo

use of org.apache.ratis.server.storage.FileInfo in project alluxio by Alluxio.

the class SnapshotReplicationManager method installDownloadedSnapshot.

/**
 * Installs a downloaded snapshot in the journal snapshot directory.
 *
 * @return the index of the installed snapshot
 */
private long installDownloadedSnapshot() {
    if (!transitionState(DownloadState.DOWNLOADED, DownloadState.INSTALLING)) {
        return RaftLog.INVALID_LOG_INDEX;
    }
    File tempFile = null;
    try (Timer.Context ctx = MetricsSystem.timer(MetricKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_INSTALL_TIMER.getName()).time()) {
        SnapshotInfo snapshot = mDownloadedSnapshot;
        if (snapshot == null) {
            throw new IllegalStateException("Snapshot is not completed");
        }
        FileInfo fileInfo = snapshot.getFiles().get(0);
        tempFile = fileInfo.getPath().toFile();
        if (!tempFile.exists()) {
            throw new FileNotFoundException(String.format("Snapshot file %s is not found", tempFile));
        }
        SnapshotInfo latestSnapshot = mStorage.getLatestSnapshot();
        TermIndex lastInstalled = latestSnapshot == null ? null : latestSnapshot.getTermIndex();
        TermIndex downloaded = snapshot.getTermIndex();
        if (lastInstalled != null && downloaded.compareTo(lastInstalled) < 0) {
            throw new AbortedException(String.format("Snapshot to be installed %s is older than current snapshot %s", downloaded, lastInstalled));
        }
        final File snapshotFile = mStorage.getSnapshotFile(downloaded.getTerm(), downloaded.getIndex());
        LOG.debug("Moving temp snapshot {} to file {}", tempFile, snapshotFile);
        MD5FileUtil.saveMD5File(snapshotFile, fileInfo.getFileDigest());
        if (!tempFile.renameTo(snapshotFile)) {
            throw new IOException(String.format("Failed to rename %s to %s", tempFile, snapshotFile));
        }
        mStorage.loadLatestSnapshot();
        LOG.info("Completed storing snapshot at {} to file {}", downloaded, snapshotFile);
        return downloaded.getIndex();
    } catch (Exception e) {
        LOG.error("Failed to install snapshot", e);
        if (tempFile != null) {
            tempFile.delete();
        }
        return RaftLog.INVALID_LOG_INDEX;
    } finally {
        transitionState(DownloadState.INSTALLING, DownloadState.IDLE);
    }
}
Also used : SnapshotInfo(org.apache.ratis.statemachine.SnapshotInfo) SingleFileSnapshotInfo(org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo) Timer(com.codahale.metrics.Timer) FileInfo(org.apache.ratis.server.storage.FileInfo) AbortedException(alluxio.exception.status.AbortedException) FileNotFoundException(java.io.FileNotFoundException) IOException(java.io.IOException) File(java.io.File) CompletionException(java.util.concurrent.CompletionException) FileNotFoundException(java.io.FileNotFoundException) AbortedException(alluxio.exception.status.AbortedException) AlluxioStatusException(alluxio.exception.status.AlluxioStatusException) IOException(java.io.IOException) NotFoundException(alluxio.exception.status.NotFoundException) TermIndex(org.apache.ratis.server.protocol.TermIndex)

Aggregations

FileInfo (org.apache.ratis.server.storage.FileInfo)3 IOException (java.io.IOException)2 MD5Hash (org.apache.ratis.io.MD5Hash)2 TermIndex (org.apache.ratis.server.protocol.TermIndex)2 SingleFileSnapshotInfo (org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo)2 AbortedException (alluxio.exception.status.AbortedException)1 AlluxioStatusException (alluxio.exception.status.AlluxioStatusException)1 NotFoundException (alluxio.exception.status.NotFoundException)1 Timer (com.codahale.metrics.Timer)1 File (java.io.File)1 FileNotFoundException (java.io.FileNotFoundException)1 FileOutputStream (java.io.FileOutputStream)1 Path (java.nio.file.Path)1 CompletionException (java.util.concurrent.CompletionException)1 Matcher (java.util.regex.Matcher)1 SnapshotInfo (org.apache.ratis.statemachine.SnapshotInfo)1