Examples with MD5Hash org.apache.ratis.io.MD5Hash used on opensource projects

Example 1 with MD5Hash

use of org.apache.ratis.io.MD5Hash in project incubator-ratis by apache.

the class SimpleStateMachineStorage method findLatestSnapshot.

public SingleFileSnapshotInfo findLatestSnapshot() throws IOException {
    SingleFileSnapshotInfo latest = null;
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(smDir.toPath())) {
        for (Path path : stream) {
            Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
            if (matcher.matches()) {
                final long endIndex = Long.parseLong(matcher.group(2));
                if (latest == null || endIndex > latest.getIndex()) {
                    final long term = Long.parseLong(matcher.group(1));
                    MD5Hash fileDigest = MD5FileUtil.readStoredMd5ForFile(path.toFile());
                    final FileInfo fileInfo = new FileInfo(path, fileDigest);
                    latest = new SingleFileSnapshotInfo(fileInfo, term, endIndex);
                }
            }
        }
    }
    return latest;
}

Example 2 with MD5Hash

use of org.apache.ratis.io.MD5Hash in project incubator-ratis by apache.

the class MD5FileUtil method readStoredMd5ForFile.

/**
 * Read the md5 checksum stored alongside the given data file.
 * @param dataFile the file containing data
 * @return the checksum stored in dataFile.md5
 */
public static MD5Hash readStoredMd5ForFile(File dataFile) throws IOException {
    final File md5File = getDigestFileForFile(dataFile);
    if (!md5File.exists()) {
        return null;
    }
    final Matcher matcher = readStoredMd5(md5File);
    String storedHash = matcher.group(1);
    File referencedFile = new File(matcher.group(2));
    // least has the same name as the file we expect
    if (!referencedFile.getName().equals(dataFile.getName())) {
        throw new IOException("MD5 file at " + md5File + " references file named " + referencedFile.getName() + " but we expected it to reference " + dataFile);
    }
    return new MD5Hash(storedHash);
}

Example 3 with MD5Hash

use of org.apache.ratis.io.MD5Hash in project incubator-ratis by apache.

the class SimpleStateMachine4Testing method takeSnapshot.

@Override
public long takeSnapshot() {
    final TermIndex termIndex = getLastAppliedTermIndex();
    if (termIndex.getTerm() <= 0 || termIndex.getIndex() <= 0) {
        return RaftServerConstants.INVALID_LOG_INDEX;
    }
    final long endIndex = termIndex.getIndex();
    // TODO: snapshot should be written to a tmp file, then renamed
    File snapshotFile = storage.getSnapshotFile(termIndex.getTerm(), termIndex.getIndex());
    LOG.debug("Taking a snapshot with t:{}, i:{}, file:{}", termIndex.getTerm(), termIndex.getIndex(), snapshotFile);
    try (LogOutputStream out = new LogOutputStream(snapshotFile, false, segmentMaxSize, preallocatedSize, bufferSize)) {
        for (final LogEntryProto entry : list) {
            if (entry.getIndex() > endIndex) {
                break;
            } else {
                out.write(entry);
            }
        }
        out.flush();
    } catch (IOException e) {
        LOG.warn("Failed to take snapshot", e);
    }
    try {
        final MD5Hash digest = MD5FileUtil.computeMd5ForFile(snapshotFile);
        MD5FileUtil.saveMD5File(snapshotFile, digest);
    } catch (IOException e) {
        LOG.warn("Hit IOException when computing MD5 for snapshot file " + snapshotFile, e);
    }
    try {
        this.storage.loadLatestSnapshot();
    } catch (IOException e) {
        LOG.warn("Hit IOException when loading latest snapshot for snapshot file " + snapshotFile, e);
    }
    // TODO: purge log segments
    return endIndex;
}

Example 4 with MD5Hash

use of org.apache.ratis.io.MD5Hash in project alluxio by Alluxio.

the class SnapshotDownloader method onNextInternal.

private void onNextInternal(R response) throws IOException {
    TermIndex termIndex = TermIndex.valueOf(mDataGetter.apply(response).getSnapshotTerm(), mDataGetter.apply(response).getSnapshotIndex());
    if (mTermIndex == null) {
        LOG.info("Downloading new snapshot {} from {}", termIndex, mSource);
        mTermIndex = termIndex;
        // start a new file
        mTempFile = RaftJournalUtils.createTempSnapshotFile(mStorage);
        mTempFile.deleteOnExit();
        mStream.onNext(mMessageBuilder.apply(0L));
    } else {
        if (!termIndex.equals(mTermIndex)) {
            throw new IOException(String.format("Mismatched term index when downloading the snapshot. expected: %s actual: %s", mTermIndex, termIndex));
        }
        if (!mDataGetter.apply(response).hasChunk()) {
            throw new IOException(String.format("A chunk for file %s is missing from the response %s.", mTempFile, response));
        }
        // write the chunk
        if (mOutputStream == null) {
            LOG.info("Start writing to temporary file {}", mTempFile.getPath());
            mOutputStream = new FileOutputStream(mTempFile);
        }
        long position = mOutputStream.getChannel().position();
        if (position != mDataGetter.apply(response).getOffset()) {
            throw new IOException(String.format("Mismatched offset in file %d, expect %d, bytes written %d", position, mDataGetter.apply(response).getOffset(), mBytesWritten));
        }
        mOutputStream.write(mDataGetter.apply(response).getChunk().toByteArray());
        mBytesWritten += mDataGetter.apply(response).getChunk().size();
        LOG.debug("Written {} bytes to snapshot file {}", mBytesWritten, mTempFile.getPath());
        if (mDataGetter.apply(response).getEof()) {
            LOG.debug("Completed writing to temporary file {} with size {}", mTempFile.getPath(), mOutputStream.getChannel().position());
            mOutputStream.close();
            mOutputStream = null;
            final MD5Hash digest = MD5FileUtil.computeMd5ForFile(mTempFile);
            mSnapshotToInstall = new SingleFileSnapshotInfo(new FileInfo(mTempFile.toPath(), digest), mTermIndex.getTerm(), mTermIndex.getIndex());
            mFuture.complete(mTermIndex);
            LOG.info("Finished copying snapshot to local file {}.", mTempFile);
            mStream.onCompleted();
        } else {
            mStream.onNext(mMessageBuilder.apply(mBytesWritten));
        }
    }
}

Example 5 with MD5Hash

use of org.apache.ratis.io.MD5Hash in project alluxio by Alluxio.

the class JournalStateMachine method takeLocalSnapshot.

/**
 * Takes a snapshot of local state machine.
 * @return the index of last included entry, or {@link RaftLog#INVALID_LOG_INDEX} if it fails
 */
public synchronized long takeLocalSnapshot() {
    // Snapshot format is [snapshotId, name1, bytes1, name2, bytes2, ...].
    if (mClosed) {
        SAMPLING_LOG.info("Skip taking snapshot because state machine is closed.");
        return RaftLog.INVALID_LOG_INDEX;
    }
    if (mServer.getLifeCycleState() != LifeCycle.State.RUNNING) {
        SAMPLING_LOG.info("Skip taking snapshot because raft server is not in running state: " + "current state is {}.", mServer.getLifeCycleState());
        return RaftLog.INVALID_LOG_INDEX;
    }
    if (mJournalApplier.isSuspended()) {
        SAMPLING_LOG.info("Skip taking snapshot while journal application is suspended.");
        return RaftLog.INVALID_LOG_INDEX;
    }
    if (!mJournalSystem.isSnapshotAllowed()) {
        SAMPLING_LOG.info("Skip taking snapshot when it is not allowed by the journal system.");
        return RaftLog.INVALID_LOG_INDEX;
    }
    LOG.debug("Calling snapshot");
    Preconditions.checkState(!mSnapshotting, "Cannot call snapshot multiple times concurrently");
    mSnapshotting = true;
    try (Timer.Context ctx = MetricsSystem.timer(MetricKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_GENERATE_TIMER.getName()).time()) {
        mLastSnapshotStartTime = System.currentTimeMillis();
        long snapshotId = mNextSequenceNumberToRead - 1;
        TermIndex last = getLastAppliedTermIndex();
        File tempFile;
        try {
            tempFile = RaftJournalUtils.createTempSnapshotFile(mStorage);
        } catch (IOException e) {
            LogUtils.warnWithException(LOG, "Failed to create temp snapshot file", e);
            return RaftLog.INVALID_LOG_INDEX;
        }
        LOG.info("Taking a snapshot to file {}", tempFile);
        final File snapshotFile = mStorage.getSnapshotFile(last.getTerm(), last.getIndex());
        try (DataOutputStream outputStream = new DataOutputStream(new FileOutputStream(tempFile))) {
            outputStream.writeLong(snapshotId);
            JournalUtils.writeToCheckpoint(outputStream, getStateMachines());
        } catch (Exception e) {
            tempFile.delete();
            LogUtils.warnWithException(LOG, "Failed to write snapshot {} to file {}", snapshotId, tempFile, e);
            return RaftLog.INVALID_LOG_INDEX;
        }
        try {
            final MD5Hash digest = MD5FileUtil.computeMd5ForFile(tempFile);
            LOG.info("Saving digest for snapshot file {}", snapshotFile);
            MD5FileUtil.saveMD5File(snapshotFile, digest);
            LOG.info("Renaming a snapshot file {} to {}", tempFile, snapshotFile);
            if (!tempFile.renameTo(snapshotFile)) {
                tempFile.delete();
                LOG.warn("Failed to rename snapshot from {} to {}", tempFile, snapshotFile);
                return RaftLog.INVALID_LOG_INDEX;
            }
            LOG.info("Completed snapshot up to SN {} in {}ms", snapshotId, System.currentTimeMillis() - mLastSnapshotStartTime);
        } catch (Exception e) {
            tempFile.delete();
            LogUtils.warnWithException(LOG, "Failed to complete snapshot: {} - {}", snapshotId, snapshotFile, e);
            return RaftLog.INVALID_LOG_INDEX;
        }
        try {
            mStorage.loadLatestSnapshot();
        } catch (Exception e) {
            snapshotFile.delete();
            LogUtils.warnWithException(LOG, "Failed to refresh latest snapshot: {}", snapshotId, e);
            return RaftLog.INVALID_LOG_INDEX;
        }
        mSnapshotLastIndex = last.getIndex();
        mLastCheckPointTime = System.currentTimeMillis();
        return last.getIndex();
    } finally {
        mSnapshotting = false;
    }
}