Examples with MD5Hash org.apache.ratis.io.MD5Hash used on opensource projects

Example 6 with MD5Hash

use of org.apache.ratis.io.MD5Hash in project alluxio by Alluxio.

the class JournalStateMachine method takeLocalSnapshot.

/**
 * Takes a snapshot of local state machine.
 * @return the index of last included entry, or {@link RaftLog#INVALID_LOG_INDEX} if it fails
 */
public synchronized long takeLocalSnapshot() {
    // Snapshot format is [snapshotId, name1, bytes1, name2, bytes2, ...].
    if (mClosed) {
        SAMPLING_LOG.info("Skip taking snapshot because state machine is closed.");
        return RaftLog.INVALID_LOG_INDEX;
    }
    if (mServer.getLifeCycleState() != LifeCycle.State.RUNNING) {
        SAMPLING_LOG.info("Skip taking snapshot because raft server is not in running state: " + "current state is {}.", mServer.getLifeCycleState());
        return RaftLog.INVALID_LOG_INDEX;
    }
    if (mJournalApplier.isSuspended()) {
        SAMPLING_LOG.info("Skip taking snapshot while journal application is suspended.");
        return RaftLog.INVALID_LOG_INDEX;
    }
    if (!mJournalSystem.isSnapshotAllowed()) {
        SAMPLING_LOG.info("Skip taking snapshot when it is not allowed by the journal system.");
        return RaftLog.INVALID_LOG_INDEX;
    }
    LOG.debug("Calling snapshot");
    Preconditions.checkState(!mSnapshotting, "Cannot call snapshot multiple times concurrently");
    mSnapshotting = true;
    try (Timer.Context ctx = MetricsSystem.timer(MetricKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_GENERATE_TIMER.getName()).time()) {
        mLastSnapshotStartTime = System.currentTimeMillis();
        long snapshotId = mNextSequenceNumberToRead - 1;
        TermIndex last = getLastAppliedTermIndex();
        File tempFile;
        try {
            tempFile = RaftJournalUtils.createTempSnapshotFile(mStorage);
        } catch (IOException e) {
            LogUtils.warnWithException(LOG, "Failed to create temp snapshot file", e);
            return RaftLog.INVALID_LOG_INDEX;
        }
        LOG.info("Taking a snapshot to file {}", tempFile);
        final File snapshotFile = mStorage.getSnapshotFile(last.getTerm(), last.getIndex());
        try (DataOutputStream outputStream = new DataOutputStream(new FileOutputStream(tempFile))) {
            outputStream.writeLong(snapshotId);
            JournalUtils.writeToCheckpoint(outputStream, getStateMachines());
        } catch (Exception e) {
            tempFile.delete();
            LogUtils.warnWithException(LOG, "Failed to write snapshot {} to file {}", snapshotId, tempFile, e);
            return RaftLog.INVALID_LOG_INDEX;
        }
        try {
            final MD5Hash digest = MD5FileUtil.computeMd5ForFile(tempFile);
            LOG.info("Saving digest for snapshot file {}", snapshotFile);
            MD5FileUtil.saveMD5File(snapshotFile, digest);
            LOG.info("Renaming a snapshot file {} to {}", tempFile, snapshotFile);
            if (!tempFile.renameTo(snapshotFile)) {
                tempFile.delete();
                LOG.warn("Failed to rename snapshot from {} to {}", tempFile, snapshotFile);
                return RaftLog.INVALID_LOG_INDEX;
            }
            LOG.info("Completed snapshot up to SN {} in {}ms", snapshotId, System.currentTimeMillis() - mLastSnapshotStartTime);
        } catch (Exception e) {
            tempFile.delete();
            LogUtils.warnWithException(LOG, "Failed to complete snapshot: {} - {}", snapshotId, snapshotFile, e);
            return RaftLog.INVALID_LOG_INDEX;
        }
        try {
            mStorage.loadLatestSnapshot();
        } catch (Exception e) {
            snapshotFile.delete();
            LogUtils.warnWithException(LOG, "Failed to refresh latest snapshot: {}", snapshotId, e);
            return RaftLog.INVALID_LOG_INDEX;
        }
        mSnapshotLastIndex = last.getIndex();
        mLastCheckPointTime = System.currentTimeMillis();
        return last.getIndex();
    } finally {
        mSnapshotting = false;
    }
}

Example 7 with MD5Hash

use of org.apache.ratis.io.MD5Hash in project incubator-ratis by apache.

the class MD5FileUtil method computeMd5ForFile.

/**
 * Read dataFile and compute its MD5 checksum.
 */
public static MD5Hash computeMd5ForFile(File dataFile) throws IOException {
    InputStream in = new FileInputStream(dataFile);
    try {
        MessageDigest digester = MD5Hash.getDigester();
        DigestInputStream dis = new DigestInputStream(in, digester);
        IOUtils.readFully(dis, 128 * 1024);
        return new MD5Hash(digester.digest());
    } finally {
        IOUtils.cleanup(LOG, in);
    }
}

Example 8 with MD5Hash

use of org.apache.ratis.io.MD5Hash in project incubator-ratis by apache.

the class SnapshotManager method installSnapshot.

@SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_BAD_PRACTICE")
public void installSnapshot(StateMachine stateMachine, InstallSnapshotRequestProto request) throws IOException {
    final InstallSnapshotRequestProto.SnapshotChunkProto snapshotChunkRequest = request.getSnapshotChunk();
    final long lastIncludedIndex = snapshotChunkRequest.getTermIndex().getIndex();
    final RaftStorageDirectory dir = storage.getStorageDir();
    // create a unique temporary directory
    final File tmpDir = new File(dir.getTmpDir(), UUID.randomUUID().toString());
    FileUtils.createDirectories(tmpDir);
    tmpDir.deleteOnExit();
    LOG.info("Installing snapshot:{}, to tmp dir:{}", request, tmpDir);
    for (FileChunkProto chunk : snapshotChunkRequest.getFileChunksList()) {
        SnapshotInfo pi = stateMachine.getLatestSnapshot();
        if (pi != null && pi.getTermIndex().getIndex() >= lastIncludedIndex) {
            throw new IOException("There exists snapshot file " + pi.getFiles() + " in " + selfId + " with endIndex >= lastIncludedIndex " + lastIncludedIndex);
        }
        // this is relative to the root dir
        String fileName = chunk.getFilename();
        // TODO: assumes flat layout inside SM dir
        File tmpSnapshotFile = new File(tmpDir, new File(dir.getRoot(), fileName).getName());
        FileOutputStream out = null;
        try {
            // same last index.
            if (chunk.getOffset() == 0) {
                if (tmpSnapshotFile.exists()) {
                    FileUtils.deleteFully(tmpSnapshotFile);
                }
                // create the temp snapshot file and put padding inside
                out = new FileOutputStream(tmpSnapshotFile);
            } else {
                Preconditions.assertTrue(tmpSnapshotFile.exists());
                out = new FileOutputStream(tmpSnapshotFile, true);
                FileChannel fc = out.getChannel();
                fc.position(chunk.getOffset());
            }
            // write data to the file
            out.write(chunk.getData().toByteArray());
        } finally {
            IOUtils.cleanup(null, out);
        }
        // the md5 digest and create the md5 meta-file.
        if (chunk.getDone()) {
            final MD5Hash expectedDigest = new MD5Hash(chunk.getFileDigest().toByteArray());
            // calculate the checksum of the snapshot file and compare it with the
            // file digest in the request
            MD5Hash digest = MD5FileUtil.computeMd5ForFile(tmpSnapshotFile);
            if (!digest.equals(expectedDigest)) {
                LOG.warn("The snapshot md5 digest {} does not match expected {}", digest, expectedDigest);
                // rename the temp snapshot file to .corrupt
                FileUtils.renameFileToCorrupt(tmpSnapshotFile);
                throw new CorruptedFileException(tmpSnapshotFile, "MD5 mismatch for snapshot-" + lastIncludedIndex + " installation");
            } else {
                MD5FileUtil.saveMD5File(tmpSnapshotFile, digest);
            }
        }
    }
    if (snapshotChunkRequest.getDone()) {
        LOG.info("Install snapshot is done, renaming tnp dir:{} to:{}", tmpDir, dir.getStateMachineDir());
        dir.getStateMachineDir().delete();
        tmpDir.renameTo(dir.getStateMachineDir());
    }
}