Examples with FileChunkProto org.apache.ratis.proto.RaftProtos.FileChunkProto used on opensource projects

Example 1 with FileChunkProto

use of org.apache.ratis.proto.RaftProtos.FileChunkProto in project incubator-ratis by apache.

the class FileChunkReader method readFileChunk.

/**
 * Read the next chunk.
 *
 * @param chunkMaxSize maximum chunk size
 * @return the chunk read from the file.
 * @throws IOException if it failed to read the file.
 */
public FileChunkProto readFileChunk(int chunkMaxSize) throws IOException {
    final long remaining = info.getFileSize() - offset;
    final int chunkLength = remaining < chunkMaxSize ? (int) remaining : chunkMaxSize;
    final ByteString data = ByteString.readFrom(in, chunkLength);
    final FileChunkProto proto = FileChunkProto.newBuilder().setFilename(relativePath.toString()).setOffset(offset).setChunkIndex(chunkIndex).setDone(offset + chunkLength == info.getFileSize()).setData(data).setFileDigest(ByteString.copyFrom(info.getFileDigest().getDigest())).build();
    chunkIndex++;
    offset += chunkLength;
    return proto;
}

Example 2 with FileChunkProto

use of org.apache.ratis.proto.RaftProtos.FileChunkProto in project incubator-ratis by apache.

the class InstallSnapshotRequests method nextInstallSnapshotRequestProto.

private InstallSnapshotRequestProto nextInstallSnapshotRequestProto() {
    final int numFiles = snapshot.getFiles().size();
    if (fileIndex >= numFiles) {
        throw new NoSuchElementException();
    }
    final FileInfo info = snapshot.getFiles().get(fileIndex);
    try {
        if (current == null) {
            current = new FileChunkReader(info, server.getRaftStorage().getStorageDir());
        }
        final FileChunkProto chunk = current.readFileChunk(snapshotChunkMaxSize);
        if (chunk.getDone()) {
            current.close();
            current = null;
            fileIndex++;
        }
        final boolean done = fileIndex == numFiles && chunk.getDone();
        return newInstallSnapshotRequest(chunk, done);
    } catch (IOException e) {
        if (current != null) {
            try {
                current.close();
                current = null;
            } catch (IOException ignored) {
            }
        }
        throw new IllegalStateException("Failed to iterate installSnapshot requests: " + this, e);
    }
}

Example 3 with FileChunkProto

use of org.apache.ratis.proto.RaftProtos.FileChunkProto in project incubator-ratis by apache.

the class SnapshotManager method installSnapshot.

@SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_BAD_PRACTICE")
public void installSnapshot(StateMachine stateMachine, InstallSnapshotRequestProto request) throws IOException {
    final InstallSnapshotRequestProto.SnapshotChunkProto snapshotChunkRequest = request.getSnapshotChunk();
    final long lastIncludedIndex = snapshotChunkRequest.getTermIndex().getIndex();
    final RaftStorageDirectory dir = storage.getStorageDir();
    // create a unique temporary directory
    final File tmpDir = new File(dir.getTmpDir(), UUID.randomUUID().toString());
    FileUtils.createDirectories(tmpDir);
    tmpDir.deleteOnExit();
    LOG.info("Installing snapshot:{}, to tmp dir:{}", request, tmpDir);
    for (FileChunkProto chunk : snapshotChunkRequest.getFileChunksList()) {
        SnapshotInfo pi = stateMachine.getLatestSnapshot();
        if (pi != null && pi.getTermIndex().getIndex() >= lastIncludedIndex) {
            throw new IOException("There exists snapshot file " + pi.getFiles() + " in " + selfId + " with endIndex >= lastIncludedIndex " + lastIncludedIndex);
        }
        // this is relative to the root dir
        String fileName = chunk.getFilename();
        // TODO: assumes flat layout inside SM dir
        File tmpSnapshotFile = new File(tmpDir, new File(dir.getRoot(), fileName).getName());
        FileOutputStream out = null;
        try {
            // same last index.
            if (chunk.getOffset() == 0) {
                if (tmpSnapshotFile.exists()) {
                    FileUtils.deleteFully(tmpSnapshotFile);
                }
                // create the temp snapshot file and put padding inside
                out = new FileOutputStream(tmpSnapshotFile);
            } else {
                Preconditions.assertTrue(tmpSnapshotFile.exists());
                out = new FileOutputStream(tmpSnapshotFile, true);
                FileChannel fc = out.getChannel();
                fc.position(chunk.getOffset());
            }
            // write data to the file
            out.write(chunk.getData().toByteArray());
        } finally {
            IOUtils.cleanup(null, out);
        }
        // the md5 digest and create the md5 meta-file.
        if (chunk.getDone()) {
            final MD5Hash expectedDigest = new MD5Hash(chunk.getFileDigest().toByteArray());
            // calculate the checksum of the snapshot file and compare it with the
            // file digest in the request
            MD5Hash digest = MD5FileUtil.computeMd5ForFile(tmpSnapshotFile);
            if (!digest.equals(expectedDigest)) {
                LOG.warn("The snapshot md5 digest {} does not match expected {}", digest, expectedDigest);
                // rename the temp snapshot file to .corrupt
                FileUtils.renameFileToCorrupt(tmpSnapshotFile);
                throw new CorruptedFileException(tmpSnapshotFile, "MD5 mismatch for snapshot-" + lastIncludedIndex + " installation");
            } else {
                MD5FileUtil.saveMD5File(tmpSnapshotFile, digest);
            }
        }
    }
    if (snapshotChunkRequest.getDone()) {
        LOG.info("Install snapshot is done, renaming tnp dir:{} to:{}", tmpDir, dir.getStateMachineDir());
        dir.getStateMachineDir().delete();
        tmpDir.renameTo(dir.getStateMachineDir());
    }
}