Examples with LocalFileMeta org.apache.ignite.raft.jraft.entity.LocalFileMetaOutter.LocalFileMeta used on opensource projects

Example 1 with LocalFileMeta

use of org.apache.ignite.raft.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project ignite-3 by apache.

the class LocalSnapshotWriter method addFile.

@Override
public boolean addFile(final String fileName, final Message fileMeta) {
    final LocalFileMetaBuilder metaBuilder = msgFactory.localFileMeta();
    if (fileMeta != null) {
        metaBuilder.source(((LocalFileMeta) fileMeta).source());
        metaBuilder.checksum(((LocalFileMeta) fileMeta).checksum());
    }
    final LocalFileMeta meta = metaBuilder.build();
    return this.metaTable.addFile(fileName, meta);
}

Example 2 with LocalFileMeta

use of org.apache.ignite.raft.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project ignite-3 by apache.

the class SnapshotFileReader method readFile.

@Override
public int readFile(final ByteBufferCollector metaBufferCollector, final String fileName, final long offset, final long maxCount) throws IOException, RetryAgainException {
    // read the whole meta file.
    if (fileName.equals(Snapshot.JRAFT_SNAPSHOT_META_FILE)) {
        final ByteBuffer metaBuf = this.metaTable.saveToByteBufferAsRemote();
        // because bufRef will flip the buffer before using, so we must set the meta buffer position to it's limit.
        metaBuf.position(metaBuf.limit());
        metaBufferCollector.setBuffer(metaBuf);
        return EOF;
    }
    final LocalFileMeta fileMeta = this.metaTable.getFileMeta(fileName);
    if (fileMeta == null) {
        throw new FileNotFoundException("LocalFileMeta not found for " + fileName);
    }
    // go through throttle
    long newMaxCount = maxCount;
    if (this.snapshotThrottle != null) {
        newMaxCount = this.snapshotThrottle.throttledByThroughput(maxCount);
        if (newMaxCount < maxCount) {
            // throughput is throttled to 0, try again.
            if (newMaxCount == 0) {
                throw new RetryAgainException("readFile throttled by throughput");
            }
        }
    }
    return readFileWithMeta(metaBufferCollector, fileName, fileMeta, offset, newMaxCount);
}

Example 3 with LocalFileMeta

use of org.apache.ignite.raft.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project ignite-3 by apache.

the class LocalSnapshotCopier method filterBeforeCopy.

boolean filterBeforeCopy(final LocalSnapshotWriter writer, final SnapshotReader lastSnapshot) throws IOException {
    final Set<String> existingFiles = writer.listFiles();
    final ArrayDeque<String> toRemove = new ArrayDeque<>();
    for (final String file : existingFiles) {
        if (this.remoteSnapshot.getFileMeta(file) == null) {
            toRemove.add(file);
            writer.removeFile(file);
        }
    }
    final Set<String> remoteFiles = this.remoteSnapshot.listFiles();
    for (final String fileName : remoteFiles) {
        final LocalFileMeta remoteMeta = (LocalFileMeta) this.remoteSnapshot.getFileMeta(fileName);
        Requires.requireNonNull(remoteMeta, "remoteMeta");
        if (remoteMeta.checksum() != null) {
            // Re-download file if this file doesn't have checksum
            writer.removeFile(fileName);
            toRemove.add(fileName);
            continue;
        }
        LocalFileMeta localMeta = (LocalFileMeta) writer.getFileMeta(fileName);
        if (localMeta != null) {
            if (localMeta.checksum() != null && localMeta.checksum().equals(remoteMeta.checksum())) {
                LOG.info("Keep file={} checksum={} in {}", fileName, remoteMeta.checksum(), writer.getPath());
                continue;
            }
            // Remove files from writer so that the file is to be copied from
            // remote_snapshot or last_snapshot
            writer.removeFile(fileName);
            toRemove.add(fileName);
        }
        // Try find files in last_snapshot
        if (lastSnapshot == null) {
            continue;
        }
        if ((localMeta = (LocalFileMeta) lastSnapshot.getFileMeta(fileName)) == null) {
            continue;
        }
        if (localMeta.checksum() == null || !localMeta.checksum().equals(remoteMeta.checksum())) {
            continue;
        }
        LOG.info("Found the same file ={} checksum={} in lastSnapshot={}", fileName, remoteMeta.checksum(), lastSnapshot.getPath());
        if (localMeta.source() == FileSource.FILE_SOURCE_LOCAL) {
            final Path sourcePath = Paths.get(lastSnapshot.getPath(), fileName);
            final Path destPath = Paths.get(writer.getPath(), fileName);
            IgniteUtils.deleteIfExists(destPath);
            try {
                Files.createLink(destPath, sourcePath);
            } catch (final IOException e) {
                LOG.error("Fail to link {} to {}", e, sourcePath, destPath);
                continue;
            }
            // Don't delete linked file
            if (!toRemove.isEmpty() && toRemove.peekLast().equals(fileName)) {
                toRemove.pollLast();
            }
        }
        // Copy file from last_snapshot
        writer.addFile(fileName, localMeta);
    }
    if (!writer.sync()) {
        LOG.error("Fail to sync writer on path={}", writer.getPath());
        return false;
    }
    for (final String fileName : toRemove) {
        final Path removePath = Paths.get(writer.getPath(), fileName);
        IgniteUtils.deleteIfExists(removePath);
        LOG.info("Deleted file: {}", removePath);
    }
    return true;
}

Example 4 with LocalFileMeta

use of org.apache.ignite.raft.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project ignite-3 by apache.

the class LocalSnapshotCopier method copyFile.

void copyFile(final String fileName) throws IOException, InterruptedException {
    if (this.writer.getFileMeta(fileName) != null) {
        LOG.info("Skipped downloading {}", fileName);
        return;
    }
    if (!checkFile(fileName)) {
        return;
    }
    final String filePath = this.writer.getPath() + File.separator + fileName;
    final Path subPath = Paths.get(filePath);
    if (!subPath.equals(subPath.getParent()) && !".".equals(subPath.getParent().getFileName().toString())) {
        final File parentDir = subPath.getParent().toFile();
        if (!parentDir.exists() && !parentDir.mkdirs()) {
            LOG.error("Fail to create directory for {}", filePath);
            setError(RaftError.EIO, "Fail to create directory");
            return;
        }
    }
    final LocalFileMeta meta = (LocalFileMeta) this.remoteSnapshot.getFileMeta(fileName);
    Session session = null;
    try {
        this.lock.lock();
        try {
            if (this.cancelled) {
                if (isOk()) {
                    setError(RaftError.ECANCELED, "ECANCELED");
                }
                return;
            }
            session = this.copier.startCopyToFile(fileName, filePath, null);
            if (session == null) {
                LOG.error("Fail to copy {}", fileName);
                setError(-1, "Fail to copy %s", fileName);
                return;
            }
            this.curSession = session;
        } finally {
            this.lock.unlock();
        }
        // join out of lock
        session.join();
        this.lock.lock();
        try {
            this.curSession = null;
        } finally {
            this.lock.unlock();
        }
        if (!session.status().isOk() && isOk()) {
            setError(session.status().getCode(), session.status().getErrorMsg());
            return;
        }
        if (!this.writer.addFile(fileName, meta)) {
            setError(RaftError.EIO, "Fail to add file to writer");
            return;
        }
        if (!this.writer.sync()) {
            setError(RaftError.EIO, "Fail to sync writer");
        }
    } finally {
        if (session != null) {
            Utils.closeQuietly(session);
        }
    }
}