Examples with LocalFileMeta com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta used on opensource projects

Example 11 with LocalFileMeta

use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.

the class CheckpointFile method save.

public synchronized boolean save(final Checkpoint checkpoint) throws IOException {
    final ProtoBufFile file = new ProtoBufFile(this.path);
    final byte[] data = checkpoint.encode();
    final LocalFileMeta meta = // 
    LocalFileMeta.newBuilder().setUserMeta(// 
    ZeroByteStringHelper.wrap(data)).build();
    return file.save(meta, true);
}

Example 12 with LocalFileMeta

use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.

the class LocalSnapshotCopier method filterBeforeCopy.

boolean filterBeforeCopy(final LocalSnapshotWriter writer, final SnapshotReader lastSnapshot) throws IOException {
    final Set<String> existingFiles = writer.listFiles();
    final ArrayDeque<String> toRemove = new ArrayDeque<>();
    for (final String file : existingFiles) {
        if (this.remoteSnapshot.getFileMeta(file) == null) {
            toRemove.add(file);
            writer.removeFile(file);
        }
    }
    final Set<String> remoteFiles = this.remoteSnapshot.listFiles();
    for (final String fileName : remoteFiles) {
        final LocalFileMeta remoteMeta = (LocalFileMeta) this.remoteSnapshot.getFileMeta(fileName);
        Requires.requireNonNull(remoteMeta, "remoteMeta");
        if (!remoteMeta.hasChecksum()) {
            // Re-download file if this file doesn't have checksum
            writer.removeFile(fileName);
            toRemove.add(fileName);
            continue;
        }
        LocalFileMeta localMeta = (LocalFileMeta) writer.getFileMeta(fileName);
        if (localMeta != null) {
            if (localMeta.hasChecksum() && localMeta.getChecksum().equals(remoteMeta.getChecksum())) {
                LOG.info("Keep file={} checksum={} in {}", fileName, remoteMeta.getChecksum(), writer.getPath());
                continue;
            }
            // Remove files from writer so that the file is to be copied from
            // remote_snapshot or last_snapshot
            writer.removeFile(fileName);
            toRemove.add(fileName);
        }
        // Try find files in last_snapshot
        if (lastSnapshot == null) {
            continue;
        }
        if ((localMeta = (LocalFileMeta) lastSnapshot.getFileMeta(fileName)) == null) {
            continue;
        }
        if (!localMeta.hasChecksum() || !localMeta.getChecksum().equals(remoteMeta.getChecksum())) {
            continue;
        }
        LOG.info("Found the same file ={} checksum={} in lastSnapshot={}", fileName, remoteMeta.getChecksum(), lastSnapshot.getPath());
        if (localMeta.getSource() == FileSource.FILE_SOURCE_LOCAL) {
            final String sourcePath = lastSnapshot.getPath() + File.separator + fileName;
            final String destPath = writer.getPath() + File.separator + fileName;
            FileUtils.deleteQuietly(new File(destPath));
            try {
                Files.createLink(Paths.get(destPath), Paths.get(sourcePath));
            } catch (final IOException e) {
                LOG.error("Fail to link {} to {}", sourcePath, destPath, e);
                continue;
            }
            // Don't delete linked file
            if (!toRemove.isEmpty() && toRemove.peekLast().equals(fileName)) {
                toRemove.pollLast();
            }
        }
        // Copy file from last_snapshot
        writer.addFile(fileName, localMeta);
    }
    if (!writer.sync()) {
        LOG.error("Fail to sync writer on path={}", writer.getPath());
        return false;
    }
    for (final String fileName : toRemove) {
        final String removePath = writer.getPath() + File.separator + fileName;
        FileUtils.deleteQuietly(new File(removePath));
        LOG.info("Deleted file: {}", removePath);
    }
    return true;
}

Example 13 with LocalFileMeta

use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.

the class LocalSnapshotMetaTable method saveToFile.

/**
 * Save metadata infos into file by path.
 */
public boolean saveToFile(String path) throws IOException {
    LocalSnapshotPbMeta.Builder pbMeta = LocalSnapshotPbMeta.newBuilder();
    if (hasMeta()) {
        pbMeta.setMeta(this.meta);
    }
    for (Map.Entry<String, LocalFileMeta> entry : this.fileMap.entrySet()) {
        File f = File.newBuilder().setName(entry.getKey()).setMeta(entry.getValue()).build();
        pbMeta.addFiles(f);
    }
    ProtoBufFile pbFile = new ProtoBufFile(path);
    return pbFile.save(pbMeta.build(), this.raftOptions.isSyncMeta());
}

Example 14 with LocalFileMeta

use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.

the class RocksKVStoreTest method snapshotTest.

public void snapshotTest() throws Exception {
    final File backupDir = new File("backup");
    if (backupDir.exists()) {
        FileUtils.deleteDirectory(backupDir);
    }
    FileUtils.forceMkdir(backupDir);
    for (int i = 0; i < 100000; i++) {
        final String v = String.valueOf(i);
        this.kvStore.put(makeKey(v), makeValue(v), null);
    }
    final Region region = new Region();
    KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 2);
    final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(backupDir.getAbsolutePath());
    final CountDownLatch latch = new CountDownLatch(1);
    final Closure done = status -> {
        assertTrue(status.isOk());
        latch.countDown();
    };
    kvStoreSnapshotFile.save(snapshotWriter, region, done, snapshotPool);
    latch.await();
    final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
    assertNotNull(meta);
    assertNotNull(get(makeKey("1")));
    this.kvStore.put(makeKey("100001"), makeValue("100001"), null);
    assertNotNull(get(makeKey("100001")));
    this.kvStore.shutdown();
    FileUtils.deleteDirectory(new File(this.tempPath));
    FileUtils.forceMkdir(new File(this.tempPath));
    this.kvStore = new RocksRawKVStore();
    this.kvStore.init(this.dbOptions);
    assertNull(get(makeKey("1")));
    final TestSnapshotReader snapshotReader = new TestSnapshotReader(snapshotWriter.metaTable, backupDir.getAbsolutePath());
    kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    final boolean ret = kvStoreSnapshotFile.load(snapshotReader, region);
    assertTrue(ret);
    for (int i = 0; i < 100000; i++) {
        final String v = String.valueOf(i);
        assertArrayEquals(makeValue(v), get(makeKey(v)));
    }
    // key[100001] is put after the snapshot, so key[100001] should not exist.
    assertNull(get(makeKey("100001")));
    FileUtils.deleteDirectory(backupDir);
    ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
}