Examples with LocalFileMeta com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta used on opensource projects

Example 6 with LocalFileMeta

use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.

the class SnapshotFileReader method readFile.

@Override
public int readFile(final ByteBufferCollector metaBufferCollector, final String fileName, final long offset, final long maxCount) throws IOException, RetryAgainException {
    // read the whole meta file.
    if (fileName.equals(Snapshot.JRAFT_SNAPSHOT_META_FILE)) {
        final ByteBuffer metaBuf = this.metaTable.saveToByteBufferAsRemote();
        // because bufRef will flip the buffer before using, so we must set the meta buffer position to it's limit.
        metaBuf.position(metaBuf.limit());
        metaBufferCollector.setBuffer(metaBuf);
        return EOF;
    }
    final LocalFileMeta fileMeta = this.metaTable.getFileMeta(fileName);
    if (fileMeta == null) {
        throw new FileNotFoundException("LocalFileMeta not found for " + fileName);
    }
    // go through throttle
    long newMaxCount = maxCount;
    if (this.snapshotThrottle != null) {
        newMaxCount = this.snapshotThrottle.throttledByThroughput(maxCount);
        if (newMaxCount < maxCount) {
            // throughput is throttled to 0, try again.
            if (newMaxCount == 0) {
                throw new RetryAgainException("readFile throttled by throughput");
            }
        }
    }
    return readFileWithMeta(metaBufferCollector, fileName, fileMeta, offset, newMaxCount);
}

Example 7 with LocalFileMeta

use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.

the class AbstractKVStoreSnapshotFile method load.

@Override
public boolean load(final SnapshotReader reader, final Region region) {
    final LocalFileMeta meta = (LocalFileMeta) reader.getFileMeta(SNAPSHOT_ARCHIVE);
    final String readerPath = reader.getPath();
    if (meta == null) {
        LOG.error("Can't find kv snapshot file, path={}.", readerPath);
        return false;
    }
    final String snapshotPath = Paths.get(readerPath, SNAPSHOT_DIR).toString();
    try {
        decompressSnapshot(readerPath, meta);
        doSnapshotLoad(snapshotPath, meta, region);
        final File tmp = new File(snapshotPath);
        // to the log information.
        if (tmp.exists()) {
            FileUtils.forceDelete(new File(snapshotPath));
        }
        return true;
    } catch (final Throwable t) {
        LOG.error("Fail to load snapshot, path={}, file list={}, {}.", readerPath, reader.listFiles(), StackTraceUtil.stackTrace(t));
        return false;
    }
}

Example 8 with LocalFileMeta

use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.

the class CheckpointFile method load.

public Checkpoint load() throws IOException {
    final ProtoBufFile file = new ProtoBufFile(this.path);
    final LocalFileMeta meta = file.load();
    if (meta != null) {
        final byte[] data = meta.getUserMeta().toByteArray();
        Checkpoint checkpoint = new Checkpoint(null, -1);
        if (checkpoint.decode(data)) {
            return checkpoint;
        }
    }
    return null;
}

Example 9 with LocalFileMeta

use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project incubator-hugegraph by apache.

the class StoreSnapshotFile method decompressSnapshot.

private String decompressSnapshot(SnapshotReader reader, String snapshotDirTar) throws IOException {
    LocalFileMeta meta = (LocalFileMeta) reader.getFileMeta(snapshotDirTar);
    if (meta == null) {
        throw new IOException("Can't find snapshot archive file, path=" + snapshotDirTar);
    }
    String diskTableKey = meta.getUserMeta().toStringUtf8();
    E.checkArgument(this.dataDisks.containsKey(diskTableKey), "The data path for '%s' should be exist", diskTableKey);
    String dataPath = this.dataDisks.get(diskTableKey);
    String parentPath = Paths.get(dataPath).getParent().toString();
    String snapshotDir = Paths.get(parentPath, StringUtils.removeEnd(snapshotDirTar, TAR)).toString();
    FileUtils.deleteDirectory(new File(snapshotDir));
    LOG.info("Delete stale snapshot dir {}", snapshotDir);
    Checksum checksum = new CRC64();
    String archiveFile = Paths.get(reader.getPath(), snapshotDirTar).toString();
    CompressUtil.decompressTar(archiveFile, parentPath, checksum);
    if (meta.hasChecksum()) {
        String expected = meta.getChecksum();
        String actual = Long.toHexString(checksum.getValue());
        E.checkArgument(expected.equals(actual), "Snapshot checksum error: '%s' != '%s'", actual, expected);
    }
    return snapshotDir;
}

Example 10 with LocalFileMeta

use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.

the class SnapshotBenchmark method snapshot.

public void snapshot(final boolean isSstSnapshot, final boolean isFastSnapshot) throws IOException {
    final File backupDir = new File("backup");
    if (backupDir.exists()) {
        FileUtils.deleteDirectory(backupDir);
    }
    FileUtils.forceMkdir(backupDir);
    final LocalFileMeta meta = doSnapshotSave(backupDir.getAbsolutePath(), isSstSnapshot, isFastSnapshot);
    this.kvStore.shutdown();
    FileUtils.deleteDirectory(new File(this.tempPath));
    FileUtils.forceMkdir(new File(this.tempPath));
    this.kvStore = new RocksRawKVStore();
    this.kvStore.init(this.dbOptions);
    final String name;
    if (isSstSnapshot) {
        name = "sst";
    } else {
        if (isFastSnapshot) {
            name = "fast";
        } else {
            name = "slow";
        }
    }
    final long decompressStart = System.nanoTime();
    final String sourceFile = Paths.get(backupDir.getAbsolutePath(), SNAPSHOT_ARCHIVE).toString();
    ZipUtil.decompress(sourceFile, backupDir.getAbsolutePath(), new CRC64());
    System.out.println(name + " decompress time cost: " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - decompressStart));
    final long loadStart = System.nanoTime();
    doSnapshotLoad(backupDir.getAbsolutePath(), meta, isFastSnapshot);
    System.out.println(name + " load snapshot time cost: " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - loadStart));
    FileUtils.deleteDirectory(backupDir);
}