Examples with Checkpoint org.rocksdb.Checkpoint used on opensource projects

Example 6 with Checkpoint

use of org.rocksdb.Checkpoint in project bookkeeper by apache.

the class RocksCheckpointerTest method testRestoreCleanupCheckpoints.

@Test
public void testRestoreCleanupCheckpoints() throws Exception {
    // create 3 checkpoints and leave them locally
    List<String> checkpointIds = createMultipleCheckpoints(3, false, false);
    store.close();
    List<String> remoteCheckpoints = checkpointStore.listFiles(RocksUtils.getDestCheckpointsPath(store.name()));
    assertEquals(checkpointIds.size(), remoteCheckpoints.size());
    for (String checkpoint : checkpointIds) {
        assertTrue(remoteCheckpoints.contains(checkpoint));
        assertTrue(new File(localCheckpointsDir, checkpoint).exists());
    }
    // restore from checkpoints
    CheckpointMetadata metadata = RocksCheckpointer.restore(store.name(), localDir, checkpointStore);
    assertNotNull(metadata);
    assertArrayEquals("checkpoint-2".getBytes(UTF_8), metadata.getTxid().toByteArray());
    for (int i = 0; i < 3; i++) {
        String checkpoint = checkpointIds.get(i);
        if (i == 2) {
            assertTrue(new File(localCheckpointsDir, checkpoint).exists());
        } else {
            assertFalse(new File(localCheckpointsDir, checkpoint).exists());
        }
        assertTrue(checkpointStore.fileExists(RocksUtils.getDestCheckpointPath(store.name(), checkpoint)));
    }
    // restore from the latest checkpoint
    store = new RocksdbKVStore<>();
    store.init(spec);
    verifyNumKvs(300);
}

Example 7 with Checkpoint

use of org.rocksdb.Checkpoint in project jstorm by alibaba.

the class RocksDbHdfsState method main.

public static void main(String[] args) {
    Map conf = new HashMap<Object, Object>();
    conf.putAll(Utils.loadConf(args[0]));
    RocksDbHdfsState<String, Integer> state = new RocksDbHdfsState<String, Integer>();
    state.setStateName(String.valueOf(1));
    // setup checkpoint
    int batchNum = JStormUtils.parseInt(conf.get("batch.num"), 100);
    state.initEnv("test", conf, "/tmp/rocksdb_test");
    String remoteCpPath = null;
    for (int i = 0; i < batchNum; i++) {
        state.put(String.valueOf(i % 20), i);
        state.checkpoint(i);
        remoteCpPath = state.backup(i);
        state.remove(i);
    }
    state.cleanup();
    state.initEnv("test", conf, "/tmp/rocksdb_test");
    state.restore(remoteCpPath);
    for (int i = 0; i < 20; i++) {
        Integer value = state.get(String.valueOf(i));
        LOG.info("key={}, value={}", String.valueOf(i), value);
    }
    state.cleanup();
}

Example 8 with Checkpoint

use of org.rocksdb.Checkpoint in project jstorm by alibaba.

the class RocksDbHdfsState method restore.

@Override
public void restore(String checkpointBackupDir) {
    LOG.info("Start restore from remote: {}", checkpointBackupDir);
    if (rocksDb != null)
        rocksDb.dispose();
    initLocalRocksDbDir();
    // Restore db files from hdfs to local disk
    try {
        if (checkpointBackupDir != null) {
            // Get dir of sst files
            int index = checkpointBackupDir.lastIndexOf("checkpoint");
            String remoteDbBackupDir = checkpointBackupDir.substring(0, index);
            // copy sstFile.list, CURRENT, MANIFEST to local disk for the specified batch
            Collection<String> files = hdfsCache.listFile(checkpointBackupDir, false);
            LOG.debug("Restore checkpoint files: {}", files);
            for (String fileName : files) hdfsCache.copyToLocal(checkpointBackupDir + "/" + fileName, rocksDbDir);
            // copy all rocksDB sst files to local disk
            String sstFileList = rocksDbDir + "/" + SST_FILE_LIST;
            File file = new File(sstFileList);
            List<String> sstFiles = FileUtils.readLines(file);
            LOG.debug("Restore sst files: {}", sstFiles);
            for (String sstFile : sstFiles) {
                hdfsCache.copyToLocal(remoteDbBackupDir + "/" + sstFile, rocksDbDir);
            }
            FileUtils.deleteQuietly(file);
        }
        initRocksDb();
    } catch (IOException e) {
        LOG.error("Failed to restore checkpoint", e);
        throw new RuntimeException(e.getMessage());
    }
}