Examples with Checkpoint org.rocksdb.Checkpoint used on opensource projects

Example 1 with Checkpoint

use of org.rocksdb.Checkpoint in project bookkeeper by apache.

the class RocksCheckpointerTest method testRestoreLocalCheckpointCorrupted.

@Test
public void testRestoreLocalCheckpointCorrupted() throws Exception {
    // create 1 checkpoints and leave them locally
    List<String> checkpointIds = createMultipleCheckpoints(1, false, false);
    store.close();
    List<String> remoteCheckpoints = checkpointStore.listFiles(RocksUtils.getDestCheckpointsPath(store.name()));
    assertEquals(checkpointIds.size(), remoteCheckpoints.size());
    for (String checkpoint : checkpointIds) {
        assertTrue(remoteCheckpoints.contains(checkpoint));
        assertTrue(new File(localCheckpointsDir, checkpoint).exists());
    }
    // remove a local checkpoint directory
    File[] files = new File(localCheckpointsDir, checkpointIds.get(0)).listFiles();
    for (int i = 0; i < files.length / 2; i++) {
        assertTrue(files[i].delete());
    }
    // restore from checkpoints
    CheckpointMetadata metadata = RocksCheckpointer.restore(store.name(), localDir, checkpointStore);
    assertNotNull(metadata);
    assertArrayEquals("checkpoint-0".getBytes(UTF_8), metadata.getTxid().toByteArray());
    String checkpoint = checkpointIds.get(0);
    assertTrue(new File(localCheckpointsDir, checkpoint).exists());
    assertTrue(checkpointStore.fileExists(RocksUtils.getDestCheckpointPath(store.name(), checkpoint)));
    // restore from the latest checkpoint
    store = new RocksdbKVStore<>();
    store.init(spec);
    verifyNumKvs(100);
}

Example 2 with Checkpoint

use of org.rocksdb.Checkpoint in project jstorm by alibaba.

the class RocksDbUnitTest method rocksDbTest.

private static void rocksDbTest(RocksDB db, List<ColumnFamilyHandle> handlers) {
    try {
        ColumnFamilyHandle handler1 = null;
        ColumnFamilyHandle handler2 = null;
        if (handlers.size() > 0) {
            // skip default column family
            handler1 = handlers.get(1);
            handler2 = handlers.get(2);
        } else {
            handler1 = db.createColumnFamily(new ColumnFamilyDescriptor("test1".getBytes()));
            handler2 = db.createColumnFamily(new ColumnFamilyDescriptor("test2".getBytes()));
        }
        int startValue1 = getStartValue(db, handler1);
        int startValue2 = getStartValue(db, handler2);
        ;
        Checkpoint cp = Checkpoint.create(db);
        if (isCompaction) {
            db.compactRange();
            LOG.info("Compaction!");
        }
        long flushWaitTime = System.currentTimeMillis() + flushInterval;
        for (int i = 0; i < putNum || putNum == -1; i++) {
            db.put(handler1, String.valueOf(i % 1000).getBytes(), String.valueOf(startValue1 + i).getBytes());
            db.put(handler2, String.valueOf(i % 1000).getBytes(), String.valueOf(startValue2 + i).getBytes());
            if (isFlush && flushWaitTime <= System.currentTimeMillis()) {
                db.flush(new FlushOptions());
                if (isCheckpoint) {
                    cp.createCheckpoint(cpPath + "/" + i);
                }
                flushWaitTime = System.currentTimeMillis() + flushInterval;
            }
        }
    } catch (RocksDBException e) {
        LOG.error("Failed to put or flush", e);
    }
}

Example 3 with Checkpoint

use of org.rocksdb.Checkpoint in project jstorm by alibaba.

the class RocksDbHdfsState method checkpoint.

/**
 * Flush the data in memtable of RocksDB into disk, and then create checkpoint
 *
 * @param batchId
 */
@Override
public void checkpoint(long batchId) {
    long startTime = System.currentTimeMillis();
    try {
        rocksDb.flush(new FlushOptions());
        Checkpoint cp = Checkpoint.create(rocksDb);
        cp.createCheckpoint(getLocalCheckpointPath(batchId));
    } catch (RocksDBException e) {
        LOG.error("Failed to create checkpoint for batch-" + batchId, e);
        throw new RuntimeException(e.getMessage());
    }
    if (JStormMetrics.enabled)
        rocksDbFlushAndCpLatency.update(System.currentTimeMillis() - startTime);
}

Example 4 with Checkpoint

use of org.rocksdb.Checkpoint in project bookkeeper by apache.

the class RocksCheckpointerTest method testRestoreCheckpointMissingLocally.

@Test
public void testRestoreCheckpointMissingLocally() throws Exception {
    // create 3 checkpoints and leave them locally
    List<String> checkpointIds = createMultipleCheckpoints(3, false, false);
    store.close();
    List<String> remoteCheckpoints = checkpointStore.listFiles(RocksUtils.getDestCheckpointsPath(store.name()));
    assertEquals(checkpointIds.size(), remoteCheckpoints.size());
    for (String checkpoint : checkpointIds) {
        assertTrue(remoteCheckpoints.contains(checkpoint));
        assertTrue(new File(localCheckpointsDir, checkpoint).exists());
    }
    // remove a local checkpoint directory
    MoreFiles.deleteRecursively(Paths.get(localCheckpointsDir.getAbsolutePath(), checkpointIds.get(2)), RecursiveDeleteOption.ALLOW_INSECURE);
    // restore from checkpoints
    CheckpointMetadata metadata = RocksCheckpointer.restore(store.name(), localDir, checkpointStore);
    assertNotNull(metadata);
    assertArrayEquals("checkpoint-2".getBytes(UTF_8), metadata.getTxid().toByteArray());
    for (int i = 0; i < 3; i++) {
        String checkpoint = checkpointIds.get(i);
        if (i == 2) {
            assertTrue(new File(localCheckpointsDir, checkpoint).exists());
        } else {
            assertFalse(new File(localCheckpointsDir, checkpoint).exists());
        }
        assertTrue(checkpointStore.fileExists(RocksUtils.getDestCheckpointPath(store.name(), checkpoint)));
    }
    // restore from the latest checkpoint
    store = new RocksdbKVStore<>();
    store.init(spec);
    verifyNumKvs(300);
}

Example 5 with Checkpoint

use of org.rocksdb.Checkpoint in project bookkeeper by apache.

the class RocksCheckpointerTest method testCheckpointRestore.

/**
 * Basic test.
 *
 * <p>- checkpoint a local state store to a remote checkpoint store
 * - restore checkpoint from the remote checkpoint store
 * - verify the restored local state store is correct
 */
@Test
public void testCheckpointRestore() throws Exception {
    final int numKvs = 100;
    final String dbName = runtime.getMethodName();
    final byte[] txid = runtime.getMethodName().getBytes(UTF_8);
    // first prepare rocksdb with 100 kvs;
    writeNumKvs(numKvs, 0);
    Checkpoint checkpoint = Checkpoint.create(store.getDb());
    // checkpoint
    RocksdbCheckpointTask checkpointTask = new RocksdbCheckpointTask(dbName, checkpoint, localCheckpointsDir, checkpointStore, false, false);
    String checkpointId = checkpointTask.checkpoint(txid);
    // checkpoint directory exists
    File checkpointedDir = new File(localCheckpointsDir, checkpointId);
    assertTrue("local checkpointed dir " + checkpointedDir + " doesn't exists when `removeLocalCheckpoints` is false", checkpointedDir.exists());
    // remote checkpoint metadata file exists
    String checkpointMetadataFile = RocksUtils.getDestCheckpointMetadataPath(store.name(), checkpointId);
    assertTrue(checkpointStore.fileExists(checkpointMetadataFile));
    int fileLen = (int) checkpointStore.getFileLength(checkpointMetadataFile);
    byte[] checkpointMetadataBytes = new byte[fileLen];
    @Cleanup InputStream fileIn = checkpointStore.openInputStream(checkpointMetadataFile);
    ByteStreams.readFully(fileIn, checkpointMetadataBytes);
    // verify the checkpointed metadata exists
    CheckpointMetadata metadata = CheckpointMetadata.parseFrom(checkpointMetadataBytes);
    assertArrayEquals(txid, metadata.getTxid().toByteArray());
    verifyCheckpointMetadata(checkpointedDir, metadata);
    verifyRemoteFiles(checkpointId, checkpointedDir);
    store.close();
    // remove local checkpointed dir
    MoreFiles.deleteRecursively(Paths.get(checkpointedDir.getAbsolutePath()), RecursiveDeleteOption.ALLOW_INSECURE);
    assertFalse(checkpointedDir.exists());
    // restore the checkpoint
    RocksdbRestoreTask restoreTask = new RocksdbRestoreTask(dbName, localCheckpointsDir, checkpointStore);
    restoreTask.restore(checkpointId, metadata);
    assertTrue(checkpointedDir.exists());
    // verify the content
    verifyCheckpointMetadata(checkpointedDir, metadata);
    verifyRemoteFiles(checkpointId, checkpointedDir);
    // make sure all the kvs are readable
    store = new RocksdbKVStore<>();
    store.init(spec);
    verifyNumKvs(numKvs);
}