Examples with CheckpointMetadata org.apache.bookkeeper.stream.proto.kv.store.CheckpointMetadata used on opensource projects

Example 1 with CheckpointMetadata

use of org.apache.bookkeeper.stream.proto.kv.store.CheckpointMetadata in project bookkeeper by apache.

the class RocksCheckpointerTest method testRestoreLocalCheckpointCorrupted.

@Test
public void testRestoreLocalCheckpointCorrupted() throws Exception {
    // create 1 checkpoints and leave them locally
    List<String> checkpointIds = createMultipleCheckpoints(1, false, false);
    store.close();
    List<String> remoteCheckpoints = checkpointStore.listFiles(RocksUtils.getDestCheckpointsPath(store.name()));
    assertEquals(checkpointIds.size(), remoteCheckpoints.size());
    for (String checkpoint : checkpointIds) {
        assertTrue(remoteCheckpoints.contains(checkpoint));
        assertTrue(new File(localCheckpointsDir, checkpoint).exists());
    }
    // remove a local checkpoint directory
    File[] files = new File(localCheckpointsDir, checkpointIds.get(0)).listFiles();
    for (int i = 0; i < files.length / 2; i++) {
        assertTrue(files[i].delete());
    }
    // restore from checkpoints
    CheckpointMetadata metadata = RocksCheckpointer.restore(store.name(), localDir, checkpointStore);
    assertNotNull(metadata);
    assertArrayEquals("checkpoint-0".getBytes(UTF_8), metadata.getTxid().toByteArray());
    String checkpoint = checkpointIds.get(0);
    assertTrue(new File(localCheckpointsDir, checkpoint).exists());
    assertTrue(checkpointStore.fileExists(RocksUtils.getDestCheckpointPath(store.name(), checkpoint)));
    // restore from the latest checkpoint
    store = new RocksdbKVStore<>();
    store.init(spec);
    verifyNumKvs(100);
}

Example 2 with CheckpointMetadata

use of org.apache.bookkeeper.stream.proto.kv.store.CheckpointMetadata in project bookkeeper by apache.

the class RocksCheckpointer method getLatestCheckpoint.

private static Pair<String, CheckpointMetadata> getLatestCheckpoint(String dbPrefix, CheckpointStore checkpointStore) throws IOException {
    String remoteCheckpointsPath = RocksUtils.getDestCheckpointsPath(dbPrefix);
    List<String> files = checkpointStore.listFiles(remoteCheckpointsPath);
    CheckpointMetadata latestCheckpoint = null;
    String latestCheckpointId = null;
    for (String checkpointId : files) {
        String metadataPath = RocksUtils.getDestCheckpointMetadataPath(dbPrefix, checkpointId);
        try (InputStream is = checkpointStore.openInputStream(metadataPath)) {
            CheckpointMetadata ckpt = CheckpointMetadata.parseFrom(is);
            if (null == latestCheckpoint) {
                latestCheckpointId = checkpointId;
                latestCheckpoint = ckpt;
            } else if (latestCheckpoint.getCreatedAt() < ckpt.getCreatedAt()) {
                latestCheckpointId = checkpointId;
                latestCheckpoint = ckpt;
            }
        }
    }
    return Pair.of(latestCheckpointId, latestCheckpoint);
}

Example 3 with CheckpointMetadata

use of org.apache.bookkeeper.stream.proto.kv.store.CheckpointMetadata in project bookkeeper by apache.

the class RocksCheckpointer method restore.

public static CheckpointMetadata restore(String dbName, File dbPath, CheckpointStore checkpointStore) throws StateStoreException {
    try {
        String dbPrefix = String.format("%s", dbName);
        Pair<String, CheckpointMetadata> latestCheckpoint = getLatestCheckpoint(dbPrefix, checkpointStore);
        File checkpointsDir = new File(dbPath, "checkpoints");
        String checkpointId = latestCheckpoint.getLeft();
        CheckpointMetadata checkpointMetadata = latestCheckpoint.getRight();
        if (checkpointId != null) {
            RocksdbRestoreTask task = new RocksdbRestoreTask(dbName, checkpointsDir, checkpointStore);
            task.restore(checkpointId, checkpointMetadata);
        } else {
            // no checkpoints available, create an empty directory
            checkpointId = UUID.randomUUID().toString();
            Files.createDirectories(Paths.get(checkpointsDir.getAbsolutePath(), checkpointId));
        }
        Path restoredCheckpointPath = Paths.get(checkpointsDir.getAbsolutePath(), checkpointId);
        log.info("Successfully restore checkpoint {} to {}", checkpointId, restoredCheckpointPath);
        File currentDir = new File(dbPath, "current");
        Files.deleteIfExists(Paths.get(currentDir.getAbsolutePath()));
        Files.createSymbolicLink(Paths.get(currentDir.getAbsolutePath()), restoredCheckpointPath);
        // after successfully restore from remote checkpoints, cleanup other unused checkpoints
        cleanupLocalCheckpoints(checkpointsDir, checkpointId);
        return checkpointMetadata;
    } catch (IOException ioe) {
        log.error("Failed to restore rocksdb {}", dbName, ioe);
        throw new StateStoreException("Failed to restore rocksdb " + dbName, ioe);
    }
}

Example 4 with CheckpointMetadata

use of org.apache.bookkeeper.stream.proto.kv.store.CheckpointMetadata in project bookkeeper by apache.

the class RocksCheckpointerTest method testRestoreCheckpointMissingLocally.

@Test
public void testRestoreCheckpointMissingLocally() throws Exception {
    // create 3 checkpoints and leave them locally
    List<String> checkpointIds = createMultipleCheckpoints(3, false, false);
    store.close();
    List<String> remoteCheckpoints = checkpointStore.listFiles(RocksUtils.getDestCheckpointsPath(store.name()));
    assertEquals(checkpointIds.size(), remoteCheckpoints.size());
    for (String checkpoint : checkpointIds) {
        assertTrue(remoteCheckpoints.contains(checkpoint));
        assertTrue(new File(localCheckpointsDir, checkpoint).exists());
    }
    // remove a local checkpoint directory
    MoreFiles.deleteRecursively(Paths.get(localCheckpointsDir.getAbsolutePath(), checkpointIds.get(2)), RecursiveDeleteOption.ALLOW_INSECURE);
    // restore from checkpoints
    CheckpointMetadata metadata = RocksCheckpointer.restore(store.name(), localDir, checkpointStore);
    assertNotNull(metadata);
    assertArrayEquals("checkpoint-2".getBytes(UTF_8), metadata.getTxid().toByteArray());
    for (int i = 0; i < 3; i++) {
        String checkpoint = checkpointIds.get(i);
        if (i == 2) {
            assertTrue(new File(localCheckpointsDir, checkpoint).exists());
        } else {
            assertFalse(new File(localCheckpointsDir, checkpoint).exists());
        }
        assertTrue(checkpointStore.fileExists(RocksUtils.getDestCheckpointPath(store.name(), checkpoint)));
    }
    // restore from the latest checkpoint
    store = new RocksdbKVStore<>();
    store.init(spec);
    verifyNumKvs(300);
}

Example 5 with CheckpointMetadata

use of org.apache.bookkeeper.stream.proto.kv.store.CheckpointMetadata in project bookkeeper by apache.

the class RocksCheckpointerTest method testCheckpointRestore.

/**
 * Basic test.
 *
 * <p>- checkpoint a local state store to a remote checkpoint store
 * - restore checkpoint from the remote checkpoint store
 * - verify the restored local state store is correct
 */
@Test
public void testCheckpointRestore() throws Exception {
    final int numKvs = 100;
    final String dbName = runtime.getMethodName();
    final byte[] txid = runtime.getMethodName().getBytes(UTF_8);
    // first prepare rocksdb with 100 kvs;
    writeNumKvs(numKvs, 0);
    Checkpoint checkpoint = Checkpoint.create(store.getDb());
    // checkpoint
    RocksdbCheckpointTask checkpointTask = new RocksdbCheckpointTask(dbName, checkpoint, localCheckpointsDir, checkpointStore, false, false);
    String checkpointId = checkpointTask.checkpoint(txid);
    // checkpoint directory exists
    File checkpointedDir = new File(localCheckpointsDir, checkpointId);
    assertTrue("local checkpointed dir " + checkpointedDir + " doesn't exists when `removeLocalCheckpoints` is false", checkpointedDir.exists());
    // remote checkpoint metadata file exists
    String checkpointMetadataFile = RocksUtils.getDestCheckpointMetadataPath(store.name(), checkpointId);
    assertTrue(checkpointStore.fileExists(checkpointMetadataFile));
    int fileLen = (int) checkpointStore.getFileLength(checkpointMetadataFile);
    byte[] checkpointMetadataBytes = new byte[fileLen];
    @Cleanup InputStream fileIn = checkpointStore.openInputStream(checkpointMetadataFile);
    ByteStreams.readFully(fileIn, checkpointMetadataBytes);
    // verify the checkpointed metadata exists
    CheckpointMetadata metadata = CheckpointMetadata.parseFrom(checkpointMetadataBytes);
    assertArrayEquals(txid, metadata.getTxid().toByteArray());
    verifyCheckpointMetadata(checkpointedDir, metadata);
    verifyRemoteFiles(checkpointId, checkpointedDir);
    store.close();
    // remove local checkpointed dir
    MoreFiles.deleteRecursively(Paths.get(checkpointedDir.getAbsolutePath()), RecursiveDeleteOption.ALLOW_INSECURE);
    assertFalse(checkpointedDir.exists());
    // restore the checkpoint
    RocksdbRestoreTask restoreTask = new RocksdbRestoreTask(dbName, localCheckpointsDir, checkpointStore);
    restoreTask.restore(checkpointId, metadata);
    assertTrue(checkpointedDir.exists());
    // verify the content
    verifyCheckpointMetadata(checkpointedDir, metadata);
    verifyRemoteFiles(checkpointId, checkpointedDir);
    // make sure all the kvs are readable
    store = new RocksdbKVStore<>();
    store.init(spec);
    verifyNumKvs(numKvs);
}