Examples with CheckpointInputStream alluxio.master.journal.checkpoint.CheckpointInputStream used on opensource projects

Example 6 with CheckpointInputStream

use of alluxio.master.journal.checkpoint.CheckpointInputStream in project alluxio by Alluxio.

the class RaftJournalDumper method readRatisSnapshotFromDir.

private void readRatisSnapshotFromDir() throws IOException {
    try (RaftStorage storage = new RaftStorageImpl(getJournalDir(), RaftServerConfigKeys.Log.CorruptionPolicy.getDefault())) {
        SimpleStateMachineStorage stateMachineStorage = new SimpleStateMachineStorage();
        stateMachineStorage.init(storage);
        SingleFileSnapshotInfo currentSnapshot = stateMachineStorage.getLatestSnapshot();
        if (currentSnapshot == null) {
            LOG.debug("No snapshot found");
            return;
        }
        final File snapshotFile = currentSnapshot.getFile().getPath().toFile();
        String checkpointPath = String.format("%s-%s-%s", mCheckpointsDir, currentSnapshot.getIndex(), snapshotFile.lastModified());
        try (DataInputStream inputStream = new DataInputStream(new FileInputStream(snapshotFile))) {
            LOG.debug("Reading snapshot-Id: {}", inputStream.readLong());
            try (CheckpointInputStream checkpointStream = new CheckpointInputStream(inputStream)) {
                readCheckpoint(checkpointStream, Paths.get(checkpointPath));
            } catch (Exception e) {
                LOG.error("Failed to read snapshot from journal.", e);
            }
        } catch (Exception e) {
            LOG.error("Failed to load snapshot {}", snapshotFile, e);
            throw e;
        }
    }
}

Example 7 with CheckpointInputStream

use of alluxio.master.journal.checkpoint.CheckpointInputStream in project alluxio by Alluxio.

the class InodeStoreBench method checkpointBenchmark.

private static void checkpointBenchmark() throws Exception {
    sStore.clear();
    System.out.printf("Writing %d million inodes ...", CHECKPOINT_INODES_MILLIONS);
    for (int i = 0; i < 1e6 * CHECKPOINT_INODES_MILLIONS; i++) {
        writeInode();
    }
    System.out.println(" done");
    File f = File.createTempFile("checkpoint", "");
    try (OutputStream out = new BufferedOutputStream(new FileOutputStream(f))) {
        long start = System.nanoTime();
        sStore.writeToCheckpoint(out);
        System.out.printf("Wrote %d million inode checkpoint in %dms%n", CHECKPOINT_INODES_MILLIONS, (System.nanoTime() - start) / Constants.MS_NANO);
    }
    try (InputStream in = new BufferedInputStream(new FileInputStream(f))) {
        long start = System.nanoTime();
        sStore.restoreFromCheckpoint(new CheckpointInputStream(in));
        System.out.printf("Restored %d million inode checkpoint in %dms%n", CHECKPOINT_INODES_MILLIONS, (System.nanoTime() - start) / Constants.MS_NANO);
    }
}

Example 8 with CheckpointInputStream

use of alluxio.master.journal.checkpoint.CheckpointInputStream in project alluxio by Alluxio.

the class CachingInodeStoreMockedBackingStoreTest method backupRestore.

@Test
public void backupRestore() throws Exception {
    MutableInodeDirectory child = createInodeDir(10, 0);
    mStore.writeNewInode(child);
    mStore.addChild(0, child);
    mStore.writeInode(MutableInodeDirectory.create(1, 1, "blah", CreateDirectoryContext.defaults()));
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    mStore.writeToCheckpoint(baos);
    mStore.restoreFromCheckpoint(new CheckpointInputStream(new ByteArrayInputStream(baos.toByteArray())));
    assertEquals(child.getName(), mStore.get(child.getId()).get().getName());
    assertEquals(child.getId(), mStore.getChild(0L, child.getName()).get().getId());
}

Example 9 with CheckpointInputStream

use of alluxio.master.journal.checkpoint.CheckpointInputStream in project alluxio by Alluxio.

the class RocksStoreTest method backupRestore.

@Test
public void backupRestore() throws Exception {
    ColumnFamilyOptions cfOpts = new ColumnFamilyOptions().setMemTableConfig(new HashLinkedListMemTableConfig()).setCompressionType(CompressionType.NO_COMPRESSION).useFixedLengthPrefixExtractor(// We always search using the initial long key
    Longs.BYTES);
    List<ColumnFamilyDescriptor> columnDescriptors = Arrays.asList(new ColumnFamilyDescriptor("test".getBytes(), cfOpts));
    String dbDir = mFolder.newFolder("rocks").getAbsolutePath();
    String backupsDir = mFolder.newFolder("rocks-backups").getAbsolutePath();
    AtomicReference<ColumnFamilyHandle> testColumn = new AtomicReference<>();
    RocksStore store = new RocksStore("test", dbDir, backupsDir, columnDescriptors, Arrays.asList(testColumn));
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    RocksDB db = store.getDb();
    int count = 10;
    for (int i = 0; i < count; i++) {
        db.put(testColumn.get(), new WriteOptions().setDisableWAL(true), ("a" + i).getBytes(), "b".getBytes());
    }
    store.writeToCheckpoint(baos);
    String newBbDir = mFolder.newFolder("rocks-new").getAbsolutePath();
    store = new RocksStore("test-new", newBbDir, backupsDir, columnDescriptors, Arrays.asList(testColumn));
    store.restoreFromCheckpoint(new CheckpointInputStream(new ByteArrayInputStream(baos.toByteArray())));
    db = store.getDb();
    for (int i = 0; i < count; i++) {
        assertArrayEquals("b".getBytes(), db.get(testColumn.get(), ("a" + i).getBytes()));
    }
}

Example 10 with CheckpointInputStream

use of alluxio.master.journal.checkpoint.CheckpointInputStream in project alluxio by Alluxio.

the class UfsJournalDumper method dumpJournal.

@Override
public void dumpJournal() throws Throwable {
    try (UfsJournal journal = new UfsJournalSystem(getJournalLocation(mInputDir), 0).createJournal(new NoopMaster(mMaster));
        PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(mJournalEntryFile)));
        JournalReader reader = new UfsJournalReader(journal, mStart, true)) {
        boolean done = false;
        while (!done && reader.getNextSequenceNumber() < mEnd) {
            JournalReader.State state = reader.advance();
            switch(state) {
                case CHECKPOINT:
                    try (CheckpointInputStream checkpoint = reader.getCheckpoint()) {
                        Path dir = Paths.get(mCheckpointsDir + "-" + reader.getNextSequenceNumber());
                        Files.createDirectories(dir);
                        readCheckpoint(checkpoint, dir);
                    }
                    break;
                case LOG:
                    Journal.JournalEntry entry = reader.getEntry();
                    out.println(ENTRY_SEPARATOR);
                    out.print(entry);
                    break;
                case DONE:
                    done = true;
                    break;
                default:
                    throw new RuntimeException("Unknown state: " + state);
            }
        }
    }
}