Examples with CheckpointInputStream alluxio.master.journal.checkpoint.CheckpointInputStream used on opensource projects

Example 1 with CheckpointInputStream

use of alluxio.master.journal.checkpoint.CheckpointInputStream in project alluxio by Alluxio.

the class JournalUtilsTest method checkpointAndRestore.

@Test
public void checkpointAndRestore() throws IOException, InterruptedException {
    Journaled journaled = new TestJournaled(0);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    JournalUtils.writeJournalEntryCheckpoint(baos, journaled);
    JournalUtils.restoreJournalEntryCheckpoint(new CheckpointInputStream(new ByteArrayInputStream(baos.toByteArray())), journaled);
}

Example 2 with CheckpointInputStream

use of alluxio.master.journal.checkpoint.CheckpointInputStream in project alluxio by Alluxio.

the class JournalUtilsTest method restoreInvalidJournalEntryCheckpoint.

@Test
public void restoreInvalidJournalEntryCheckpoint() throws IOException {
    Journaled journaled = new TestJournaled(0);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    // Checkpoint version doesn't match Constants.JOURNAL_ENTRY_CHECKPOINT_VERSION.
    CheckpointOutputStream cos = new CheckpointOutputStream(baos, CheckpointType.COMPOUND);
    cos.flush();
    mThrown.expect(IllegalStateException.class);
    mThrown.expectMessage("Unrecognized checkpoint type");
    JournalUtils.restoreJournalEntryCheckpoint(new CheckpointInputStream(new ByteArrayInputStream(baos.toByteArray())), journaled);
}

Example 3 with CheckpointInputStream

use of alluxio.master.journal.checkpoint.CheckpointInputStream in project alluxio by Alluxio.

the class JournalStateMachine method install.

private void install(File snapshotFile) {
    if (mClosed) {
        return;
    }
    if (mIgnoreApplys) {
        LOG.warn("Unexpected request to install a snapshot on a read-only journal state machine");
        return;
    }
    long snapshotId = 0L;
    try (Timer.Context ctx = MetricsSystem.timer(MetricKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_REPLAY_TIMER.getName()).time();
        DataInputStream stream = new DataInputStream(new FileInputStream(snapshotFile))) {
        snapshotId = stream.readLong();
        JournalUtils.restoreFromCheckpoint(new CheckpointInputStream(stream), getStateMachines());
    } catch (Exception e) {
        JournalUtils.handleJournalReplayFailure(LOG, e, "Failed to install snapshot: %s", snapshotId);
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_JOURNAL_TOLERATE_CORRUPTION)) {
            return;
        }
    }
    if (snapshotId < mNextSequenceNumberToRead - 1) {
        LOG.warn("Installed snapshot for SN {} but next SN to read is {}", snapshotId, mNextSequenceNumberToRead);
    }
    mNextSequenceNumberToRead = snapshotId + 1;
    LOG.info("Successfully installed snapshot up to SN {}", snapshotId);
}

Example 4 with CheckpointInputStream

use of alluxio.master.journal.checkpoint.CheckpointInputStream in project alluxio by Alluxio.

the class UfsJournalReader method updateInputStream.

/**
 * Updates the journal input stream by closing the current journal input stream if it is done and
 * opening a new one.
 */
private void updateInputStream() throws IOException {
    if (mInputStream != null && (mInputStream.mFile.isIncompleteLog() || !mInputStream.isDone(mNextSequenceNumber))) {
        return;
    }
    if (mInputStream != null) {
        mInputStream.close();
        mInputStream = null;
    }
    if (mFilesToProcess.isEmpty()) {
        UfsJournalSnapshot snapshot = UfsJournalSnapshot.getSnapshot(mJournal);
        if (snapshot.getCheckpoints().isEmpty() && snapshot.getLogs().isEmpty()) {
            return;
        }
        int index = 0;
        if (!snapshot.getCheckpoints().isEmpty()) {
            UfsJournalFile checkpoint = snapshot.getLatestCheckpoint();
            if (mNextSequenceNumber < checkpoint.getEnd()) {
                String location = checkpoint.getLocation().toString();
                LOG.info("Reading checkpoint {}", location);
                mCheckpointStream = new CheckpointInputStream(mUfs.open(location, OpenOptions.defaults().setRecoverFailedOpen(true)));
                mNextSequenceNumber = checkpoint.getEnd();
            }
            for (; index < snapshot.getLogs().size(); index++) {
                UfsJournalFile file = snapshot.getLogs().get(index);
                if (file.getEnd() > checkpoint.getEnd()) {
                    break;
                }
            }
        // index now points to the first log with mEnd > checkpoint.mEnd.
        }
        for (; index < snapshot.getLogs().size(); index++) {
            UfsJournalFile file = snapshot.getLogs().get(index);
            if ((!mReadIncompleteLog && file.isIncompleteLog()) || mNextSequenceNumber >= file.getEnd()) {
                continue;
            }
            mFilesToProcess.add(snapshot.getLogs().get(index));
        }
    }
    if (!mFilesToProcess.isEmpty()) {
        mInputStream = new JournalInputStream(mFilesToProcess.poll(), mUfs);
    }
}

Example 5 with CheckpointInputStream

use of alluxio.master.journal.checkpoint.CheckpointInputStream in project alluxio by Alluxio.

the class JournalUtilsTest method checkpointAndRestoreComponents.

@Test
public void checkpointAndRestoreComponents() throws Exception {
    List<TestJournaled> components = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        components.add(new TestJournaled(i));
    }
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    JournalUtils.writeToCheckpoint(baos, components);
    components.forEach(c -> assertEquals(0, c.getNumEntriesProcessed()));
    JournalUtils.restoreFromCheckpoint(new CheckpointInputStream(new ByteArrayInputStream(baos.toByteArray())), components);
    components.forEach(c -> assertEquals(1, c.getNumEntriesProcessed()));
}