Examples with JournalOutputStream alluxio.master.journal.JournalOutputStream used on opensource projects

Example 1 with JournalOutputStream

use of alluxio.master.journal.JournalOutputStream in project alluxio by Alluxio.

the class InodeTreeTest method verifyJournal.

// helper for verifying that correct objects were journaled to the output stream
private static void verifyJournal(InodeTree root, List<Inode<?>> journaled) throws Exception {
    JournalOutputStream mockOutputStream = Mockito.mock(JournalOutputStream.class);
    root.streamToJournalCheckpoint(mockOutputStream);
    for (Inode<?> node : journaled) {
        Mockito.verify(mockOutputStream).writeEntry(node.toJournalEntry());
    }
    Mockito.verifyNoMoreInteractions(mockOutputStream);
}

Example 2 with JournalOutputStream

use of alluxio.master.journal.JournalOutputStream in project alluxio by Alluxio.

the class AbstractMaster method start.

@Override
public void start(boolean isLeader) throws IOException {
    Preconditions.checkState(mExecutorService == null);
    mExecutorService = mExecutorServiceFactory.create();
    mIsLeader = isLeader;
    LOG.info("{}: Starting {} master.", getName(), mIsLeader ? "leader" : "standby");
    if (mIsLeader) {
        Preconditions.checkState(mJournal instanceof ReadWriteJournal);
        mJournalWriter = ((ReadWriteJournal) mJournal).getNewWriter();
        /**
       * The sequence for dealing with the journal before starting as the leader:
       *
       * Phase 1. Recover from a backup checkpoint if the last startup failed while writing the
       * checkpoint.
       *
       * Phase 2. Mark all the logs as completed. Since this master is the leader, it is allowed to
       * write the journal, so it can mark the current log as completed. After this step, the
       * current log file will not exist, and all logs will be complete.
       *
       * Phase 3. Reconstruct the state from the journal. This uses the JournalTailer to process all
       * of the checkpoint and the complete log files. Since all logs are complete, after this step,
       * the master will reflect the state of all of the journal entries.
       *
       * Phase 4. Write out the checkpoint file. Since this master is completely up-to-date, it
       * writes out the checkpoint file. When the checkpoint file is closed, it will then delete the
       * complete log files.
       *
       * Since this method is called before the master RPC server starts serving, there is no
       * concurrent access to the master during these phases.
       */
        // Phase 1: Recover from a backup checkpoint if necessary.
        mJournalWriter.recoverCheckpoint();
        // Phase 2: Mark all logs as complete, including the current log. After this call, the current
        // log should not exist, and all the log files will be complete.
        mJournalWriter.completeAllLogs();
        // Phase 3: Replay all the state of the checkpoint and the completed log files.
        JournalTailer catchupTailer;
        if (mStandbyJournalTailer != null && mStandbyJournalTailer.getLatestJournalTailer() != null && mStandbyJournalTailer.getLatestJournalTailer().isValid()) {
            // This master was previously in standby mode, and processed some of the journal. Re-use the
            // same tailer (still valid) to continue processing any remaining journal entries.
            LOG.info("{}: finish processing remaining journal entries (standby -> master).", getName());
            catchupTailer = mStandbyJournalTailer.getLatestJournalTailer();
            catchupTailer.processNextJournalLogFiles();
        } else {
            // This master has not successfully processed any of the journal, so create a fresh tailer
            // to process the entire journal.
            catchupTailer = new JournalTailer(this, mJournal);
            if (catchupTailer.checkpointExists()) {
                LOG.info("{}: process entire journal before becoming leader master.", getName());
                catchupTailer.processJournalCheckpoint(true);
                catchupTailer.processNextJournalLogFiles();
            } else {
                LOG.info("{}: journal checkpoint does not exist, nothing to process.", getName());
            }
        }
        long latestSequenceNumber = catchupTailer.getLatestSequenceNumber();
        // Phase 4: initialize the journal and write out the checkpoint file (the state of all
        // completed logs).
        JournalOutputStream checkpointStream = mJournalWriter.getCheckpointOutputStream(latestSequenceNumber);
        LOG.info("{}: start writing checkpoint.", getName());
        streamToJournalCheckpoint(checkpointStream);
        checkpointStream.close();
        LOG.info("{}: done with writing checkpoint.", getName());
        mAsyncJournalWriter = new AsyncJournalWriter(mJournalWriter);
    } else {
        // This master is in standby mode. Start the journal tailer thread. Since the master is in
        // standby mode, its RPC server is NOT serving. Therefore, the only thread modifying the
        // master is this journal tailer thread (no concurrent access).
        mStandbyJournalTailer = new JournalTailerThread(this, mJournal);
        mStandbyJournalTailer.start();
    }
}

Example 3 with JournalOutputStream

use of alluxio.master.journal.JournalOutputStream in project alluxio by Alluxio.

the class LineageStoreTest method journalEntrySerialization.

/**
   * Tests the {@link LineageStore#streamToJournalCheckpoint(JournalOutputStream)} method.
   */
@Test
public void journalEntrySerialization() throws Exception {
    long l1 = mLineageStore.createLineage(new ArrayList<Long>(), Lists.newArrayList(1L), mJob);
    long l2 = mLineageStore.createLineage(Lists.newArrayList(1L), Lists.newArrayList(2L), mJob);
    JournalOutputStream outputStream = Mockito.mock(JournalOutputStream.class);
    mLineageStore.streamToJournalCheckpoint(outputStream);
    Mockito.verify(outputStream).writeEntry(mLineageStore.getLineage(l1).toJournalEntry());
    Mockito.verify(outputStream).writeEntry(mLineageStore.getLineage(l2).toJournalEntry());
}