Examples with ReadWriteJournal alluxio.master.journal.ReadWriteJournal used on opensource projects

Example 1 with ReadWriteJournal

use of alluxio.master.journal.ReadWriteJournal in project alluxio by Alluxio.

the class AbstractMaster method start.

@Override
public void start(boolean isLeader) throws IOException {
    Preconditions.checkState(mExecutorService == null);
    mExecutorService = mExecutorServiceFactory.create();
    mIsLeader = isLeader;
    LOG.info("{}: Starting {} master.", getName(), mIsLeader ? "leader" : "standby");
    if (mIsLeader) {
        Preconditions.checkState(mJournal instanceof ReadWriteJournal);
        mJournalWriter = ((ReadWriteJournal) mJournal).getNewWriter();
        /**
       * The sequence for dealing with the journal before starting as the leader:
       *
       * Phase 1. Recover from a backup checkpoint if the last startup failed while writing the
       * checkpoint.
       *
       * Phase 2. Mark all the logs as completed. Since this master is the leader, it is allowed to
       * write the journal, so it can mark the current log as completed. After this step, the
       * current log file will not exist, and all logs will be complete.
       *
       * Phase 3. Reconstruct the state from the journal. This uses the JournalTailer to process all
       * of the checkpoint and the complete log files. Since all logs are complete, after this step,
       * the master will reflect the state of all of the journal entries.
       *
       * Phase 4. Write out the checkpoint file. Since this master is completely up-to-date, it
       * writes out the checkpoint file. When the checkpoint file is closed, it will then delete the
       * complete log files.
       *
       * Since this method is called before the master RPC server starts serving, there is no
       * concurrent access to the master during these phases.
       */
        // Phase 1: Recover from a backup checkpoint if necessary.
        mJournalWriter.recoverCheckpoint();
        // Phase 2: Mark all logs as complete, including the current log. After this call, the current
        // log should not exist, and all the log files will be complete.
        mJournalWriter.completeAllLogs();
        // Phase 3: Replay all the state of the checkpoint and the completed log files.
        JournalTailer catchupTailer;
        if (mStandbyJournalTailer != null && mStandbyJournalTailer.getLatestJournalTailer() != null && mStandbyJournalTailer.getLatestJournalTailer().isValid()) {
            // This master was previously in standby mode, and processed some of the journal. Re-use the
            // same tailer (still valid) to continue processing any remaining journal entries.
            LOG.info("{}: finish processing remaining journal entries (standby -> master).", getName());
            catchupTailer = mStandbyJournalTailer.getLatestJournalTailer();
            catchupTailer.processNextJournalLogFiles();
        } else {
            // This master has not successfully processed any of the journal, so create a fresh tailer
            // to process the entire journal.
            catchupTailer = new JournalTailer(this, mJournal);
            if (catchupTailer.checkpointExists()) {
                LOG.info("{}: process entire journal before becoming leader master.", getName());
                catchupTailer.processJournalCheckpoint(true);
                catchupTailer.processNextJournalLogFiles();
            } else {
                LOG.info("{}: journal checkpoint does not exist, nothing to process.", getName());
            }
        }
        long latestSequenceNumber = catchupTailer.getLatestSequenceNumber();
        // Phase 4: initialize the journal and write out the checkpoint file (the state of all
        // completed logs).
        JournalOutputStream checkpointStream = mJournalWriter.getCheckpointOutputStream(latestSequenceNumber);
        LOG.info("{}: start writing checkpoint.", getName());
        streamToJournalCheckpoint(checkpointStream);
        checkpointStream.close();
        LOG.info("{}: done with writing checkpoint.", getName());
        mAsyncJournalWriter = new AsyncJournalWriter(mJournalWriter);
    } else {
        // This master is in standby mode. Start the journal tailer thread. Since the master is in
        // standby mode, its RPC server is NOT serving. Therefore, the only thread modifying the
        // master is this journal tailer thread (no concurrent access).
        mStandbyJournalTailer = new JournalTailerThread(this, mJournal);
        mStandbyJournalTailer.start();
    }
}

Example 2 with ReadWriteJournal

use of alluxio.master.journal.ReadWriteJournal in project alluxio by Alluxio.

the class JournalIntegrationTest method completedEditLogDeletion.

/**
   * Tests completed edit log deletion.
   */
@Test
public void completedEditLogDeletion() throws Exception {
    for (int i = 0; i < 124; i++) {
        mFileSystem.createFile(new AlluxioURI("/a" + i), CreateFileOptions.defaults().setBlockSizeBytes((i + 10) / 10 * 64)).close();
    }
    mLocalAlluxioCluster.stopFS();
    String journalFolder = PathUtils.concatPath(mLocalAlluxioCluster.getMaster().getJournalFolder(), Constants.FILE_SYSTEM_MASTER_NAME);
    Journal journal = new ReadWriteJournal(journalFolder);
    String completedPath = journal.getCompletedDirectory();
    Assert.assertTrue(UnderFileSystem.Factory.get(completedPath).listStatus(completedPath).length > 1);
    multiEditLogTestUtil();
    Assert.assertTrue(UnderFileSystem.Factory.get(completedPath).listStatus(completedPath).length <= 1);
    multiEditLogTestUtil();
}

Example 3 with ReadWriteJournal

use of alluxio.master.journal.ReadWriteJournal in project alluxio by Alluxio.

the class JournalIntegrationTest method deleteFsMasterJournalLogs.

private void deleteFsMasterJournalLogs() throws IOException {
    String journalFolder = mLocalAlluxioCluster.getMaster().getJournalFolder();
    Journal journal = new ReadWriteJournal(PathUtils.concatPath(journalFolder, Constants.FILE_SYSTEM_MASTER_NAME));
    UnderFileSystem.Factory.get(journalFolder).deleteFile(journal.getCurrentLogFilePath());
}

Example 4 with ReadWriteJournal

use of alluxio.master.journal.ReadWriteJournal in project alluxio by Alluxio.

the class JournalIntegrationTest method multipleFlush.

/**
   * Tests flushing the journal multiple times, without writing any data.
   */
@Test
public void multipleFlush() throws Exception {
    // Set the max log size to 0 to force a flush to write a new file.
    String existingMax = Configuration.get(PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX);
    Configuration.set(PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX, "0");
    try {
        String journalFolder = mLocalAlluxioCluster.getMaster().getJournalFolder();
        ReadWriteJournal journal = new ReadWriteJournal(PathUtils.concatPath(journalFolder, Constants.FILE_SYSTEM_MASTER_NAME));
        JournalWriter writer = journal.getNewWriter();
        writer.getCheckpointOutputStream(0).close();
        // Flush multiple times, without writing to the log.
        writer.flushEntryStream();
        writer.flushEntryStream();
        writer.flushEntryStream();
        UnderFileStatus[] paths = UnderFileSystem.Factory.get(journalFolder).listStatus(journal.getCompletedDirectory());
        // Make sure no new empty files were created.
        Assert.assertTrue(paths == null || paths.length == 0);
    } finally {
        // Reset the max log size.
        Configuration.set(PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX, existingMax);
    }
}