Examples with NoopMaster alluxio.master.NoopMaster used on opensource projects

Example 16 with NoopMaster

use of alluxio.master.NoopMaster in project alluxio by Alluxio.

the class RaftJournalTest method gainPrimacyDuringCatchup.

@Test
public void gainPrimacyDuringCatchup() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    // Using a large entry count for catching transition while in-progress.
    final int entryCount = 100;
    // Suspend follower journal system.
    mFollowerJournalSystem.suspend(null);
    // Create entries on the leader journal context.
    // These will be replicated to follower journal context once resumed.
    ForkJoinPool.commonPool().submit(() -> {
        try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
            for (int i = 0; i < entryCount; i++) {
                journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
            }
        } catch (Exception e) {
            Assert.fail(String.format("Failed while writing entries: %s", e.toString()));
        }
    }).get();
    // Catch up follower journal to a large index to be able to transition while in progress.
    Map<String, Long> backupSequences = new HashMap<>();
    backupSequences.put("FileSystemMaster", (long) entryCount);
    // Set delay for each internal processing of entries before initiating catch-up.
    countingMaster.setApplyDelay(100);
    CatchupFuture catchupFuture = mFollowerJournalSystem.catchup(backupSequences);
    // Wait until advancing starts.
    CommonUtils.waitFor("Advancing to start.", () -> countingMaster.getApplyCount() > 0, mWaitOptions);
    // Gain primacy in follower journal
    promoteFollower();
    // Validate it catches up.
    CommonUtils.waitFor("Old olf follower to catch up.", () -> countingMaster.getApplyCount() == entryCount, mWaitOptions);
    // Follower should no longer be suspended after becoming primary.
    Assert.assertFalse(mFollowerJournalSystem.isSuspended());
}

Example 17 with NoopMaster

use of alluxio.master.NoopMaster in project alluxio by Alluxio.

the class UfsJournalDumper method dumpJournal.

@Override
public void dumpJournal() throws Throwable {
    try (UfsJournal journal = new UfsJournalSystem(getJournalLocation(mInputDir), 0).createJournal(new NoopMaster(mMaster));
        PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(mJournalEntryFile)));
        JournalReader reader = new UfsJournalReader(journal, mStart, true)) {
        boolean done = false;
        while (!done && reader.getNextSequenceNumber() < mEnd) {
            JournalReader.State state = reader.advance();
            switch(state) {
                case CHECKPOINT:
                    try (CheckpointInputStream checkpoint = reader.getCheckpoint()) {
                        Path dir = Paths.get(mCheckpointsDir + "-" + reader.getNextSequenceNumber());
                        Files.createDirectories(dir);
                        readCheckpoint(checkpoint, dir);
                    }
                    break;
                case LOG:
                    Journal.JournalEntry entry = reader.getEntry();
                    out.println(ENTRY_SEPARATOR);
                    out.print(entry);
                    break;
                case DONE:
                    done = true;
                    break;
                default:
                    throw new RuntimeException("Unknown state: " + state);
            }
        }
    }
}

Example 18 with NoopMaster

use of alluxio.master.NoopMaster in project alluxio by Alluxio.

the class UfsJournalIntegrationTest method multipleFlush.

/**
 * Tests flushing the journal multiple times, without writing any data.
 */
@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX, "0" })
public void multipleFlush() throws Exception {
    String journalFolder = mLocalAlluxioCluster.getLocalAlluxioMaster().getJournalFolder();
    mLocalAlluxioCluster.stop();
    UfsJournal journal = new UfsJournal(new URI(PathUtils.concatPath(journalFolder, Constants.FILE_SYSTEM_MASTER_NAME)), new NoopMaster(), 0, Collections::emptySet);
    journal.start();
    journal.gainPrimacy();
    UfsStatus[] paths = UnderFileSystem.Factory.create(journalFolder, ServerConfiguration.global()).listStatus(journal.getLogDir().toString());
    int expectedSize = paths == null ? 0 : paths.length;
    journal.flush();
    journal.flush();
    journal.flush();
    journal.close();
    paths = UnderFileSystem.Factory.create(journalFolder, ServerConfiguration.global()).listStatus(journal.getLogDir().toString());
    int actualSize = paths == null ? 0 : paths.length;
    // No new files are created.
    Assert.assertEquals(expectedSize, actualSize);
}

Example 19 with NoopMaster

use of alluxio.master.NoopMaster in project alluxio by Alluxio.

the class IntegrationTestUtils method waitForUfsJournalCheckpoint.

/**
 * Waits for a checkpoint to be written in the specified master's journal.
 *
 * @param masterName the name of the master
 * @param journalLocation the location of the journal
 */
public static void waitForUfsJournalCheckpoint(String masterName, URI journalLocation) throws TimeoutException, InterruptedException {
    UfsJournal journal = new UfsJournal(URIUtils.appendPathOrDie(journalLocation, masterName), new NoopMaster(""), 0, Collections::emptySet);
    CommonUtils.waitFor("checkpoint to be written", () -> {
        UfsJournalSnapshot snapshot;
        try {
            snapshot = UfsJournalSnapshot.getSnapshot(journal);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return !snapshot.getCheckpoints().isEmpty();
    });
}

Example 20 with NoopMaster

use of alluxio.master.NoopMaster in project alluxio by Alluxio.

the class UfsJournalIntegrationTest method completedEditLogDeletion.

/**
 * Tests completed edit log deletion.
 */
@Test
public void completedEditLogDeletion() throws Exception {
    for (int i = 0; i < 124; i++) {
        mFileSystem.createFile(new AlluxioURI("/a" + i), CreateFilePOptions.newBuilder().setBlockSizeBytes((i + 10) / 10 * 64).build()).close();
    }
    mLocalAlluxioCluster.stopFS();
    String journalFolder = PathUtils.concatPath(mLocalAlluxioCluster.getLocalAlluxioMaster().getJournalFolder(), Constants.FILE_SYSTEM_MASTER_NAME);
    UfsJournal journal = new UfsJournal(new URI(journalFolder), new NoopMaster(), 0, Collections::emptySet);
    URI completedLocation = journal.getLogDir();
    Assert.assertTrue(UnderFileSystem.Factory.create(completedLocation.toString(), UnderFileSystemConfiguration.defaults(ServerConfiguration.global())).listStatus(completedLocation.toString()).length > 1);
    multiEditLogTestUtil();
    Assert.assertTrue(UnderFileSystem.Factory.create(completedLocation.toString(), UnderFileSystemConfiguration.defaults(ServerConfiguration.global())).listStatus(completedLocation.toString()).length > 1);
    multiEditLogTestUtil();
}