Examples with NoopMaster alluxio.master.NoopMaster used on opensource projects

Example 1 with NoopMaster

use of alluxio.master.NoopMaster in project alluxio by Alluxio.

the class Format method format.

/**
 * Formats the Alluxio file system.
 *
 * @param mode either {@code MASTER} or {@code WORKER}
 * @param alluxioConf Alluxio configuration
 */
public static void format(Mode mode, AlluxioConfiguration alluxioConf) throws IOException {
    NoopUfsManager noopUfsManager = new NoopUfsManager();
    switch(mode) {
        case MASTER:
            URI journalLocation = JournalUtils.getJournalLocation();
            LOG.info("Formatting master journal: {}", journalLocation);
            JournalSystem journalSystem = new JournalSystem.Builder().setLocation(journalLocation).build(CommonUtils.ProcessType.MASTER);
            for (String masterServiceName : ServiceUtils.getMasterServiceNames()) {
                journalSystem.createJournal(new NoopMaster(masterServiceName, noopUfsManager));
            }
            journalSystem.format();
            break;
        case WORKER:
            String workerDataFolder = ServerConfiguration.getString(PropertyKey.WORKER_DATA_FOLDER);
            LOG.info("Formatting worker data folder: {}", workerDataFolder);
            int storageLevels = ServerConfiguration.getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS);
            for (int level = 0; level < storageLevels; level++) {
                PropertyKey tierLevelDirPath = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(level);
                String[] dirPaths = ServerConfiguration.getString(tierLevelDirPath).split(",");
                String name = "Data path for tier " + level;
                for (String dirPath : dirPaths) {
                    String dirWorkerDataFolder = CommonUtils.getWorkerDataDirectory(dirPath, alluxioConf);
                    LOG.info("Formatting {}:{}", name, dirWorkerDataFolder);
                    formatWorkerDataFolder(dirWorkerDataFolder);
                }
            }
            break;
        default:
            throw new RuntimeException(String.format("Unrecognized format mode: %s", mode));
    }
}

Example 2 with NoopMaster

use of alluxio.master.NoopMaster in project alluxio by Alluxio.

the class RaftJournalTest method subsequentCatchups.

@Test
public void subsequentCatchups() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    // Suspend follower journal system.
    mFollowerJournalSystem.suspend(null);
    final int entryBatchCount = 5;
    // Create 2 batches of entries on the leader journal context.
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryBatchCount * 2; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    Map<String, Long> backupSequences = new HashMap<>();
    // Catch up follower journal system to first batch of entries.
    backupSequences.put("FileSystemMaster", (long) entryBatchCount - 1);
    mFollowerJournalSystem.catchup(backupSequences).waitTermination();
    // Catch up follower journal system to second batch of entries.
    backupSequences.put("FileSystemMaster", (long) (2 * entryBatchCount) - 1);
    mFollowerJournalSystem.catchup(backupSequences).waitTermination();
    // Verify master has caught up after advancing.
    Assert.assertEquals(entryBatchCount * 2, countingMaster.getApplyCount());
}

Example 3 with NoopMaster

use of alluxio.master.NoopMaster in project alluxio by Alluxio.

the class RaftJournalTest method gainPrimacyAfterSuspend.

@Test
public void gainPrimacyAfterSuspend() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    // Suspend follower journal system.
    mFollowerJournalSystem.suspend(null);
    // Create entries on the leader journal context.
    // These will be replicated to follower journal context.
    final int entryCount = 10;
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryCount; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    // Assert that no entries applied by suspended journal system.
    Assert.assertEquals(0, countingMaster.getApplyCount());
    // Gain primacy in follower journal and validate it catches up.
    promoteFollower();
    CommonUtils.waitFor("full state acquired after resume", () -> mFollowerJournalSystem.getCurrentSequenceNumbers().values().stream().distinct().collect(Collectors.toList()).get(0) == entryCount - 1, mWaitOptions);
    // Follower should no longer be suspended after becoming primary.
    Assert.assertFalse(mFollowerJournalSystem.isSuspended());
}

Example 4 with NoopMaster

use of alluxio.master.NoopMaster in project alluxio by Alluxio.

the class RaftJournalTest method suspendSnapshotRestart.

@Test
public void suspendSnapshotRestart() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    final int entryCount = 10;
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryCount; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    // Suspend follower journal system.
    mFollowerJournalSystem.suspend(null);
    // Write more entries which are not applied due to suspension.
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(entryCount).build()).build());
    }
    // Ask the follower to do a snapshot.
    mFollowerJournalSystem.checkpoint();
    // Restart the follower.
    mFollowerJournalSystem.stop();
    mFollowerJournalSystem.start();
    Thread.sleep(ServerConfiguration.getMs(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT));
    // Verify that all entries are replayed despite the snapshot was requested while some entries
    // are queued up during suspension.
    CommonUtils.waitFor("full state acquired after restart", () -> countingMaster.getApplyCount() == entryCount + 1, mWaitOptions);
}

Example 5 with NoopMaster

use of alluxio.master.NoopMaster in project alluxio by Alluxio.

the class RaftJournalTest method gainPrimacyAfterCatchup.

@Test
public void gainPrimacyAfterCatchup() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    // Suspend follower journal system.
    mFollowerJournalSystem.suspend(null);
    // Catch up follower journal system to target-index:5.
    final long catchupIndex = 5;
    Map<String, Long> backupSequences = new HashMap<>();
    backupSequences.put("FileSystemMaster", catchupIndex);
    CatchupFuture catchupFuture = mFollowerJournalSystem.catchup(backupSequences);
    // Create entries on the leader journal context.
    // These will be replicated to follower journal context.
    final int entryCount = 10;
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryCount; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    // Wait until caught up.
    catchupFuture.waitTermination();
    Assert.assertEquals(catchupIndex + 1, countingMaster.getApplyCount());
    // Gain primacy in follower journal and validate it catches up.
    promoteFollower();
    CommonUtils.waitFor("full state acquired after resume", () -> countingMaster.getApplyCount() == entryCount, mWaitOptions);
    // Follower should no longer be suspended after becoming primary.
    Assert.assertFalse(mFollowerJournalSystem.isSuspended());
}