Examples with JournalContext alluxio.master.journal.JournalContext used on opensource projects

Example 6 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class RaftJournalTest method subsequentCatchups.

@Test
public void subsequentCatchups() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    // Suspend follower journal system.
    mFollowerJournalSystem.suspend(null);
    final int entryBatchCount = 5;
    // Create 2 batches of entries on the leader journal context.
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryBatchCount * 2; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    Map<String, Long> backupSequences = new HashMap<>();
    // Catch up follower journal system to first batch of entries.
    backupSequences.put("FileSystemMaster", (long) entryBatchCount - 1);
    mFollowerJournalSystem.catchup(backupSequences).waitTermination();
    // Catch up follower journal system to second batch of entries.
    backupSequences.put("FileSystemMaster", (long) (2 * entryBatchCount) - 1);
    mFollowerJournalSystem.catchup(backupSequences).waitTermination();
    // Verify master has caught up after advancing.
    Assert.assertEquals(entryBatchCount * 2, countingMaster.getApplyCount());
}

Example 7 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class RaftJournalTest method gainPrimacyAfterSuspend.

@Test
public void gainPrimacyAfterSuspend() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    // Suspend follower journal system.
    mFollowerJournalSystem.suspend(null);
    // Create entries on the leader journal context.
    // These will be replicated to follower journal context.
    final int entryCount = 10;
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryCount; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    // Assert that no entries applied by suspended journal system.
    Assert.assertEquals(0, countingMaster.getApplyCount());
    // Gain primacy in follower journal and validate it catches up.
    promoteFollower();
    CommonUtils.waitFor("full state acquired after resume", () -> mFollowerJournalSystem.getCurrentSequenceNumbers().values().stream().distinct().collect(Collectors.toList()).get(0) == entryCount - 1, mWaitOptions);
    // Follower should no longer be suspended after becoming primary.
    Assert.assertFalse(mFollowerJournalSystem.isSuspended());
}

Example 8 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class RaftJournalTest method suspendSnapshotRestart.

@Test
public void suspendSnapshotRestart() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    final int entryCount = 10;
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryCount; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    // Suspend follower journal system.
    mFollowerJournalSystem.suspend(null);
    // Write more entries which are not applied due to suspension.
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(entryCount).build()).build());
    }
    // Ask the follower to do a snapshot.
    mFollowerJournalSystem.checkpoint();
    // Restart the follower.
    mFollowerJournalSystem.stop();
    mFollowerJournalSystem.start();
    Thread.sleep(ServerConfiguration.getMs(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT));
    // Verify that all entries are replayed despite the snapshot was requested while some entries
    // are queued up during suspension.
    CommonUtils.waitFor("full state acquired after restart", () -> countingMaster.getApplyCount() == entryCount + 1, mWaitOptions);
}

Example 9 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class RaftJournalTest method gainPrimacyAfterCatchup.

@Test
public void gainPrimacyAfterCatchup() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    // Suspend follower journal system.
    mFollowerJournalSystem.suspend(null);
    // Catch up follower journal system to target-index:5.
    final long catchupIndex = 5;
    Map<String, Long> backupSequences = new HashMap<>();
    backupSequences.put("FileSystemMaster", catchupIndex);
    CatchupFuture catchupFuture = mFollowerJournalSystem.catchup(backupSequences);
    // Create entries on the leader journal context.
    // These will be replicated to follower journal context.
    final int entryCount = 10;
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryCount; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    // Wait until caught up.
    catchupFuture.waitTermination();
    Assert.assertEquals(catchupIndex + 1, countingMaster.getApplyCount());
    // Gain primacy in follower journal and validate it catches up.
    promoteFollower();
    CommonUtils.waitFor("full state acquired after resume", () -> countingMaster.getApplyCount() == entryCount, mWaitOptions);
    // Follower should no longer be suspended after becoming primary.
    Assert.assertFalse(mFollowerJournalSystem.isSuspended());
}

Example 10 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class DefaultMetaMaster method start.

@Override
public void start(Boolean isPrimary) throws IOException {
    super.start(isPrimary);
    mWorkerConfigStore.reset();
    mMasterConfigStore.reset();
    if (isPrimary) {
        // Add the configuration of the current leader master
        mMasterConfigStore.registerNewConf(mMasterAddress, ConfigurationUtils.getConfiguration(ServerConfiguration.global(), Scope.MASTER));
        // The service that detects lost standby master nodes
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_LOST_MASTER_DETECTION, new LostMasterDetectionHeartbeatExecutor(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_STANDBY_HEARTBEAT_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_LOG_CONFIG_REPORT_SCHEDULING, new LogConfigReportHeartbeatExecutor(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_LOG_CONFIG_REPORT_HEARTBEAT_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_DAILY_BACKUP_ENABLED)) {
            mDailyBackup = new DailyMetadataBackup(this, Executors.newSingleThreadScheduledExecutor(ThreadFactoryUtils.build("DailyMetadataBackup-%d", true)), mUfsManager);
            mDailyBackup.start();
        }
        if (mJournalSpaceMonitor != null) {
            getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_JOURNAL_SPACE_MONITOR, mJournalSpaceMonitor, ServerConfiguration.getMs(PropertyKey.MASTER_JOURNAL_SPACE_MONITOR_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        }
        if (mState.getClusterID().equals(INVALID_CLUSTER_ID)) {
            try (JournalContext context = createJournalContext()) {
                String clusterID = java.util.UUID.randomUUID().toString();
                mState.applyAndJournal(context, clusterID);
                LOG.info("Created new cluster ID {}", clusterID);
            }
            if (ServerConfiguration.getBoolean(PropertyKey.MASTER_UPDATE_CHECK_ENABLED) && !ServerConfiguration.getBoolean(PropertyKey.TEST_MODE)) {
                getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_UPDATE_CHECK, new UpdateChecker(this), (int) ServerConfiguration.getMs(PropertyKey.MASTER_UPDATE_CHECK_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
            }
        } else {
            LOG.info("Detected existing cluster ID {}", mState.getClusterID());
        }
        mBackupRole = new BackupLeaderRole(mCoreMasterContext);
    } else {
        if (ConfigurationUtils.isHaMode(ServerConfiguration.global())) {
            // Standby master should setup MetaMasterSync to communicate with the leader master
            RetryHandlingMetaMasterMasterClient metaMasterClient = new RetryHandlingMetaMasterMasterClient(MasterClientContext.newBuilder(ClientContext.create(ServerConfiguration.global())).build());
            getExecutorService().submit(new HeartbeatThread(HeartbeatContext.META_MASTER_SYNC, new MetaMasterSync(mMasterAddress, metaMasterClient), (int) ServerConfiguration.getMs(PropertyKey.MASTER_STANDBY_HEARTBEAT_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
            LOG.info("Standby master with address {} starts sending heartbeat to leader master.", mMasterAddress);
        }
        // Enable worker role if backup delegation is enabled.
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_BACKUP_DELEGATION_ENABLED)) {
            mBackupRole = new BackupWorkerRole(mCoreMasterContext);
        }
    }
}