Examples with Journal alluxio.master.journal.Journal used on opensource projects

Example 1 with Journal

use of alluxio.master.journal.Journal in project alluxio by Alluxio.

the class RaftJournalSystem method catchUp.

/**
 * Attempts to catch up. If the master loses leadership during this method, it will return early.
 *
 * The caller is responsible for detecting and responding to leadership changes.
 */
private void catchUp(JournalStateMachine stateMachine, RaftJournalAppender client) throws TimeoutException, InterruptedException {
    long startTime = System.currentTimeMillis();
    long waitBeforeRetry = ServerConfiguration.global().getMs(PropertyKey.MASTER_EMBEDDED_JOURNAL_CATCHUP_RETRY_WAIT);
    // Wait for any outstanding snapshot to complete.
    CommonUtils.waitFor("snapshotting to finish", () -> !stateMachine.isSnapshotting(), WaitForOptions.defaults().setTimeoutMs(10 * Constants.MINUTE_MS));
    OptionalLong endCommitIndex = OptionalLong.empty();
    try {
        // affects the completion time estimate in the logs.
        synchronized (this) {
            // synchronized to appease findbugs; shouldn't make any difference
            RaftPeerId serverId = mServer.getId();
            Optional<RaftProtos.CommitInfoProto> commitInfo = getGroupInfo().getCommitInfos().stream().filter(commit -> serverId.equals(RaftPeerId.valueOf(commit.getServer().getId()))).findFirst();
            if (commitInfo.isPresent()) {
                endCommitIndex = OptionalLong.of(commitInfo.get().getCommitIndex());
            } else {
                throw new IOException("Commit info was not present. Couldn't find the current server's " + "latest commit");
            }
        }
    } catch (IOException e) {
        LogUtils.warnWithException(LOG, "Failed to get raft log information before replay." + " Replay statistics will not be available", e);
    }
    RaftJournalProgressLogger progressLogger = new RaftJournalProgressLogger(mStateMachine, endCommitIndex);
    // leader before trying again.
    while (true) {
        if (mPrimarySelector.getState() != PrimarySelector.State.PRIMARY) {
            return;
        }
        long lastAppliedSN = stateMachine.getLastAppliedSequenceNumber();
        long gainPrimacySN = ThreadLocalRandom.current().nextLong(Long.MIN_VALUE, 0);
        LOG.info("Performing catchup. Last applied SN: {}. Catchup ID: {}", lastAppliedSN, gainPrimacySN);
        Exception ex;
        try {
            CompletableFuture<RaftClientReply> future = client.sendAsync(toRaftMessage(JournalEntry.newBuilder().setSequenceNumber(gainPrimacySN).build()), TimeDuration.valueOf(5, TimeUnit.SECONDS));
            RaftClientReply reply = future.get(5, TimeUnit.SECONDS);
            ex = reply.getException();
        } catch (TimeoutException | ExecutionException | IOException e) {
            ex = e;
        }
        if (ex != null) {
            // LeaderNotReadyException typically indicates Ratis is still replaying the journal.
            if (ex instanceof LeaderNotReadyException) {
                progressLogger.logProgress();
            } else {
                LOG.info("Exception submitting term start entry: {}", ex.toString());
            }
            // avoid excessive retries when server is not ready
            Thread.sleep(waitBeforeRetry);
            continue;
        }
        try {
            CommonUtils.waitFor("term start entry " + gainPrimacySN + " to be applied to state machine", () -> stateMachine.getLastPrimaryStartSequenceNumber() == gainPrimacySN, WaitForOptions.defaults().setInterval(Constants.SECOND_MS).setTimeoutMs(5 * Constants.SECOND_MS));
        } catch (TimeoutException e) {
            LOG.info(e.toString());
            continue;
        }
        // are not leader.
        try {
            CommonUtils.waitFor("check primacySN " + gainPrimacySN + " and lastAppliedSN " + lastAppliedSN + " to be applied to leader", () -> stateMachine.getLastAppliedSequenceNumber() == lastAppliedSN && stateMachine.getLastPrimaryStartSequenceNumber() == gainPrimacySN, WaitForOptions.defaults().setInterval(Constants.SECOND_MS).setTimeoutMs((int) mConf.getMaxElectionTimeoutMs()));
        } catch (TimeoutException e) {
            // Restart the catchup process.
            continue;
        }
        LOG.info("Caught up in {}ms. Last sequence number from previous term: {}.", System.currentTimeMillis() - startTime, stateMachine.getLastAppliedSequenceNumber());
        return;
    }
}

Example 2 with Journal

use of alluxio.master.journal.Journal in project alluxio by Alluxio.

the class RaftJournalSystem method startInternal.

@Override
public synchronized void startInternal() throws InterruptedException, IOException {
    LOG.info("Initializing Raft Journal System");
    InetSocketAddress localAddress = mConf.getLocalAddress();
    mPeerId = RaftJournalUtils.getPeerId(localAddress);
    List<InetSocketAddress> addresses = mConf.getClusterAddresses();
    Set<RaftPeer> peers = addresses.stream().map(addr -> RaftPeer.newBuilder().setId(RaftJournalUtils.getPeerId(addr)).setAddress(addr).build()).collect(Collectors.toSet());
    mRaftGroup = RaftGroup.valueOf(RAFT_GROUP_ID, peers);
    initServer();
    super.registerMetrics();
    List<InetSocketAddress> clusterAddresses = mConf.getClusterAddresses();
    LOG.info("Starting Raft journal system. Cluster addresses: {}. Local address: {}", clusterAddresses, mConf.getLocalAddress());
    long startTime = System.currentTimeMillis();
    try {
        mServer.start();
    } catch (IOException e) {
        String errorMessage = ExceptionMessage.FAILED_RAFT_BOOTSTRAP.getMessage(Arrays.toString(clusterAddresses.toArray()), e.getCause() == null ? e : e.getCause().toString());
        throw new IOException(errorMessage, e.getCause());
    }
    LOG.info("Started Raft Journal System in {}ms", System.currentTimeMillis() - startTime);
    joinQuorum();
}

Example 3 with Journal

use of alluxio.master.journal.Journal in project alluxio by Alluxio.

the class JournalIntegrationTest method completedEditLogDeletion.

/**
   * Tests completed edit log deletion.
   */
@Test
public void completedEditLogDeletion() throws Exception {
    for (int i = 0; i < 124; i++) {
        mFileSystem.createFile(new AlluxioURI("/a" + i), CreateFileOptions.defaults().setBlockSizeBytes((i + 10) / 10 * 64)).close();
    }
    mLocalAlluxioCluster.stopFS();
    String journalFolder = PathUtils.concatPath(mLocalAlluxioCluster.getMaster().getJournalFolder(), Constants.FILE_SYSTEM_MASTER_NAME);
    Journal journal = new ReadWriteJournal(journalFolder);
    String completedPath = journal.getCompletedDirectory();
    Assert.assertTrue(UnderFileSystem.Factory.get(completedPath).listStatus(completedPath).length > 1);
    multiEditLogTestUtil();
    Assert.assertTrue(UnderFileSystem.Factory.get(completedPath).listStatus(completedPath).length <= 1);
    multiEditLogTestUtil();
}

Example 4 with Journal

use of alluxio.master.journal.Journal in project alluxio by Alluxio.

the class JournalIntegrationTest method deleteFsMasterJournalLogs.

private void deleteFsMasterJournalLogs() throws IOException {
    String journalFolder = mLocalAlluxioCluster.getMaster().getJournalFolder();
    Journal journal = new ReadWriteJournal(PathUtils.concatPath(journalFolder, Constants.FILE_SYSTEM_MASTER_NAME));
    UnderFileSystem.Factory.get(journalFolder).deleteFile(journal.getCurrentLogFilePath());
}

Example 5 with Journal

use of alluxio.master.journal.Journal in project alluxio by Alluxio.

the class KeyValueMasterFactory method create.

@Override
public KeyValueMaster create(List<? extends Master> masters, JournalFactory journalFactory) {
    if (!isEnabled()) {
        return null;
    }
    Preconditions.checkArgument(journalFactory != null, "journal factory may not be null");
    LOG.info("Creating {} ", KeyValueMaster.class.getName());
    Journal journal = journalFactory.get(getName());
    for (Master master : masters) {
        if (master instanceof FileSystemMaster) {
            LOG.info("{} is created", KeyValueMaster.class.getName());
            return new KeyValueMaster((FileSystemMaster) master, journal);
        }
    }
    LOG.error("Fail to create {} due to missing {}", KeyValueMaster.class.getName(), FileSystemMaster.class.getName());
    return null;
}