Examples with UfsJournal alluxio.master.journal.ufs.UfsJournal used on opensource projects

Example 1 with UfsJournal

use of alluxio.master.journal.ufs.UfsJournal in project alluxio by Alluxio.

the class UfsJournalIntegrationTest method deleteFsMasterJournalLogs.

private void deleteFsMasterJournalLogs() throws Exception {
    String journalFolder = mLocalAlluxioCluster.getLocalAlluxioMaster().getJournalFolder();
    UfsJournal journal = new UfsJournal(new URI(PathUtils.concatPath(journalFolder, Constants.FILE_SYSTEM_MASTER_NAME)), new NoopMaster(), 0, Collections::emptySet);
    if (UfsJournalSnapshot.getCurrentLog(journal) != null) {
        UnderFileSystem.Factory.create(journalFolder, ServerConfiguration.global()).deleteFile(UfsJournalSnapshot.getCurrentLog(journal).getLocation().toString());
    }
}

Example 2 with UfsJournal

use of alluxio.master.journal.ufs.UfsJournal in project alluxio by Alluxio.

the class TriggeredCheckpointTest method ufsJournal.

@Test
public void ufsJournal() throws Exception {
    int numFiles = 100;
    MultiProcessCluster cluster = MultiProcessCluster.newBuilder(PortCoordination.TRIGGERED_UFS_CHECKPOINT).setClusterName("TriggeredUfsCheckpointTest").addProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.UFS.toString()).addProperty(PropertyKey.MASTER_JOURNAL_CHECKPOINT_PERIOD_ENTRIES, String.valueOf(numFiles)).setNumMasters(1).setNumWorkers(1).build();
    try {
        cluster.start();
        cluster.waitForAllNodesRegistered(20 * Constants.SECOND_MS);
        // Get enough journal entries
        createFiles(cluster, numFiles);
        // Trigger checkpoint
        Assert.assertEquals(cluster.getMasterAddresses().get(0).getHostname(), cluster.getMetaMasterClient().checkpoint());
        String journalLocation = cluster.getJournalDir();
        UfsJournal ufsJournal = new UfsJournal(URIUtils.appendPathOrDie(new URI(journalLocation), Constants.FILE_SYSTEM_MASTER_NAME), new NoopMaster(""), 0, Collections::emptySet);
        Assert.assertEquals(1, UfsJournalSnapshot.getSnapshot(ufsJournal).getCheckpoints().size());
        validateCheckpointInClusterRestart(cluster);
        cluster.notifySuccess();
    } finally {
        cluster.destroy();
    }
}

Example 3 with UfsJournal

use of alluxio.master.journal.ufs.UfsJournal in project alluxio by Alluxio.

the class UfsJournalDumper method dumpJournal.

@Override
public void dumpJournal() throws Throwable {
    try (UfsJournal journal = new UfsJournalSystem(getJournalLocation(mInputDir), 0).createJournal(new NoopMaster(mMaster));
        PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(mJournalEntryFile)));
        JournalReader reader = new UfsJournalReader(journal, mStart, true)) {
        boolean done = false;
        while (!done && reader.getNextSequenceNumber() < mEnd) {
            JournalReader.State state = reader.advance();
            switch(state) {
                case CHECKPOINT:
                    try (CheckpointInputStream checkpoint = reader.getCheckpoint()) {
                        Path dir = Paths.get(mCheckpointsDir + "-" + reader.getNextSequenceNumber());
                        Files.createDirectories(dir);
                        readCheckpoint(checkpoint, dir);
                    }
                    break;
                case LOG:
                    Journal.JournalEntry entry = reader.getEntry();
                    out.println(ENTRY_SEPARATOR);
                    out.print(entry);
                    break;
                case DONE:
                    done = true;
                    break;
                default:
                    throw new RuntimeException("Unknown state: " + state);
            }
        }
    }
}

Example 4 with UfsJournal

use of alluxio.master.journal.ufs.UfsJournal in project alluxio by Alluxio.

the class UfsJournalIntegrationTest method multipleFlush.

/**
 * Tests flushing the journal multiple times, without writing any data.
 */
@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX, "0" })
public void multipleFlush() throws Exception {
    String journalFolder = mLocalAlluxioCluster.getLocalAlluxioMaster().getJournalFolder();
    mLocalAlluxioCluster.stop();
    UfsJournal journal = new UfsJournal(new URI(PathUtils.concatPath(journalFolder, Constants.FILE_SYSTEM_MASTER_NAME)), new NoopMaster(), 0, Collections::emptySet);
    journal.start();
    journal.gainPrimacy();
    UfsStatus[] paths = UnderFileSystem.Factory.create(journalFolder, ServerConfiguration.global()).listStatus(journal.getLogDir().toString());
    int expectedSize = paths == null ? 0 : paths.length;
    journal.flush();
    journal.flush();
    journal.flush();
    journal.close();
    paths = UnderFileSystem.Factory.create(journalFolder, ServerConfiguration.global()).listStatus(journal.getLogDir().toString());
    int actualSize = paths == null ? 0 : paths.length;
    // No new files are created.
    Assert.assertEquals(expectedSize, actualSize);
}

Example 5 with UfsJournal

use of alluxio.master.journal.ufs.UfsJournal in project alluxio by Alluxio.

the class IntegrationTestUtils method waitForUfsJournalCheckpoint.

/**
 * Waits for a checkpoint to be written in the specified master's journal.
 *
 * @param masterName the name of the master
 * @param journalLocation the location of the journal
 */
public static void waitForUfsJournalCheckpoint(String masterName, URI journalLocation) throws TimeoutException, InterruptedException {
    UfsJournal journal = new UfsJournal(URIUtils.appendPathOrDie(journalLocation, masterName), new NoopMaster(""), 0, Collections::emptySet);
    CommonUtils.waitFor("checkpoint to be written", () -> {
        UfsJournalSnapshot snapshot;
        try {
            snapshot = UfsJournalSnapshot.getSnapshot(journal);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return !snapshot.getCheckpoints().isEmpty();
    });
}