Examples with JournalEntryStreamReader alluxio.master.journal.JournalEntryStreamReader used on opensource projects

Example 1 with JournalEntryStreamReader

use of alluxio.master.journal.JournalEntryStreamReader in project alluxio by Alluxio.

the class BackupManager method initFromBackup.

/**
 * Restores master state from the specified backup.
 *
 * @param is an input stream to read from the backup
 */
public void initFromBackup(InputStream is) throws IOException {
    try (GzipCompressorInputStream gzIn = new GzipCompressorInputStream(is);
        JournalEntryStreamReader reader = new JournalEntryStreamReader(gzIn)) {
        List<Master> masters = mRegistry.getServers();
        // Executor for applying backup.
        CompletionService<Boolean> completionService = new ExecutorCompletionService<>(Executors.newFixedThreadPool(2, ThreadFactoryUtils.build("master-backup-%d", true)));
        // List of active tasks.
        Set<Future<?>> activeTasks = new HashSet<>();
        // Entry queue will be used as a buffer and synchronization between readers and appliers.
        LinkedBlockingQueue<JournalEntry> journalEntryQueue = new LinkedBlockingQueue<>(ServerConfiguration.getInt(PropertyKey.MASTER_BACKUP_ENTRY_BUFFER_COUNT));
        // Whether still reading from backup.
        AtomicBoolean readingActive = new AtomicBoolean(true);
        // Index masters by name.
        Map<String, Master> mastersByName = Maps.uniqueIndex(masters, Master::getName);
        // Shows how many entries have been applied.
        AtomicLong appliedEntryCount = new AtomicLong(0);
        // Progress executor
        ScheduledExecutorService traceExecutor = Executors.newScheduledThreadPool(1, ThreadFactoryUtils.build("master-backup-tracer-%d", true));
        traceExecutor.scheduleAtFixedRate(() -> {
            LOG.info("{} entries from backup applied so far...", appliedEntryCount.get());
        }, 30, 30, TimeUnit.SECONDS);
        // Start the timer for backup metrics.
        long startRestoreTime = System.currentTimeMillis();
        // Create backup reader task.
        activeTasks.add(completionService.submit(() -> {
            try {
                JournalEntry entry;
                while ((entry = reader.readEntry()) != null) {
                    journalEntryQueue.put(entry);
                }
                // Put termination entry for signaling the applier.
                journalEntryQueue.put(JournalEntry.newBuilder().setSequenceNumber(TERMINATION_SEQ).build());
                return true;
            } catch (InterruptedException ie) {
                // Continue interrupt chain.
                Thread.currentThread().interrupt();
                throw new RuntimeException("Thread interrupted while reading from backup stream.", ie);
            } finally {
                readingActive.set(false);
            }
        }));
        // Create applier task.
        activeTasks.add(completionService.submit(() -> {
            try {
                // Read entries from backup.
                while (readingActive.get() || journalEntryQueue.size() > 0) {
                    // Drain current elements.
                    // Draining entries makes it possible to allow writes while current ones are
                    // being applied.
                    List<JournalEntry> drainedEntries = new LinkedList<>();
                    if (0 == journalEntryQueue.drainTo(drainedEntries)) {
                        // No elements at the moment. Fall back to polling.
                        JournalEntry entry = journalEntryQueue.poll(10, TimeUnit.MILLISECONDS);
                        if (entry == null) {
                            // No entry yet.
                            continue;
                        }
                        drainedEntries.add(entry);
                    }
                    // Apply drained entries.
                    // Map for storing journal contexts.
                    Map<Master, JournalContext> masterJCMap = new HashMap<>();
                    try {
                        // They should be closed after applying drained entries.
                        for (Master master : masters) {
                            masterJCMap.put(master, master.createJournalContext());
                        }
                        // Apply entries.
                        for (JournalEntry entry : drainedEntries) {
                            // Check for termination entry.
                            if (entry.getSequenceNumber() == TERMINATION_SEQ) {
                                // Reading finished.
                                return true;
                            }
                            String masterName;
                            try {
                                masterName = JournalEntryAssociation.getMasterForEntry(entry);
                            } catch (IllegalStateException ise) {
                                ProcessUtils.fatalError(LOG, ise, "Unrecognized journal entry: %s", entry);
                                throw ise;
                            }
                            try {
                                Master master = mastersByName.get(masterName);
                                master.applyAndJournal(masterJCMap.get(master), entry);
                                appliedEntryCount.incrementAndGet();
                            } catch (Exception e) {
                                JournalUtils.handleJournalReplayFailure(LOG, e, "Failed to apply " + "journal entry to master %s. Entry: %s", masterName, entry);
                            }
                        }
                    } finally {
                        // before next round.
                        for (JournalContext journalContext : masterJCMap.values()) {
                            journalContext.close();
                        }
                    }
                }
                return true;
            } catch (InterruptedException ie) {
                // Continue interrupt chain.
                Thread.currentThread().interrupt();
                throw new RuntimeException("Thread interrupted while applying backup content.", ie);
            }
        }));
        // Wait until backup tasks are completed and stop metrics timer.
        try {
            safeWaitTasks(activeTasks, completionService);
        } finally {
            mRestoreTimeMs = System.currentTimeMillis() - startRestoreTime;
            mRestoreEntriesCount = appliedEntryCount.get();
            traceExecutor.shutdownNow();
        }
        LOG.info("Restored {} entries from backup", appliedEntryCount.get());
    }
}

Example 2 with JournalEntryStreamReader

use of alluxio.master.journal.JournalEntryStreamReader in project alluxio by Alluxio.

the class UfsJournalLogWriter method recoverLastPersistedJournalEntry.

/**
 * Examine the UFS to determine the most recent journal entry, and return its sequence number.
 *
 * 1. Locate the most recent incomplete journal file, i.e. journal file that starts with
 *    a valid sequence number S (hex), and ends with 0x7fffffffffffffff. The journal file
 *    name encodes this information, i.e. S-0x7fffffffffffffff.
 * 2. Sequentially scan the incomplete journal file, and identify the last journal
 *    entry that has been persisted in UFS. Suppose it is X.
 * 3. Rename the incomplete journal file to S-<X+1>. Future journal writes will write to
 *    a new file named <X+1>-0x7fffffffffffffff.
 * 4. If the incomplete journal does not exist or no entry can be found in the incomplete
 *    journal, check the last complete journal file for the last persisted journal entry.
 *
 * @return sequence number of the last persisted journal entry, or -1 if no entry can be found
 */
private long recoverLastPersistedJournalEntry() throws IOException {
    UfsJournalSnapshot snapshot = UfsJournalSnapshot.getSnapshot(mJournal);
    long lastPersistSeq = -1;
    UfsJournalFile currentLog = snapshot.getCurrentLog(mJournal);
    if (currentLog != null) {
        LOG.info("Recovering from previous UFS journal write failure." + " Scanning for the last persisted journal entry. currentLog: " + currentLog.toString());
        try (JournalEntryStreamReader reader = new JournalEntryStreamReader(mUfs.open(currentLog.getLocation().toString(), OpenOptions.defaults().setRecoverFailedOpen(true)))) {
            JournalEntry entry;
            while ((entry = reader.readEntry()) != null) {
                if (entry.getSequenceNumber() > lastPersistSeq) {
                    lastPersistSeq = entry.getSequenceNumber();
                }
            }
        } catch (IOException e) {
            throw e;
        }
        if (lastPersistSeq != -1) {
            // If the current log is an empty file, do not complete with SN: 0
            completeLog(currentLog, lastPersistSeq + 1);
        }
    }
    // last persisted journal entry, in case no entry has been found in the INCOMPLETE journal.
    if (lastPersistSeq < 0) {
        // Re-evaluate snapshot because the incomplete journal will be destroyed if
        // it does not contain any valid entry.
        snapshot = UfsJournalSnapshot.getSnapshot(mJournal);
        // journalFiles[journalFiles.size()-1] is the latest complete journal file.
        List<UfsJournalFile> journalFiles = snapshot.getLogs();
        if (!journalFiles.isEmpty()) {
            for (int i = journalFiles.size() - 1; i >= 0; i--) {
                UfsJournalFile journal = journalFiles.get(i);
                if (!journal.isIncompleteLog()) {
                    // Do not consider incomplete logs (handled above)
                    lastPersistSeq = journal.getEnd() - 1;
                    LOG.info("Found last persisted journal entry with seq {} in {}.", lastPersistSeq, journal.getLocation().toString());
                    break;
                }
            }
        }
    }
    return lastPersistSeq;
}