Examples with JournalEntry alluxio.proto.journal.Journal.JournalEntry used on opensource projects

Example 11 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class JournalUtils method restoreJournalEntryCheckpoint.

/**
 * Restores the given journaled object from the journal entries in the input stream.
 *
 * This is the complement of
 * {@link #writeJournalEntryCheckpoint(OutputStream, JournalEntryIterable)}.
 *
 * @param input the stream to read from
 * @param journaled the object to restore
 */
public static void restoreJournalEntryCheckpoint(CheckpointInputStream input, Journaled journaled) throws IOException {
    Preconditions.checkState(input.getType() == CheckpointType.JOURNAL_ENTRY, "Unrecognized checkpoint type when restoring %s: %s", journaled.getCheckpointName(), input.getType());
    journaled.resetState();
    LOG.info("Reading journal entries");
    JournalEntryStreamReader reader = new JournalEntryStreamReader(input);
    JournalEntry entry;
    while ((entry = reader.readEntry()) != null) {
        try {
            journaled.processJournalEntry(entry);
        } catch (Throwable t) {
            handleJournalReplayFailure(LOG, t, "Failed to process journal entry %s from a journal checkpoint", entry);
        }
    }
}

Example 12 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class JournalUtils method writeJournalEntryCheckpoint.

/**
 * Writes a checkpoint of the entries in the given iterable.
 *
 * This is the complement of
 * {@link #restoreJournalEntryCheckpoint(CheckpointInputStream, Journaled)}.
 *
 * @param output the stream to write to
 * @param iterable the iterable for fetching journal entries
 */
public static void writeJournalEntryCheckpoint(OutputStream output, JournalEntryIterable iterable) throws IOException, InterruptedException {
    output = new CheckpointOutputStream(output, CheckpointType.JOURNAL_ENTRY);
    try (CloseableIterator<JournalEntry> it = iterable.getJournalEntryIterator()) {
        LOG.info("Write journal entry checkpoint");
        while (it.get().hasNext()) {
            if (Thread.interrupted()) {
                throw new InterruptedException();
            }
            it.get().next().writeDelimitedTo(output);
        }
    }
    output.flush();
}

Example 13 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class AsyncJournalWriter method doFlush.

/**
 * A dedicated thread that goes over outstanding queue items and writes/flushes them. Other
 * threads can track progress by submitting tickets via ::flush() call.
 */
private void doFlush() {
    // Runs the loop until ::stop() is called.
    while (!mStopFlushing) {
        /**
         * Stand still unless;
         * - queue has items
         * - permit is given by:
         *   - clients
         *   -::stop()
         */
        while (mQueue.isEmpty() && !mStopFlushing) {
            try {
                // queued entries proactively.
                if (mFlushSemaphore.tryAcquire(mFlushBatchTimeNs, TimeUnit.NANOSECONDS)) {
                    break;
                }
            } catch (InterruptedException ie) {
                break;
            }
        }
        try {
            long startTime = System.nanoTime();
            // Write pending entries to journal.
            while (!mQueue.isEmpty()) {
                // Get, but do not remove, the head entry.
                JournalEntry entry = mQueue.peek();
                if (entry == null) {
                    // No more entries in the queue. Break write session.
                    break;
                }
                mJournalWriter.write(entry);
                JournalUtils.sinkAppend(mJournalSinks, entry);
                // Remove the head entry, after the entry was successfully written.
                mQueue.poll();
                mWriteCounter++;
                if (((System.nanoTime() - startTime) >= mFlushBatchTimeNs) && !mStopFlushing) {
                    // infinite while-loop.
                    break;
                }
            }
            // Either written new entries or previous flush had been failed.
            if (mFlushCounter.get() < mWriteCounter) {
                try (Timer.Context ctx = MetricsSystem.timer(MetricKey.MASTER_JOURNAL_FLUSH_TIMER.getName()).time()) {
                    mJournalWriter.flush();
                }
                JournalUtils.sinkFlush(mJournalSinks);
                mFlushCounter.set(mWriteCounter);
            }
            // Notify tickets that have been served to wake up.
            Iterator<FlushTicket> ticketIterator = mTicketSet.iterator();
            while (ticketIterator.hasNext()) {
                FlushTicket ticket = ticketIterator.next();
                if (ticket.getTargetCounter() <= mFlushCounter.get()) {
                    ticket.setCompleted();
                    ticketIterator.remove();
                }
            }
        } catch (IOException | JournalClosedException exc) {
            // Add the error logging here since the actual flush error may be overwritten
            // by the future meaningless ratis.protocol.AlreadyClosedException
            SAMPLING_LOG.warn("Failed to flush journal entry: " + exc.getMessage(), exc);
            Metrics.JOURNAL_FLUSH_FAILURE.inc();
            // Release only tickets that have been flushed. Fail the rest.
            Iterator<FlushTicket> ticketIterator = mTicketSet.iterator();
            while (ticketIterator.hasNext()) {
                FlushTicket ticket = ticketIterator.next();
                ticketIterator.remove();
                if (ticket.getTargetCounter() <= mFlushCounter.get()) {
                    ticket.setCompleted();
                } else {
                    ticket.setError(exc);
                }
            }
        }
    }
}

Example 14 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class UfsJournalLogWriter method recoverLastPersistedJournalEntry.

/**
 * Examine the UFS to determine the most recent journal entry, and return its sequence number.
 *
 * 1. Locate the most recent incomplete journal file, i.e. journal file that starts with
 *    a valid sequence number S (hex), and ends with 0x7fffffffffffffff. The journal file
 *    name encodes this information, i.e. S-0x7fffffffffffffff.
 * 2. Sequentially scan the incomplete journal file, and identify the last journal
 *    entry that has been persisted in UFS. Suppose it is X.
 * 3. Rename the incomplete journal file to S-<X+1>. Future journal writes will write to
 *    a new file named <X+1>-0x7fffffffffffffff.
 * 4. If the incomplete journal does not exist or no entry can be found in the incomplete
 *    journal, check the last complete journal file for the last persisted journal entry.
 *
 * @return sequence number of the last persisted journal entry, or -1 if no entry can be found
 */
private long recoverLastPersistedJournalEntry() throws IOException {
    UfsJournalSnapshot snapshot = UfsJournalSnapshot.getSnapshot(mJournal);
    long lastPersistSeq = -1;
    UfsJournalFile currentLog = snapshot.getCurrentLog(mJournal);
    if (currentLog != null) {
        LOG.info("Recovering from previous UFS journal write failure." + " Scanning for the last persisted journal entry. currentLog: " + currentLog.toString());
        try (JournalEntryStreamReader reader = new JournalEntryStreamReader(mUfs.open(currentLog.getLocation().toString(), OpenOptions.defaults().setRecoverFailedOpen(true)))) {
            JournalEntry entry;
            while ((entry = reader.readEntry()) != null) {
                if (entry.getSequenceNumber() > lastPersistSeq) {
                    lastPersistSeq = entry.getSequenceNumber();
                }
            }
        } catch (IOException e) {
            throw e;
        }
        if (lastPersistSeq != -1) {
            // If the current log is an empty file, do not complete with SN: 0
            completeLog(currentLog, lastPersistSeq + 1);
        }
    }
    // last persisted journal entry, in case no entry has been found in the INCOMPLETE journal.
    if (lastPersistSeq < 0) {
        // Re-evaluate snapshot because the incomplete journal will be destroyed if
        // it does not contain any valid entry.
        snapshot = UfsJournalSnapshot.getSnapshot(mJournal);
        // journalFiles[journalFiles.size()-1] is the latest complete journal file.
        List<UfsJournalFile> journalFiles = snapshot.getLogs();
        if (!journalFiles.isEmpty()) {
            for (int i = journalFiles.size() - 1; i >= 0; i--) {
                UfsJournalFile journal = journalFiles.get(i);
                if (!journal.isIncompleteLog()) {
                    // Do not consider incomplete logs (handled above)
                    lastPersistSeq = journal.getEnd() - 1;
                    LOG.info("Found last persisted journal entry with seq {} in {}.", lastPersistSeq, journal.getLocation().toString());
                    break;
                }
            }
        }
    }
    return lastPersistSeq;
}

Example 15 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class UfsJournalLogWriter method write.

public synchronized void write(JournalEntry entry) throws IOException, JournalClosedException {
    checkIsWritable();
    try {
        maybeRecoverFromUfsFailures();
        maybeRotateLog();
    } catch (IOJournalClosedException e) {
        throw e.toJournalClosedException();
    }
    try {
        JournalEntry entryToWrite = entry.toBuilder().setSequenceNumber(mNextSequenceNumber).build();
        entryToWrite.writeDelimitedTo(mJournalOutputStream);
        LOG.debug("Adding journal entry (seq={}) to retryList with {} entries. currentLog: {}", entryToWrite.getSequenceNumber(), mEntriesToFlush.size(), currentLogName());
        mEntriesToFlush.add(entryToWrite);
        mNextSequenceNumber++;
    } catch (IOJournalClosedException e) {
        throw e.toJournalClosedException();
    } catch (IOException e) {
        // Set mNeedsRecovery to true so that {@code maybeRecoverFromUfsFailures}
        // can know a UFS failure has occurred.
        mNeedsRecovery = true;
        throw new IOException(ExceptionMessage.JOURNAL_WRITE_FAILURE.getMessageWithUrl(RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, mJournalOutputStream.currentLog(), e.getMessage()), e);
    }
}