Examples with JournalEntry alluxio.proto.journal.Journal.JournalEntry used on opensource projects

Example 26 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class ProtoBufJournalFormatter method deserialize.

@Override
public JournalInputStream deserialize(final InputStream inputStream) throws IOException {
    return new JournalInputStream() {

        private final byte[] mBuffer = new byte[1024];

        private long mLatestSequenceNumber;

        @Override
        public JournalEntry getNextEntry() throws IOException {
            int firstByte = inputStream.read();
            if (firstByte == -1) {
                return null;
            }
            // All journal entries start with their size in bytes written as a varint.
            int size = ProtoUtils.readRawVarint32(firstByte, inputStream);
            byte[] buffer = size <= mBuffer.length ? mBuffer : new byte[size];
            // Total bytes read so far for journal entry.
            int totalBytesRead = 0;
            while (totalBytesRead < size) {
                // Bytes read in last read request.
                int latestBytesRead = inputStream.read(buffer, totalBytesRead, size - totalBytesRead);
                if (latestBytesRead < 0) {
                    break;
                }
                totalBytesRead += latestBytesRead;
            }
            if (totalBytesRead < size) {
                LOG.warn("Journal entry was truncated. Expected to read " + size + " bytes but only got " + totalBytesRead);
                return null;
            }
            JournalEntry entry = JournalEntry.parseFrom(new ByteArrayInputStream(buffer, 0, size));
            if (entry != null) {
                mLatestSequenceNumber = entry.getSequenceNumber();
            }
            return entry;
        }

        @Override
        public void close() throws IOException {
            inputStream.close();
        }

        @Override
        public long getLatestSequenceNumber() {
            return mLatestSequenceNumber;
        }
    };
}

Example 27 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class LineageIdGeneratorTest method journalEntrySerialization.

/**
   * Tests the {@link LineageIdGenerator#initFromJournalEntry(
   * alluxio.proto.journal.Lineage.LineageIdGeneratorEntry)} method.
   */
@Test
public void journalEntrySerialization() {
    LineageIdGenerator generator = new LineageIdGenerator();
    long id = generator.generateId();
    JournalEntry entry = generator.toJournalEntry();
    generator = new LineageIdGenerator();
    generator.initFromJournalEntry(entry.getLineageIdGenerator());
    Assert.assertEquals(id + 1, generator.generateId());
}

Example 28 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class JournalStateMachine method applyJournalEntryCommand.

/**
 * Applies a journal entry commit to the state machine.
 * @param commit the commit
 */
public synchronized void applyJournalEntryCommand(TransactionContext commit) {
    JournalEntry entry;
    try {
        entry = JournalEntry.parseFrom(commit.getStateMachineLogEntry().getLogData().asReadOnlyByteBuffer());
    } catch (Exception e) {
        ProcessUtils.fatalError(LOG, e, "Encountered invalid journal entry in commit: %s.", commit);
        System.exit(-1);
        // We should never reach here.
        throw new IllegalStateException(e);
    }
    try {
        applyEntry(entry);
    } finally {
        Preconditions.checkState(commit.getLogEntry().getIndex() > mLastAppliedCommitIndex);
        mLastAppliedCommitIndex = commit.getLogEntry().getIndex();
    }
}

Example 29 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class ProtoBufJournalFormatter method deserialize.

@Override
public JournalInputStream deserialize(final InputStream inputStream) throws IOException {
    return new JournalInputStream() {

        private final byte[] mBuffer = new byte[1024];

        private long mLatestSequenceNumber;

        @Override
        public JournalEntry read() throws IOException {
            int firstByte = inputStream.read();
            if (firstByte == -1) {
                return null;
            }
            // All journal entries start with their size in bytes written as a varint.
            int size = ProtoUtils.readRawVarint32(firstByte, inputStream);
            byte[] buffer = size <= mBuffer.length ? mBuffer : new byte[size];
            // Total bytes read so far for journal entry.
            int totalBytesRead = 0;
            while (totalBytesRead < size) {
                // Bytes read in last read request.
                int latestBytesRead = inputStream.read(buffer, totalBytesRead, size - totalBytesRead);
                if (latestBytesRead < 0) {
                    break;
                }
                totalBytesRead += latestBytesRead;
            }
            if (totalBytesRead < size) {
                LOG.warn("Journal entry was truncated. Expected to read " + size + " bytes but only got " + totalBytesRead);
                return null;
            }
            JournalEntry entry = JournalEntry.parseFrom(new ByteArrayInputStream(buffer, 0, size));
            if (entry != null) {
                mLatestSequenceNumber = entry.getSequenceNumber();
            }
            return entry;
        }

        @Override
        public void close() throws IOException {
            inputStream.close();
        }

        @Override
        public long getLatestSequenceNumber() {
            return mLatestSequenceNumber;
        }
    };
}

Example 30 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class UfsJournalLogWriter method maybeRecoverFromUfsFailures.

/**
 * Core logic of UFS journal recovery from UFS failures.
 *
 * If Alluxio stores its journals in UFS, then Alluxio needs to handle UFS failures.
 * When UFS is dead, there is nothing Alluxio can do because Alluxio relies on UFS to
 * persist journal entries. Consequently any metadata operation will block because Alluxio
 * cannot flush their journal entries.
 * Once UFS comes back online, Alluxio needs to perform the following operations:
 * 1. Find out the sequence number of the last persisted journal entry, say X. Then the first
 *    non-persisted entry has sequence number Y = X + 1.
 * 2. Check whether there is any missing journal entry between Y (inclusive) and the oldest
 *    entry in mEntriesToFlush, say Z. If Z > Y, then it means journal entries in [Y, Z) are
 *    missing, and Alluxio cannot recover. Otherwise, for each journal entry in
 *    {@link #mEntriesToFlush}, if its sequence number is larger than or equal to Y, retry
 *    writing it to UFS by calling the {@code UfsJournalLogWriter#write} method.
 */
private void maybeRecoverFromUfsFailures() throws IOException, JournalClosedException {
    checkIsWritable();
    if (!mNeedsRecovery) {
        return;
    }
    try (Timer.Context ctx = MetricsSystem.timer(MetricKey.MASTER_UFS_JOURNAL_FAILURE_RECOVER_TIMER.getName()).time()) {
        long lastPersistSeq = recoverLastPersistedJournalEntry();
        if (lastPersistSeq == -1) {
            throw new RuntimeException("Cannot find any journal entry to recover. location: " + mJournal.getLocation());
        }
        createNewLogFile(lastPersistSeq + 1);
        if (!mEntriesToFlush.isEmpty()) {
            JournalEntry firstEntryToFlush = mEntriesToFlush.peek();
            if (firstEntryToFlush.getSequenceNumber() > lastPersistSeq + 1) {
                throw new RuntimeException(ExceptionMessage.JOURNAL_ENTRY_MISSING.getMessageWithUrl(RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, lastPersistSeq + 1, firstEntryToFlush.getSequenceNumber()));
            }
            long retryEndSeq = lastPersistSeq;
            LOG.info("Retry writing unwritten journal entries from seq {} to currentLog {}", lastPersistSeq + 1, currentLogName());
            for (JournalEntry entry : mEntriesToFlush) {
                if (entry.getSequenceNumber() > lastPersistSeq) {
                    try {
                        entry.toBuilder().build().writeDelimitedTo(mJournalOutputStream);
                        retryEndSeq = entry.getSequenceNumber();
                    } catch (IOJournalClosedException e) {
                        throw e.toJournalClosedException();
                    } catch (IOException e) {
                        throw new IOException(ExceptionMessage.JOURNAL_WRITE_FAILURE.getMessageWithUrl(RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, mJournalOutputStream.currentLog(), e.getMessage()), e);
                    }
                }
            }
            LOG.info("Finished writing unwritten journal entries from {} to {}. currentLog: {}", lastPersistSeq + 1, retryEndSeq, currentLogName());
            if (retryEndSeq != mNextSequenceNumber - 1) {
                throw new RuntimeException("Failed to recover all entries to flush, expecting " + (mNextSequenceNumber - 1) + " but only found entry " + retryEndSeq + " currentLog: " + currentLogName());
            }
        }
    }
    mNeedsRecovery = false;
}