Examples with JournalEntry alluxio.proto.journal.Journal.JournalEntry used on opensource projects

Example 6 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class BackupManager method backup.

/**
 * Writes a backup to the specified stream.
 *
 * @param os the stream to write to
 * @param entryCount will receive total entry count that are backed up
 */
public void backup(OutputStream os, AtomicLong entryCount) throws IOException {
    // Create gZIP compressed stream as back-up stream.
    GzipCompressorOutputStream zipStream = new GzipCompressorOutputStream(os);
    // Executor for taking backup.
    CompletionService<Boolean> completionService = new ExecutorCompletionService<>(Executors.newFixedThreadPool(4, ThreadFactoryUtils.build("master-backup-%d", true)));
    // List of active tasks.
    Set<Future<?>> activeTasks = new HashSet<>();
    // Entry queue will be used as a buffer and synchronization between readers and writer.
    // Use of {@link LinkedBlockingQueue} is preferred because of {@code #drainTo()} method,
    // using which all existing entries can be drained while allowing writes.
    // Processing/draining one-by-one using {@link ConcurrentLinkedQueue} proved to be
    // inefficient compared to draining with dedicated method.
    LinkedBlockingQueue<JournalEntry> journalEntryQueue = new LinkedBlockingQueue<>(ServerConfiguration.getInt(PropertyKey.MASTER_BACKUP_ENTRY_BUFFER_COUNT));
    // Whether buffering is still active.
    AtomicBoolean bufferingActive = new AtomicBoolean(true);
    // Start the timer for backup metrics.
    long startBackupTime = System.currentTimeMillis();
    // Submit master reader task.
    activeTasks.add(completionService.submit(() -> {
        try {
            for (Master master : mRegistry.getServers()) {
                try (CloseableIterator<JournalEntry> it = master.getJournalEntryIterator()) {
                    while (it.hasNext()) {
                        journalEntryQueue.put(it.next());
                        if (Thread.interrupted()) {
                            throw new InterruptedException();
                        }
                    }
                }
            }
            // Put termination entry for signaling the writer.
            journalEntryQueue.put(JournalEntry.newBuilder().setSequenceNumber(TERMINATION_SEQ).build());
            return true;
        } catch (InterruptedException ie) {
            LOG.info("Backup reader task interrupted");
            Thread.currentThread().interrupt();
            throw new RuntimeException("Thread interrupted while reading master state.", ie);
        } finally {
            // Signal reader completion.
            bufferingActive.set(false);
        }
    }));
    // Submit writer task.
    activeTasks.add(completionService.submit(() -> {
        try {
            List<JournalEntry> pendingEntries = new LinkedList<>();
            while (bufferingActive.get() || journalEntryQueue.size() > 0) {
                // Drain pending entries.
                if (0 == journalEntryQueue.drainTo(pendingEntries)) {
                    // No elements at the moment. Fall-back to blocking mode.
                    pendingEntries.add(journalEntryQueue.take());
                }
                if (Thread.interrupted()) {
                    throw new InterruptedException();
                }
                // Write entries to back-up stream.
                for (JournalEntry journalEntry : pendingEntries) {
                    // Check for termination entry.
                    if (journalEntry.getSequenceNumber() == TERMINATION_SEQ) {
                        // Reading finished.
                        return true;
                    }
                    journalEntry.writeDelimitedTo(zipStream);
                    entryCount.incrementAndGet();
                }
                pendingEntries.clear();
            }
            return true;
        } catch (InterruptedException ie) {
            LOG.info("Backup writer task interrupted");
            // Continue interrupt chain.
            Thread.currentThread().interrupt();
            throw new RuntimeException("Thread interrupted while writing to backup stream.", ie);
        }
    }));
    // Wait until backup tasks are completed.
    safeWaitTasks(activeTasks, completionService);
    // Close timer and update entry count.
    mBackupTimeMs = System.currentTimeMillis() - startBackupTime;
    mBackupEntriesCount = entryCount.get();
    // finish() instead of close() since close would close os, which is owned by the caller.
    zipStream.finish();
    LOG.info("Created backup with {} entries", entryCount.get());
}

Example 7 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class DefaultBlockMaster method getJournalEntryIterator.

@Override
public CloseableIterator<JournalEntry> getJournalEntryIterator() {
    Iterator<Block> blockStoreIterator = mBlockStore.iterator();
    Iterator<JournalEntry> blockIterator = new Iterator<JournalEntry>() {

        @Override
        public boolean hasNext() {
            return blockStoreIterator.hasNext();
        }

        @Override
        public JournalEntry next() {
            if (!hasNext()) {
                throw new NoSuchElementException();
            }
            Block block = blockStoreIterator.next();
            BlockInfoEntry blockInfoEntry = BlockInfoEntry.newBuilder().setBlockId(block.getId()).setLength(block.getMeta().getLength()).build();
            return JournalEntry.newBuilder().setBlockInfo(blockInfoEntry).build();
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException("BlockMaster#Iterator#remove is not supported.");
        }
    };
    CloseableIterator<JournalEntry> closeableIterator = CloseableIterator.create(blockIterator, (whatever) -> {
        if (blockStoreIterator instanceof CloseableIterator) {
            final CloseableIterator<Block> c = (CloseableIterator<Block>) blockStoreIterator;
            c.close();
        } else {
        // no op
        }
    });
    return CloseableIterator.concat(CloseableIterator.noopCloseable(CommonUtils.singleElementIterator(getContainerIdJournalEntry())), closeableIterator);
}

Example 8 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class RaftJournalWriter method flush.

@Override
public void flush() throws IOException, JournalClosedException {
    if (mClosed) {
        throw new JournalClosedException("Cannot flush. Journal writer has been closed");
    }
    if (mJournalEntryBuilder != null) {
        long flushSN = mNextSequenceNumberToWrite.get() - 1;
        try {
            // It is ok to submit the same entries multiple times because we de-duplicate by sequence
            // number when applying them. This could happen if submit fails and we re-submit the same
            // entry on retry.
            JournalEntry entry = mJournalEntryBuilder.build();
            Message message = RaftJournalSystem.toRaftMessage(entry);
            mLastSubmittedSequenceNumber.set(flushSN);
            LOG.trace("Flushing entry {} ({})", entry, message);
            RaftClientReply reply = mClient.sendAsync(message, TimeDuration.valueOf(mWriteTimeoutMs, TimeUnit.MILLISECONDS)).get(mWriteTimeoutMs, TimeUnit.MILLISECONDS);
            mLastCommittedSequenceNumber.set(flushSN);
            if (reply.getException() != null) {
                throw reply.getException();
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new IOException(e);
        } catch (ExecutionException e) {
            throw new IOException(e.getCause());
        } catch (TimeoutException e) {
            throw new IOException(String.format("Timed out after waiting %s milliseconds for journal entries to be processed", mWriteTimeoutMs), e);
        }
        mJournalEntryBuilder = null;
    }
}

Example 9 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class UfsJournalCheckpointThread method runInternal.

private void runInternal() {
    // Keeps reading journal entries. If none is found, sleep for sometime. Periodically write
    // checkpoints if some conditions are met. When a shutdown signal is received, wait until
    // no new journal entries.
    LOG.info("{}: Journal checkpoint thread started.", mMaster.getName());
    // Set to true if it has waited for a quiet period. Reset if a valid journal entry is read.
    boolean quietPeriodWaited = false;
    mCatchupState = CatchupState.IN_PROGRESS;
    while (true) {
        JournalEntry entry = null;
        try {
            switch(mJournalReader.advance()) {
                case CHECKPOINT:
                    LOG.debug("{}: Restoring from checkpoint", mMaster.getName());
                    mMaster.restoreFromCheckpoint(mJournalReader.getCheckpoint());
                    LOG.debug("{}: Finished restoring from checkpoint", mMaster.getName());
                    break;
                case LOG:
                    entry = mJournalReader.getEntry();
                    try {
                        if (!mMaster.processJournalEntry(entry)) {
                            JournalUtils.handleJournalReplayFailure(LOG, null, "%s: Unrecognized journal entry: %s", mMaster.getName(), entry);
                        } else {
                            JournalUtils.sinkAppend(mJournalSinks, entry);
                        }
                    } catch (Throwable t) {
                        JournalUtils.handleJournalReplayFailure(LOG, t, "%s: Failed to read or process journal entry %s.", mMaster.getName(), entry);
                    }
                    if (quietPeriodWaited) {
                        LOG.info("Quiet period interrupted by new journal entry");
                        quietPeriodWaited = false;
                    }
                    mLastAppliedSN = entry.getSequenceNumber();
                    break;
                default:
                    mCatchupState = CatchupState.DONE;
                    break;
            }
        } catch (IOException e) {
            LOG.error("{}: Failed to read or process a journal entry.", mMaster.getName(), e);
            try {
                mJournalReader.close();
            } catch (IOException ee) {
                LOG.warn("{}: Failed to close the journal reader with error {}.", mMaster.getName(), ee.toString());
            }
            long nextSequenceNumber = mJournalReader.getNextSequenceNumber();
            mJournalReader = new UfsJournalReader(mJournal, nextSequenceNumber, false);
            quietPeriodWaited = false;
            continue;
        }
        // Sleep for a while if no entry is found.
        if (entry == null) {
            // Only try to checkpoint when it can keep up.
            maybeCheckpoint();
            if (mShutdownInitiated) {
                if (quietPeriodWaited || !mWaitQuietPeriod) {
                    mCatchupState = CatchupState.DONE;
                    LOG.info("{}: Journal checkpoint thread has been shutdown. No new logs have been found " + "during the quiet period.", mMaster.getName());
                    if (mJournalReader != null) {
                        try {
                            mJournalReader.close();
                        } catch (IOException e) {
                            LOG.warn("{}: Failed to close the journal reader with error {}.", mMaster.getName(), e.toString());
                        }
                    }
                    return;
                }
                CommonUtils.sleepMs(LOG, mShutdownQuietWaitTimeMs);
                quietPeriodWaited = true;
            } else {
                CommonUtils.sleepMs(LOG, mJournalCheckpointSleepTimeMs);
            }
        }
        if (Thread.interrupted() && !mShutdownInitiated) {
            LOG.info("{}: Checkpoint thread interrupted, shutting down", mMaster.getName());
            return;
        }
    }
}

Example 10 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class AsyncJournalWriter method flush.

/**
 * Flushes and waits until the specified counter is flushed to the journal. If the specified
 * counter is already flushed, this is essentially a no-op.
 *
 * @param targetCounter the counter to flush
 */
public void flush(final long targetCounter) throws IOException {
    if (targetCounter <= mFlushCounter.get()) {
        return;
    }
    // Using reentrant lock, since it seems to result in higher throughput than using 'synchronized'
    mFlushLock.lock();
    try {
        long startTime = System.nanoTime();
        long flushCounter = mFlushCounter.get();
        if (targetCounter <= flushCounter) {
            // The specified counter is already flushed, so just return.
            return;
        }
        long writeCounter = mWriteCounter.get();
        while (targetCounter > writeCounter) {
            for (; ; ) {
                // Get, but do not remove, the head entry.
                JournalEntry entry = mQueue.peek();
                if (entry == null) {
                    // No more entries in the queue. Break out of the infinite for-loop.
                    break;
                }
                mJournalWriter.write(entry);
                // Remove the head entry, after the entry was successfully written.
                mQueue.poll();
                writeCounter = mWriteCounter.incrementAndGet();
                if (writeCounter >= targetCounter) {
                    if ((System.nanoTime() - startTime) >= mFlushBatchTimeNs) {
                        // infinite for-loop.
                        break;
                    }
                }
            }
        }
        mJournalWriter.flush();
        mFlushCounter.set(writeCounter);
    } finally {
        mFlushLock.unlock();
    }
}