Examples with CloseableIterator alluxio.resource.CloseableIterator used on opensource projects

Example 1 with CloseableIterator

use of alluxio.resource.CloseableIterator in project alluxio by Alluxio.

the class BackupManager method backup.

/**
 * Writes a backup to the specified stream.
 *
 * @param os the stream to write to
 * @param entryCount will receive total entry count that are backed up
 */
public void backup(OutputStream os, AtomicLong entryCount) throws IOException {
    // Create gZIP compressed stream as back-up stream.
    GzipCompressorOutputStream zipStream = new GzipCompressorOutputStream(os);
    // Executor for taking backup.
    CompletionService<Boolean> completionService = new ExecutorCompletionService<>(Executors.newFixedThreadPool(4, ThreadFactoryUtils.build("master-backup-%d", true)));
    // List of active tasks.
    Set<Future<?>> activeTasks = new HashSet<>();
    // Entry queue will be used as a buffer and synchronization between readers and writer.
    // Use of {@link LinkedBlockingQueue} is preferred because of {@code #drainTo()} method,
    // using which all existing entries can be drained while allowing writes.
    // Processing/draining one-by-one using {@link ConcurrentLinkedQueue} proved to be
    // inefficient compared to draining with dedicated method.
    LinkedBlockingQueue<JournalEntry> journalEntryQueue = new LinkedBlockingQueue<>(ServerConfiguration.getInt(PropertyKey.MASTER_BACKUP_ENTRY_BUFFER_COUNT));
    // Whether buffering is still active.
    AtomicBoolean bufferingActive = new AtomicBoolean(true);
    // Start the timer for backup metrics.
    long startBackupTime = System.currentTimeMillis();
    // Submit master reader task.
    activeTasks.add(completionService.submit(() -> {
        try {
            for (Master master : mRegistry.getServers()) {
                try (CloseableIterator<JournalEntry> it = master.getJournalEntryIterator()) {
                    while (it.hasNext()) {
                        journalEntryQueue.put(it.next());
                        if (Thread.interrupted()) {
                            throw new InterruptedException();
                        }
                    }
                }
            }
            // Put termination entry for signaling the writer.
            journalEntryQueue.put(JournalEntry.newBuilder().setSequenceNumber(TERMINATION_SEQ).build());
            return true;
        } catch (InterruptedException ie) {
            LOG.info("Backup reader task interrupted");
            Thread.currentThread().interrupt();
            throw new RuntimeException("Thread interrupted while reading master state.", ie);
        } finally {
            // Signal reader completion.
            bufferingActive.set(false);
        }
    }));
    // Submit writer task.
    activeTasks.add(completionService.submit(() -> {
        try {
            List<JournalEntry> pendingEntries = new LinkedList<>();
            while (bufferingActive.get() || journalEntryQueue.size() > 0) {
                // Drain pending entries.
                if (0 == journalEntryQueue.drainTo(pendingEntries)) {
                    // No elements at the moment. Fall-back to blocking mode.
                    pendingEntries.add(journalEntryQueue.take());
                }
                if (Thread.interrupted()) {
                    throw new InterruptedException();
                }
                // Write entries to back-up stream.
                for (JournalEntry journalEntry : pendingEntries) {
                    // Check for termination entry.
                    if (journalEntry.getSequenceNumber() == TERMINATION_SEQ) {
                        // Reading finished.
                        return true;
                    }
                    journalEntry.writeDelimitedTo(zipStream);
                    entryCount.incrementAndGet();
                }
                pendingEntries.clear();
            }
            return true;
        } catch (InterruptedException ie) {
            LOG.info("Backup writer task interrupted");
            // Continue interrupt chain.
            Thread.currentThread().interrupt();
            throw new RuntimeException("Thread interrupted while writing to backup stream.", ie);
        }
    }));
    // Wait until backup tasks are completed.
    safeWaitTasks(activeTasks, completionService);
    // Close timer and update entry count.
    mBackupTimeMs = System.currentTimeMillis() - startBackupTime;
    mBackupEntriesCount = entryCount.get();
    // finish() instead of close() since close would close os, which is owned by the caller.
    zipStream.finish();
    LOG.info("Created backup with {} entries", entryCount.get());
}

Example 2 with CloseableIterator

use of alluxio.resource.CloseableIterator in project alluxio by Alluxio.

the class DefaultBlockMaster method getJournalEntryIterator.

@Override
public CloseableIterator<JournalEntry> getJournalEntryIterator() {
    Iterator<Block> blockStoreIterator = mBlockStore.iterator();
    Iterator<JournalEntry> blockIterator = new Iterator<JournalEntry>() {

        @Override
        public boolean hasNext() {
            return blockStoreIterator.hasNext();
        }

        @Override
        public JournalEntry next() {
            if (!hasNext()) {
                throw new NoSuchElementException();
            }
            Block block = blockStoreIterator.next();
            BlockInfoEntry blockInfoEntry = BlockInfoEntry.newBuilder().setBlockId(block.getId()).setLength(block.getMeta().getLength()).build();
            return JournalEntry.newBuilder().setBlockInfo(blockInfoEntry).build();
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException("BlockMaster#Iterator#remove is not supported.");
        }
    };
    CloseableIterator<JournalEntry> closeableIterator = CloseableIterator.create(blockIterator, (whatever) -> {
        if (blockStoreIterator instanceof CloseableIterator) {
            final CloseableIterator<Block> c = (CloseableIterator<Block>) blockStoreIterator;
            c.close();
        } else {
        // no op
        }
    });
    return CloseableIterator.concat(CloseableIterator.noopCloseable(CommonUtils.singleElementIterator(getContainerIdJournalEntry())), closeableIterator);
}

Example 3 with CloseableIterator

use of alluxio.resource.CloseableIterator in project alluxio by Alluxio.

the class RocksUtils method createCloseableIterator.

/**
 * Used to wrap an {@link CloseableIterator} over {@link RocksIterator}.
 * It seeks given iterator to first entry before returning the iterator.
 *
 * @param rocksIterator the rocks iterator
 * @param parser parser to produce iterated values from rocks key-value
 * @param <T> iterator value type
 * @return wrapped iterator
 */
public static <T> CloseableIterator<T> createCloseableIterator(RocksIterator rocksIterator, RocksIteratorParser<T> parser) {
    rocksIterator.seekToFirst();
    AtomicBoolean valid = new AtomicBoolean(true);
    Iterator<T> iter = new Iterator<T>() {

        @Override
        public boolean hasNext() {
            return valid.get() && rocksIterator.isValid();
        }

        @Override
        public T next() {
            try {
                return parser.next(rocksIterator);
            } catch (Exception exc) {
                LOG.warn("Iteration aborted because of error", exc);
                rocksIterator.close();
                valid.set(false);
                throw new RuntimeException(exc);
            } finally {
                rocksIterator.next();
                if (!rocksIterator.isValid()) {
                    rocksIterator.close();
                    valid.set(false);
                }
            }
        }
    };
    return CloseableIterator.create(iter, (whatever) -> {
        rocksIterator.close();
    });
}