Examples with CompletableOperation io.pravega.segmentstore.server.logs.operations.CompletableOperation used on opensource projects

Example 1 with CompletableOperation

use of io.pravega.segmentstore.server.logs.operations.CompletableOperation in project pravega by pravega.

the class OperationProcessor method processOperations.

/**
 * Processes a set of pending operations (essentially a single iteration of the QueueProcessor).
 * Steps:
 * <ol>
 * <li> Picks the next items from the queue
 * <li> Creates a DataFrameBuilder and starts appending items to it.
 * <li> As the DataFrameBuilder acknowledges DataFrames being published, acknowledge the corresponding Operations as well.
 * <li> If at the end, the Queue still has items to process, processes those as well.
 * </ol>
 *
 * @param operations The initial set of operations to process (in order). Multiple operations may be processed eventually
 *                   depending on how the operationQueue changes while this is processing.
 */
private void processOperations(Queue<CompletableOperation> operations) {
    log.debug("{}: processOperations (OperationCount = {}).", this.traceObjectId, operations.size());
    // Process the operations in the queue. This loop will ensure we do continuous processing in case new items
    // arrived while we were busy handling the current items.
    Timer processTimer = new Timer();
    int count = 0;
    while (!operations.isEmpty()) {
        try {
            // Process the current set of operations.
            while (!operations.isEmpty()) {
                CompletableOperation o = operations.poll();
                this.metrics.operationQueueWaitTime(o.getTimer().getElapsedMillis());
                try {
                    processOperation(o);
                    this.state.addPending(o);
                    count++;
                } catch (Throwable ex) {
                    ex = Exceptions.unwrap(ex);
                    this.state.failOperation(o, ex);
                    if (isFatalException(ex)) {
                        // operation as its failure is isolated to itself (most likely it's invalid).
                        throw ex;
                    }
                }
            }
            // context switching, better DataFrame occupancy optimization) rather than by going back to run().
            if (operations.isEmpty()) {
                // We have processed all operations in the queue: this is a good time to report metrics.
                this.metrics.currentState(this.operationQueue.size(), this.state.getPendingCount());
                this.metrics.processOperations(count, processTimer.getElapsedMillis());
                // Reset this timer since we may be pulling in new operations.
                processTimer = new Timer();
                count = 0;
                if (!this.throttlerCalculator.isThrottlingRequired()) {
                    // Only pull in new operations if we do not require throttling. If we do, we need to go back to
                    // the main OperationProcessor loop and delay processing the next batch of operations.
                    operations = this.operationQueue.poll(MAX_READ_AT_ONCE);
                }
                if (operations.isEmpty()) {
                    log.debug("{}: processOperations (Flush).", this.traceObjectId);
                    synchronized (this.stateLock) {
                        this.dataFrameBuilder.flush();
                    }
                } else {
                    log.debug("{}: processOperations (Add OperationCount = {}).", this.traceObjectId, operations.size());
                }
            }
        } catch (Throwable ex) {
            // Fail ALL the operations that haven't been acknowledged yet.
            ex = Exceptions.unwrap(ex);
            this.state.fail(ex, null);
            if (isFatalException(ex)) {
                // If we encountered a fatal exception, it means we detected something that we cannot possibly recover from.
                // We need to shutdown right away (this will be done by the main loop).
                // But first, fail any Operations that we did not have a chance to process yet.
                cancelIncompleteOperations(operations, ex);
                throw Lombok.sneakyThrow(ex);
            }
        }
    }
}

Example 2 with CompletableOperation

use of io.pravega.segmentstore.server.logs.operations.CompletableOperation in project pravega by pravega.

the class OperationProcessor method cancelIncompleteOperations.

/**
 * Cancels those Operations in the given list that have not yet completed with the given exception.
 */
private void cancelIncompleteOperations(Iterable<CompletableOperation> operations, Throwable failException) {
    assert failException != null : "no exception to set";
    int cancelCount = 0;
    for (CompletableOperation o : operations) {
        if (!o.isDone()) {
            this.state.failOperation(o, failException);
            cancelCount++;
        }
    }
    log.warn("{}: Cancelling {} operations with exception: {}.", this.traceObjectId, cancelCount, failException.toString());
}

Example 3 with CompletableOperation

use of io.pravega.segmentstore.server.logs.operations.CompletableOperation in project pravega by pravega.

the class OperationProcessor method processOperation.

/**
 * Processes a single operation.
 * Steps:
 * <ol>
 * <li> Pre-processes operation (in MetadataUpdater).
 * <li> Assigns Sequence Number.
 * <li> Appends to DataFrameBuilder.
 * <li> Accepts operation in MetadataUpdater.
 * </ol>
 *
 * @param operation        The operation to process.
 * @throws Exception If an exception occurred while processing this operation. Depending on the type of the exception,
 * this could be due to the operation itself being invalid, or because we are unable to process any more operations.
 */
private void processOperation(CompletableOperation operation) throws Exception {
    Preconditions.checkState(!operation.isDone(), "The Operation has already been processed.");
    Operation entry = operation.getOperation();
    if (!entry.canSerialize()) {
        // This operation cannot be serialized, so don't bother doing anything with it.
        return;
    }
    synchronized (this.stateLock) {
        // Update Metadata and Operations with any missing data (offsets, lengths, etc) - the Metadata Updater
        // has all the knowledge for that task.
        this.metadataUpdater.preProcessOperation(entry);
        // Entry is ready to be serialized; assign a sequence number.
        entry.setSequenceNumber(this.metadataUpdater.nextOperationSequenceNumber());
        this.dataFrameBuilder.append(entry);
        this.metadataUpdater.acceptOperation(entry);
    }
    log.trace("{}: DataFrameBuilder.Append {}.", this.traceObjectId, entry);
}

Example 4 with CompletableOperation

use of io.pravega.segmentstore.server.logs.operations.CompletableOperation in project pravega by pravega.

the class OperationProcessor method closeQueue.

/**
 * Closes the Operation Queue and fails all Operations in it with the given exception.
 *
 * @param causingException The exception to fail with. If null, it will default to ObjectClosedException.
 */
private void closeQueue(Throwable causingException) {
    // Close the operation queue and extract any outstanding Operations from it.
    Collection<CompletableOperation> remainingOperations = this.operationQueue.close();
    if (remainingOperations != null && remainingOperations.size() > 0) {
        // If any outstanding Operations were left in the queue, they need to be failed.
        // If no other cause was passed, assume we are closing the queue because we are shutting down.
        Throwable failException = causingException != null ? causingException : new CancellationException();
        cancelIncompleteOperations(remainingOperations, failException);
    }
    // The commit queue will auto-close when we are done and it itself is empty. We just need to unblock it in case
    // it was idle and waiting on a pending take() operation.
    this.commitQueue.cancelPendingTake();
}