Examples with OperationContext io.pravega.controller.store.stream.OperationContext used on opensource projects

Example 31 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class StreamMetadataTasks method getTruncationStreamCutByTimeLimit.

private CompletableFuture<Map<Long, Long>> getTruncationStreamCutByTimeLimit(String scope, String stream, OperationContext context, RetentionPolicy policy, RetentionSet retentionSet, Map<Long, Long> lowerBound) {
    long currentTime = retentionClock.get().get();
    // we get the streamcuts from retentionset that satisfy the min and max bounds with min pointing to most recent
    // streamcut to satisfy both min and max bounds while max refering to oldest such streamcut in retention set.
    // limits.key will refer to max and limit.value will refer to min.
    Map.Entry<StreamCutReferenceRecord, StreamCutReferenceRecord> limits = getBoundStreamCuts(policy, retentionSet, x -> currentTime - x.getRecordingTime());
    // if subscriber lowerbound is greater than (ahead of/after) streamcut corresponding to the max time and is less than
    // (behind/before) stream cut for min time  from the retention set then we can safely truncate at lowerbound.
    // Else we will truncate at the max time bound if it exists
    // 1. if LB is greater than (ahead of/after) min => truncate at min
    // 2. if LB is less than (behind/before) max => truncate at max
    // 3. if LB is less than (behind/before) min && LB is greater than (ahead of/after) max => truncate at LB
    // 4. if LB is less than (behind/before) min && overlaps max => truncate at max
    // 5. if LB overlaps with min and max ==> so its got both recent data and older data.
    // we will truncate at a streamcut less than (behind/before) max in this case.
    CompletableFuture<StreamCutRecord> limitMinFuture = limits.getValue() == null ? CompletableFuture.completedFuture(null) : streamMetadataStore.getStreamCutRecord(scope, stream, limits.getValue(), context, executor);
    // if lowerbound is empty simply return min
    if (lowerBound == null || lowerBound.isEmpty()) {
        return limitMinFuture.thenApply(min -> Optional.ofNullable(min).map(StreamCutRecord::getStreamCut).orElse(null));
    }
    Optional<StreamCutReferenceRecord> maxBoundRef = retentionSet.getRetentionRecords().stream().filter(x -> currentTime - x.getRecordingTime() >= policy.getRetentionMax()).max(Comparator.comparingLong(StreamCutReferenceRecord::getRecordingTime));
    CompletableFuture<StreamCutRecord> limitMaxFuture = limits.getKey() == null ? CompletableFuture.completedFuture(null) : streamMetadataStore.getStreamCutRecord(scope, stream, limits.getKey(), context, executor);
    CompletableFuture<StreamCutRecord> maxBoundFuture = maxBoundRef.map(x -> streamMetadataStore.getStreamCutRecord(scope, stream, x, context, executor)).orElse(CompletableFuture.completedFuture(null));
    return CompletableFuture.allOf(limitMaxFuture, limitMinFuture, maxBoundFuture).thenCompose(v -> {
        StreamCutRecord limitMax = limitMaxFuture.join();
        StreamCutRecord limitMin = limitMinFuture.join();
        StreamCutRecord maxBound = maxBoundFuture.join();
        if (limitMin != null) {
            return streamMetadataStore.compareStreamCut(scope, stream, limitMin.getStreamCut(), lowerBound, context, executor).thenCompose(compareWithMin -> {
                switch(compareWithMin) {
                    case EqualOrAfter:
                        // if it overlaps with limitmax, then we truncate at maxbound
                        return truncateAtLowerBoundOrMax(scope, stream, context, lowerBound, limitMax, maxBound);
                    case Overlaps:
                        // and we are choosing from retention set.
                        return getStreamcutBeforeLowerbound(scope, stream, context, retentionSet, lowerBound);
                    case Before:
                        // min is less than (behind/before) lb. truncate at min
                        return CompletableFuture.completedFuture(limitMin.getStreamCut());
                    default:
                        throw new IllegalArgumentException("Invalid Compare streamcut response");
                }
            });
        } else {
            return CompletableFuture.completedFuture(null);
        }
    });
}

Example 32 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class StreamMetadataTasks method createStream.

/**
 * Create stream.
 *
 * @param scope           scope.
 * @param stream          stream name.
 * @param config          stream configuration.
 * @param createTimestamp creation timestamp.
 * @param requestId       requestId.
 * @return creation status.
 */
@Task(name = "createStream", version = "1.0", resource = "{scope}/{stream}")
public CompletableFuture<CreateStreamStatus.Status> createStream(String scope, String stream, StreamConfiguration config, long createTimestamp, long requestId) {
    log.debug(requestId, "createStream with resource called.");
    OperationContext context = streamMetadataStore.createStreamContext(scope, stream, requestId);
    return execute(new Resource(scope, stream), new Serializable[] { scope, stream, config, createTimestamp, requestId }, () -> createStreamBody(scope, stream, config, createTimestamp, context));
}

Example 33 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class StreamMetadataTasks method sealStream.

@VisibleForTesting
CompletableFuture<UpdateStreamStatus.Status> sealStream(String scope, String stream, OperationContext context, int retryCount) {
    long requestId = context.getRequestId();
    // 1. post event for seal.
    SealStreamEvent event = new SealStreamEvent(scope, stream, requestId);
    return eventHelperFuture.thenCompose(eventHelper -> eventHelper.addIndexAndSubmitTask(event, // 2. set state to sealing
    () -> RetryHelper.withRetriesAsync(() -> streamMetadataStore.getVersionedState(scope, stream, context, executor).thenCompose(state -> {
        if (state.getObject().equals(State.SEALED)) {
            return CompletableFuture.completedFuture(state);
        } else {
            return streamMetadataStore.updateVersionedState(scope, stream, State.SEALING, state, context, executor);
        }
    }), RetryHelper.RETRYABLE_PREDICATE.or(e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException), retryCount, executor)).thenCompose(result -> {
        if (result.getObject().equals(State.SEALED) || result.getObject().equals(State.SEALING)) {
            return eventHelper.checkDone(() -> isSealed(scope, stream, context)).thenApply(x -> UpdateStreamStatus.Status.SUCCESS);
        } else {
            return CompletableFuture.completedFuture(UpdateStreamStatus.Status.FAILURE);
        }
    })).exceptionally(ex -> {
        final String message = "Exception thrown in trying to notify sealed segments.";
        return handleUpdateStreamError(ex, requestId, message, NameUtils.getScopedStreamName(scope, stream));
    });
}

Example 34 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class StreamMetadataTasks method startTruncation.

public CompletableFuture<Boolean> startTruncation(String scope, String stream, Map<Long, Long> streamCut, OperationContext contextOpt) {
    final OperationContext context = contextOpt != null ? contextOpt : streamMetadataStore.createStreamContext(scope, stream, ControllerService.nextRequestId());
    long requestId = context.getRequestId();
    return streamMetadataStore.getTruncationRecord(scope, stream, context, executor).thenCompose(property -> {
        if (!property.getObject().isUpdating()) {
            // 2. post event with new stream cut if no truncation is ongoing
            return eventHelperFuture.thenCompose(eventHelper -> eventHelper.addIndexAndSubmitTask(new TruncateStreamEvent(scope, stream, requestId), // 3. start truncation by updating the metadata
            () -> streamMetadataStore.startTruncation(scope, stream, streamCut, context, executor)).thenApply(x -> {
                log.debug(requestId, "Started truncation request for stream {}/{}", scope, stream);
                return true;
            }));
        } else {
            log.error(requestId, "Another truncation in progress for {}/{}", scope, stream);
            return CompletableFuture.completedFuture(false);
        }
    });
}

Example 35 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class StreamTransactionMetadataTasks method sealTxnBody.

/**
 * Seals a txn and transitions it to COMMITTING (resp. ABORTING) state if commit param is true (resp. false).
 *
 * Post-condition:
 * 1. If seal completes successfully, then
 *     (a) txn state is COMMITTING/ABORTING,
 *     (b) CommitEvent/AbortEvent is present in the commit stream/abort stream,
 *     (c) txn is removed from host-txn index,
 *     (d) txn is removed from the timeout service.
 *
 * 2. If process fails after transitioning txn to COMMITTING/ABORTING state, but before responding to client, then
 * since txn is present in the host-txn index, some other controller process shall put CommitEvent/AbortEvent to
 * commit stream/abort stream.
 *
 * @param host    host id. It is different from hostId iff invoked from TxnSweeper for aborting orphaned txn.
 * @param scope   scope name.
 * @param stream  stream name.
 * @param commit  boolean indicating whether to commit txn.
 * @param txnId   txn id.
 * @param version expected version of txn node in store.
 * @param ctx     context.
 * @return        Txn status after sealing it.
 */
CompletableFuture<TxnStatus> sealTxnBody(final String host, final String scope, final String stream, final boolean commit, final UUID txnId, final Version version, final String writerId, final long timestamp, final OperationContext ctx) {
    Preconditions.checkNotNull(ctx, "Operation context cannot be null");
    long requestId = ctx.getRequestId();
    TxnResource resource = new TxnResource(scope, stream, txnId);
    Optional<Version> versionOpt = Optional.ofNullable(version);
    // Step 1. Add txn to current host's index, if it is not already present
    CompletableFuture<Void> addIndex = host.equals(hostId) && !timeoutService.containsTxn(scope, stream, txnId) ? // then txn would no longer be open.
    streamMetadataStore.addTxnToIndex(hostId, resource, version) : CompletableFuture.completedFuture(null);
    addIndex.whenComplete((v, e) -> {
        if (e != null) {
            log.debug(requestId, "Txn={}, already present/newly added to host-txn index of host={}", txnId, hostId);
        } else {
            log.debug(requestId, "Txn={}, added txn to host-txn index of host={}", txnId, hostId);
        }
    });
    // Step 2. Seal txn
    CompletableFuture<AbstractMap.SimpleEntry<TxnStatus, Integer>> sealFuture = addIndex.thenComposeAsync(x -> streamMetadataStore.sealTransaction(scope, stream, txnId, commit, versionOpt, writerId, timestamp, ctx, executor), executor).whenComplete((v, e) -> {
        if (e != null) {
            log.debug(requestId, "Txn={}, failed sealing txn", txnId);
        } else {
            log.debug(requestId, "Txn={}, sealed successfully, commit={}", txnId, commit);
        }
    });
    // Step 3. write event to corresponding stream.
    return sealFuture.thenComposeAsync(pair -> {
        TxnStatus status = pair.getKey();
        switch(status) {
            case COMMITTING:
                return writeCommitEvent(scope, stream, pair.getValue(), txnId, status, requestId);
            case ABORTING:
                return writeAbortEvent(scope, stream, pair.getValue(), txnId, status, requestId);
            case ABORTED:
            case COMMITTED:
                return CompletableFuture.completedFuture(status);
            case OPEN:
            case UNKNOWN:
            default:
                // exception would be thrown.
                return CompletableFuture.completedFuture(status);
        }
    }, executor).thenComposeAsync(status -> {
        // Step 4. Remove txn from timeoutService, and from the index.
        timeoutService.removeTxn(scope, stream, txnId);
        log.debug(requestId, "Txn={}, removed from timeout service", txnId);
        return streamMetadataStore.removeTxnFromIndex(host, resource, true).whenComplete((v, e) -> {
            if (e != null) {
                log.debug(requestId, "Txn={}, failed removing txn from host-txn index of host={}", txnId, hostId);
            } else {
                log.debug(requestId, "Txn={}, removed txn from host-txn index of host={}", txnId, hostId);
            }
        }).thenApply(x -> status);
    }, executor);
}