Examples with OperationContext io.pravega.controller.store.stream.OperationContext used on opensource projects

Example 11 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class ControllerServiceTest method setup.

@Before
public void setup() throws ExecutionException, InterruptedException {
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final ScalingPolicy policy2 = ScalingPolicy.fixed(3);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    final StreamConfiguration configuration2 = StreamConfiguration.builder().scope(SCOPE).streamName(stream2).scalingPolicy(policy2).build();
    // createScope
    streamStore.createScope(SCOPE).get();
    // region createStream
    startTs = System.currentTimeMillis();
    OperationContext context = streamStore.createContext(SCOPE, stream1);
    streamStore.createStream(SCOPE, stream1, configuration1, startTs, context, executor).get();
    streamStore.setState(SCOPE, stream1, State.ACTIVE, context, executor);
    OperationContext context2 = streamStore.createContext(SCOPE, stream2);
    streamStore.createStream(SCOPE, stream2, configuration2, startTs, context2, executor).get();
    streamStore.setState(SCOPE, stream2, State.ACTIVE, context2, executor);
    // endregion
    // region scaleSegments
    SimpleEntry<Double, Double> segment1 = new SimpleEntry<>(0.5, 0.75);
    SimpleEntry<Double, Double> segment2 = new SimpleEntry<>(0.75, 1.0);
    List<Integer> sealedSegments = Collections.singletonList(1);
    scaleTs = System.currentTimeMillis();
    StartScaleResponse startScaleResponse = streamStore.startScale(SCOPE, stream1, sealedSegments, Arrays.asList(segment1, segment2), startTs + 20, false, null, executor).get();
    List<Segment> segmentCreated = startScaleResponse.getSegmentsCreated();
    streamStore.setState(SCOPE, stream1, State.SCALING, null, executor).get();
    streamStore.scaleNewSegmentsCreated(SCOPE, stream1, sealedSegments, segmentCreated, startScaleResponse.getActiveEpoch(), scaleTs, null, executor).get();
    streamStore.scaleSegmentsSealed(SCOPE, stream1, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), segmentCreated, startScaleResponse.getActiveEpoch(), scaleTs, null, executor).get();
    SimpleEntry<Double, Double> segment3 = new SimpleEntry<>(0.0, 0.5);
    SimpleEntry<Double, Double> segment4 = new SimpleEntry<>(0.5, 0.75);
    SimpleEntry<Double, Double> segment5 = new SimpleEntry<>(0.75, 1.0);
    sealedSegments = Arrays.asList(0, 1, 2);
    startScaleResponse = streamStore.startScale(SCOPE, stream2, sealedSegments, Arrays.asList(segment3, segment4, segment5), startTs + 20, false, null, executor).get();
    segmentCreated = startScaleResponse.getSegmentsCreated();
    streamStore.setState(SCOPE, stream2, State.SCALING, null, executor).get();
    streamStore.scaleNewSegmentsCreated(SCOPE, stream2, sealedSegments, segmentCreated, startScaleResponse.getActiveEpoch(), scaleTs, null, executor).get();
    streamStore.scaleSegmentsSealed(SCOPE, stream2, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), segmentCreated, startScaleResponse.getActiveEpoch(), scaleTs, null, executor).get();
// endregion
}

Example 12 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class StreamMetadataTasks method retention.

/**
 * Method to check retention policy and generate new periodic cuts and/or truncate stream at an existing stream cut.
 *
 * @param scope scope
 * @param stream stream
 * @param policy retention policy
 * @param recordingTime time of recording
 * @param contextOpt operation context
 * @param delegationToken token to be sent to segmentstore to authorize this operation.
 * @return future.
 */
public CompletableFuture<Void> retention(final String scope, final String stream, final RetentionPolicy policy, final long recordingTime, final OperationContext contextOpt, final String delegationToken) {
    Preconditions.checkNotNull(policy);
    final OperationContext context = contextOpt == null ? streamMetadataStore.createContext(scope, stream) : contextOpt;
    return streamMetadataStore.getStreamCutsFromRetentionSet(scope, stream, context, executor).thenCompose(retentionSet -> {
        StreamCutRecord latestCut = retentionSet.stream().max(Comparator.comparingLong(StreamCutRecord::getRecordingTime)).orElse(null);
        return checkGenerateStreamCut(scope, stream, context, latestCut, recordingTime, delegationToken).thenCompose(newRecord -> truncate(scope, stream, policy, context, retentionSet, newRecord, recordingTime));
    }).thenAccept(x -> DYNAMIC_LOGGER.recordMeterEvents(nameFromStream(RETENTION_FREQUENCY, scope, stream), 1));
}

Example 13 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class StreamTransactionMetadataTasks method sealTxnBody.

/**
 * Seals a txn and transitions it to COMMITTING (resp. ABORTING) state if commit param is true (resp. false).
 *
 * Post-condition:
 * 1. If seal completes successfully, then
 *     (a) txn state is COMMITTING/ABORTING,
 *     (b) CommitEvent/AbortEvent is present in the commit stream/abort stream,
 *     (c) txn is removed from host-txn index,
 *     (d) txn is removed from the timeout service.
 *
 * 2. If process fails after transitioning txn to COMMITTING/ABORTING state, but before responding to client, then
 * since txn is present in the host-txn index, some other controller process shall put CommitEvent/AbortEvent to
 * commit stream/abort stream.
 *
 * @param host    host id. It is different from hostId iff invoked from TxnSweeper for aborting orphaned txn.
 * @param scope   scope name.
 * @param stream  stream name.
 * @param commit  boolean indicating whether to commit txn.
 * @param txnId   txn id.
 * @param version expected version of txn node in store.
 * @param ctx     context.
 * @return        Txn status after sealing it.
 */
CompletableFuture<TxnStatus> sealTxnBody(final String host, final String scope, final String stream, final boolean commit, final UUID txnId, final Integer version, final OperationContext ctx) {
    TxnResource resource = new TxnResource(scope, stream, txnId);
    Optional<Integer> versionOpt = Optional.ofNullable(version);
    // Step 1. Add txn to current host's index, if it is not already present
    CompletableFuture<Void> addIndex = host.equals(hostId) && !timeoutService.containsTxn(scope, stream, txnId) ? // then txn would no longer be open.
    streamMetadataStore.addTxnToIndex(hostId, resource, Integer.MAX_VALUE) : CompletableFuture.completedFuture(null);
    addIndex.whenComplete((v, e) -> {
        if (e != null) {
            log.debug("Txn={}, already present/newly added to host-txn index of host={}", txnId, hostId);
        } else {
            log.debug("Txn={}, added txn to host-txn index of host={}", txnId, hostId);
        }
    });
    // Step 2. Seal txn
    CompletableFuture<AbstractMap.SimpleEntry<TxnStatus, Integer>> sealFuture = addIndex.thenComposeAsync(x -> streamMetadataStore.sealTransaction(scope, stream, txnId, commit, versionOpt, ctx, executor), executor).whenComplete((v, e) -> {
        if (e != null) {
            log.debug("Txn={}, failed sealing txn", txnId);
        } else {
            log.debug("Txn={}, sealed successfully, commit={}", txnId, commit);
        }
    });
    // Step 3. write event to corresponding stream.
    return sealFuture.thenComposeAsync(pair -> {
        TxnStatus status = pair.getKey();
        switch(status) {
            case COMMITTING:
                return writeCommitEvent(scope, stream, pair.getValue(), txnId, status);
            case ABORTING:
                return writeAbortEvent(scope, stream, pair.getValue(), txnId, status);
            case ABORTED:
            case COMMITTED:
                return CompletableFuture.completedFuture(status);
            case OPEN:
            case UNKNOWN:
            default:
                // exception would be thrown.
                return CompletableFuture.completedFuture(status);
        }
    }, executor).thenComposeAsync(status -> {
        // Step 4. Remove txn from timeoutService, and from the index.
        timeoutService.removeTxn(scope, stream, txnId);
        log.debug("Txn={}, removed from timeout service", txnId);
        return streamMetadataStore.removeTxnFromIndex(host, resource, true).whenComplete((v, e) -> {
            if (e != null) {
                log.debug("Txn={}, failed removing txn from host-txn index of host={}", txnId, hostId);
            } else {
                log.debug("Txn={}, removed txn from host-txn index of host={}", txnId, hostId);
            }
        }).thenApply(x -> status);
    }, executor);
}

Example 14 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class ControllerService method getSegmentsImmediatelyFollowing.

public CompletableFuture<Map<SegmentRange, List<Integer>>> getSegmentsImmediatelyFollowing(SegmentId segment) {
    Preconditions.checkNotNull(segment, "segment");
    OperationContext context = streamStore.createContext(segment.getStreamInfo().getScope(), segment.getStreamInfo().getStream());
    return streamStore.getSuccessors(segment.getStreamInfo().getScope(), segment.getStreamInfo().getStream(), segment.getSegmentNumber(), context, executor).thenComposeAsync(successors -> Futures.keysAllOfWithResults(successors.entrySet().stream().collect(Collectors.toMap(entry -> streamStore.getSegment(segment.getStreamInfo().getScope(), segment.getStreamInfo().getStream(), entry.getKey(), context, executor).thenApply(seg -> ModelHelper.createSegmentRange(segment.getStreamInfo().getScope(), segment.getStreamInfo().getStream(), seg.getNumber(), seg.getKeyStart(), seg.getKeyEnd())), Map.Entry::getValue))), executor);
}

Example 15 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class AbortRequestHandler method processEvent.

@Override
public CompletableFuture<Void> processEvent(AbortEvent event) {
    String scope = event.getScope();
    String stream = event.getStream();
    int epoch = event.getEpoch();
    UUID txId = event.getTxid();
    OperationContext context = streamMetadataStore.createContext(scope, stream);
    log.debug("Aborting transaction {} on stream {}/{}", event.getTxid(), event.getScope(), event.getStream());
    return streamMetadataStore.getActiveSegmentIds(event.getScope(), event.getStream(), epoch, context, executor).thenCompose(segments -> Futures.allOfWithResults(segments.stream().parallel().map(segment -> notifyAbortToHost(scope, stream, segment, txId)).collect(Collectors.toList()))).thenCompose(x -> streamMetadataStore.abortTransaction(scope, stream, epoch, txId, context, executor)).thenCompose(x -> Futures.toVoid(streamMetadataTasks.tryCompleteScale(scope, stream, epoch, context, this.streamMetadataTasks.retrieveDelegationToken()))).whenComplete((result, error) -> {
        if (error != null) {
            log.error("Failed aborting transaction {} on stream {}/{}", event.getTxid(), event.getScope(), event.getStream());
        } else {
            log.debug("Successfully aborted transaction {} on stream {}/{}", event.getTxid(), event.getScope(), event.getStream());
            if (processedEvents != null) {
                processedEvents.offer(event);
            }
        }
    });
}