Examples with OperationContext io.pravega.controller.store.stream.OperationContext used on opensource projects

Example 36 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class PravegaTablesKVTMetadataStore method recordLastKVTableSegment.

@Override
CompletableFuture<Void> recordLastKVTableSegment(final String scope, final String kvtable, int lastActiveSegment, OperationContext ctx, final Executor executor) {
    OperationContext context = getOperationContext(ctx);
    final String key = getScopedKVTName(scope, kvtable);
    return Futures.completeOn(storeHelper.createTable(DELETED_KVTABLES_TABLE, context.getRequestId()).thenCompose(created -> {
        return storeHelper.expectingDataNotFound(storeHelper.getCachedOrLoad(DELETED_KVTABLES_TABLE, key, BYTES_TO_INTEGER_FUNCTION, System.currentTimeMillis(), context.getRequestId()), null).thenCompose(existing -> {
            log.debug(context.getRequestId(), "Recording last segment {} for KeyValueTable {}/{} on deletion.", lastActiveSegment, scope, kvtable);
            if (existing != null) {
                final int oldLastActiveSegment = existing.getObject();
                Preconditions.checkArgument(lastActiveSegment >= oldLastActiveSegment, "Old last active segment ({}) for {}/{} is higher than current one {}.", oldLastActiveSegment, scope, kvtable, lastActiveSegment);
                return Futures.toVoid(storeHelper.updateEntry(DELETED_KVTABLES_TABLE, key, lastActiveSegment, INTEGER_TO_BYTES_FUNCTION, existing.getVersion(), context.getRequestId()));
            } else {
                return Futures.toVoid(storeHelper.addNewEntryIfAbsent(DELETED_KVTABLES_TABLE, key, lastActiveSegment, INTEGER_TO_BYTES_FUNCTION, context.getRequestId()));
            }
        });
    }), executor);
}

Example 37 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class RequestHandlersTest method testDeleteScopeRecursive.

@Test
public void testDeleteScopeRecursive() {
    StreamMetadataStore streamStoreSpied = spy(getStore());
    KVTableMetadataStore kvtStoreSpied = spy(getKvtStore());
    OperationContext ctx = new OperationContext() {

        @Override
        public long getOperationStartTime() {
            return 0;
        }

        @Override
        public long getRequestId() {
            return 0;
        }
    };
    UUID scopeId = streamStoreSpied.getScopeId(scope, ctx, executor).join();
    doAnswer(x -> {
        CompletableFuture<UUID> cf = new CompletableFuture<>();
        cf.complete(scopeId);
        return cf;
    }).when(streamStoreSpied).getScopeId(eq(scope), eq(ctx), eq(executor));
    doAnswer(invocation -> {
        CompletableFuture<Boolean> cf = new CompletableFuture<>();
        cf.complete(false);
        return cf;
    }).when(streamStoreSpied).isScopeSealed(eq(scope), any(), any());
    DeleteScopeTask requestHandler = new DeleteScopeTask(streamMetadataTasks, streamStoreSpied, kvtStoreSpied, kvtTasks, executor);
    DeleteScopeEvent event = new DeleteScopeEvent(scope, System.currentTimeMillis(), scopeId);
    CompletableFuture<Void> future = CompletableFuture.completedFuture(null).thenComposeAsync(v -> requestHandler.execute(event), executor);
    future.join();
}

Example 38 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class ControllerEventProcessorPravegaTablesStreamTest method testTxnPartialCommitRetry.

@Test(timeout = 10000)
public void testTxnPartialCommitRetry() {
    PravegaTablesStoreHelper storeHelper = spy(new PravegaTablesStoreHelper(SegmentHelperMock.getSegmentHelperMockForTables(executor), GrpcAuthHelper.getDisabledAuthHelper(), executor));
    this.streamStore = new PravegaTablesStreamMetadataStore(PRAVEGA_ZK_CURATOR_RESOURCE.client, executor, Duration.ofHours(Config.COMPLETED_TRANSACTION_TTL_IN_HOURS), storeHelper);
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    EventHelper eventHelperMock = EventHelperMock.getEventHelperMock(executor, "1", ((AbstractStreamMetadataStore) this.streamStore).getHostTaskIndex());
    StreamMetadataTasks streamMetadataTasks = new StreamMetadataTasks(streamStore, this.bucketStore, TaskStoreFactory.createInMemoryStore(executor), segmentHelperMock, executor, "1", GrpcAuthHelper.getDisabledAuthHelper(), eventHelperMock);
    StreamTransactionMetadataTasks streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(this.streamStore, segmentHelperMock, executor, "host", GrpcAuthHelper.getDisabledAuthHelper());
    streamTransactionMetadataTasks.initializeStreamWriters(new EventStreamWriterMock<>(), new EventStreamWriterMock<>());
    String scope = "scope";
    String stream = "stream";
    // region createStream
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scalingPolicy(policy1).build();
    streamStore.createScope(scope, null, executor).join();
    long start = System.currentTimeMillis();
    streamStore.createStream(scope, stream, configuration1, start, null, executor).join();
    streamStore.setState(scope, stream, State.ACTIVE, null, executor).join();
    StreamMetadataTasks spyStreamMetadataTasks = spy(streamMetadataTasks);
    List<VersionedTransactionData> txnDataList = createAndCommitTransactions(3);
    int epoch = txnDataList.get(0).getEpoch();
    spyStreamMetadataTasks.setRequestEventWriter(new EventStreamWriterMock<>());
    CommitRequestHandler commitEventProcessor = new CommitRequestHandler(streamStore, spyStreamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
    final String committingTxnsRecordKey = "committingTxns";
    long failingClientRequestId = 123L;
    doReturn(failingClientRequestId).when(spyStreamMetadataTasks).getRequestId(any());
    OperationContext context = this.streamStore.createStreamContext(scope, stream, failingClientRequestId);
    streamStore.startCommitTransactions(scope, stream, 100, context, executor).join();
    doReturn(Futures.failedFuture(new RuntimeException())).when(storeHelper).updateEntry(anyString(), eq(committingTxnsRecordKey), any(), ArgumentMatchers.<Function<String, byte[]>>any(), any(), eq(failingClientRequestId));
    AssertExtensions.assertFutureThrows("Updating CommittingTxnRecord fails", commitEventProcessor.processEvent(new CommitEvent(scope, stream, epoch)), e -> Exceptions.unwrap(e) instanceof RuntimeException);
    verify(storeHelper, times(1)).removeEntries(anyString(), any(), eq(failingClientRequestId));
    VersionedMetadata<CommittingTransactionsRecord> versionedCommitRecord = this.streamStore.getVersionedCommittingTransactionsRecord(scope, stream, context, executor).join();
    CommittingTransactionsRecord commitRecord = versionedCommitRecord.getObject();
    assertFalse(CommittingTransactionsRecord.EMPTY.equals(commitRecord));
    for (VersionedTransactionData txnData : txnDataList) {
        checkTransactionState(scope, stream, txnData.getId(), TxnStatus.COMMITTED);
    }
    long goodClientRequestId = 4567L;
    doReturn(goodClientRequestId).when(spyStreamMetadataTasks).getRequestId(any());
    commitEventProcessor.processEvent(new CommitEvent(scope, stream, epoch)).join();
    versionedCommitRecord = this.streamStore.getVersionedCommittingTransactionsRecord(scope, stream, context, executor).join();
    commitRecord = versionedCommitRecord.getObject();
    assertTrue(CommittingTransactionsRecord.EMPTY.equals(commitRecord));
    for (VersionedTransactionData txnData : txnDataList) {
        checkTransactionState(scope, stream, txnData.getId(), TxnStatus.COMMITTED);
    }
}

Example 39 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class MockStreamTransactionMetadataTasks method createTxn.

@Override
@Synchronized
public CompletableFuture<Pair<VersionedTransactionData, List<Segment>>> createTxn(final String scope, final String stream, final long lease, final long scaleGracePeriod, final OperationContext contextOpt) {
    final OperationContext context = contextOpt == null ? streamMetadataStore.createContext(scope, stream) : contextOpt;
    final UUID txnId = UUID.randomUUID();
    return streamMetadataStore.createTransaction(scope, stream, txnId, lease, 10 * lease, scaleGracePeriod, context, executor).thenCompose(txData -> {
        log.info("Created transaction {} with version {}", txData.getId(), txData.getVersion());
        return streamMetadataStore.getActiveSegments(scope, stream, context, executor).thenApply(segmentList -> new ImmutablePair<>(txData, segmentList));
    });
}

Example 40 with OperationContext

use of io.pravega.controller.store.stream.OperationContext in project pravega by pravega.

the class StreamTransactionMetadataTasks method sealTxnBody.

/**
 * Seals a txn and transitions it to COMMITTING (resp. ABORTING) state if commit param is true (resp. false).
 *
 * Post-condition:
 * 1. If seal completes successfully, then
 *     (a) txn state is COMMITTING/ABORTING,
 *     (b) CommitEvent/AbortEvent is present in the commit stream/abort stream,
 *     (c) txn is removed from host-txn index,
 *     (d) txn is removed from the timeout service.
 *
 * 2. If process fails after transitioning txn to COMMITTING/ABORTING state, but before responding to client, then
 * since txn is present in the host-txn index, some other controller process shall put CommitEvent/AbortEvent to
 * commit stream/abort stream.
 *
 * @param host    host id. It is different from hostId iff invoked from TxnSweeper for aborting orphaned txn.
 * @param scope   scope name.
 * @param stream  stream name.
 * @param commit  boolean indicating whether to commit txn.
 * @param txnId   txn id.
 * @param version expected version of txn node in store.
 * @param ctx     context.
 * @return        Txn status after sealing it.
 */
CompletableFuture<TxnStatus> sealTxnBody(final String host, final String scope, final String stream, final boolean commit, final UUID txnId, final Integer version, final OperationContext ctx) {
    TxnResource resource = new TxnResource(scope, stream, txnId);
    Optional<Integer> versionOpt = Optional.ofNullable(version);
    // Step 1. Add txn to current host's index, if it is not already present
    CompletableFuture<Void> addIndex = host.equals(hostId) && !timeoutService.containsTxn(scope, stream, txnId) ? // then txn would no longer be open.
    streamMetadataStore.addTxnToIndex(hostId, resource, Integer.MAX_VALUE) : CompletableFuture.completedFuture(null);
    addIndex.whenComplete((v, e) -> {
        if (e != null) {
            log.debug("Txn={}, already present/newly added to host-txn index of host={}", txnId, hostId);
        } else {
            log.debug("Txn={}, added txn to host-txn index of host={}", txnId, hostId);
        }
    });
    // Step 2. Seal txn
    CompletableFuture<AbstractMap.SimpleEntry<TxnStatus, Integer>> sealFuture = addIndex.thenComposeAsync(x -> streamMetadataStore.sealTransaction(scope, stream, txnId, commit, versionOpt, ctx, executor), executor).whenComplete((v, e) -> {
        if (e != null) {
            log.debug("Txn={}, failed sealing txn", txnId);
        } else {
            log.debug("Txn={}, sealed successfully, commit={}", txnId, commit);
        }
    });
    // Step 3. write event to corresponding stream.
    return sealFuture.thenComposeAsync(pair -> {
        TxnStatus status = pair.getKey();
        switch(status) {
            case COMMITTING:
                return writeCommitEvent(scope, stream, pair.getValue(), txnId, status);
            case ABORTING:
                return writeAbortEvent(scope, stream, pair.getValue(), txnId, status);
            case ABORTED:
            case COMMITTED:
                return CompletableFuture.completedFuture(status);
            case OPEN:
            case UNKNOWN:
            default:
                // exception would be thrown.
                return CompletableFuture.completedFuture(status);
        }
    }, executor).thenComposeAsync(status -> {
        // Step 4. Remove txn from timeoutService, and from the index.
        timeoutService.removeTxn(scope, stream, txnId);
        log.debug("Txn={}, removed from timeout service", txnId);
        return streamMetadataStore.removeTxnFromIndex(host, resource, true).whenComplete((v, e) -> {
            if (e != null) {
                log.debug("Txn={}, failed removing txn from host-txn index of host={}", txnId, hostId);
            } else {
                log.debug("Txn={}, removed txn from host-txn index of host={}", txnId, hostId);
            }
        }).thenApply(x -> status);
    }, executor);
}