Examples with TxnWriterMark io.pravega.controller.store.stream.TxnWriterMark used on opensource projects

Example 1 with TxnWriterMark

use of io.pravega.controller.store.stream.TxnWriterMark in project pravega by pravega.

the class CommitRequestHandler method tryCommitTransactions.

/**
 * Try creating txn commit list first. if node already exists and doesn't match the processing in the event, throw
 * operation not allowed.
 * This will result in event being posted back in the stream and retried later. Generally if a transaction commit starts,
 * it will come to an end. However, during failover, once we have created the node, we are guaranteed that it will
 * be only that transaction that will be getting committed at that time.
 * @return CompletableFuture which when completed will contain the epoch on which transactions were committed.
 */
private CompletableFuture<Integer> tryCommitTransactions(final String scope, final String stream, final OperationContext context) {
    Timer timer = new Timer();
    Map<String, TxnWriterMark> writerMarks = new HashMap<>();
    Map<UUID, String> txnIdToWriterId = new HashMap<>();
    return streamMetadataStore.getVersionedState(scope, stream, context, executor).thenComposeAsync(state -> {
        final AtomicReference<VersionedMetadata<State>> stateRecord = new AtomicReference<>(state);
        CompletableFuture<VersionedMetadata<CommittingTransactionsRecord>> commitFuture = streamMetadataStore.startCommitTransactions(scope, stream, MAX_TRANSACTION_COMMIT_BATCH_SIZE, context, executor).thenComposeAsync(txnsTuple -> {
            VersionedMetadata<CommittingTransactionsRecord> committingTxnsRecord = txnsTuple.getKey();
            if (committingTxnsRecord.getObject().equals(CommittingTransactionsRecord.EMPTY)) {
                // completed all the work.
                return CompletableFuture.completedFuture(committingTxnsRecord);
            } else {
                int txnEpoch = committingTxnsRecord.getObject().getEpoch();
                List<UUID> txnList = committingTxnsRecord.getObject().getTransactionsToCommit();
                log.info(context.getRequestId(), "Committing {} transactions on epoch {} on stream {}/{}", txnList, txnEpoch, scope, stream);
                // Once state is set to committing, we are guaranteed that this will be the only
                // processing that can happen on the stream and we can proceed with committing
                // outstanding transactions collected in the txnList step.
                CompletableFuture<Void> future;
                // completion of transactions in commit state.
                if (state.getObject().equals(State.SEALING)) {
                    future = CompletableFuture.completedFuture(null);
                } else {
                    // If state is not SEALING, try to set the state to COMMITTING_TXN before proceeding.
                    // If we are unable to set the state to COMMITTING_TXN, it will get OPERATION_NOT_ALLOWED
                    // and the processing will be retried later.
                    future = streamMetadataStore.updateVersionedState(scope, stream, State.COMMITTING_TXN, state, context, executor).thenAccept(stateRecord::set);
                }
                txnsTuple.getValue().forEach(txn -> {
                    if (!Strings.isNullOrEmpty(txn.getWriterId())) {
                        txnIdToWriterId.put(txn.getId(), txn.getWriterId());
                        if (!writerMarks.containsKey(txn.getWriterId()) || writerMarks.get(txn.getWriterId()).getTimestamp() < txn.getCommitTime()) {
                            writerMarks.put(txn.getWriterId(), new TxnWriterMark(txn.getCommitTime(), ImmutableMap.of(), txn.getId()));
                        }
                    }
                });
                // TxnCommittingRecord ensures no other rollingTxn can run concurrently
                return future.thenCompose(v -> getEpochRecords(scope, stream, txnEpoch, context).thenCompose(records -> {
                    EpochRecord txnEpochRecord = records.get(0);
                    EpochRecord activeEpochRecord = records.get(1);
                    CommitTxnContext commitContext = new CommitTxnContext(scope, stream, context, txnIdToWriterId, writerMarks);
                    if (activeEpochRecord.getEpoch() == txnEpoch || activeEpochRecord.getReferenceEpoch() == txnEpochRecord.getReferenceEpoch()) {
                        // we can commit transactions immediately
                        return commitTransactions(commitContext, committingTxnsRecord, activeEpochRecord.getSegmentIds().stream().collect(Collectors.toList())).thenApply(txnOffsets -> committingTxnsRecord);
                    } else {
                        return rollTransactions(commitContext, committingTxnsRecord, txnEpochRecord, activeEpochRecord);
                    }
                }));
            }
        }, executor);
        // reset state to ACTIVE if it was COMMITTING_TXN
        return commitFuture.thenCompose(committingTxnsRecord -> streamMetadataStore.completeCommitTransactions(scope, stream, committingTxnsRecord, context, executor, writerMarks).thenCompose(v -> resetStateConditionally(scope, stream, stateRecord.get(), context)).thenRun(() -> TransactionMetrics.getInstance().commitTransaction(scope, stream, timer.getElapsed())).thenApply(v -> committingTxnsRecord.getObject().getEpoch()));
    }, executor);
}

Example 2 with TxnWriterMark

use of io.pravega.controller.store.stream.TxnWriterMark in project pravega by pravega.

the class CommitRequestHandler method commitTransactions.

/**
 * This method loops over each transaction in the list, and commits them in order
 * At the end of this method's execution, all transactions in the list would have committed into given list of segments.
 */
private CompletableFuture<Void> commitTransactions(CommitTxnContext commitContext, VersionedMetadata<CommittingTransactionsRecord> txnRecord, List<Long> segments) {
    // Chain all transaction commit futures one after the other. This will ensure that order of commit
    // if honoured and is based on the order in the list.
    List<UUID> transactionsToCommit = txnRecord.getObject().getTransactionsToCommit();
    boolean noteTime = commitContext.writerMarks.size() > 0;
    Timer segMergeTimer = new Timer();
    return streamMetadataTasks.mergeTxnSegmentsIntoStreamSegments(commitContext.scope, commitContext.stream, segments, transactionsToCommit, commitContext.context.getRequestId()).thenCompose(segmentOffsets -> {
        TransactionMetrics.getInstance().commitTransactionSegments(segMergeTimer.getElapsed());
        if (noteTime) {
            for (int i = 0; i < transactionsToCommit.size(); i++) {
                int index = i;
                val txnOffsets = segmentOffsets.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, x -> x.getValue().get(index)));
                val txnId = transactionsToCommit.get(i);
                String writerId = commitContext.txnIdToWriterId.get(txnId);
                if (!Strings.isNullOrEmpty(writerId) && commitContext.writerMarks.get(writerId).getTransactionId().equals(txnId)) {
                    TxnWriterMark mark = commitContext.writerMarks.get(writerId);
                    commitContext.writerMarks.put(writerId, new TxnWriterMark(mark.getTimestamp(), txnOffsets, mark.getTransactionId()));
                }
            }
            return bucketStore.addStreamToBucketStore(BucketStore.ServiceType.WatermarkingService, commitContext.scope, commitContext.stream, executor);
        }
        return CompletableFuture.completedFuture(null);
    });
}