Examples with EpochRecord io.pravega.controller.store.stream.records.EpochRecord used on opensource projects

Example 1 with EpochRecord

use of io.pravega.controller.store.stream.records.EpochRecord in project pravega by pravega.

the class CommitRequestHandler method tryCommitTransactions.

/**
 * Try creating txn commit list first. if node already exists and doesn't match the processing in the event, throw
 * operation not allowed.
 * This will result in event being posted back in the stream and retried later. Generally if a transaction commit starts,
 * it will come to an end. However, during failover, once we have created the node, we are guaranteed that it will
 * be only that transaction that will be getting committed at that time.
 * @return CompletableFuture which when completed will contain the epoch on which transactions were committed.
 */
private CompletableFuture<Integer> tryCommitTransactions(final String scope, final String stream, final OperationContext context) {
    Timer timer = new Timer();
    Map<String, TxnWriterMark> writerMarks = new HashMap<>();
    Map<UUID, String> txnIdToWriterId = new HashMap<>();
    return streamMetadataStore.getVersionedState(scope, stream, context, executor).thenComposeAsync(state -> {
        final AtomicReference<VersionedMetadata<State>> stateRecord = new AtomicReference<>(state);
        CompletableFuture<VersionedMetadata<CommittingTransactionsRecord>> commitFuture = streamMetadataStore.startCommitTransactions(scope, stream, MAX_TRANSACTION_COMMIT_BATCH_SIZE, context, executor).thenComposeAsync(txnsTuple -> {
            VersionedMetadata<CommittingTransactionsRecord> committingTxnsRecord = txnsTuple.getKey();
            if (committingTxnsRecord.getObject().equals(CommittingTransactionsRecord.EMPTY)) {
                // completed all the work.
                return CompletableFuture.completedFuture(committingTxnsRecord);
            } else {
                int txnEpoch = committingTxnsRecord.getObject().getEpoch();
                List<UUID> txnList = committingTxnsRecord.getObject().getTransactionsToCommit();
                log.info(context.getRequestId(), "Committing {} transactions on epoch {} on stream {}/{}", txnList, txnEpoch, scope, stream);
                // Once state is set to committing, we are guaranteed that this will be the only
                // processing that can happen on the stream and we can proceed with committing
                // outstanding transactions collected in the txnList step.
                CompletableFuture<Void> future;
                // completion of transactions in commit state.
                if (state.getObject().equals(State.SEALING)) {
                    future = CompletableFuture.completedFuture(null);
                } else {
                    // If state is not SEALING, try to set the state to COMMITTING_TXN before proceeding.
                    // If we are unable to set the state to COMMITTING_TXN, it will get OPERATION_NOT_ALLOWED
                    // and the processing will be retried later.
                    future = streamMetadataStore.updateVersionedState(scope, stream, State.COMMITTING_TXN, state, context, executor).thenAccept(stateRecord::set);
                }
                txnsTuple.getValue().forEach(txn -> {
                    if (!Strings.isNullOrEmpty(txn.getWriterId())) {
                        txnIdToWriterId.put(txn.getId(), txn.getWriterId());
                        if (!writerMarks.containsKey(txn.getWriterId()) || writerMarks.get(txn.getWriterId()).getTimestamp() < txn.getCommitTime()) {
                            writerMarks.put(txn.getWriterId(), new TxnWriterMark(txn.getCommitTime(), ImmutableMap.of(), txn.getId()));
                        }
                    }
                });
                // TxnCommittingRecord ensures no other rollingTxn can run concurrently
                return future.thenCompose(v -> getEpochRecords(scope, stream, txnEpoch, context).thenCompose(records -> {
                    EpochRecord txnEpochRecord = records.get(0);
                    EpochRecord activeEpochRecord = records.get(1);
                    CommitTxnContext commitContext = new CommitTxnContext(scope, stream, context, txnIdToWriterId, writerMarks);
                    if (activeEpochRecord.getEpoch() == txnEpoch || activeEpochRecord.getReferenceEpoch() == txnEpochRecord.getReferenceEpoch()) {
                        // we can commit transactions immediately
                        return commitTransactions(commitContext, committingTxnsRecord, activeEpochRecord.getSegmentIds().stream().collect(Collectors.toList())).thenApply(txnOffsets -> committingTxnsRecord);
                    } else {
                        return rollTransactions(commitContext, committingTxnsRecord, txnEpochRecord, activeEpochRecord);
                    }
                }));
            }
        }, executor);
        // reset state to ACTIVE if it was COMMITTING_TXN
        return commitFuture.thenCompose(committingTxnsRecord -> streamMetadataStore.completeCommitTransactions(scope, stream, committingTxnsRecord, context, executor, writerMarks).thenCompose(v -> resetStateConditionally(scope, stream, stateRecord.get(), context)).thenRun(() -> TransactionMetrics.getInstance().commitTransaction(scope, stream, timer.getElapsed())).thenApply(v -> committingTxnsRecord.getObject().getEpoch()));
    }, executor);
}

Example 2 with EpochRecord

use of io.pravega.controller.store.stream.records.EpochRecord in project pravega by pravega.

the class PersistentStreamBase method computeStreamCutSpanInternal.

private ImmutableMap<StreamSegmentRecord, Integer> computeStreamCutSpanInternal(Map<Long, Long> streamCut, int epochLow, int epochHigh, List<EpochRecord> epochs) {
    List<Long> toFind = new ArrayList<>(streamCut.keySet());
    ImmutableMap.Builder<StreamSegmentRecord, Integer> resultSet = ImmutableMap.builder();
    for (int i = epochHigh - epochLow; i >= 0; i--) {
        if (toFind.isEmpty()) {
            break;
        }
        EpochRecord epochRecord = epochs.get(i);
        Set<Long> epochSegments = epochRecord.getSegmentIds();
        List<Long> found = toFind.stream().filter(epochSegments::contains).collect(Collectors.toList());
        resultSet.putAll(found.stream().collect(Collectors.toMap(x -> epochRecord.getSegments().stream().filter(z -> z.segmentId() == x).findFirst().get(), x -> epochRecord.getEpoch())));
        toFind.removeAll(epochSegments);
    }
    return resultSet.build();
}

Example 3 with EpochRecord

use of io.pravega.controller.store.stream.records.EpochRecord in project pravega by pravega.

the class PersistentStreamBase method rollingTxnCreateDuplicateEpochs.

@Override
public CompletableFuture<Void> rollingTxnCreateDuplicateEpochs(Map<Long, Long> sealedTxnEpochSegments, long time, VersionedMetadata<CommittingTransactionsRecord> record, OperationContext context) {
    Preconditions.checkNotNull(context, "Operation context cannot be null");
    Preconditions.checkArgument(record.getObject().isRollingTxnRecord());
    CommittingTransactionsRecord committingTxnRecord = record.getObject();
    return getActiveEpoch(true, context).thenCompose(activeEpochRecord -> getEpochRecord(committingTxnRecord.getEpoch(), context).thenCompose(transactionEpochRecord -> {
        if (activeEpochRecord.getEpoch() > committingTxnRecord.getCurrentEpoch()) {
            log.debug(context.getRequestId(), "Duplicate Epochs {} already created. Ignore.", committingTxnRecord.getNewActiveEpoch());
            return CompletableFuture.completedFuture(null);
        }
        long timeStamp = Math.max(activeEpochRecord.getCreationTime() + 1, time);
        ImmutableList.Builder<StreamSegmentRecord> duplicateTxnSegmentsBuilder = ImmutableList.builder();
        transactionEpochRecord.getSegments().stream().forEach(x -> duplicateTxnSegmentsBuilder.add(newSegmentRecord(computeSegmentId(getSegmentNumber(x.segmentId()), committingTxnRecord.getNewTxnEpoch()), timeStamp, x.getKeyStart(), x.getKeyEnd())));
        ImmutableList.Builder<StreamSegmentRecord> duplicateActiveSegmentsBuilder = ImmutableList.builder();
        activeEpochRecord.getSegments().stream().forEach(x -> duplicateActiveSegmentsBuilder.add(newSegmentRecord(computeSegmentId(getSegmentNumber(x.segmentId()), committingTxnRecord.getNewActiveEpoch()), timeStamp + 1, x.getKeyStart(), x.getKeyEnd())));
        CompletableFuture<EpochRecord> txnEpochFuture = getSplitMergeCountsTillEpoch(activeEpochRecord, context).thenCompose(txnSplitMergeCount -> {
            ImmutableList<StreamSegmentRecord> duplicateTxnEpochSegments = duplicateTxnSegmentsBuilder.build();
            EpochRecord duplicateTxnEpoch = new EpochRecord(committingTxnRecord.getNewTxnEpoch(), transactionEpochRecord.getReferenceEpoch(), duplicateTxnEpochSegments, timeStamp, getNewEpochSplitCount(txnSplitMergeCount.getKey(), activeEpochRecord.getSegments(), duplicateTxnEpochSegments), getNewEpochMergeCount(txnSplitMergeCount.getValue(), activeEpochRecord.getSegments(), duplicateTxnEpochSegments));
            return CompletableFuture.completedFuture(duplicateTxnEpoch);
        });
        CompletableFuture<EpochRecord> activeEpochFuture = txnEpochFuture.thenCompose(previousEpoch -> getSplitMergeCountsTillEpoch(previousEpoch, context).thenCompose(prevSplitMergeCounts -> {
            ImmutableList<StreamSegmentRecord> activeEpochSegments = duplicateActiveSegmentsBuilder.build();
            EpochRecord duplicateActiveEpoch = new EpochRecord(committingTxnRecord.getNewActiveEpoch(), activeEpochRecord.getReferenceEpoch(), activeEpochSegments, timeStamp + 1, getNewEpochSplitCount(prevSplitMergeCounts.getKey(), previousEpoch.getSegments(), activeEpochSegments), getNewEpochMergeCount(prevSplitMergeCounts.getValue(), previousEpoch.getSegments(), activeEpochSegments));
            return CompletableFuture.completedFuture(duplicateActiveEpoch);
        }));
        return CompletableFuture.allOf(txnEpochFuture, activeEpochFuture).thenCompose(v -> {
            EpochRecord duplicateTxnEpoch = txnEpochFuture.join();
            EpochRecord duplicateActiveEpoch = activeEpochFuture.join();
            HistoryTimeSeriesRecord timeSeriesRecordTxnEpoch = new HistoryTimeSeriesRecord(duplicateTxnEpoch.getEpoch(), duplicateTxnEpoch.getReferenceEpoch(), ImmutableList.of(), ImmutableList.of(), timeStamp);
            HistoryTimeSeriesRecord timeSeriesRecordActiveEpoch = new HistoryTimeSeriesRecord(duplicateActiveEpoch.getEpoch(), duplicateActiveEpoch.getReferenceEpoch(), ImmutableList.of(), ImmutableList.of(), timeStamp + 1);
            return createEpochRecord(duplicateTxnEpoch, context).thenCompose(x -> updateHistoryTimeSeries(timeSeriesRecordTxnEpoch, context)).thenCompose(x -> createEpochRecord(duplicateActiveEpoch, context)).thenCompose(x -> updateHistoryTimeSeries(timeSeriesRecordActiveEpoch, context)).thenCompose(x -> createSegmentSealedEpochRecords(activeEpochRecord.getSegments().stream().map(StreamSegmentRecord::segmentId).collect(Collectors.toList()), duplicateTxnEpoch.getEpoch(), context)).thenCompose(x -> createSegmentSealedEpochRecords(duplicateTxnEpoch.getSegments().stream().map(StreamSegmentRecord::segmentId).collect(Collectors.toList()), duplicateActiveEpoch.getEpoch(), context));
        }).thenCompose(r -> updateSealedSegmentSizes(sealedTxnEpochSegments, context));
    }));
}

Example 4 with EpochRecord

use of io.pravega.controller.store.stream.records.EpochRecord in project pravega by pravega.

the class PersistentStreamBase method createHistoryRecords.

private CompletionStage<Void> createHistoryRecords(int startingSegmentNumber, CreateStreamResponse createStreamResponse, OperationContext context) {
    Preconditions.checkNotNull(context, "operation context cannot be null");
    final int numSegments = createStreamResponse.getConfiguration().getScalingPolicy().getMinNumSegments();
    // create epoch 0 record
    final double keyRangeChunk = 1.0 / numSegments;
    long creationTime = createStreamResponse.getTimestamp();
    final ImmutableList.Builder<StreamSegmentRecord> builder = ImmutableList.builder();
    IntStream.range(0, numSegments).boxed().forEach(x -> builder.add(newSegmentRecord(0, startingSegmentNumber + x, creationTime, x * keyRangeChunk, (x + 1) * keyRangeChunk)));
    EpochRecord epoch0 = new EpochRecord(0, 0, builder.build(), creationTime, 0L, 0L);
    return createEpochRecord(epoch0, context).thenCompose(r -> createHistoryChunk(epoch0, context)).thenCompose(r -> createSealedSegmentSizeMapShardIfAbsent(0, context)).thenCompose(r -> createRetentionSetDataIfAbsent(new RetentionSet(ImmutableList.of()), context)).thenCompose(r -> createCurrentEpochRecordDataIfAbsent(epoch0, context));
}

Example 5 with EpochRecord

use of io.pravega.controller.store.stream.records.EpochRecord in project pravega by pravega.

the class StreamMetadataTasks method checkScale.

/**
 * Helper method to check if scale operation against an epoch completed or not.
 *
 * @param scope          scope.
 * @param stream         stream name.
 * @param epoch          stream epoch.
 * @param requestId      request id.
 * @return returns the newly created segments.
 */
public CompletableFuture<ScaleStatusResponse> checkScale(String scope, String stream, int epoch, long requestId) {
    OperationContext context = streamMetadataStore.createStreamContext(scope, stream, requestId);
    CompletableFuture<EpochRecord> activeEpochFuture = streamMetadataStore.getActiveEpoch(scope, stream, context, true, executor);
    CompletableFuture<State> stateFuture = streamMetadataStore.getState(scope, stream, true, context, executor);
    CompletableFuture<EpochTransitionRecord> etrFuture = streamMetadataStore.getEpochTransition(scope, stream, context, executor).thenApply(VersionedMetadata::getObject);
    return CompletableFuture.allOf(stateFuture, activeEpochFuture, etrFuture).handle((r, ex) -> {
        ScaleStatusResponse.Builder response = ScaleStatusResponse.newBuilder();
        if (ex != null) {
            Throwable e = Exceptions.unwrap(ex);
            if (e instanceof StoreException.DataNotFoundException) {
                response.setStatus(ScaleStatusResponse.ScaleStatus.INVALID_INPUT);
            } else {
                response.setStatus(ScaleStatusResponse.ScaleStatus.INTERNAL_ERROR);
            }
        } else {
            EpochRecord activeEpoch = activeEpochFuture.join();
            State state = stateFuture.join();
            EpochTransitionRecord etr = etrFuture.join();
            if (epoch > activeEpoch.getEpoch()) {
                response.setStatus(ScaleStatusResponse.ScaleStatus.INVALID_INPUT);
            } else if (activeEpoch.getEpoch() == epoch || activeEpoch.getReferenceEpoch() == epoch) {
                response.setStatus(ScaleStatusResponse.ScaleStatus.IN_PROGRESS);
            } else {
                // has not completed.
                if (epoch + 1 == activeEpoch.getReferenceEpoch() && state.equals(State.SCALING) && (etr.equals(EpochTransitionRecord.EMPTY) || etr.getNewEpoch() == activeEpoch.getEpoch())) {
                    response.setStatus(ScaleStatusResponse.ScaleStatus.IN_PROGRESS);
                } else {
                    response.setStatus(ScaleStatusResponse.ScaleStatus.SUCCESS);
                }
            }
        }
        return response.build();
    });
}