Examples with CommittingTransactionsRecord io.pravega.controller.store.stream.records.CommittingTransactionsRecord used on opensource projects

Example 11 with CommittingTransactionsRecord

use of io.pravega.controller.store.stream.records.CommittingTransactionsRecord in project pravega by pravega.

the class StreamTestBase method segmentQueriesDuringRollingTxn.

@Test(timeout = 30000L)
public void segmentQueriesDuringRollingTxn() {
    OperationContext context = getContext();
    // start scale and perform `getSegment`, `getActiveEpoch` and `getEpoch` during different phases of scale
    int startingSegmentNumber = new Random().nextInt(2000);
    long time = System.currentTimeMillis();
    Stream stream = createStream("scope", "stream" + startingSegmentNumber, time, 5, startingSegmentNumber);
    StreamSegmentRecord segment = stream.getSegment(startingSegmentNumber, context).join();
    assertEquals(segment.segmentId(), startingSegmentNumber + 0L);
    assertEquals(segment.getKeyStart(), 0, 0);
    assertEquals(segment.getKeyEnd(), 1.0 / 5, 0);
    createAndCommitTransaction(stream, 0, 0L);
    List<Map.Entry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.0, 0.5));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.5, 1.0));
    time = time + 1;
    scaleStream(stream, time, Lists.newArrayList(startingSegmentNumber + 0L, startingSegmentNumber + 1L, startingSegmentNumber + 2L, startingSegmentNumber + 3L, startingSegmentNumber + 4L), newRanges, Collections.emptyMap());
    List<StreamSegmentRecord> activeSegmentsBefore = stream.getActiveSegments(context).join();
    // start commit transactions
    VersionedMetadata<CommittingTransactionsRecord> ctr = stream.startCommittingTransactions(100, context).join().getKey();
    stream.getVersionedState(context).thenCompose(s -> stream.updateVersionedState(s, State.COMMITTING_TXN, context)).join();
    // start rolling transaction
    ctr = stream.startRollingTxn(1, ctr, context).join();
    List<StreamSegmentRecord> activeSegments1 = stream.getActiveSegments(context).join();
    assertEquals(activeSegments1, activeSegmentsBefore);
    Map<StreamSegmentRecord, List<Long>> successorsWithPredecessors = stream.getSuccessorsWithPredecessors(computeSegmentId(startingSegmentNumber + 5, 1), context).join();
    assertTrue(successorsWithPredecessors.isEmpty());
    // rolling txn create duplicate epochs. We should be able to get successors and predecessors after this step.
    time = time + 1;
    stream.rollingTxnCreateDuplicateEpochs(Collections.emptyMap(), time, ctr, context).join();
    activeSegments1 = stream.getActiveSegments(context).join();
    assertEquals(activeSegments1, activeSegmentsBefore);
    successorsWithPredecessors = stream.getSuccessorsWithPredecessors(computeSegmentId(startingSegmentNumber + 5, 1), context).join();
    Set<StreamSegmentRecord> successors = successorsWithPredecessors.keySet();
    assertEquals(3, successors.size());
    assertTrue(successors.stream().allMatch(x -> x.getCreationEpoch() == 2));
    assertTrue(successors.stream().anyMatch(x -> x.getSegmentNumber() == startingSegmentNumber + 0));
    assertTrue(successors.stream().anyMatch(x -> x.getSegmentNumber() == startingSegmentNumber + 1));
    assertTrue(successors.stream().anyMatch(x -> x.getSegmentNumber() == startingSegmentNumber + 2));
    successorsWithPredecessors = stream.getSuccessorsWithPredecessors(computeSegmentId(startingSegmentNumber + 0, 2), context).join();
    successors = successorsWithPredecessors.keySet();
    assertEquals(1, successors.size());
    assertTrue(successors.stream().allMatch(x -> x.segmentId() == computeSegmentId(startingSegmentNumber + 5, 3)));
    stream.completeRollingTxn(Collections.emptyMap(), ctr, context).join();
    stream.completeCommittingTransactions(ctr, context, Collections.emptyMap()).join();
}

Example 12 with CommittingTransactionsRecord

use of io.pravega.controller.store.stream.records.CommittingTransactionsRecord in project pravega by pravega.

the class RequestHandlersTest method concurrentRollingTxnCommit.

private void concurrentRollingTxnCommit(String stream, String func, boolean expectFailureOnFirstJob, Predicate<Throwable> firstExceptionPredicate, Map<String, Integer> invocationCount, int expectedVersion) {
    StreamMetadataStore streamStore1 = getStore();
    StreamMetadataStore streamStore1Spied = spy(getStore());
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, 1)).build();
    streamStore1.createStream(scope, stream, config, System.currentTimeMillis(), null, executor).join();
    streamStore1.setState(scope, stream, State.ACTIVE, null, executor).join();
    StreamMetadataStore streamStore2 = getStore();
    CommitRequestHandler requestHandler1 = new CommitRequestHandler(streamStore1Spied, streamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
    CommitRequestHandler requestHandler2 = new CommitRequestHandler(streamStore2, streamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
    ScaleOperationTask scaleRequesthandler = new ScaleOperationTask(streamMetadataTasks, streamStore2, executor);
    // create txn on epoch 0 and set it to committing
    UUID txnId = streamStore1.generateTransactionId(scope, stream, null, executor).join();
    VersionedTransactionData txnEpoch0 = streamStore1.createTransaction(scope, stream, txnId, 1000L, 10000L, null, executor).join();
    streamStore1.sealTransaction(scope, stream, txnId, true, Optional.of(txnEpoch0.getVersion()), "", Long.MIN_VALUE, null, executor).join();
    // perform scale
    ScaleOpEvent event = new ScaleOpEvent(scope, stream, Lists.newArrayList(0L), Lists.newArrayList(new AbstractMap.SimpleEntry<>(0.0, 1.0)), false, System.currentTimeMillis(), System.currentTimeMillis());
    scaleRequesthandler.execute(event).join();
    // regular commit
    // start commit transactions
    CompletableFuture<Void> wait = new CompletableFuture<>();
    CompletableFuture<Void> signal = new CompletableFuture<>();
    // test rolling transaction --> since transaction on epoch 0 is committing, it will get committed first.
    CommitEvent commitOnEpoch0 = new CommitEvent(scope, stream, 0);
    setMockCommitTxnLatch(streamStore1, streamStore1Spied, func, signal, wait);
    // start rolling txn
    // stall rolling transaction in different stages
    CompletableFuture<Void> future1Rolling = CompletableFuture.completedFuture(null).thenComposeAsync(v -> requestHandler1.execute(commitOnEpoch0), executor);
    signal.join();
    requestHandler2.execute(commitOnEpoch0).join();
    wait.complete(null);
    if (expectFailureOnFirstJob) {
        AssertExtensions.assertSuppliedFutureThrows("first commit should fail", () -> future1Rolling, firstExceptionPredicate);
        verify(streamStore1Spied, times(invocationCount.get("startCommitTransactions"))).startCommitTransactions(anyString(), anyString(), anyInt(), any(), any());
        verify(streamStore1Spied, times(invocationCount.get("startRollingTxn"))).startRollingTxn(anyString(), anyString(), anyInt(), any(), any(), any());
        verify(streamStore1Spied, times(invocationCount.get("rollingTxnCreateDuplicateEpochs"))).rollingTxnCreateDuplicateEpochs(anyString(), anyString(), any(), anyLong(), any(), any(), any());
        verify(streamStore1Spied, times(invocationCount.get("completeRollingTxn"))).completeRollingTxn(anyString(), anyString(), any(), any(), any(), any());
        verify(streamStore1Spied, times(invocationCount.get("completeCommitTransactions"))).completeCommitTransactions(anyString(), anyString(), any(), any(), any(), any());
        verify(streamStore1Spied, times(invocationCount.get("updateVersionedState"))).updateVersionedState(anyString(), anyString(), any(), any(), any(), any());
    } else {
        future1Rolling.join();
    }
    // validate rolling txn done and first job has updated the CTR with new txn record
    VersionedMetadata<CommittingTransactionsRecord> versioned = streamStore1.getVersionedCommittingTransactionsRecord(scope, stream, null, executor).join();
    assertEquals(CommittingTransactionsRecord.EMPTY, versioned.getObject());
    assertEquals(expectedVersion, getVersionNumber(versioned.getVersion()));
    assertEquals(3, streamStore1.getActiveEpoch(scope, stream, null, true, executor).join().getEpoch());
    assertEquals(State.ACTIVE, streamStore1.getState(scope, stream, true, null, executor).join());
}

Example 13 with CommittingTransactionsRecord

use of io.pravega.controller.store.stream.records.CommittingTransactionsRecord in project pravega by pravega.

the class CommitRequestHandler method commitTransactions.

/**
 * This method loops over each transaction in the list, and commits them in order
 * At the end of this method's execution, all transactions in the list would have committed into given list of segments.
 */
private CompletableFuture<Void> commitTransactions(CommitTxnContext commitContext, VersionedMetadata<CommittingTransactionsRecord> txnRecord, List<Long> segments) {
    // Chain all transaction commit futures one after the other. This will ensure that order of commit
    // if honoured and is based on the order in the list.
    List<UUID> transactionsToCommit = txnRecord.getObject().getTransactionsToCommit();
    boolean noteTime = commitContext.writerMarks.size() > 0;
    Timer segMergeTimer = new Timer();
    return streamMetadataTasks.mergeTxnSegmentsIntoStreamSegments(commitContext.scope, commitContext.stream, segments, transactionsToCommit, commitContext.context.getRequestId()).thenCompose(segmentOffsets -> {
        TransactionMetrics.getInstance().commitTransactionSegments(segMergeTimer.getElapsed());
        if (noteTime) {
            for (int i = 0; i < transactionsToCommit.size(); i++) {
                int index = i;
                val txnOffsets = segmentOffsets.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, x -> x.getValue().get(index)));
                val txnId = transactionsToCommit.get(i);
                String writerId = commitContext.txnIdToWriterId.get(txnId);
                if (!Strings.isNullOrEmpty(writerId) && commitContext.writerMarks.get(writerId).getTransactionId().equals(txnId)) {
                    TxnWriterMark mark = commitContext.writerMarks.get(writerId);
                    commitContext.writerMarks.put(writerId, new TxnWriterMark(mark.getTimestamp(), txnOffsets, mark.getTransactionId()));
                }
            }
            return bucketStore.addStreamToBucketStore(BucketStore.ServiceType.WatermarkingService, commitContext.scope, commitContext.stream, executor);
        }
        return CompletableFuture.completedFuture(null);
    });
}

Example 14 with CommittingTransactionsRecord

use of io.pravega.controller.store.stream.records.CommittingTransactionsRecord in project pravega by pravega.

the class PersistentStreamBase method startCommittingTransactions.

@Override
public CompletableFuture<Map.Entry<VersionedMetadata<CommittingTransactionsRecord>, List<VersionedTransactionData>>> startCommittingTransactions(int limit, OperationContext context) {
    Preconditions.checkNotNull(context, "Operation context cannot be null");
    return getVersionedCommitTransactionsRecord(context).thenCompose(versioned -> {
        if (versioned.getObject().equals(CommittingTransactionsRecord.EMPTY)) {
            return getOrderedCommittingTxnInLowestEpoch(limit, context).thenCompose(list -> {
                if (list.isEmpty()) {
                    List<VersionedTransactionData> emptyTransactionData = new LinkedList<>();
                    return CompletableFuture.completedFuture(new SimpleEntry<>(versioned, emptyTransactionData));
                } else {
                    ImmutableList.Builder<UUID> txIdList = ImmutableList.builder();
                    list.forEach(x -> {
                        txIdList.add(x.getId());
                    });
                    List<Long> positions = list.stream().map(VersionedTransactionData::getCommitOrder).collect(Collectors.toList());
                    int epoch = RecordHelper.getTransactionEpoch(list.get(0).getId());
                    CommittingTransactionsRecord record = new CommittingTransactionsRecord(epoch, txIdList.build());
                    return updateCommittingTxnRecord(new VersionedMetadata<>(record, versioned.getVersion()), context).thenCompose(version -> removeTxnsFromCommitOrder(positions, context).thenApply(v -> new SimpleEntry<>(new VersionedMetadata<>(record, version), list)));
                }
            });
        } else {
            List<String> transactionsToCommit = versioned.getObject().getTransactionsToCommit().stream().map(UUID::toString).collect(Collectors.toList());
            return getVersionedTransactionRecords(versioned.getObject().getEpoch(), transactionsToCommit, context).thenApply(x -> new SimpleEntry<>(versioned, x));
        }
    });
}

Example 15 with CommittingTransactionsRecord

use of io.pravega.controller.store.stream.records.CommittingTransactionsRecord in project pravega by pravega.

the class StreamMetadataStoreTest method scaleWithTxTest.

@Test(timeout = 30000)
public void scaleWithTxTest() throws Exception {
    final String scope = "ScopeScaleWithTx";
    final String stream = "StreamScaleWithTx";
    final ScalingPolicy policy = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration = StreamConfiguration.builder().scalingPolicy(policy).build();
    long start = System.currentTimeMillis();
    store.createScope(scope, null, executor).get();
    store.createStream(scope, stream, configuration, start, null, executor).get();
    store.setState(scope, stream, State.ACTIVE, null, executor).get();
    long scaleTs = System.currentTimeMillis();
    SimpleEntry<Double, Double> segment2 = new SimpleEntry<>(0.5, 0.75);
    SimpleEntry<Double, Double> segment3 = new SimpleEntry<>(0.75, 1.0);
    List<Long> scale1SealedSegments = Collections.singletonList(1L);
    // region Txn created before scale and during scale
    // scale with transaction test
    // first txn created before-scale
    UUID txnId = store.generateTransactionId(scope, stream, null, executor).join();
    VersionedTransactionData tx01 = store.createTransaction(scope, stream, txnId, 100, 100, null, executor).get();
    assertEquals(0, tx01.getEpoch());
    VersionedMetadata<EpochTransitionRecord> versioned = store.submitScale(scope, stream, scale1SealedSegments, Arrays.asList(segment2, segment3), scaleTs, null, null, executor).join();
    EpochTransitionRecord response = versioned.getObject();
    Map<Long, Map.Entry<Double, Double>> scale1SegmentsCreated = response.getNewSegmentsWithRange();
    final int epoch = response.getActiveEpoch();
    assertEquals(0, epoch);
    assertNotNull(scale1SegmentsCreated);
    VersionedMetadata<State> state = store.getVersionedState(scope, stream, null, executor).join();
    state = store.updateVersionedState(scope, stream, State.SCALING, state, null, executor).join();
    versioned = store.startScale(scope, stream, false, versioned, state, null, executor).join();
    // second txn created after new segments are created in segment table but not yet in history table
    // assert that txn is created on old epoch
    store.scaleCreateNewEpochs(scope, stream, versioned, null, executor).join();
    txnId = store.generateTransactionId(scope, stream, null, executor).join();
    VersionedTransactionData tx02 = store.createTransaction(scope, stream, txnId, 100, 100, null, executor).get();
    assertEquals(0, tx02.getEpoch());
    assertEquals(0, (int) (tx02.getId().getMostSignificantBits() >> 32));
    // third transaction created after new epoch created
    txnId = store.generateTransactionId(scope, stream, null, executor).join();
    store.sealTransaction(scope, stream, tx02.getId(), true, Optional.of(tx02.getVersion()), "", Long.MIN_VALUE, null, executor).get();
    store.sealTransaction(scope, stream, tx01.getId(), true, Optional.of(tx01.getVersion()), "", Long.MIN_VALUE, null, executor).get();
    store.scaleSegmentsSealed(scope, stream, scale1SealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), versioned, null, executor).join();
    store.completeScale(scope, stream, versioned, null, executor).join();
    VersionedTransactionData tx03 = store.createTransaction(scope, stream, txnId, 100, 100, null, executor).get();
    assertEquals(0, tx03.getEpoch());
    assertEquals(0, (int) (tx03.getId().getMostSignificantBits() >> 32));
    store.setState(scope, stream, State.ACTIVE, null, executor).join();
    // ensure that we can commit transactions on old epoch and roll over.
    EpochRecord activeEpoch = store.getActiveEpoch(scope, stream, null, true, executor).join();
    // submit another scale request without starting the scale
    List<Long> scale2SealedSegments = Collections.singletonList(0L);
    long scaleTs2 = System.currentTimeMillis();
    SimpleEntry<Double, Double> segment4 = new SimpleEntry<>(0.0, 0.25);
    SimpleEntry<Double, Double> segment5 = new SimpleEntry<>(0.25, 0.5);
    VersionedMetadata<EpochTransitionRecord> versioned2 = store.submitScale(scope, stream, scale2SealedSegments, Arrays.asList(segment4, segment5), scaleTs2, null, null, executor).join();
    EpochTransitionRecord response2 = versioned2.getObject();
    assertEquals(activeEpoch.getEpoch(), response2.getActiveEpoch());
    VersionedMetadata<CommittingTransactionsRecord> record = store.startCommitTransactions(scope, stream, 100, null, executor).join().getKey();
    store.setState(scope, stream, State.COMMITTING_TXN, null, executor).join();
    record = store.startRollingTxn(scope, stream, activeEpoch.getEpoch(), record, null, executor).join();
    store.rollingTxnCreateDuplicateEpochs(scope, stream, Collections.emptyMap(), System.currentTimeMillis(), record, null, executor).join();
    store.completeRollingTxn(scope, stream, Collections.emptyMap(), record, null, executor).join();
    store.completeCommitTransactions(scope, stream, record, null, executor, Collections.emptyMap()).join();
    store.setState(scope, stream, State.ACTIVE, null, executor).join();
    activeEpoch = store.getActiveEpoch(scope, stream, null, true, executor).join();
    assertEquals(3, activeEpoch.getEpoch());
    assertEquals(1, activeEpoch.getReferenceEpoch());
    assertEquals(3, activeEpoch.getSegments().size());
    List<StreamSegmentRecord> txnDuplicate = store.getSegmentsInEpoch(scope, stream, 2, null, executor).join();
    assertEquals(2, txnDuplicate.size());
    List<StreamSegmentRecord> activeEpochDuplicate = store.getSegmentsInEpoch(scope, stream, 3, null, executor).join();
    assertEquals(3, activeEpochDuplicate.size());
    EpochRecord txnCommittedEpoch = store.getEpoch(scope, stream, 2, null, executor).join();
    assertEquals(0, txnCommittedEpoch.getReferenceEpoch());
    assertEquals(store.transactionStatus(scope, stream, tx01.getId(), null, executor).join(), TxnStatus.COMMITTED);
    assertEquals(store.transactionStatus(scope, stream, tx02.getId(), null, executor).join(), TxnStatus.COMMITTED);
    assertEquals(store.transactionStatus(scope, stream, tx03.getId(), null, executor).join(), TxnStatus.OPEN);
    store.sealTransaction(scope, stream, tx03.getId(), true, Optional.of(tx03.getVersion()), "", Long.MIN_VALUE, null, executor).get();
    // endregion
    // region verify migrate request for manual scale
    // now start manual scale against previously submitted scale request that was on old epoch from before rolling txn.
    // verify that it gets migrated to latest duplicate epoch
    state = store.getVersionedState(scope, stream, null, executor).join();
    state = store.updateVersionedState(scope, stream, State.SCALING, state, null, executor).join();
    versioned2 = store.submitScale(scope, stream, scale2SealedSegments, Arrays.asList(segment4, segment5), scaleTs2, null, null, executor).join();
    versioned2 = store.startScale(scope, stream, true, versioned2, state, null, executor).join();
    store.scaleCreateNewEpochs(scope, stream, versioned2, null, executor).join();
    txnId = store.generateTransactionId(scope, stream, null, executor).join();
    VersionedTransactionData tx14 = store.createTransaction(scope, stream, txnId, 100, 100, null, executor).get();
    assertEquals(1, tx14.getEpoch());
    store.sealTransaction(scope, stream, tx14.getId(), true, Optional.of(tx14.getVersion()), "", Long.MIN_VALUE, null, executor).get();
    // verify that new txns can be created and are created on original epoch
    txnId = store.generateTransactionId(scope, stream, null, executor).join();
    VersionedTransactionData tx15 = store.createTransaction(scope, stream, txnId, 100, 100, null, executor).get();
    assertEquals(1, tx15.getEpoch());
    store.scaleCreateNewEpochs(scope, stream, versioned2, null, executor).join();
    store.scaleSegmentsSealed(scope, stream, Collections.emptyMap(), versioned2, null, executor).join();
    store.completeScale(scope, stream, versioned2, null, executor).join();
    store.setState(scope, stream, State.ACTIVE, null, executor).get();
    activeEpoch = store.getActiveEpoch(scope, stream, null, true, executor).join();
    assertEquals(4, activeEpoch.getEpoch());
    assertEquals(4, activeEpoch.getReferenceEpoch());
    store.sealTransaction(scope, stream, tx15.getId(), true, Optional.of(tx15.getVersion()), "", Long.MIN_VALUE, null, executor).get();
    record = store.startCommitTransactions(scope, stream, 100, null, executor).join().getKey();
    store.setState(scope, stream, State.COMMITTING_TXN, null, executor).get();
    record = store.startRollingTxn(scope, stream, activeEpoch.getEpoch(), record, null, executor).join();
    store.rollingTxnCreateDuplicateEpochs(scope, stream, Collections.emptyMap(), System.currentTimeMillis(), record, null, executor).join();
    store.completeRollingTxn(scope, stream, Collections.emptyMap(), record, null, executor).join();
    store.completeCommitTransactions(scope, stream, record, null, executor, Collections.emptyMap()).join();
    store.setState(scope, stream, State.ACTIVE, null, executor).join();
    activeEpoch = store.getActiveEpoch(scope, stream, null, true, executor).join();
    assertEquals(6, activeEpoch.getEpoch());
    assertEquals(4, activeEpoch.getReferenceEpoch());
// endregion
}