Examples with CommitRequestHandler io.pravega.controller.server.eventProcessor.requesthandlers.CommitRequestHandler used on opensource projects

Example 11 with CommitRequestHandler

use of io.pravega.controller.server.eventProcessor.requesthandlers.CommitRequestHandler in project pravega by pravega.

the class RequestHandlersTest method concurrentRollingTxnCommit.

private void concurrentRollingTxnCommit(String stream, String func, boolean expectFailureOnFirstJob, Predicate<Throwable> firstExceptionPredicate, Map<String, Integer> invocationCount, int expectedVersion) {
    StreamMetadataStore streamStore1 = getStore();
    StreamMetadataStore streamStore1Spied = spy(getStore());
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, 1)).build();
    streamStore1.createStream(scope, stream, config, System.currentTimeMillis(), null, executor).join();
    streamStore1.setState(scope, stream, State.ACTIVE, null, executor).join();
    StreamMetadataStore streamStore2 = getStore();
    CommitRequestHandler requestHandler1 = new CommitRequestHandler(streamStore1Spied, streamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
    CommitRequestHandler requestHandler2 = new CommitRequestHandler(streamStore2, streamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
    ScaleOperationTask scaleRequesthandler = new ScaleOperationTask(streamMetadataTasks, streamStore2, executor);
    // create txn on epoch 0 and set it to committing
    UUID txnId = streamStore1.generateTransactionId(scope, stream, null, executor).join();
    VersionedTransactionData txnEpoch0 = streamStore1.createTransaction(scope, stream, txnId, 1000L, 10000L, null, executor).join();
    streamStore1.sealTransaction(scope, stream, txnId, true, Optional.of(txnEpoch0.getVersion()), "", Long.MIN_VALUE, null, executor).join();
    // perform scale
    ScaleOpEvent event = new ScaleOpEvent(scope, stream, Lists.newArrayList(0L), Lists.newArrayList(new AbstractMap.SimpleEntry<>(0.0, 1.0)), false, System.currentTimeMillis(), System.currentTimeMillis());
    scaleRequesthandler.execute(event).join();
    // regular commit
    // start commit transactions
    CompletableFuture<Void> wait = new CompletableFuture<>();
    CompletableFuture<Void> signal = new CompletableFuture<>();
    // test rolling transaction --> since transaction on epoch 0 is committing, it will get committed first.
    CommitEvent commitOnEpoch0 = new CommitEvent(scope, stream, 0);
    setMockCommitTxnLatch(streamStore1, streamStore1Spied, func, signal, wait);
    // start rolling txn
    // stall rolling transaction in different stages
    CompletableFuture<Void> future1Rolling = CompletableFuture.completedFuture(null).thenComposeAsync(v -> requestHandler1.execute(commitOnEpoch0), executor);
    signal.join();
    requestHandler2.execute(commitOnEpoch0).join();
    wait.complete(null);
    if (expectFailureOnFirstJob) {
        AssertExtensions.assertSuppliedFutureThrows("first commit should fail", () -> future1Rolling, firstExceptionPredicate);
        verify(streamStore1Spied, times(invocationCount.get("startCommitTransactions"))).startCommitTransactions(anyString(), anyString(), anyInt(), any(), any());
        verify(streamStore1Spied, times(invocationCount.get("startRollingTxn"))).startRollingTxn(anyString(), anyString(), anyInt(), any(), any(), any());
        verify(streamStore1Spied, times(invocationCount.get("rollingTxnCreateDuplicateEpochs"))).rollingTxnCreateDuplicateEpochs(anyString(), anyString(), any(), anyLong(), any(), any(), any());
        verify(streamStore1Spied, times(invocationCount.get("completeRollingTxn"))).completeRollingTxn(anyString(), anyString(), any(), any(), any(), any());
        verify(streamStore1Spied, times(invocationCount.get("completeCommitTransactions"))).completeCommitTransactions(anyString(), anyString(), any(), any(), any(), any());
        verify(streamStore1Spied, times(invocationCount.get("updateVersionedState"))).updateVersionedState(anyString(), anyString(), any(), any(), any(), any());
    } else {
        future1Rolling.join();
    }
    // validate rolling txn done and first job has updated the CTR with new txn record
    VersionedMetadata<CommittingTransactionsRecord> versioned = streamStore1.getVersionedCommittingTransactionsRecord(scope, stream, null, executor).join();
    assertEquals(CommittingTransactionsRecord.EMPTY, versioned.getObject());
    assertEquals(expectedVersion, getVersionNumber(versioned.getVersion()));
    assertEquals(3, streamStore1.getActiveEpoch(scope, stream, null, true, executor).join().getEpoch());
    assertEquals(State.ACTIVE, streamStore1.getState(scope, stream, true, null, executor).join());
}

Example 12 with CommitRequestHandler

use of io.pravega.controller.server.eventProcessor.requesthandlers.CommitRequestHandler in project pravega by pravega.

the class ScaleRequestHandlerTest method testInconsistentScaleRequestAfterRollingTxn.

@Test(timeout = 30000)
public void testInconsistentScaleRequestAfterRollingTxn() throws Exception {
    // This test checks a scenario where after rolling txn, if an outstanding scale request
    // was present, its epoch consistency should fail
    String stream = "newStream";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, 2)).build();
    streamMetadataTasks.createStream(scope, stream, config, System.currentTimeMillis(), 0L).get();
    EventWriterMock writer = new EventWriterMock();
    streamMetadataTasks.setRequestEventWriter(writer);
    ScaleOperationTask scaleRequestHandler = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    StreamRequestHandler requestHandler = new StreamRequestHandler(null, scaleRequestHandler, null, null, null, null, null, null, null, streamStore, null, executor);
    CommitRequestHandler commitRequestHandler = new CommitRequestHandler(streamStore, streamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
    // 1 create transaction on old epoch and set it to committing
    UUID txnIdOldEpoch = streamStore.generateTransactionId(scope, stream, null, executor).join();
    VersionedTransactionData txnData = streamStore.createTransaction(scope, stream, txnIdOldEpoch, 10000, 10000, null, executor).join();
    streamStore.sealTransaction(scope, stream, txnData.getId(), true, Optional.empty(), "", Long.MIN_VALUE, null, executor).join();
    UUID txnIdOldEpoch2 = streamStore.generateTransactionId(scope, stream, null, executor).join();
    VersionedTransactionData txnData2 = streamStore.createTransaction(scope, stream, txnIdOldEpoch2, 10000, 10000, null, executor).join();
    streamStore.sealTransaction(scope, stream, txnData2.getId(), true, Optional.empty(), "", Long.MIN_VALUE, null, executor).join();
    EpochRecord epochZero = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    assertEquals(0, epochZero.getEpoch());
    // 2. start scale
    requestHandler.process(new ScaleOpEvent(scope, stream, Lists.newArrayList(0L), Lists.newArrayList(new AbstractMap.SimpleEntry<>(0.0, 0.25), new AbstractMap.SimpleEntry<>(0.25, 0.5)), false, System.currentTimeMillis(), System.currentTimeMillis()), () -> false).join();
    // 3. verify that scale is complete
    State state = streamStore.getState(scope, stream, true, null, executor).join();
    assertEquals(State.ACTIVE, state);
    // 4. just submit a new scale. don't let it run. this should create an epoch transition. state should still be active
    streamStore.submitScale(scope, stream, Lists.newArrayList(1L), Lists.newArrayList(new AbstractMap.SimpleEntry<>(0.5, 0.75), new AbstractMap.SimpleEntry<>(0.75, 1.0)), System.currentTimeMillis(), null, null, executor).join();
    // 5. commit on old epoch. this should roll over.
    assertTrue(Futures.await(commitRequestHandler.processEvent(new CommitEvent(scope, stream, txnData.getEpoch()))));
    TxnStatus txnStatus = streamStore.transactionStatus(scope, stream, txnIdOldEpoch, null, executor).join();
    assertEquals(TxnStatus.COMMITTED, txnStatus);
    // 6. run scale. this should fail in scaleCreateNewEpochs with IllegalArgumentException with epochTransitionConsistent
    AssertExtensions.assertFutureThrows("epoch transition should be inconsistent", requestHandler.process(new ScaleOpEvent(scope, stream, Lists.newArrayList(1L), Lists.newArrayList(new AbstractMap.SimpleEntry<>(0.5, 0.75), new AbstractMap.SimpleEntry<>(0.75, 1.0)), false, System.currentTimeMillis(), System.currentTimeMillis()), () -> false), e -> Exceptions.unwrap(e) instanceof IllegalStateException);
    state = streamStore.getState(scope, stream, true, null, executor).join();
    assertEquals(State.ACTIVE, state);
}

Example 13 with CommitRequestHandler

use of io.pravega.controller.server.eventProcessor.requesthandlers.CommitRequestHandler in project pravega by pravega.

the class ControllerEventProcessorTest method testCommitEventForSealingStream.

@Test(timeout = 60000)
public void testCommitEventForSealingStream() {
    ScaleOperationTask scaleTask = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    SealStreamTask sealStreamTask = new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, executor);
    String stream = "commitWithSeal";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    streamStore.createStream(SCOPE, stream, config, System.currentTimeMillis(), null, executor).join();
    streamStore.setState(SCOPE, stream, State.ACTIVE, null, executor).join();
    UUID txnOnEpoch0 = streamStore.generateTransactionId(SCOPE, stream, null, executor).join();
    VersionedTransactionData txnData0 = streamStore.createTransaction(SCOPE, stream, txnOnEpoch0, 10000, 10000, null, executor).join();
    Assert.assertNotNull(txnData0);
    checkTransactionState(SCOPE, stream, txnOnEpoch0, TxnStatus.OPEN);
    streamStore.sealTransaction(SCOPE, stream, txnData0.getId(), true, Optional.empty(), "", Long.MIN_VALUE, null, executor).join();
    checkTransactionState(SCOPE, stream, txnData0.getId(), TxnStatus.COMMITTING);
    // scale stream
    List<Map.Entry<Double, Double>> newRange = new LinkedList<>();
    newRange.add(new AbstractMap.SimpleEntry<>(0.0, 1.0));
    scaleTask.execute(new ScaleOpEvent(SCOPE, stream, Collections.singletonList(0L), newRange, false, System.currentTimeMillis(), 0L)).join();
    UUID txnOnEpoch1 = streamStore.generateTransactionId(SCOPE, stream, null, executor).join();
    VersionedTransactionData txnData1 = streamStore.createTransaction(SCOPE, stream, txnOnEpoch1, 10000, 10000, null, executor).join();
    Assert.assertNotNull(txnData1);
    checkTransactionState(SCOPE, stream, txnOnEpoch1, TxnStatus.OPEN);
    streamStore.sealTransaction(SCOPE, stream, txnData1.getId(), true, Optional.empty(), "", Long.MIN_VALUE, null, executor).join();
    checkTransactionState(SCOPE, stream, txnData1.getId(), TxnStatus.COMMITTING);
    // set the stream to SEALING
    streamStore.setState(SCOPE, stream, State.SEALING, null, executor).join();
    // attempt to seal the stream. This should fail with postponement.
    AssertExtensions.assertFutureThrows("Seal stream should fail with operation not allowed as their are outstanding transactions", sealStreamTask.execute(new SealStreamEvent(SCOPE, stream, 0L)), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    // now attempt to commit the transaction on epoch 1. epoch in commit event is ignored and transactions on lowest epoch
    // should be committed first.
    CommitRequestHandler commitEventProcessor = new CommitRequestHandler(streamStore, streamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
    commitEventProcessor.processEvent(new CommitEvent(SCOPE, stream, txnData1.getEpoch())).join();
    checkTransactionState(SCOPE, stream, txnData0.getId(), TxnStatus.COMMITTED);
    checkTransactionState(SCOPE, stream, txnData1.getId(), TxnStatus.COMMITTING);
    EpochRecord activeEpoch = streamStore.getActiveEpoch(SCOPE, stream, null, true, executor).join();
    assertEquals(3, activeEpoch.getEpoch());
    assertEquals(1, activeEpoch.getReferenceEpoch());
    // attempt to seal the stream. This should still fail with postponement.
    AssertExtensions.assertFutureThrows("Seal stream should fail with operation not allowed as their are outstanding transactions", sealStreamTask.execute(new SealStreamEvent(SCOPE, stream, 0L)), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    // now attempt to commit the transaction on epoch 1.
    commitEventProcessor.processEvent(new CommitEvent(SCOPE, stream, txnData1.getEpoch())).join();
    checkTransactionState(SCOPE, stream, txnData1.getId(), TxnStatus.COMMITTED);
    // verify transaction has rolled over
    activeEpoch = streamStore.getActiveEpoch(SCOPE, stream, null, true, executor).join();
    assertEquals(3, activeEpoch.getEpoch());
    assertEquals(1, activeEpoch.getReferenceEpoch());
    // now attempt to seal the stream. it should complete.
    sealStreamTask.execute(new SealStreamEvent(SCOPE, stream, 0L)).join();
}

Example 14 with CommitRequestHandler

use of io.pravega.controller.server.eventProcessor.requesthandlers.CommitRequestHandler in project pravega by pravega.

the class ControllerEventProcessorTest method testTransactionOutstandingCommit.

@Test(timeout = 10000)
public void testTransactionOutstandingCommit() {
    // keep a committxnlist in the store
    // same epoch --> commit txn list should be cleared first
    // subsequent events should complete remainder txns that are in committing state
    // lower epoch --> rolling transaction
    // higher epoch --> no transactions can exist on higher epoch. nothing to do. ignore.
    List<VersionedTransactionData> txnDataList1 = createAndCommitTransactions(3);
    int epoch = txnDataList1.get(0).getEpoch();
    streamStore.startCommitTransactions(SCOPE, STREAM, 100, null, executor).join();
    List<VersionedTransactionData> txnDataList2 = createAndCommitTransactions(3);
    streamMetadataTasks.setRequestEventWriter(new EventStreamWriterMock<>());
    CommitRequestHandler commitEventProcessor = new CommitRequestHandler(streamStore, streamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
    commitEventProcessor.processEvent(new CommitEvent(SCOPE, STREAM, epoch)).join();
    for (VersionedTransactionData txnData : txnDataList1) {
        checkTransactionState(SCOPE, STREAM, txnData.getId(), TxnStatus.COMMITTED);
    }
    for (VersionedTransactionData txnData : txnDataList2) {
        checkTransactionState(SCOPE, STREAM, txnData.getId(), TxnStatus.COMMITTING);
    }
    commitEventProcessor.processEvent(new CommitEvent(SCOPE, STREAM, epoch)).join();
    for (VersionedTransactionData txnData : txnDataList2) {
        checkTransactionState(SCOPE, STREAM, txnData.getId(), TxnStatus.COMMITTED);
    }
}

Example 15 with CommitRequestHandler

use of io.pravega.controller.server.eventProcessor.requesthandlers.CommitRequestHandler in project pravega by pravega.

the class ControllerEventProcessorTest method testCommitEventProcessor.

@Test(timeout = 10000)
public void testCommitEventProcessor() {
    UUID txnId = streamStore.generateTransactionId(SCOPE, STREAM, null, executor).join();
    VersionedTransactionData txnData = streamStore.createTransaction(SCOPE, STREAM, txnId, 10000, 10000, null, executor).join();
    Assert.assertNotNull(txnData);
    checkTransactionState(SCOPE, STREAM, txnId, TxnStatus.OPEN);
    streamStore.sealTransaction(SCOPE, STREAM, txnData.getId(), true, Optional.empty(), "", Long.MIN_VALUE, null, executor).join();
    checkTransactionState(SCOPE, STREAM, txnData.getId(), TxnStatus.COMMITTING);
    CommitRequestHandler commitEventProcessor = new CommitRequestHandler(streamStore, streamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
    commitEventProcessor.processEvent(new CommitEvent(SCOPE, STREAM, txnData.getEpoch())).join();
    checkTransactionState(SCOPE, STREAM, txnData.getId(), TxnStatus.COMMITTED);
}