Examples with EpochRecord io.pravega.controller.store.stream.records.EpochRecord used on opensource projects

Example 11 with EpochRecord

use of io.pravega.controller.store.stream.records.EpochRecord in project pravega by pravega.

the class StreamMetadataStoreTest method scaleTest.

@Test(timeout = 30000)
public void scaleTest() throws Exception {
    final String scope = "ScopeScale";
    final String stream = "StreamScale";
    final ScalingPolicy policy = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration = StreamConfiguration.builder().scalingPolicy(policy).build();
    long start = System.currentTimeMillis();
    store.createScope(scope, null, executor).get();
    store.createStream(scope, stream, configuration, start, null, executor).get();
    store.setState(scope, stream, State.ACTIVE, null, executor).get();
    // set minimum number of segments to 1 so that we can also test scale downs
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    store.startUpdateConfiguration(scope, stream, config, null, executor).join();
    VersionedMetadata<StreamConfigurationRecord> configRecord = store.getConfigurationRecord(scope, stream, null, executor).join();
    store.completeUpdateConfiguration(scope, stream, configRecord, null, executor).join();
    // region idempotent
    long scaleTs = System.currentTimeMillis();
    SimpleEntry<Double, Double> segment1 = new SimpleEntry<>(0.5, 0.75);
    SimpleEntry<Double, Double> segment2 = new SimpleEntry<>(0.75, 1.0);
    List<Long> scale1SealedSegments = Collections.singletonList(computeSegmentId(1, 0));
    // 1. submit scale
    VersionedMetadata<EpochTransitionRecord> empty = store.getEpochTransition(scope, stream, null, executor).join();
    VersionedMetadata<EpochTransitionRecord> response = store.submitScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, null, null, executor).join();
    Map<Long, Map.Entry<Double, Double>> scale1SegmentsCreated = response.getObject().getNewSegmentsWithRange();
    final int scale1ActiveEpoch = response.getObject().getActiveEpoch();
    assertEquals(0, scale1ActiveEpoch);
    // rerun start scale with old epoch transition. should throw write conflict
    AssertExtensions.assertSuppliedFutureThrows("", () -> store.submitScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, empty, null, executor), e -> Exceptions.unwrap(e) instanceof StoreException.WriteConflictException);
    // rerun start scale with null epoch transition, should be idempotent
    response = store.submitScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, null, null, executor).join();
    assertEquals(response.getObject().getNewSegmentsWithRange(), scale1SegmentsCreated);
    VersionedMetadata<State> state = store.getVersionedState(scope, stream, null, executor).join();
    state = store.updateVersionedState(scope, stream, State.SCALING, state, null, executor).get();
    response = store.startScale(scope, stream, false, response, state, null, executor).join();
    // 2. scale new segments created
    store.scaleCreateNewEpochs(scope, stream, response, null, executor).join();
    // rerun start scale and new segments created
    response = store.submitScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, null, null, executor).join();
    assertEquals(response.getObject().getNewSegmentsWithRange(), scale1SegmentsCreated);
    response = store.startScale(scope, stream, false, response, state, null, executor).join();
    store.scaleCreateNewEpochs(scope, stream, response, null, executor).join();
    // 3. scale segments sealed -- this will complete scale
    store.scaleSegmentsSealed(scope, stream, scale1SealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), response, null, executor).join();
    store.completeScale(scope, stream, response, null, executor).join();
    store.setState(scope, stream, State.ACTIVE, null, executor).get();
    // rerun -- idempotent
    store.scaleCreateNewEpochs(scope, stream, response, null, executor).join();
    EpochRecord activeEpoch = store.getActiveEpoch(scope, stream, null, true, executor).join();
    assertEquals(1, activeEpoch.getEpoch());
    store.scaleSegmentsSealed(scope, stream, scale1SealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), response, null, executor).join();
    store.getActiveEpoch(scope, stream, null, true, executor).join();
    assertEquals(1, activeEpoch.getEpoch());
    // rerun submit scale -- should fail with precondition failure
    VersionedMetadata<EpochTransitionRecord> etr = store.getEpochTransition(scope, stream, null, executor).join();
    assertEquals(EpochTransitionRecord.EMPTY, empty.getObject());
    AssertExtensions.assertThrows("Submit scale with old data with old etr", () -> store.submitScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, empty, null, executor).join(), e -> Exceptions.unwrap(e) instanceof StoreException.WriteConflictException);
    AssertExtensions.assertThrows("Submit scale with old data with latest etr", () -> store.submitScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, etr, null, executor).join(), e -> Exceptions.unwrap(e) instanceof EpochTransitionOperationExceptions.PreConditionFailureException);
    AssertExtensions.assertThrows("Submit scale with null etr", () -> store.submitScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, null, null, executor).join(), e -> Exceptions.unwrap(e) instanceof EpochTransitionOperationExceptions.PreConditionFailureException);
    // endregion
    // 2 different conflicting scale operations
    // region run concurrent conflicting scale
    SimpleEntry<Double, Double> segment3 = new SimpleEntry<>(0.0, 0.5);
    SimpleEntry<Double, Double> segment4 = new SimpleEntry<>(0.5, 0.75);
    SimpleEntry<Double, Double> segment5 = new SimpleEntry<>(0.75, 1.0);
    List<Long> scale2SealedSegments = Arrays.asList(computeSegmentId(0, 0), computeSegmentId(2, 1), computeSegmentId(3, 1));
    long scaleTs2 = System.currentTimeMillis();
    response = store.submitScale(scope, stream, scale2SealedSegments, Arrays.asList(segment3, segment4, segment5), scaleTs2, null, null, executor).get();
    Map<Long, Map.Entry<Double, Double>> scale2SegmentsCreated = response.getObject().getNewSegmentsWithRange();
    final int scale2ActiveEpoch = response.getObject().getActiveEpoch();
    store.setState(scope, stream, State.SCALING, null, executor).get();
    // rerun of scale 1 -- should fail with conflict
    AssertExtensions.assertThrows("Concurrent conflicting scale", () -> store.submitScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, null, null, executor).join(), e -> Exceptions.unwrap(e) instanceof EpochTransitionOperationExceptions.ConflictException);
    store.scaleCreateNewEpochs(scope, stream, response, null, executor).get();
    store.scaleSegmentsSealed(scope, stream, scale1SealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), response, null, executor).get();
    store.completeScale(scope, stream, response, null, executor).join();
    store.setState(scope, stream, State.ACTIVE, null, executor).get();
    // endregion
    // region concurrent submit scale requests
    // run two concurrent runScale operations such that after doing a getEpochTransition, we create a new epoch
    // transition node. We should get ScaleConflict in such a case.
    // mock createEpochTransition
    SimpleEntry<Double, Double> segment6 = new SimpleEntry<>(0.0, 1.0);
    List<Long> scale3SealedSegments = Arrays.asList(computeSegmentId(4, 2), computeSegmentId(5, 2), computeSegmentId(6, 2));
    long scaleTs3 = System.currentTimeMillis();
    @SuppressWarnings("unchecked") PersistentStreamBase streamObj = (PersistentStreamBase) ((AbstractStreamMetadataStore) store).getStream(scope, stream, null);
    PersistentStreamBase streamObjSpied = spy(streamObj);
    CompletableFuture<Void> latch = new CompletableFuture<>();
    CompletableFuture<Void> updateEpochTransitionCalled = new CompletableFuture<>();
    doAnswer(x -> CompletableFuture.runAsync(() -> {
        // wait until we create epoch transition outside of this method
        updateEpochTransitionCalled.complete(null);
        latch.join();
    }).thenCompose(v -> streamObj.updateEpochTransitionNode(x.getArgument(0), x.getArgument(1)))).when(streamObjSpied).updateEpochTransitionNode(any(), any());
    doAnswer(x -> streamObj.getEpochTransitionNode(x.getArgument(0))).when(streamObjSpied).getEpochTransitionNode(any());
    OperationContext context = new StreamOperationContext(((AbstractStreamMetadataStore) store).getScope(scope, null), streamObjSpied, 0L);
    // the following should be stuck at createEpochTransition
    CompletableFuture<VersionedMetadata<EpochTransitionRecord>> resp = store.submitScale(scope, stream, scale3SealedSegments, Collections.singletonList(segment6), scaleTs3, null, context, executor);
    updateEpochTransitionCalled.join();
    VersionedMetadata<EpochTransitionRecord> epochRecord = streamObj.getEpochTransition(context).join();
    streamObj.updateEpochTransitionNode(new VersionedMetadata<>(EpochTransitionRecord.EMPTY, epochRecord.getVersion()), context).join();
    latch.complete(null);
    AssertExtensions.assertFutureThrows("", resp, e -> Exceptions.unwrap(e) instanceof StoreException.WriteConflictException);
// endregion
}

Example 12 with EpochRecord

use of io.pravega.controller.store.stream.records.EpochRecord in project pravega by pravega.

the class StreamMetadataTasksTest method updateStreamSegmentCountScalingPolicyTest.

@Test(timeout = 30000)
public void updateStreamSegmentCountScalingPolicyTest() throws Exception {
    int initialSegments = streamStorePartialMock.getActiveSegments(SCOPE, stream1, null, executor).join().size();
    WriterMock requestEventWriter = new WriterMock(streamMetadataTasks, executor);
    streamMetadataTasks.setRequestEventWriter(requestEventWriter);
    // scaleup
    StreamConfiguration streamConfiguration = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, initialSegments + 1)).build();
    updateConfigVerifyScale(requestEventWriter, streamConfiguration, initialSegments + 1);
    // now reduce the number of segments (=1). no scale should happen as we are already more than that.
    streamConfiguration = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, 1)).build();
    updateConfigVerifyScale(requestEventWriter, streamConfiguration, initialSegments + 1);
    EpochRecord activeEpoch = streamStorePartialMock.getActiveEpoch(SCOPE, stream1, null, true, executor).join();
    // now create an epoch transition record (store.submit scale)
    VersionedMetadata<EpochTransitionRecord> etr = streamStorePartialMock.submitScale(SCOPE, stream1, new ArrayList<>(activeEpoch.getSegmentIds()), Collections.singletonList(new AbstractMap.SimpleEntry<>(0.0, 1.0)), System.currentTimeMillis(), null, null, executor).join();
    // update the stream. the epoch transition should be reset and should have no effect.
    streamConfiguration = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, initialSegments + 5)).build();
    updateConfigVerifyScale(requestEventWriter, streamConfiguration, initialSegments + 5);
    assertEquals(streamMetadataTasks.checkScale(SCOPE, stream1, etr.getObject().getActiveEpoch(), 0L).join().getStatus(), Controller.ScaleStatusResponse.ScaleStatus.SUCCESS);
}

Example 13 with EpochRecord

use of io.pravega.controller.store.stream.records.EpochRecord in project pravega by pravega.

the class PravegaTablesScaleRequestHandlerTest method testEpochMigration.

@Test(timeout = 30000)
public void testEpochMigration() throws ExecutionException, InterruptedException {
    final String scope = "scopeEpoch";
    streamStore.createScope(scope, null, executor).get();
    final String testStream = "streamEpoch";
    final String epoch0Key = "epochRecord-0";
    long creationTime = System.currentTimeMillis();
    StreamSegmentRecord segRecord = new StreamSegmentRecord(0, 0, creationTime, 0.0, 1.0);
    EpochRecord firstEpochInOldFormat = new EpochRecord(0, 0, ImmutableList.of(segRecord), creationTime, EpochRecord.DEFAULT_COUNT_VALUE, EpochRecord.DEFAULT_COUNT_VALUE);
    VersionedMetadata<EpochRecord> expectedEpochRecord = new VersionedMetadata<>(firstEpochInOldFormat, new Version.IntVersion(0));
    doReturn(CompletableFuture.completedFuture(expectedEpochRecord)).when(storeHelper).getCachedOrLoad(anyString(), eq(epoch0Key), any(), anyLong(), anyLong());
    ScaleOperationTask scaleRequestHandler = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, 1)).build();
    streamStore.createStream(scope, testStream, config, System.currentTimeMillis(), null, executor).join();
    streamStore.setState(scope, testStream, State.ACTIVE, null, executor).join();
    assertEquals(firstEpochInOldFormat, streamStore.getEpoch(scope, testStream, 0, null, executor).join());
    ArrayList<Map.Entry<Double, Double>> newRange = new ArrayList<>();
    newRange.add(new AbstractMap.SimpleEntry<>(0.0, 1.0));
    // start with manual scale
    ScaleOpEvent event = new ScaleOpEvent(scope, testStream, Lists.newArrayList(0L), newRange, true, System.currentTimeMillis(), System.currentTimeMillis());
    streamStore.submitScale(scope, testStream, Lists.newArrayList(0L), new ArrayList<>(newRange), System.currentTimeMillis(), null, null, executor).join();
    // perform scaling
    scaleRequestHandler.execute(event).join();
    assertEquals(State.ACTIVE, streamStore.getState(scope, testStream, true, null, executor).join());
    assertEquals(1, streamStore.getActiveEpoch(scope, testStream, null, true, executor).join().getEpoch());
}

Example 14 with EpochRecord

use of io.pravega.controller.store.stream.records.EpochRecord in project pravega by pravega.

the class ScaleRequestHandlerTest method testMigrateManualScaleRequestAfterRollingTxn.

@Test(timeout = 30000)
public void testMigrateManualScaleRequestAfterRollingTxn() throws Exception {
    // This test checks a scenario where after rolling txn, if an outstanding scale request
    // was present, its epoch consistency should fail
    String stream = "newStream";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, 2)).build();
    streamMetadataTasks.createStream(scope, stream, config, System.currentTimeMillis(), 0L).get();
    EventWriterMock writer = new EventWriterMock();
    streamMetadataTasks.setRequestEventWriter(writer);
    ScaleOperationTask scaleRequestHandler = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    StreamRequestHandler requestHandler = new StreamRequestHandler(null, scaleRequestHandler, null, null, null, null, null, null, null, streamStore, null, executor);
    CommitRequestHandler commitRequestHandler = new CommitRequestHandler(streamStore, streamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
    // 1 create transaction on old epoch and set it to committing
    UUID txnIdOldEpoch = streamStore.generateTransactionId(scope, stream, null, executor).join();
    VersionedTransactionData txnData = streamStore.createTransaction(scope, stream, txnIdOldEpoch, 10000, 10000, null, executor).join();
    streamStore.sealTransaction(scope, stream, txnData.getId(), true, Optional.empty(), "", Long.MIN_VALUE, null, executor).join();
    UUID txnIdOldEpoch2 = streamStore.generateTransactionId(scope, stream, null, executor).join();
    VersionedTransactionData txnData2 = streamStore.createTransaction(scope, stream, txnIdOldEpoch2, 10000, 10000, null, executor).join();
    streamStore.sealTransaction(scope, stream, txnData2.getId(), true, Optional.empty(), "", Long.MIN_VALUE, null, executor).join();
    EpochRecord epochZero = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    assertEquals(0, epochZero.getEpoch());
    // 2. start scale
    requestHandler.process(new ScaleOpEvent(scope, stream, Lists.newArrayList(0L), Lists.newArrayList(new AbstractMap.SimpleEntry<>(0.0, 0.25), new AbstractMap.SimpleEntry<>(0.25, 0.5)), false, System.currentTimeMillis(), System.currentTimeMillis()), () -> false).join();
    // 3. verify that scale is complete
    State state = streamStore.getState(scope, stream, true, null, executor).join();
    assertEquals(State.ACTIVE, state);
    // 4. just submit a new scale. don't let it run. this should create an epoch transition. state should still be active
    streamStore.submitScale(scope, stream, Lists.newArrayList(1L), Lists.newArrayList(new AbstractMap.SimpleEntry<>(0.5, 0.75), new AbstractMap.SimpleEntry<>(0.75, 1.0)), System.currentTimeMillis(), null, null, executor).join();
    // 5. commit on old epoch. this should roll over.
    assertTrue(Futures.await(commitRequestHandler.processEvent(new CommitEvent(scope, stream, txnData.getEpoch()))));
    TxnStatus txnStatus = streamStore.transactionStatus(scope, stream, txnIdOldEpoch, null, executor).join();
    assertEquals(TxnStatus.COMMITTED, txnStatus);
    // 6. run scale against old record but with manual scale flag set to true. This should be migrated to new epoch and processed.
    requestHandler.process(new ScaleOpEvent(scope, stream, Lists.newArrayList(1L), Lists.newArrayList(new AbstractMap.SimpleEntry<>(0.5, 0.75), new AbstractMap.SimpleEntry<>(0.75, 1.0)), true, System.currentTimeMillis(), System.currentTimeMillis()), () -> false).join();
    state = streamStore.getState(scope, stream, true, null, executor).join();
    assertEquals(State.ACTIVE, state);
    EpochRecord epoch = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    assertEquals(4, epoch.getEpoch());
}

Example 15 with EpochRecord

use of io.pravega.controller.store.stream.records.EpochRecord in project pravega by pravega.

the class ScaleRequestHandlerTest method testScaleWithTransactionRequest.

@Test(timeout = 30000)
public void testScaleWithTransactionRequest() throws InterruptedException {
    EventWriterMock writer = new EventWriterMock();
    streamMetadataTasks.setRequestEventWriter(writer);
    ScaleOperationTask scaleRequestHandler = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    StreamRequestHandler requestHandler = new StreamRequestHandler(null, scaleRequestHandler, null, null, null, null, null, null, null, streamStore, null, executor);
    CommitRequestHandler commitRequestHandler = new CommitRequestHandler(streamStore, streamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
    // 1 create transaction on old epoch and set it to committing
    UUID txnIdOldEpoch = streamStore.generateTransactionId(scope, stream, null, executor).join();
    VersionedTransactionData txnData = streamStore.createTransaction(scope, stream, txnIdOldEpoch, 10000, 10000, null, executor).join();
    streamStore.sealTransaction(scope, stream, txnData.getId(), true, Optional.empty(), "", Long.MIN_VALUE, null, executor).join();
    EpochRecord epochZero = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    assertEquals(0, epochZero.getEpoch());
    // 2. start scale
    requestHandler.process(new ScaleOpEvent(scope, stream, Lists.newArrayList(0L, 1L, 2L), Lists.newArrayList(new AbstractMap.SimpleEntry<>(0.0, 1.0)), false, System.currentTimeMillis(), System.currentTimeMillis()), () -> false).join();
    // 3. verify that scale is complete
    State state = streamStore.getState(scope, stream, true, null, executor).join();
    assertEquals(State.ACTIVE, state);
    EpochRecord epochOne = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    assertEquals(1, epochOne.getEpoch());
    // 4. create transaction -> verify that this is created on new epoch
    UUID txnIdNewEpoch = streamStore.generateTransactionId(scope, stream, null, executor).join();
    VersionedTransactionData txnDataNew = streamStore.createTransaction(scope, stream, txnIdNewEpoch, 10000, 10000, null, executor).join();
    streamStore.sealTransaction(scope, stream, txnDataNew.getId(), true, Optional.empty(), "", Long.MIN_VALUE, null, executor).join();
    // 5. commit on old epoch. this should roll over
    assertTrue(Futures.await(commitRequestHandler.processEvent(new CommitEvent(scope, stream, txnData.getEpoch()))));
    TxnStatus txnStatus = streamStore.transactionStatus(scope, stream, txnIdOldEpoch, null, executor).join();
    assertEquals(TxnStatus.COMMITTED, txnStatus);
    EpochRecord epochTwo = streamStore.getEpoch(scope, stream, 2, null, executor).join();
    EpochRecord epochThree = streamStore.getEpoch(scope, stream, 3, null, executor).join();
    assertEquals(0, epochTwo.getReferenceEpoch());
    assertEquals(epochZero.getSegments().size(), epochTwo.getSegments().size());
    assertEquals(epochZero.getSegments().stream().map(x -> NameUtils.getSegmentNumber(x.segmentId())).collect(Collectors.toSet()), epochTwo.getSegments().stream().map(x -> NameUtils.getSegmentNumber(x.segmentId())).collect(Collectors.toSet()));
    assertEquals(1, epochThree.getReferenceEpoch());
    assertEquals(epochOne.getSegments().size(), epochThree.getSegments().size());
    assertEquals(epochOne.getSegments().stream().map(x -> NameUtils.getSegmentNumber(x.segmentId())).collect(Collectors.toSet()), epochThree.getSegments().stream().map(x -> NameUtils.getSegmentNumber(x.segmentId())).collect(Collectors.toSet()));
    EpochRecord activeEpoch = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    assertEquals(epochThree, activeEpoch);
    // 6. commit on new epoch. This should happen on duplicate of new epoch successfully
    assertTrue(Futures.await(commitRequestHandler.processEvent(new CommitEvent(scope, stream, txnDataNew.getEpoch()))));
    txnStatus = streamStore.transactionStatus(scope, stream, txnIdNewEpoch, null, executor).join();
    assertEquals(TxnStatus.COMMITTED, txnStatus);
    activeEpoch = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    assertEquals(epochThree, activeEpoch);
}