Examples with VersionedMetadata io.pravega.controller.store.VersionedMetadata used on opensource projects

Example 21 with VersionedMetadata

use of io.pravega.controller.store.VersionedMetadata in project pravega by pravega.

the class StreamTestBase method segmentQueriesDuringRollingTxn.

@Test(timeout = 30000L)
public void segmentQueriesDuringRollingTxn() {
    OperationContext context = getContext();
    // start scale and perform `getSegment`, `getActiveEpoch` and `getEpoch` during different phases of scale
    int startingSegmentNumber = new Random().nextInt(2000);
    long time = System.currentTimeMillis();
    Stream stream = createStream("scope", "stream" + startingSegmentNumber, time, 5, startingSegmentNumber);
    StreamSegmentRecord segment = stream.getSegment(startingSegmentNumber, context).join();
    assertEquals(segment.segmentId(), startingSegmentNumber + 0L);
    assertEquals(segment.getKeyStart(), 0, 0);
    assertEquals(segment.getKeyEnd(), 1.0 / 5, 0);
    createAndCommitTransaction(stream, 0, 0L);
    List<Map.Entry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.0, 0.5));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.5, 1.0));
    time = time + 1;
    scaleStream(stream, time, Lists.newArrayList(startingSegmentNumber + 0L, startingSegmentNumber + 1L, startingSegmentNumber + 2L, startingSegmentNumber + 3L, startingSegmentNumber + 4L), newRanges, Collections.emptyMap());
    List<StreamSegmentRecord> activeSegmentsBefore = stream.getActiveSegments(context).join();
    // start commit transactions
    VersionedMetadata<CommittingTransactionsRecord> ctr = stream.startCommittingTransactions(100, context).join().getKey();
    stream.getVersionedState(context).thenCompose(s -> stream.updateVersionedState(s, State.COMMITTING_TXN, context)).join();
    // start rolling transaction
    ctr = stream.startRollingTxn(1, ctr, context).join();
    List<StreamSegmentRecord> activeSegments1 = stream.getActiveSegments(context).join();
    assertEquals(activeSegments1, activeSegmentsBefore);
    Map<StreamSegmentRecord, List<Long>> successorsWithPredecessors = stream.getSuccessorsWithPredecessors(computeSegmentId(startingSegmentNumber + 5, 1), context).join();
    assertTrue(successorsWithPredecessors.isEmpty());
    // rolling txn create duplicate epochs. We should be able to get successors and predecessors after this step.
    time = time + 1;
    stream.rollingTxnCreateDuplicateEpochs(Collections.emptyMap(), time, ctr, context).join();
    activeSegments1 = stream.getActiveSegments(context).join();
    assertEquals(activeSegments1, activeSegmentsBefore);
    successorsWithPredecessors = stream.getSuccessorsWithPredecessors(computeSegmentId(startingSegmentNumber + 5, 1), context).join();
    Set<StreamSegmentRecord> successors = successorsWithPredecessors.keySet();
    assertEquals(3, successors.size());
    assertTrue(successors.stream().allMatch(x -> x.getCreationEpoch() == 2));
    assertTrue(successors.stream().anyMatch(x -> x.getSegmentNumber() == startingSegmentNumber + 0));
    assertTrue(successors.stream().anyMatch(x -> x.getSegmentNumber() == startingSegmentNumber + 1));
    assertTrue(successors.stream().anyMatch(x -> x.getSegmentNumber() == startingSegmentNumber + 2));
    successorsWithPredecessors = stream.getSuccessorsWithPredecessors(computeSegmentId(startingSegmentNumber + 0, 2), context).join();
    successors = successorsWithPredecessors.keySet();
    assertEquals(1, successors.size());
    assertTrue(successors.stream().allMatch(x -> x.segmentId() == computeSegmentId(startingSegmentNumber + 5, 3)));
    stream.completeRollingTxn(Collections.emptyMap(), ctr, context).join();
    stream.completeCommittingTransactions(ctr, context, Collections.emptyMap()).join();
}

Example 22 with VersionedMetadata

use of io.pravega.controller.store.VersionedMetadata in project pravega by pravega.

the class StreamTest method testConcurrentGetSuccessorScale.

private void testConcurrentGetSuccessorScale(StreamMetadataStore store, BiFunction<String, String, Stream> createStream) throws Exception {
    final ScalingPolicy policy = ScalingPolicy.fixed(1);
    final String streamName = "test";
    String scopeName = "test";
    store.createScope(scopeName, null, executorService()).join();
    Stream stream = spy(createStream.apply(scopeName, streamName));
    StreamConfiguration streamConfig = StreamConfiguration.builder().scalingPolicy(policy).build();
    store.createStream(scopeName, streamName, streamConfig, System.currentTimeMillis(), null, executorService()).join();
    store.setState(scopeName, streamName, State.ACTIVE, null, executorService()).join();
    List<Map.Entry<Double, Double>> newRanges;
    newRanges = Arrays.asList(new AbstractMap.SimpleEntry<>(0.0, 0.5), new AbstractMap.SimpleEntry<>(0.5, 1.0));
    long scale = System.currentTimeMillis();
    ArrayList<Long> sealedSegments = Lists.newArrayList(0L);
    long one = NameUtils.computeSegmentId(1, 1);
    long two = NameUtils.computeSegmentId(2, 1);
    OperationContext context = getContext();
    VersionedMetadata<EpochTransitionRecord> response = stream.submitScale(sealedSegments, newRanges, scale, null, context).join();
    Map<Long, Map.Entry<Double, Double>> newSegments = response.getObject().getNewSegmentsWithRange();
    VersionedMetadata<State> state = stream.getVersionedState(context).join();
    state = stream.updateVersionedState(state, State.SCALING, context).join();
    stream.startScale(false, response, state, context).join();
    stream.scaleCreateNewEpoch(response, context).join();
    // history table has a partial record at this point.
    // now we could have sealed the segments so get successors could be called.
    Map<Long, List<Long>> successors = stream.getSuccessorsWithPredecessors(0, context).join().entrySet().stream().collect(Collectors.toMap(x -> x.getKey().segmentId(), Map.Entry::getValue));
    assertTrue(successors.containsKey(one) && successors.containsKey(two));
    // reset mock so that we can resume scale operation
    stream.scaleOldSegmentsSealed(sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), response, context).join();
    stream.completeScale(response, context).join();
    successors = stream.getSuccessorsWithPredecessors(0, context).join().entrySet().stream().collect(Collectors.toMap(x -> x.getKey().segmentId(), Map.Entry::getValue));
    assertTrue(successors.containsKey(one) && successors.containsKey(two));
}

Example 23 with VersionedMetadata

use of io.pravega.controller.store.VersionedMetadata in project pravega by pravega.

the class TaskTest method setUp.

@Before
public void setUp() throws Exception {
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    cli = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new RetryOneTime(2000));
    cli.start();
    streamStore = getStream();
    taskMetadataStore = TaskStoreFactory.createZKStore(cli, executor);
    segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    streamMetadataTasks = new StreamMetadataTasks(streamStore, StreamStoreFactory.createInMemoryBucketStore(), taskMetadataStore, segmentHelperMock, executor, HOSTNAME, GrpcAuthHelper.getDisabledAuthHelper());
    final String stream2 = "stream2";
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final ScalingPolicy policy2 = ScalingPolicy.fixed(3);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scalingPolicy(policy1).build();
    final StreamConfiguration configuration2 = StreamConfiguration.builder().scalingPolicy(policy2).build();
    // region createStream
    streamStore.createScope(SCOPE, null, executor).join();
    long start = System.currentTimeMillis();
    streamStore.createStream(SCOPE, stream1, configuration1, start, null, executor).join();
    streamStore.setState(SCOPE, stream1, State.ACTIVE, null, executor).join();
    streamStore.createStream(SCOPE, stream2, configuration2, start, null, executor).join();
    streamStore.setState(SCOPE, stream2, State.ACTIVE, null, executor).join();
    // endregion
    // region scaleSegments
    AbstractMap.SimpleEntry<Double, Double> segment1 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment2 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Long> sealedSegments = Collections.singletonList(1L);
    VersionedMetadata<EpochTransitionRecord> versioned = streamStore.submitScale(SCOPE, stream1, sealedSegments, Arrays.asList(segment1, segment2), start + 20, null, null, executor).get();
    EpochTransitionRecord response = versioned.getObject();
    Map<Long, Map.Entry<Double, Double>> segmentsCreated = response.getNewSegmentsWithRange();
    VersionedMetadata<State> state = streamStore.getVersionedState(SCOPE, stream1, null, executor).join();
    state = streamStore.updateVersionedState(SCOPE, stream1, State.SCALING, state, null, executor).get();
    versioned = streamStore.startScale(SCOPE, stream1, false, versioned, state, null, executor).join();
    streamStore.scaleCreateNewEpochs(SCOPE, stream1, versioned, null, executor).get();
    streamStore.scaleSegmentsSealed(SCOPE, stream1, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), versioned, null, executor).get();
    streamStore.completeScale(SCOPE, stream1, versioned, null, executor).join();
    streamStore.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
    AbstractMap.SimpleEntry<Double, Double> segment3 = new AbstractMap.SimpleEntry<>(0.0, 0.5);
    AbstractMap.SimpleEntry<Double, Double> segment4 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment5 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Long> sealedSegments1 = Arrays.asList(0L, 1L, 2L);
    versioned = streamStore.submitScale(SCOPE, stream2, sealedSegments1, Arrays.asList(segment3, segment4, segment5), start + 20, null, null, executor).get();
    response = versioned.getObject();
    segmentsCreated = response.getNewSegmentsWithRange();
    state = streamStore.getVersionedState(SCOPE, stream2, null, executor).join();
    state = streamStore.updateVersionedState(SCOPE, stream2, State.SCALING, state, null, executor).get();
    versioned = streamStore.startScale(SCOPE, stream2, false, versioned, state, null, executor).join();
    streamStore.scaleCreateNewEpochs(SCOPE, stream2, versioned, null, executor).get();
    streamStore.scaleSegmentsSealed(SCOPE, stream2, sealedSegments1.stream().collect(Collectors.toMap(x -> x, x -> 0L)), versioned, null, executor).get();
    streamStore.completeScale(SCOPE, stream2, versioned, null, executor).join();
    streamStore.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
// endregion
}

Example 24 with VersionedMetadata

use of io.pravega.controller.store.VersionedMetadata in project pravega by pravega.

the class StreamMetadataTasks method isTruncated.

@VisibleForTesting
CompletableFuture<Boolean> isTruncated(String scope, String stream, Map<Long, Long> streamCut, OperationContext context) {
    CompletableFuture<State> stateFuture = streamMetadataStore.getState(scope, stream, true, context, executor);
    CompletableFuture<StreamTruncationRecord> configPropertyFuture = streamMetadataStore.getTruncationRecord(scope, stream, context, executor).thenApply(VersionedMetadata::getObject);
    return CompletableFuture.allOf(stateFuture, configPropertyFuture).thenApply(v -> {
        State state = stateFuture.join();
        StreamTruncationRecord truncationRecord = configPropertyFuture.join();
        // if property is updating and doesn't match our request, it's a subsequent update
        if (truncationRecord.isUpdating()) {
            return !truncationRecord.getStreamCut().equals(streamCut);
        } else {
            // if stream is sealed then truncate should not be allowed
            if (state.equals(State.SEALED)) {
                log.error("Cannot truncate a sealed stream {}/{}", scope, stream);
                throw new UnsupportedOperationException("Cannot truncate a sealed stream: " + NameUtils.getScopedStreamName(scope, stream));
            }
            // matches our expectation and state is not updating
            return !(truncationRecord.getStreamCut().equals(streamCut) && state.equals(State.TRUNCATING));
        }
    });
}

Example 25 with VersionedMetadata

use of io.pravega.controller.store.VersionedMetadata in project pravega by pravega.

the class ScaleRequestHandlerTest method testScaleRequestWithMinimumSegment.

@Test(timeout = 30000)
public void testScaleRequestWithMinimumSegment() throws ExecutionException, InterruptedException {
    AutoScaleTask requestHandler = new AutoScaleTask(streamMetadataTasks, streamStore, executor);
    ScaleOperationTask scaleRequestHandler = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    StreamRequestHandler multiplexer = new StreamRequestHandler(requestHandler, scaleRequestHandler, null, null, null, null, null, null, null, streamStore, null, executor);
    EventWriterMock writer = new EventWriterMock();
    streamMetadataTasks.setRequestEventWriter(writer);
    String stream = "mystream";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, 5)).build();
    streamMetadataTasks.createStream(scope, stream, config, System.currentTimeMillis(), 0L).get();
    // change stream configuration to min segment count = 4
    config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, 4)).build();
    streamStore.startUpdateConfiguration(scope, stream, config, null, executor).join();
    VersionedMetadata<StreamConfigurationRecord> configRecord = streamStore.getConfigurationRecord(scope, stream, null, executor).join();
    streamStore.completeUpdateConfiguration(scope, stream, configRecord, null, executor).join();
    // process first auto scale down event. it should only mark the segment as cold
    multiplexer.process(new AutoScaleEvent(scope, stream, 1L, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis()), () -> false).join();
    assertTrue(writer.queue.isEmpty());
    assertTrue(streamStore.isCold(scope, stream, 1L, null, executor).join());
    // process second auto scale down event. since its not for an immediate neighbour so it should only mark the segment as cold
    multiplexer.process(new AutoScaleEvent(scope, stream, 3L, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis()), () -> false).join();
    assertTrue(streamStore.isCold(scope, stream, 3L, null, executor).join());
    // no scale event should be posted
    assertTrue(writer.queue.isEmpty());
    // process third auto scale down event. This should result in a scale op event being posted to merge segments 0, 1
    multiplexer.process(new AutoScaleEvent(scope, stream, 0L, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis()), () -> false).join();
    assertTrue(streamStore.isCold(scope, stream, 0L, null, executor).join());
    // verify that a new event has been posted
    assertEquals(1, writer.queue.size());
    ControllerEvent event = writer.queue.take();
    assertTrue(event instanceof ScaleOpEvent);
    ScaleOpEvent scaleDownEvent1 = (ScaleOpEvent) event;
    assertEquals(1, scaleDownEvent1.getNewRanges().size());
    assertEquals(2, scaleDownEvent1.getSegmentsToSeal().size());
    assertTrue(scaleDownEvent1.getSegmentsToSeal().contains(0L));
    assertTrue(scaleDownEvent1.getSegmentsToSeal().contains(1L));
    // process fourth auto scale down event. This should result in a scale op event being posted to merge segments 3, 4
    multiplexer.process(new AutoScaleEvent(scope, stream, 4L, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis()), () -> false).join();
    assertTrue(streamStore.isCold(scope, stream, 4L, null, executor).join());
    // verify that a new event has been posted
    assertEquals(1, writer.queue.size());
    event = writer.queue.take();
    assertTrue(event instanceof ScaleOpEvent);
    ScaleOpEvent scaleDownEvent2 = (ScaleOpEvent) event;
    assertEquals(1, scaleDownEvent2.getNewRanges().size());
    assertEquals(2, scaleDownEvent2.getSegmentsToSeal().size());
    assertTrue(scaleDownEvent2.getSegmentsToSeal().contains(3L));
    assertTrue(scaleDownEvent2.getSegmentsToSeal().contains(4L));
    // process first scale down event, this should submit scale and scale the stream down to 4 segments
    multiplexer.process(scaleDownEvent1, () -> false).join();
    EpochRecord activeEpoch = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    List<StreamSegmentRecord> segments = activeEpoch.getSegments();
    assertEquals(1, activeEpoch.getEpoch());
    assertEquals(4, segments.size());
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 2));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 3));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 4));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 5));
    // process second scale down event, this should submit scale and scale the stream down to 4 segments
    multiplexer.process(scaleDownEvent2, () -> false).join();
    // verify that no scale has happened
    activeEpoch = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    // verify that no scale has happened.
    assertEquals(1, activeEpoch.getEpoch());
    assertEquals(4, segments.size());
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 2));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 3));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 4));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 5));
}