Examples with StreamSegmentRecord io.pravega.controller.store.stream.records.StreamSegmentRecord used on opensource projects

Example 16 with StreamSegmentRecord

use of io.pravega.controller.store.stream.records.StreamSegmentRecord in project pravega by pravega.

the class LocalControllerTest method testGetSegmentsBetween.

@Test(timeout = 10000)
public void testGetSegmentsBetween() throws ExecutionException, InterruptedException {
    List<StreamSegmentRecord> list = new ArrayList<>();
    when(this.mockControllerService.getSegmentsBetweenStreamCuts(any(), anyLong())).thenReturn(CompletableFuture.completedFuture(list));
    Assert.assertTrue(Futures.await(this.testController.getSegments(new StreamCutImpl(new StreamImpl("scope", "stream"), Collections.emptyMap()), new StreamCutImpl(new StreamImpl("scope", "stream"), Collections.emptyMap()))));
}

Example 17 with StreamSegmentRecord

use of io.pravega.controller.store.stream.records.StreamSegmentRecord in project pravega by pravega.

the class ScaleRequestHandlerTest method testScaleRange.

@Test(timeout = 30000)
public void testScaleRange() throws ExecutionException, InterruptedException {
    // key range values taken from issue #2543
    StreamSegmentRecord segment = new StreamSegmentRecord(2, 1, 100L, 0.1706574888245243, 0.7085170563088633);
    doReturn(CompletableFuture.completedFuture(segment)).when(streamStore).getSegment(any(), any(), anyLong(), any(), any());
    AutoScaleTask requestHandler = new AutoScaleTask(streamMetadataTasks, streamStore, executor);
    ScaleOperationTask scaleRequestHandler = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    StreamRequestHandler multiplexer = new StreamRequestHandler(requestHandler, scaleRequestHandler, null, null, null, null, null, null, null, streamStore, null, executor);
    // Send number of splits = 1
    EventWriterMock writer = new EventWriterMock();
    streamMetadataTasks.setRequestEventWriter(writer);
    AutoScaleEvent scaleUpEvent = new AutoScaleEvent(scope, stream, NameUtils.computeSegmentId(2, 1), AutoScaleEvent.UP, System.currentTimeMillis(), 1, false, System.currentTimeMillis());
    assertTrue(Futures.await(multiplexer.process(scaleUpEvent, () -> false)));
    reset(streamStore);
    // verify that one scaleOp event is written into the stream
    assertEquals(1, writer.queue.size());
    ControllerEvent event = writer.queue.take();
    assertTrue(event instanceof ScaleOpEvent);
    ScaleOpEvent scaleOpEvent = (ScaleOpEvent) event;
    assertEquals(2, scaleOpEvent.getNewRanges().size());
    assertEquals(0.1706574888245243, scaleOpEvent.getNewRanges().get(0).getKey(), 0.0);
    assertEquals(0.7085170563088633, scaleOpEvent.getNewRanges().get(1).getValue(), 0.0);
    assertTrue(scaleOpEvent.getNewRanges().get(0).getValue().doubleValue() == scaleOpEvent.getNewRanges().get(1).getKey().doubleValue());
}

Example 18 with StreamSegmentRecord

use of io.pravega.controller.store.stream.records.StreamSegmentRecord in project pravega by pravega.

the class ScaleRequestHandlerTest method testScaleRequestWithMinimumSegment.

@Test(timeout = 30000)
public void testScaleRequestWithMinimumSegment() throws ExecutionException, InterruptedException {
    AutoScaleTask requestHandler = new AutoScaleTask(streamMetadataTasks, streamStore, executor);
    ScaleOperationTask scaleRequestHandler = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    StreamRequestHandler multiplexer = new StreamRequestHandler(requestHandler, scaleRequestHandler, null, null, null, null, null, null, null, streamStore, null, executor);
    EventWriterMock writer = new EventWriterMock();
    streamMetadataTasks.setRequestEventWriter(writer);
    String stream = "mystream";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, 5)).build();
    streamMetadataTasks.createStream(scope, stream, config, System.currentTimeMillis(), 0L).get();
    // change stream configuration to min segment count = 4
    config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, 4)).build();
    streamStore.startUpdateConfiguration(scope, stream, config, null, executor).join();
    VersionedMetadata<StreamConfigurationRecord> configRecord = streamStore.getConfigurationRecord(scope, stream, null, executor).join();
    streamStore.completeUpdateConfiguration(scope, stream, configRecord, null, executor).join();
    // process first auto scale down event. it should only mark the segment as cold
    multiplexer.process(new AutoScaleEvent(scope, stream, 1L, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis()), () -> false).join();
    assertTrue(writer.queue.isEmpty());
    assertTrue(streamStore.isCold(scope, stream, 1L, null, executor).join());
    // process second auto scale down event. since its not for an immediate neighbour so it should only mark the segment as cold
    multiplexer.process(new AutoScaleEvent(scope, stream, 3L, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis()), () -> false).join();
    assertTrue(streamStore.isCold(scope, stream, 3L, null, executor).join());
    // no scale event should be posted
    assertTrue(writer.queue.isEmpty());
    // process third auto scale down event. This should result in a scale op event being posted to merge segments 0, 1
    multiplexer.process(new AutoScaleEvent(scope, stream, 0L, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis()), () -> false).join();
    assertTrue(streamStore.isCold(scope, stream, 0L, null, executor).join());
    // verify that a new event has been posted
    assertEquals(1, writer.queue.size());
    ControllerEvent event = writer.queue.take();
    assertTrue(event instanceof ScaleOpEvent);
    ScaleOpEvent scaleDownEvent1 = (ScaleOpEvent) event;
    assertEquals(1, scaleDownEvent1.getNewRanges().size());
    assertEquals(2, scaleDownEvent1.getSegmentsToSeal().size());
    assertTrue(scaleDownEvent1.getSegmentsToSeal().contains(0L));
    assertTrue(scaleDownEvent1.getSegmentsToSeal().contains(1L));
    // process fourth auto scale down event. This should result in a scale op event being posted to merge segments 3, 4
    multiplexer.process(new AutoScaleEvent(scope, stream, 4L, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis()), () -> false).join();
    assertTrue(streamStore.isCold(scope, stream, 4L, null, executor).join());
    // verify that a new event has been posted
    assertEquals(1, writer.queue.size());
    event = writer.queue.take();
    assertTrue(event instanceof ScaleOpEvent);
    ScaleOpEvent scaleDownEvent2 = (ScaleOpEvent) event;
    assertEquals(1, scaleDownEvent2.getNewRanges().size());
    assertEquals(2, scaleDownEvent2.getSegmentsToSeal().size());
    assertTrue(scaleDownEvent2.getSegmentsToSeal().contains(3L));
    assertTrue(scaleDownEvent2.getSegmentsToSeal().contains(4L));
    // process first scale down event, this should submit scale and scale the stream down to 4 segments
    multiplexer.process(scaleDownEvent1, () -> false).join();
    EpochRecord activeEpoch = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    List<StreamSegmentRecord> segments = activeEpoch.getSegments();
    assertEquals(1, activeEpoch.getEpoch());
    assertEquals(4, segments.size());
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 2));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 3));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 4));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 5));
    // process second scale down event, this should submit scale and scale the stream down to 4 segments
    multiplexer.process(scaleDownEvent2, () -> false).join();
    // verify that no scale has happened
    activeEpoch = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    // verify that no scale has happened.
    assertEquals(1, activeEpoch.getEpoch());
    assertEquals(4, segments.size());
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 2));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 3));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 4));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 5));
}

Example 19 with StreamSegmentRecord

use of io.pravega.controller.store.stream.records.StreamSegmentRecord in project pravega by pravega.

the class AbortRequestHandler method processEvent.

@Override
public CompletableFuture<Void> processEvent(AbortEvent event) {
    String scope = event.getScope();
    String stream = event.getStream();
    int epoch = event.getEpoch();
    UUID txId = event.getTxid();
    long requestId = event.getRequestId();
    if (requestId == 0L) {
        requestId = ControllerService.nextRequestId();
    }
    Timer timer = new Timer();
    OperationContext context = streamMetadataStore.createStreamContext(scope, stream, requestId);
    log.info(requestId, "Aborting transaction {} on stream {}/{}", event.getTxid(), event.getScope(), event.getStream());
    return Futures.toVoid(streamMetadataStore.getSegmentsInEpoch(event.getScope(), event.getStream(), epoch, context, executor).thenApply(segments -> segments.stream().map(StreamSegmentRecord::segmentId).collect(Collectors.toList())).thenCompose(segments -> streamMetadataTasks.notifyTxnAbort(scope, stream, segments, txId, context.getRequestId())).thenCompose(x -> streamMetadataStore.abortTransaction(scope, stream, txId, context, executor)).whenComplete((result, error) -> {
        if (error != null) {
            log.warn(context.getRequestId(), "Failed aborting transaction {} on stream {}/{}", event.getTxid(), event.getScope(), event.getStream());
            TransactionMetrics.getInstance().abortTransactionFailed(scope, stream);
        } else {
            log.info(context.getRequestId(), "Successfully aborted transaction {} on stream {}/{}", event.getTxid(), event.getScope(), event.getStream());
            if (processedEvents != null) {
                processedEvents.offer(event);
            }
            TransactionMetrics.getInstance().abortTransaction(scope, stream, timer.getElapsed());
        }
    }));
}

Example 20 with StreamSegmentRecord

use of io.pravega.controller.store.stream.records.StreamSegmentRecord in project pravega by pravega.

the class PeriodicWatermarking method computeWatermark.

/**
 * This method takes marks (time + position) of active writers and finds greatest lower bound on time and
 * least upper bound on positions and returns the watermark object composed of the two.
 * The least upper bound computed from positions may not result in a consistent and complete stream cut.
 * So, a positional upper bound is then converted into a stream cut by including segments from higher epoch.
 * Also, it is possible that in an effort to fill missing range, we may end up creating an upper bound that
 * is composed of segments from highest epoch. In next iteration, from new writer positions, we may be able to
 * compute a tighter upper bound. But since watermark has to advance position and time, we will take the upper bound
 * of previous stream cut and new stream cut.
 *
 * @param scope scope
 * @param streamName stream name
 * @param context operation context
 * @param activeWriters marks for all active writers.
 * @param previousWatermark previous watermark that was emitted.
 * @return CompletableFuture which when completed will contain watermark to be emitted.
 */
private CompletableFuture<Watermark> computeWatermark(String scope, String streamName, OperationContext context, List<Map.Entry<String, WriterMark>> activeWriters, Watermark previousWatermark) {
    long requestId = context.getRequestId();
    Watermark.WatermarkBuilder builder = Watermark.builder();
    ConcurrentHashMap<SegmentWithRange, Long> upperBound = new ConcurrentHashMap<>();
    // We are deliberately making two passes over writers - first to find lowest time. Second loop will convert writer
    // positions to StreamSegmentRecord objects by retrieving ranges from store. And then perform computation on those
    // objects.
    LongSummaryStatistics summarized = activeWriters.stream().collect(Collectors.summarizingLong(x -> x.getValue().getTimestamp()));
    long lowerBoundOnTime = summarized.getMin();
    long upperBoundOnTime = summarized.getMax();
    if (lowerBoundOnTime > previousWatermark.getLowerTimeBound()) {
        CompletableFuture<List<Map<SegmentWithRange, Long>>> positionsFuture = Futures.allOfWithResults(activeWriters.stream().map(x -> {
            return Futures.keysAllOfWithResults(x.getValue().getPosition().entrySet().stream().collect(Collectors.toMap(y -> getSegmentWithRange(scope, streamName, context, y.getKey()), Entry::getValue)));
        }).collect(Collectors.toList()));
        log.debug(requestId, "Emitting watermark for stream {}/{} with time {}", scope, streamName, lowerBoundOnTime);
        return positionsFuture.thenAccept(listOfPositions -> listOfPositions.forEach(position -> {
            // add writer positions to upperBound map.
            addToUpperBound(position, upperBound);
        })).thenCompose(v -> computeStreamCut(scope, streamName, context, upperBound, previousWatermark).thenApply(streamCut -> builder.lowerTimeBound(lowerBoundOnTime).upperTimeBound(upperBoundOnTime).streamCut(ImmutableMap.copyOf(streamCut)).build()));
    } else {
        // new time is not advanced. No watermark to be emitted.
        return CompletableFuture.completedFuture(null);
    }
}