Examples with Watermark io.pravega.shared.watermarks.Watermark used on opensource projects

Example 16 with Watermark

use of io.pravega.shared.watermarks.Watermark in project pravega by pravega.

the class PeriodicWatermarking method computeWatermark.

/**
 * This method takes marks (time + position) of active writers and finds greatest lower bound on time and
 * least upper bound on positions and returns the watermark object composed of the two.
 * The least upper bound computed from positions may not result in a consistent and complete stream cut.
 * So, a positional upper bound is then converted into a stream cut by including segments from higher epoch.
 * Also, it is possible that in an effort to fill missing range, we may end up creating an upper bound that
 * is composed of segments from highest epoch. In next iteration, from new writer positions, we may be able to
 * compute a tighter upper bound. But since watermark has to advance position and time, we will take the upper bound
 * of previous stream cut and new stream cut.
 *
 * @param scope scope
 * @param streamName stream name
 * @param context operation context
 * @param activeWriters marks for all active writers.
 * @param previousWatermark previous watermark that was emitted.
 * @return CompletableFuture which when completed will contain watermark to be emitted.
 */
private CompletableFuture<Watermark> computeWatermark(String scope, String streamName, OperationContext context, List<Map.Entry<String, WriterMark>> activeWriters, Watermark previousWatermark) {
    long requestId = context.getRequestId();
    Watermark.WatermarkBuilder builder = Watermark.builder();
    ConcurrentHashMap<SegmentWithRange, Long> upperBound = new ConcurrentHashMap<>();
    // We are deliberately making two passes over writers - first to find lowest time. Second loop will convert writer
    // positions to StreamSegmentRecord objects by retrieving ranges from store. And then perform computation on those
    // objects.
    LongSummaryStatistics summarized = activeWriters.stream().collect(Collectors.summarizingLong(x -> x.getValue().getTimestamp()));
    long lowerBoundOnTime = summarized.getMin();
    long upperBoundOnTime = summarized.getMax();
    if (lowerBoundOnTime > previousWatermark.getLowerTimeBound()) {
        CompletableFuture<List<Map<SegmentWithRange, Long>>> positionsFuture = Futures.allOfWithResults(activeWriters.stream().map(x -> {
            return Futures.keysAllOfWithResults(x.getValue().getPosition().entrySet().stream().collect(Collectors.toMap(y -> getSegmentWithRange(scope, streamName, context, y.getKey()), Entry::getValue)));
        }).collect(Collectors.toList()));
        log.debug(requestId, "Emitting watermark for stream {}/{} with time {}", scope, streamName, lowerBoundOnTime);
        return positionsFuture.thenAccept(listOfPositions -> listOfPositions.forEach(position -> {
            // add writer positions to upperBound map.
            addToUpperBound(position, upperBound);
        })).thenCompose(v -> computeStreamCut(scope, streamName, context, upperBound, previousWatermark).thenApply(streamCut -> builder.lowerTimeBound(lowerBoundOnTime).upperTimeBound(upperBoundOnTime).streamCut(ImmutableMap.copyOf(streamCut)).build()));
    } else {
        // new time is not advanced. No watermark to be emitted.
        return CompletableFuture.completedFuture(null);
    }
}

Example 17 with Watermark

use of io.pravega.shared.watermarks.Watermark in project pravega by pravega.

the class WatermarkReaderImplTest method testUpdates.

@Test
public void testUpdates() {
    Stream stream = new StreamImpl("Scope", "streamName");
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    @Cleanup MockClientFactory clientFactory = new MockClientFactory("Scope", segmentStreamFactory);
    String markStream = NameUtils.getMarkStreamForStream("streamName");
    createScopeAndStream("Scope", markStream, clientFactory.getController());
    RevisionedStreamClient<Watermark> writer = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    InlineExecutor executor = new InlineExecutor();
    @Cleanup WatermarkReaderImpl impl = new WatermarkReaderImpl(stream, writer, executor);
    SegmentWithRange s0 = new SegmentWithRange(new Segment(stream.getScope(), stream.getStreamName(), 0), 0, 0.5);
    SegmentWithRange s1 = new SegmentWithRange(new Segment(stream.getScope(), stream.getStreamName(), 1), 0.5, 1);
    SegmentWithRange s2 = new SegmentWithRange(new Segment(stream.getScope(), stream.getStreamName(), 2), 0, 0.5);
    SegmentWithRange s3 = new SegmentWithRange(new Segment(stream.getScope(), stream.getStreamName(), 3), 0.5, 1);
    Map<SegmentWithRange, Long> m1 = ImmutableMap.of(s0, 0L, s1, 0L);
    Map<SegmentWithRange, Long> m2 = ImmutableMap.of(s0, 2L, s1, 0L);
    Map<SegmentWithRange, Long> m3 = ImmutableMap.of(s0, 2L, s1, 2L);
    Map<SegmentWithRange, Long> m4 = ImmutableMap.of(s2, 0L, s1, 2L);
    Map<SegmentWithRange, Long> m5 = ImmutableMap.of(s2, 4L, s1, 2L);
    Map<SegmentWithRange, Long> m6 = ImmutableMap.of(s2, 4L, s1, 4L);
    Map<SegmentWithRange, Long> m7 = ImmutableMap.of(s2, 4L, s3, 0L);
    Map<SegmentWithRange, Long> m8 = ImmutableMap.of(s2, 6L, s3, 4L);
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m1)).lowerTimeBound(10).upperTimeBound(19).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m2)).lowerTimeBound(20).upperTimeBound(29).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m3)).lowerTimeBound(30).upperTimeBound(39).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m4)).lowerTimeBound(40).upperTimeBound(49).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m5)).lowerTimeBound(50).upperTimeBound(59).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m6)).lowerTimeBound(60).upperTimeBound(69).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m7)).lowerTimeBound(70).upperTimeBound(79).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m8)).lowerTimeBound(80).upperTimeBound(89).build());
    assertEquals(null, impl.getTimeWindow().getLowerTimeBound());
    assertEquals(null, impl.getTimeWindow().getUpperTimeBound());
    impl.advanceTo(ImmutableMap.of(s0, 1L, s1, 0L));
    assertEquals(10, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(29, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s0, 3L, s1, 0L));
    assertEquals(20, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(49, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s0, 5L, s1, 0L));
    assertEquals(20, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(49, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s0, 6L, s1, 0L));
    assertEquals(20, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(49, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s0, 6L, s1, 1L));
    assertEquals(20, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(49, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s0, 6L, s1, 3L));
    assertEquals(30, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(69, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 0L, s1, 3L));
    assertEquals(40, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(69, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 4L, s1, 3L));
    assertEquals(50, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(69, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 4L, s1, 5L));
    assertEquals(60, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(79, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 4L, s3, 1L));
    assertEquals(70, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(89, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 5L, s3, 1L));
    assertEquals(70, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(89, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 5L, s3, 5L));
    assertEquals(70, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(89, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 6L, s3, 5L));
    assertEquals(80, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(null, impl.getTimeWindow().getUpperTimeBound());
    impl.advanceTo(ImmutableMap.of(s2, 7L, s3, 7L));
    assertEquals(80, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(null, impl.getTimeWindow().getUpperTimeBound());
}

Example 18 with Watermark

use of io.pravega.shared.watermarks.Watermark in project pravega by pravega.

the class WatermarkReaderImplTest method testCompare.

@Test
public void testCompare() {
    Stream stream = new StreamImpl("Scope", "streamName");
    SegmentWithRange s0 = new SegmentWithRange(new Segment(stream.getScope(), stream.getStreamName(), 0), 0, 0.5);
    SegmentWithRange s1 = new SegmentWithRange(new Segment(stream.getScope(), stream.getStreamName(), 1), 0.5, 1);
    SegmentWithRange s2 = new SegmentWithRange(new Segment(stream.getScope(), stream.getStreamName(), 2), 0, 0.5);
    SegmentWithRange s3 = new SegmentWithRange(new Segment(stream.getScope(), stream.getStreamName(), 3), 0.5, 1);
    Map<SegmentWithRange, Long> readerGroupPosition = ImmutableMap.of(s1, 1L, s2, 2L);
    Watermark watermark = Watermark.builder().streamCut(convert(readerGroupPosition)).build();
    assertEquals(1, WatermarkReaderImpl.compare(stream, readerGroupPosition, watermark));
    Map<SegmentWithRange, Long> before = ImmutableMap.of(s0, 0L, s1, 1L);
    assertEquals(-1, WatermarkReaderImpl.compare(stream, before, watermark));
    before = ImmutableMap.of(s1, 0L, s2, 1L);
    assertEquals(-1, WatermarkReaderImpl.compare(stream, before, watermark));
    before = ImmutableMap.of(s1, 1L, s2, 1L);
    assertEquals(-1, WatermarkReaderImpl.compare(stream, before, watermark));
    Map<SegmentWithRange, Long> after = ImmutableMap.of(s2, 2L, s3, 3L);
    assertEquals(1, WatermarkReaderImpl.compare(stream, after, watermark));
    after = ImmutableMap.of(s1, 2L, s2, 3L);
    assertEquals(1, WatermarkReaderImpl.compare(stream, after, watermark));
    after = ImmutableMap.of(s1, 1L, s2, 3L);
    assertEquals(1, WatermarkReaderImpl.compare(stream, after, watermark));
    Map<SegmentWithRange, Long> overlap = ImmutableMap.of(s0, 0L, s3, 3L);
    assertEquals(0, WatermarkReaderImpl.compare(stream, overlap, watermark));
    overlap = ImmutableMap.of(s1, 0L, s2, 3L);
    assertEquals(0, WatermarkReaderImpl.compare(stream, overlap, watermark));
    overlap = ImmutableMap.of(s1, 3L, s2, 0L);
    assertEquals(0, WatermarkReaderImpl.compare(stream, overlap, watermark));
}

Example 19 with Watermark

use of io.pravega.shared.watermarks.Watermark in project pravega by pravega.

the class WatermarkingTest method recreateStreamWatermarkTest.

@Test(timeout = 120000)
public void recreateStreamWatermarkTest() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    // then delete stream and move to next iteration and verify that watermarks are generated.
    for (int i = 0; i < 2; i++) {
        String scope = "scope";
        String stream = "recreateStreamWatermarkTest";
        streamManager.createScope(scope);
        streamManager.createStream(scope, stream, config);
        // create writer
        @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
        JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
        @Cleanup EventStreamWriter<Long> writer = clientFactory.createEventWriter(stream, javaSerializer, EventWriterConfig.builder().build());
        AtomicBoolean stopFlag = new AtomicBoolean(false);
        // write events
        CompletableFuture<Void> writerFuture = writeEvents(writer, stopFlag);
        @Cleanup SynchronizerClientFactory syncClientFactory = SynchronizerClientFactory.withScope(scope, clientConfig);
        String markStream = NameUtils.getMarkStreamForStream(stream);
        @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
        LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
        fetchWatermarks(watermarkReader, watermarks, stopFlag);
        AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
        // stop run and seal and delete stream
        stopFlag.set(true);
        writerFuture.join();
        streamManager.sealStream(scope, stream);
        streamManager.deleteStream(scope, stream);
    }
}

Example 20 with Watermark

use of io.pravega.shared.watermarks.Watermark in project pravega by pravega.

the class ControllerWatermarkingTest method watermarkTest.

@Test(timeout = 60000)
public void watermarkTest() throws Exception {
    Controller controller = controllerWrapper.getController();
    String scope = "scope";
    String stream = "stream";
    controller.createScope(scope).join();
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    controller.createStream(scope, stream, config).join();
    String markStream = NameUtils.getMarkStreamForStream(stream);
    Stream streamObj = new StreamImpl(scope, stream);
    WriterPosition pos1 = WriterPosition.builder().segments(Collections.singletonMap(new Segment(scope, stream, 0L), 10L)).build();
    WriterPosition pos2 = WriterPosition.builder().segments(Collections.singletonMap(new Segment(scope, stream, 0L), 20L)).build();
    controller.noteTimestampFromWriter("1", streamObj, 1L, pos1).join();
    controller.noteTimestampFromWriter("2", streamObj, 2L, pos2).join();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    @Cleanup RevisionedStreamClient<Watermark> reader = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    AssertExtensions.assertEventuallyEquals(true, () -> {
        Iterator<Entry<Revision, Watermark>> watermarks = reader.readFrom(reader.fetchOldestRevision());
        return watermarks.hasNext();
    }, 30000);
    Iterator<Entry<Revision, Watermark>> watermarks = reader.readFrom(reader.fetchOldestRevision());
    Watermark watermark = watermarks.next().getValue();
    assertEquals(watermark.getLowerTimeBound(), 1L);
    assertTrue(watermark.getStreamCut().entrySet().stream().anyMatch(x -> x.getKey().getSegmentId() == 0L && x.getValue() == 20L));
    controller.sealStream(scope, stream).join();
    controller.deleteStream(scope, stream).join();
    AssertExtensions.assertFutureThrows("Mark Stream should not exist", controller.getCurrentSegments(scope, markStream), e -> Exceptions.unwrap(e) instanceof StoreException.DataNotFoundException);
}