Examples with WatermarkSerializer io.pravega.client.watermark.WatermarkSerializer used on opensource projects

Example 6 with WatermarkSerializer

use of io.pravega.client.watermark.WatermarkSerializer in project pravega by pravega.

the class WatermarkingTest method watermarkingTests.

@Test
public void watermarkingTests() throws Exception {
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
    // create 2 writers
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, clientConfig);
    JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<Long> writer1 = clientFactory.createEventWriter(STREAM, javaSerializer, EventWriterConfig.builder().build());
    @Cleanup EventStreamWriter<Long> writer2 = clientFactory.createEventWriter(STREAM, javaSerializer, EventWriterConfig.builder().build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    writeEvents(writer1, stopFlag);
    writeEvents(writer2, stopFlag);
    // scale the stream several times so that we get complex positions
    Stream streamObj = Stream.of(SCOPE, STREAM);
    scale(controller, streamObj);
    @Cleanup ClientFactoryImpl syncClientFactory = new ClientFactoryImpl(SCOPE, new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor()), connectionFactory);
    String markStream = NameUtils.getMarkStreamForStream(STREAM);
    RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
    // scale down one controller instance.
    Futures.getAndHandleExceptions(controllerInstance.scaleService(1), ExecutionException::new);
    // wait until at least 2 more watermarks are emitted
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 4, 100000);
    stopFlag.set(true);
    Watermark watermark0 = watermarks.take();
    Watermark watermark1 = watermarks.take();
    Watermark watermark2 = watermarks.take();
    Watermark watermark3 = watermarks.take();
    assertTrue(watermark0.getLowerTimeBound() <= watermark0.getUpperTimeBound());
    assertTrue(watermark1.getLowerTimeBound() <= watermark1.getUpperTimeBound());
    assertTrue(watermark2.getLowerTimeBound() <= watermark2.getUpperTimeBound());
    assertTrue(watermark3.getLowerTimeBound() <= watermark3.getUpperTimeBound());
    // verify that watermarks are increasing in time.
    assertTrue(watermark0.getLowerTimeBound() < watermark1.getLowerTimeBound());
    assertTrue(watermark1.getLowerTimeBound() < watermark2.getLowerTimeBound());
    assertTrue(watermark2.getLowerTimeBound() < watermark3.getLowerTimeBound());
    // use watermark as lower and upper bounds.
    Map<Segment, Long> positionMap0 = watermark0.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(SCOPE, STREAM, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutStart = new StreamCutImpl(streamObj, positionMap0);
    Map<Stream, StreamCut> start = Collections.singletonMap(streamObj, streamCutStart);
    Map<Segment, Long> positionMap2 = watermark2.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(SCOPE, STREAM, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutEnd = new StreamCutImpl(streamObj, positionMap2);
    Map<Stream, StreamCut> end = Collections.singletonMap(streamObj, streamCutEnd);
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(SCOPE, controller, syncClientFactory);
    String readerGroup = "rg";
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(streamObj).startingStreamCuts(start).endingStreamCuts(end).build());
    // create reader on the stream
    @Cleanup final EventStreamReader<Long> reader = clientFactory.createReader("myreader", readerGroup, javaSerializer, ReaderConfig.builder().build());
    // read events from the reader.
    // verify that events read belong to the bound
    EventRead<Long> event = reader.readNextEvent(10000L);
    AtomicReference<TimeWindow> currentTimeWindow = new AtomicReference<>();
    AssertExtensions.assertEventuallyEquals(true, () -> {
        currentTimeWindow.set(reader.getCurrentTimeWindow(streamObj));
        return currentTimeWindow.get() != null && currentTimeWindow.get().getLowerTimeBound() != null && currentTimeWindow.get().getUpperTimeBound() != null;
    }, 100000);
    log.info("current time window = {}", currentTimeWindow.get());
    while (event.getEvent() != null) {
        Long time = event.getEvent();
        log.info("event read = {}", time);
        event.getPosition();
        assertTrue(time >= currentTimeWindow.get().getLowerTimeBound());
        event = reader.readNextEvent(10000L);
        if (event.isCheckpoint()) {
            event = reader.readNextEvent(10000L);
        }
    }
}

Example 7 with WatermarkSerializer

use of io.pravega.client.watermark.WatermarkSerializer in project pravega by pravega.

the class EventStreamReaderTest method testTimeWindow.

@Test
public void testTimeWindow() throws SegmentSealedException {
    String scope = "scope";
    String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    String groupName = "readerGroup";
    String readerGroupStream = NameUtils.getStreamForReaderGroup(groupName);
    String markStream = NameUtils.getMarkStreamForStream(streamName);
    // Create factories
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    @Cleanup MockClientFactory clientFactory = new MockClientFactory(scope, segmentStreamFactory);
    MockController controller = (MockController) clientFactory.getController();
    @Cleanup InlineExecutor executor = new InlineExecutor();
    // Create streams
    controller.createScope(scope).join();
    controller.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).build());
    controller.createStream(scope, readerGroupStream, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
    // Reader group state synchronizer
    ReaderGroupConfig config = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(stream).build();
    StateSynchronizer<ReaderGroupState> sync = clientFactory.createStateSynchronizer(readerGroupStream, new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
    // Watermark reader/writer
    @Cleanup RevisionedStreamClient<Watermark> markWriter = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    @Cleanup WatermarkReaderImpl markReader = new WatermarkReaderImpl(stream, markWriter, executor);
    // Initialize reader group state
    Map<SegmentWithRange, Long> segments = ReaderGroupImpl.getSegmentsForStreams(controller, config);
    sync.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, getEndSegmentsForStreams(config), false));
    // Data segment writers
    Segment segment1 = new Segment(scope, streamName, 0);
    Segment segment2 = new Segment(scope, streamName, 1);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    @Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    // Write stream data
    writeInt(stream1, 1);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    // Write mark data
    val r1 = new SegmentWithRange(segment1, 0, 0.5).convert();
    val r2 = new SegmentWithRange(segment2, 0.5, 1).convert();
    markWriter.writeUnconditionally(new Watermark(0L, 99L, ImmutableMap.of(r1, 0L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(100L, 199L, ImmutableMap.of(r1, 12L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(200L, 299L, ImmutableMap.of(r1, 12L, r2, 12L)));
    markWriter.writeUnconditionally(new Watermark(300L, 399L, ImmutableMap.of(r1, 12L, r2, 24L)));
    markWriter.writeUnconditionally(new Watermark(400L, 499L, ImmutableMap.of(r1, 12L, r2, 36L)));
    // Create reader
    AtomicLong clock = new AtomicLong();
    ReaderGroupStateManager groupState = new ReaderGroupStateManager(scope, groupName, "reader1", sync, controller, clock::get);
    groupState.initializeReader(0);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, new Orderer(), clock::get, ReaderConfig.builder().build(), ImmutableMap.of(stream, markReader), Mockito.mock(Controller.class));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    EventRead<byte[]> event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    TimeWindow timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(199, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(1, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(299, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(200, timeWindow.getLowerTimeBound().longValue());
    assertEquals(399, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(300, timeWindow.getLowerTimeBound().longValue());
    assertEquals(499, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(null, event.getEvent());
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(400, timeWindow.getLowerTimeBound().longValue());
    assertEquals(null, timeWindow.getUpperTimeBound());
}

Example 8 with WatermarkSerializer

use of io.pravega.client.watermark.WatermarkSerializer in project pravega by pravega.

the class WatermarkReaderImplTest method testUpdates.

@Test
public void testUpdates() {
    Stream stream = new StreamImpl("Scope", "streamName");
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    @Cleanup MockClientFactory clientFactory = new MockClientFactory("Scope", segmentStreamFactory);
    String markStream = NameUtils.getMarkStreamForStream("streamName");
    createScopeAndStream("Scope", markStream, clientFactory.getController());
    RevisionedStreamClient<Watermark> writer = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    InlineExecutor executor = new InlineExecutor();
    @Cleanup WatermarkReaderImpl impl = new WatermarkReaderImpl(stream, writer, executor);
    SegmentWithRange s0 = new SegmentWithRange(new Segment(stream.getScope(), stream.getStreamName(), 0), 0, 0.5);
    SegmentWithRange s1 = new SegmentWithRange(new Segment(stream.getScope(), stream.getStreamName(), 1), 0.5, 1);
    SegmentWithRange s2 = new SegmentWithRange(new Segment(stream.getScope(), stream.getStreamName(), 2), 0, 0.5);
    SegmentWithRange s3 = new SegmentWithRange(new Segment(stream.getScope(), stream.getStreamName(), 3), 0.5, 1);
    Map<SegmentWithRange, Long> m1 = ImmutableMap.of(s0, 0L, s1, 0L);
    Map<SegmentWithRange, Long> m2 = ImmutableMap.of(s0, 2L, s1, 0L);
    Map<SegmentWithRange, Long> m3 = ImmutableMap.of(s0, 2L, s1, 2L);
    Map<SegmentWithRange, Long> m4 = ImmutableMap.of(s2, 0L, s1, 2L);
    Map<SegmentWithRange, Long> m5 = ImmutableMap.of(s2, 4L, s1, 2L);
    Map<SegmentWithRange, Long> m6 = ImmutableMap.of(s2, 4L, s1, 4L);
    Map<SegmentWithRange, Long> m7 = ImmutableMap.of(s2, 4L, s3, 0L);
    Map<SegmentWithRange, Long> m8 = ImmutableMap.of(s2, 6L, s3, 4L);
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m1)).lowerTimeBound(10).upperTimeBound(19).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m2)).lowerTimeBound(20).upperTimeBound(29).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m3)).lowerTimeBound(30).upperTimeBound(39).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m4)).lowerTimeBound(40).upperTimeBound(49).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m5)).lowerTimeBound(50).upperTimeBound(59).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m6)).lowerTimeBound(60).upperTimeBound(69).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m7)).lowerTimeBound(70).upperTimeBound(79).build());
    writer.writeUnconditionally(Watermark.builder().streamCut(convert(m8)).lowerTimeBound(80).upperTimeBound(89).build());
    assertEquals(null, impl.getTimeWindow().getLowerTimeBound());
    assertEquals(null, impl.getTimeWindow().getUpperTimeBound());
    impl.advanceTo(ImmutableMap.of(s0, 1L, s1, 0L));
    assertEquals(10, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(29, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s0, 3L, s1, 0L));
    assertEquals(20, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(49, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s0, 5L, s1, 0L));
    assertEquals(20, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(49, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s0, 6L, s1, 0L));
    assertEquals(20, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(49, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s0, 6L, s1, 1L));
    assertEquals(20, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(49, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s0, 6L, s1, 3L));
    assertEquals(30, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(69, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 0L, s1, 3L));
    assertEquals(40, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(69, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 4L, s1, 3L));
    assertEquals(50, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(69, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 4L, s1, 5L));
    assertEquals(60, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(79, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 4L, s3, 1L));
    assertEquals(70, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(89, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 5L, s3, 1L));
    assertEquals(70, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(89, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 5L, s3, 5L));
    assertEquals(70, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(89, impl.getTimeWindow().getUpperTimeBound().longValue());
    impl.advanceTo(ImmutableMap.of(s2, 6L, s3, 5L));
    assertEquals(80, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(null, impl.getTimeWindow().getUpperTimeBound());
    impl.advanceTo(ImmutableMap.of(s2, 7L, s3, 7L));
    assertEquals(80, impl.getTimeWindow().getLowerTimeBound().longValue());
    assertEquals(null, impl.getTimeWindow().getUpperTimeBound());
}

Example 9 with WatermarkSerializer

use of io.pravega.client.watermark.WatermarkSerializer in project pravega by pravega.

the class WatermarkingTest method recreateStreamWatermarkTest.

@Test(timeout = 120000)
public void recreateStreamWatermarkTest() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    // then delete stream and move to next iteration and verify that watermarks are generated.
    for (int i = 0; i < 2; i++) {
        String scope = "scope";
        String stream = "recreateStreamWatermarkTest";
        streamManager.createScope(scope);
        streamManager.createStream(scope, stream, config);
        // create writer
        @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
        JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
        @Cleanup EventStreamWriter<Long> writer = clientFactory.createEventWriter(stream, javaSerializer, EventWriterConfig.builder().build());
        AtomicBoolean stopFlag = new AtomicBoolean(false);
        // write events
        CompletableFuture<Void> writerFuture = writeEvents(writer, stopFlag);
        @Cleanup SynchronizerClientFactory syncClientFactory = SynchronizerClientFactory.withScope(scope, clientConfig);
        String markStream = NameUtils.getMarkStreamForStream(stream);
        @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
        LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
        fetchWatermarks(watermarkReader, watermarks, stopFlag);
        AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
        // stop run and seal and delete stream
        stopFlag.set(true);
        writerFuture.join();
        streamManager.sealStream(scope, stream);
        streamManager.deleteStream(scope, stream);
    }
}

Example 10 with WatermarkSerializer

use of io.pravega.client.watermark.WatermarkSerializer in project pravega by pravega.

the class ControllerWatermarkingTest method watermarkTest.

@Test(timeout = 60000)
public void watermarkTest() throws Exception {
    Controller controller = controllerWrapper.getController();
    String scope = "scope";
    String stream = "stream";
    controller.createScope(scope).join();
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    controller.createStream(scope, stream, config).join();
    String markStream = NameUtils.getMarkStreamForStream(stream);
    Stream streamObj = new StreamImpl(scope, stream);
    WriterPosition pos1 = WriterPosition.builder().segments(Collections.singletonMap(new Segment(scope, stream, 0L), 10L)).build();
    WriterPosition pos2 = WriterPosition.builder().segments(Collections.singletonMap(new Segment(scope, stream, 0L), 20L)).build();
    controller.noteTimestampFromWriter("1", streamObj, 1L, pos1).join();
    controller.noteTimestampFromWriter("2", streamObj, 2L, pos2).join();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    @Cleanup RevisionedStreamClient<Watermark> reader = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    AssertExtensions.assertEventuallyEquals(true, () -> {
        Iterator<Entry<Revision, Watermark>> watermarks = reader.readFrom(reader.fetchOldestRevision());
        return watermarks.hasNext();
    }, 30000);
    Iterator<Entry<Revision, Watermark>> watermarks = reader.readFrom(reader.fetchOldestRevision());
    Watermark watermark = watermarks.next().getValue();
    assertEquals(watermark.getLowerTimeBound(), 1L);
    assertTrue(watermark.getStreamCut().entrySet().stream().anyMatch(x -> x.getKey().getSegmentId() == 0L && x.getValue() == 20L));
    controller.sealStream(scope, stream).join();
    controller.deleteStream(scope, stream).join();
    AssertExtensions.assertFutureThrows("Mark Stream should not exist", controller.getCurrentSegments(scope, markStream), e -> Exceptions.unwrap(e) instanceof StoreException.DataNotFoundException);
}