Examples with Watermark io.pravega.shared.watermarks.Watermark used on opensource projects

Example 6 with Watermark

use of io.pravega.shared.watermarks.Watermark in project pravega by pravega.

the class RestoreBackUpDataRecoveryTest method testDurableDataLogFailRecoveryWatermarking.

/**
 * Tests the data recovery scenario with watermarking events.
 *  What test does, step by step:
 *  1. Starts Pravega locally with just 4 segment containers.
 *  2. Writes {@link #TOTAL_NUM_EVENTS} events to a segment with watermarks.
 *  3. Waits for all segments created to be flushed to the long term storage.
 *  4. Shuts down the controller, segment store and bookeeper/zookeeper.
 *  5. Creates back up of container metadata segment and its attribute segment before deleting them from the Long Term Storage .
 *  6. Starts 4 debug segment containers using a new bookeeper/zookeeper and the Long Term Storage.
 *  7. Re-creates the container metadata segments in DurableLog and lets them to be flushed to the Long Term Storage.
 *  8. Starts segment store and controller.
 *  9. Read all events and verify that all events are below the bounds.
 * @throws Exception    In case of an exception occurred while execution.
 */
@Test(timeout = 180000)
public void testDurableDataLogFailRecoveryWatermarking() throws Exception {
    int instanceId = 0;
    int bookieCount = 1;
    int containerCount = 4;
    String readerGroup = "rgTx";
    // Creating a long term storage only once here.
    this.storageFactory = new InMemoryStorageFactory(executorService());
    log.info("Created a long term storage.");
    // Start a new BK & ZK, segment store and controller
    @Cleanup PravegaRunner pravegaRunner = new PravegaRunner(instanceId++, bookieCount, containerCount, this.storageFactory);
    // Create a scope and a stream
    createScopeStream(pravegaRunner.controllerRunner.controller, SCOPE, STREAM1);
    // Create a client to write events.
    @Cleanup ClientRunner clientRunner = new ClientRunner(pravegaRunner.controllerRunner);
    // Create a writer
    @Cleanup TransactionalEventStreamWriter<Long> writer = clientRunner.clientFactory.createTransactionalEventWriter("writer1", STREAM1, new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutTime(TRANSACTION_TIMEOUT.toMillis()).build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    CompletableFuture<Void> writerFuture = writeTxEvents(writer, stopFlag);
    // scale the stream several times so that we get complex positions
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    Stream streamObj = Stream.of(SCOPE, STREAM1);
    scale(pravegaRunner.controllerRunner.controller, streamObj, config);
    // get watermarks
    LinkedBlockingQueue<Watermark> watermarks = getWatermarks(pravegaRunner, stopFlag, writerFuture);
    // Shut down the controller
    pravegaRunner.controllerRunner.close();
    // Flush DurableLog to Long Term Storage
    flushToStorage(pravegaRunner.segmentStoreRunner.serviceBuilder);
    // Shutdown SegmentStore
    pravegaRunner.segmentStoreRunner.close();
    // Shutdown BookKeeper & ZooKeeper
    pravegaRunner.bookKeeperRunner.close();
    log.info("SegmentStore, BookKeeper & ZooKeeper shutdown");
    // Get the long term storage from the running pravega instance
    @Cleanup Storage storage = new AsyncStorageWrapper(new RollingStorage(this.storageFactory.createSyncStorage(), new SegmentRollingPolicy(DEFAULT_ROLLING_SIZE)), executorService());
    Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).join();
    // start a new BookKeeper and ZooKeeper.
    pravegaRunner.bookKeeperRunner = new BookKeeperRunner(instanceId++, bookieCount);
    createBookKeeperLogFactory();
    log.info("Started a new BookKeeper and ZooKeeper.");
    // Recover segments
    runRecovery(containerCount, storage, backUpMetadataSegments);
    // Start a new segment store and controller
    pravegaRunner.restartControllerAndSegmentStore(this.storageFactory, this.dataLogFactory);
    log.info("Started segment store and controller again.");
    // Create the client with new controller.
    @Cleanup ClientRunner newClientRunner = new ClientRunner(pravegaRunner.controllerRunner);
    // read events and verify
    readVerifyEventsWithWatermarks(readerGroup, newClientRunner, streamObj, watermarks);
}

Example 7 with Watermark

use of io.pravega.shared.watermarks.Watermark in project pravega by pravega.

the class WatermarkingTest method watermarkTest.

@Test(timeout = 120000)
public void watermarkTest() throws Exception {
    Controller controller = PRAVEGA.getLocalController();
    String scope = "scope";
    String stream = "watermarkTest";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    streamManager.createScope(scope);
    streamManager.createStream(scope, stream, config);
    Stream streamObj = Stream.of(scope, stream);
    // create 2 writers
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
    JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<Long> writer1 = clientFactory.createEventWriter(stream, javaSerializer, EventWriterConfig.builder().build());
    @Cleanup EventStreamWriter<Long> writer2 = clientFactory.createEventWriter(stream, javaSerializer, EventWriterConfig.builder().build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    CompletableFuture<Void> writer1Future = writeEvents(writer1, stopFlag);
    CompletableFuture<Void> writer2Future = writeEvents(writer2, stopFlag);
    // scale the stream several times so that we get complex positions
    scale(controller, streamObj, config);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    @Cleanup ClientFactoryImpl syncClientFactory = new ClientFactoryImpl(scope, new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor()), connectionFactory);
    String markStream = NameUtils.getMarkStreamForStream(stream);
    @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
    stopFlag.set(true);
    writer1Future.join();
    writer2Future.join();
    // read events from the stream
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, controller, syncClientFactory);
    Watermark watermark0 = watermarks.take();
    Watermark watermark1 = watermarks.take();
    assertTrue(watermark0.getLowerTimeBound() <= watermark0.getUpperTimeBound());
    assertTrue(watermark1.getLowerTimeBound() <= watermark1.getUpperTimeBound());
    assertTrue(watermark0.getLowerTimeBound() < watermark1.getLowerTimeBound());
    Map<Segment, Long> positionMap0 = watermark0.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    Map<Segment, Long> positionMap1 = watermark1.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutFirst = new StreamCutImpl(streamObj, positionMap0);
    StreamCut streamCutSecond = new StreamCutImpl(streamObj, positionMap1);
    Map<Stream, StreamCut> firstMarkStreamCut = Collections.singletonMap(streamObj, streamCutFirst);
    Map<Stream, StreamCut> secondMarkStreamCut = Collections.singletonMap(streamObj, streamCutSecond);
    // read from stream cut of first watermark
    String readerGroup = "watermarkTest-group";
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(streamObj).startingStreamCuts(firstMarkStreamCut).endingStreamCuts(secondMarkStreamCut).disableAutomaticCheckpoints().build());
    @Cleanup final EventStreamReader<Long> reader = clientFactory.createReader("myreader", readerGroup, javaSerializer, ReaderConfig.builder().build());
    EventRead<Long> event = reader.readNextEvent(10000L);
    TimeWindow currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    while (event.getEvent() != null && currentTimeWindow.getLowerTimeBound() == null && currentTimeWindow.getUpperTimeBound() == null) {
        event = reader.readNextEvent(10000L);
        currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    }
    assertNotNull(currentTimeWindow.getUpperTimeBound());
    // read all events and verify that all events are below the bounds
    while (event.getEvent() != null) {
        Long time = event.getEvent();
        log.info("timewindow = {} event = {}", currentTimeWindow, time);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || time >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || time <= currentTimeWindow.getUpperTimeBound());
        TimeWindow nextTimeWindow = reader.getCurrentTimeWindow(streamObj);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || nextTimeWindow.getLowerTimeBound() >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || nextTimeWindow.getUpperTimeBound() >= currentTimeWindow.getUpperTimeBound());
        currentTimeWindow = nextTimeWindow;
        event = reader.readNextEvent(10000L);
        if (event.isCheckpoint()) {
            event = reader.readNextEvent(10000L);
        }
    }
    assertNotNull(currentTimeWindow.getLowerTimeBound());
}

Example 8 with Watermark

use of io.pravega.shared.watermarks.Watermark in project pravega by pravega.

the class WatermarkingTest method watermarkTxnTest.

@Test(timeout = 120000)
public void watermarkTxnTest() throws Exception {
    Controller controller = PRAVEGA.getLocalController();
    String scope = "scopeTx";
    String stream = "watermarkTxnTest";
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    streamManager.createScope(scope);
    streamManager.createStream(scope, stream, config);
    Stream streamObj = Stream.of(scope, stream);
    // create 2 writers
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
    JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
    @Cleanup TransactionalEventStreamWriter<Long> writer1 = clientFactory.createTransactionalEventWriter("writer1", stream, new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutTime(10000).build());
    @Cleanup TransactionalEventStreamWriter<Long> writer2 = clientFactory.createTransactionalEventWriter("writer2", stream, new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutTime(10000).build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    CompletableFuture<Void> writer1Future = writeTxEvents(writer1, stopFlag);
    CompletableFuture<Void> writer2Future = writeTxEvents(writer2, stopFlag);
    // scale the stream several times so that we get complex positions
    scale(controller, streamObj, config);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    @Cleanup ClientFactoryImpl syncClientFactory = new ClientFactoryImpl(scope, new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor()), connectionFactory);
    String markStream = NameUtils.getMarkStreamForStream(stream);
    @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
    stopFlag.set(true);
    writer1Future.join();
    writer2Future.join();
    // read events from the stream
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, controller, syncClientFactory);
    Watermark watermark0 = watermarks.take();
    Watermark watermark1 = watermarks.take();
    assertTrue(watermark0.getLowerTimeBound() <= watermark0.getUpperTimeBound());
    assertTrue(watermark1.getLowerTimeBound() <= watermark1.getUpperTimeBound());
    assertTrue(watermark0.getLowerTimeBound() < watermark1.getLowerTimeBound());
    Map<Segment, Long> positionMap0 = watermark0.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    Map<Segment, Long> positionMap1 = watermark1.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutFirst = new StreamCutImpl(streamObj, positionMap0);
    StreamCut streamCutSecond = new StreamCutImpl(streamObj, positionMap1);
    Map<Stream, StreamCut> firstMarkStreamCut = Collections.singletonMap(streamObj, streamCutFirst);
    Map<Stream, StreamCut> secondMarkStreamCut = Collections.singletonMap(streamObj, streamCutSecond);
    // read from stream cut of first watermark
    String readerGroup = "watermarkTxnTest-group";
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(streamObj).startingStreamCuts(firstMarkStreamCut).endingStreamCuts(secondMarkStreamCut).disableAutomaticCheckpoints().build());
    @Cleanup final EventStreamReader<Long> reader = clientFactory.createReader("myreaderTx", readerGroup, javaSerializer, ReaderConfig.builder().build());
    EventRead<Long> event = reader.readNextEvent(10000L);
    TimeWindow currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    while (event.getEvent() != null && currentTimeWindow.getLowerTimeBound() == null && currentTimeWindow.getUpperTimeBound() == null) {
        event = reader.readNextEvent(10000L);
        currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    }
    assertNotNull(currentTimeWindow.getUpperTimeBound());
    // read all events and verify that all events are below the bounds
    while (event.getEvent() != null) {
        Long time = event.getEvent();
        log.info("timewindow = {} event = {}", currentTimeWindow, time);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || time >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || time <= currentTimeWindow.getUpperTimeBound());
        TimeWindow nextTimeWindow = reader.getCurrentTimeWindow(streamObj);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || nextTimeWindow.getLowerTimeBound() >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || nextTimeWindow.getUpperTimeBound() >= currentTimeWindow.getUpperTimeBound());
        currentTimeWindow = nextTimeWindow;
        event = reader.readNextEvent(10000L);
        if (event.isCheckpoint()) {
            event = reader.readNextEvent(10000L);
        }
    }
    assertNotNull(currentTimeWindow.getLowerTimeBound());
}

Example 9 with Watermark

use of io.pravega.shared.watermarks.Watermark in project pravega by pravega.

the class RestoreBackUpDataRecoveryTest method getWatermarks.

/**
 * Gets watermarks used while writing the events
 */
private LinkedBlockingQueue<Watermark> getWatermarks(PravegaRunner pravegaRunner, AtomicBoolean stopFlag, CompletableFuture<Void> writerFuture) throws Exception {
    @Cleanup SynchronizerClientFactory syncClientFactory = SynchronizerClientFactory.withScope(SCOPE, ClientConfig.builder().controllerURI(pravegaRunner.controllerRunner.controllerURI).build());
    String markStream = NameUtils.getMarkStreamForStream(STREAM1);
    RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    CompletableFuture<Void> fetchWaterMarksFuture = fetchWatermarks(watermarkReader, watermarks, stopFlag);
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
    stopFlag.set(true);
    fetchWaterMarksFuture.join();
    writerFuture.join();
    return watermarks;
}

Example 10 with Watermark

use of io.pravega.shared.watermarks.Watermark in project pravega by pravega.

the class WatermarkingTest method progressingWatermarkWithWriterTimeouts.

@Test(timeout = 60000)
public void progressingWatermarkWithWriterTimeouts() throws Exception {
    String scope = "Timeout";
    String streamName = "progressingWatermarkWithWriterTimeouts";
    int numSegments = 1;
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    assertNotNull(streamManager);
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(numSegments)).build());
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
    @Cleanup SynchronizerClientFactory syncClientFactory = SynchronizerClientFactory.withScope(scope, clientConfig);
    String markStream = NameUtils.getMarkStreamForStream(streamName);
    @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    // create two writers and write two sevent and call note time for each writer.
    @Cleanup EventStreamWriter<String> writer1 = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer1.writeEvent("1").get();
    writer1.noteTime(100L);
    @Cleanup EventStreamWriter<String> writer2 = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer2.writeEvent("2").get();
    writer2.noteTime(102L);
    // writer0 should timeout. writer1 and writer2 should result in two more watermarks with following times:
    // 1: 100L-101L 2: 101-101
    // then first writer should timeout and be discarded. But second writer should continue to be active as its time
    // is higher than first watermark. This should result in a second watermark to be emitted.
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() == 2, 100000);
    Watermark watermark1 = watermarks.poll();
    Watermark watermark2 = watermarks.poll();
    assertEquals(100L, watermark1.getLowerTimeBound());
    assertEquals(102L, watermark1.getUpperTimeBound());
    assertEquals(102L, watermark2.getLowerTimeBound());
    assertEquals(102L, watermark2.getUpperTimeBound());
    // stream cut should be same
    assertTrue(watermark2.getStreamCut().entrySet().stream().allMatch(x -> watermark1.getStreamCut().get(x.getKey()).equals(x.getValue())));
    // bring back writer1 and post an event with note time smaller than current watermark
    writer1.writeEvent("3").get();
    writer1.noteTime(101L);
    // no watermark should be emitted.
    Watermark nullMark = watermarks.poll(10, TimeUnit.SECONDS);
    assertNull(nullMark);
}