Examples with EventStreamWriter io.pravega.client.stream.EventStreamWriter used on opensource projects

Example 1 with EventStreamWriter

use of io.pravega.client.stream.EventStreamWriter in project pravega by pravega.

the class WatermarkingTest method progressingWatermarkWithWriterTimeouts.

@Test(timeout = 60000)
public void progressingWatermarkWithWriterTimeouts() throws Exception {
    String scope = "Timeout";
    String streamName = "progressingWatermarkWithWriterTimeouts";
    int numSegments = 1;
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    assertNotNull(streamManager);
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(numSegments)).build());
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
    @Cleanup SynchronizerClientFactory syncClientFactory = SynchronizerClientFactory.withScope(scope, clientConfig);
    String markStream = NameUtils.getMarkStreamForStream(streamName);
    @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    // create two writers and write two sevent and call note time for each writer.
    @Cleanup EventStreamWriter<String> writer1 = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer1.writeEvent("1").get();
    writer1.noteTime(100L);
    @Cleanup EventStreamWriter<String> writer2 = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer2.writeEvent("2").get();
    writer2.noteTime(102L);
    // writer0 should timeout. writer1 and writer2 should result in two more watermarks with following times:
    // 1: 100L-101L 2: 101-101
    // then first writer should timeout and be discarded. But second writer should continue to be active as its time
    // is higher than first watermark. This should result in a second watermark to be emitted.
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() == 2, 100000);
    Watermark watermark1 = watermarks.poll();
    Watermark watermark2 = watermarks.poll();
    assertEquals(100L, watermark1.getLowerTimeBound());
    assertEquals(102L, watermark1.getUpperTimeBound());
    assertEquals(102L, watermark2.getLowerTimeBound());
    assertEquals(102L, watermark2.getUpperTimeBound());
    // stream cut should be same
    assertTrue(watermark2.getStreamCut().entrySet().stream().allMatch(x -> watermark1.getStreamCut().get(x.getKey()).equals(x.getValue())));
    // bring back writer1 and post an event with note time smaller than current watermark
    writer1.writeEvent("3").get();
    writer1.noteTime(101L);
    // no watermark should be emitted.
    Watermark nullMark = watermarks.poll(10, TimeUnit.SECONDS);
    assertNull(nullMark);
}

Example 2 with EventStreamWriter

use of io.pravega.client.stream.EventStreamWriter in project pravega by pravega.

the class WatermarkingTest method watermarkTest.

@Test(timeout = 120000)
public void watermarkTest() throws Exception {
    Controller controller = PRAVEGA.getLocalController();
    String scope = "scope";
    String stream = "watermarkTest";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    streamManager.createScope(scope);
    streamManager.createStream(scope, stream, config);
    Stream streamObj = Stream.of(scope, stream);
    // create 2 writers
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
    JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<Long> writer1 = clientFactory.createEventWriter(stream, javaSerializer, EventWriterConfig.builder().build());
    @Cleanup EventStreamWriter<Long> writer2 = clientFactory.createEventWriter(stream, javaSerializer, EventWriterConfig.builder().build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    CompletableFuture<Void> writer1Future = writeEvents(writer1, stopFlag);
    CompletableFuture<Void> writer2Future = writeEvents(writer2, stopFlag);
    // scale the stream several times so that we get complex positions
    scale(controller, streamObj, config);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    @Cleanup ClientFactoryImpl syncClientFactory = new ClientFactoryImpl(scope, new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor()), connectionFactory);
    String markStream = NameUtils.getMarkStreamForStream(stream);
    @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
    stopFlag.set(true);
    writer1Future.join();
    writer2Future.join();
    // read events from the stream
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, controller, syncClientFactory);
    Watermark watermark0 = watermarks.take();
    Watermark watermark1 = watermarks.take();
    assertTrue(watermark0.getLowerTimeBound() <= watermark0.getUpperTimeBound());
    assertTrue(watermark1.getLowerTimeBound() <= watermark1.getUpperTimeBound());
    assertTrue(watermark0.getLowerTimeBound() < watermark1.getLowerTimeBound());
    Map<Segment, Long> positionMap0 = watermark0.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    Map<Segment, Long> positionMap1 = watermark1.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutFirst = new StreamCutImpl(streamObj, positionMap0);
    StreamCut streamCutSecond = new StreamCutImpl(streamObj, positionMap1);
    Map<Stream, StreamCut> firstMarkStreamCut = Collections.singletonMap(streamObj, streamCutFirst);
    Map<Stream, StreamCut> secondMarkStreamCut = Collections.singletonMap(streamObj, streamCutSecond);
    // read from stream cut of first watermark
    String readerGroup = "watermarkTest-group";
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(streamObj).startingStreamCuts(firstMarkStreamCut).endingStreamCuts(secondMarkStreamCut).disableAutomaticCheckpoints().build());
    @Cleanup final EventStreamReader<Long> reader = clientFactory.createReader("myreader", readerGroup, javaSerializer, ReaderConfig.builder().build());
    EventRead<Long> event = reader.readNextEvent(10000L);
    TimeWindow currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    while (event.getEvent() != null && currentTimeWindow.getLowerTimeBound() == null && currentTimeWindow.getUpperTimeBound() == null) {
        event = reader.readNextEvent(10000L);
        currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    }
    assertNotNull(currentTimeWindow.getUpperTimeBound());
    // read all events and verify that all events are below the bounds
    while (event.getEvent() != null) {
        Long time = event.getEvent();
        log.info("timewindow = {} event = {}", currentTimeWindow, time);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || time >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || time <= currentTimeWindow.getUpperTimeBound());
        TimeWindow nextTimeWindow = reader.getCurrentTimeWindow(streamObj);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || nextTimeWindow.getLowerTimeBound() >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || nextTimeWindow.getUpperTimeBound() >= currentTimeWindow.getUpperTimeBound());
        currentTimeWindow = nextTimeWindow;
        event = reader.readNextEvent(10000L);
        if (event.isCheckpoint()) {
            event = reader.readNextEvent(10000L);
        }
    }
    assertNotNull(currentTimeWindow.getLowerTimeBound());
}

Example 3 with EventStreamWriter

use of io.pravega.client.stream.EventStreamWriter in project pravega by pravega.

the class DelegationTokenTest method testDelegationTokenGetsRenewedAfterExpiry.

/**
 * This test verifies that a event stream reader continues to read events as a result of automatic delegation token
 * renewal, after the initial delegation token it uses expires.
 *
 * We use an extraordinarily high test timeout and read timeouts to account for any inordinate delays that may be
 * encountered in testing environments.
 */
@Test(timeout = 50000)
public void testDelegationTokenGetsRenewedAfterExpiry() throws InterruptedException {
    // Delegation token renewal threshold is 5 seconds, so we are using 6 seconds as Token TTL so that token doesn't
    // get renewed before each use.
    ClusterWrapper pravegaCluster = ClusterWrapper.builder().authEnabled(true).tokenTtlInSeconds(6).build();
    try {
        pravegaCluster.start();
        final String scope = "testscope";
        final String streamName = "teststream";
        final int numSegments = 1;
        final ClientConfig clientConfig = ClientConfig.builder().controllerURI(URI.create(pravegaCluster.controllerUri())).credentials(new DefaultCredentials("1111_aaaa", "admin")).build();
        log.debug("Done creating client config.");
        createScopeStream(scope, streamName, numSegments, clientConfig);
        @Cleanup final EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
        // Perform writes on a separate thread.
        Runnable runnable = () -> {
            @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<String>(), EventWriterConfig.builder().build());
            for (int i = 0; i < 10; i++) {
                String msg = "message: " + i;
                writer.writeEvent(msg).join();
                log.debug("Done writing message '{}' to stream '{} / {}'", msg, scope, streamName);
            }
        };
        @Cleanup("interrupt") Thread writerThread = new Thread(runnable);
        writerThread.start();
        // Now, read the events from the stream.
        String readerGroup = UUID.randomUUID().toString().replace("-", "");
        ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).disableAutomaticCheckpoints().build();
        @Cleanup ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
        readerGroupManager.createReaderGroup(readerGroup, readerGroupConfig);
        @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", readerGroup, new JavaSerializer<String>(), ReaderConfig.builder().build());
        int j = 0;
        EventRead<String> event = null;
        do {
            event = reader.readNextEvent(2000);
            if (event.getEvent() != null) {
                log.info("Done reading event: {}", event.getEvent());
                j++;
            }
            // We are keeping sleep time relatively large, just to make sure that the delegation token expires
            // midway.
            Thread.sleep(500);
        } while (event.getEvent() != null);
        // Assert that we end up reading 10 events even though delegation token must have expired midway.
        // 
        // To look for evidence of delegation token renewal check the logs for the following message:
        // - "Token is nearing expiry, so refreshing it"
        assertSame(10, j);
    } finally {
        pravegaCluster.close();
    }
}

Example 4 with EventStreamWriter

use of io.pravega.client.stream.EventStreamWriter in project pravega by pravega.

the class WatermarkingTest method watermarkingTests.

@Test
public void watermarkingTests() throws Exception {
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
    // create 2 writers
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, clientConfig);
    JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<Long> writer1 = clientFactory.createEventWriter(STREAM, javaSerializer, EventWriterConfig.builder().build());
    @Cleanup EventStreamWriter<Long> writer2 = clientFactory.createEventWriter(STREAM, javaSerializer, EventWriterConfig.builder().build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    writeEvents(writer1, stopFlag);
    writeEvents(writer2, stopFlag);
    // scale the stream several times so that we get complex positions
    Stream streamObj = Stream.of(SCOPE, STREAM);
    scale(controller, streamObj);
    @Cleanup ClientFactoryImpl syncClientFactory = new ClientFactoryImpl(SCOPE, new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor()), connectionFactory);
    String markStream = NameUtils.getMarkStreamForStream(STREAM);
    RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
    // scale down one controller instance.
    Futures.getAndHandleExceptions(controllerInstance.scaleService(1), ExecutionException::new);
    // wait until at least 2 more watermarks are emitted
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 4, 100000);
    stopFlag.set(true);
    Watermark watermark0 = watermarks.take();
    Watermark watermark1 = watermarks.take();
    Watermark watermark2 = watermarks.take();
    Watermark watermark3 = watermarks.take();
    assertTrue(watermark0.getLowerTimeBound() <= watermark0.getUpperTimeBound());
    assertTrue(watermark1.getLowerTimeBound() <= watermark1.getUpperTimeBound());
    assertTrue(watermark2.getLowerTimeBound() <= watermark2.getUpperTimeBound());
    assertTrue(watermark3.getLowerTimeBound() <= watermark3.getUpperTimeBound());
    // verify that watermarks are increasing in time.
    assertTrue(watermark0.getLowerTimeBound() < watermark1.getLowerTimeBound());
    assertTrue(watermark1.getLowerTimeBound() < watermark2.getLowerTimeBound());
    assertTrue(watermark2.getLowerTimeBound() < watermark3.getLowerTimeBound());
    // use watermark as lower and upper bounds.
    Map<Segment, Long> positionMap0 = watermark0.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(SCOPE, STREAM, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutStart = new StreamCutImpl(streamObj, positionMap0);
    Map<Stream, StreamCut> start = Collections.singletonMap(streamObj, streamCutStart);
    Map<Segment, Long> positionMap2 = watermark2.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(SCOPE, STREAM, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutEnd = new StreamCutImpl(streamObj, positionMap2);
    Map<Stream, StreamCut> end = Collections.singletonMap(streamObj, streamCutEnd);
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(SCOPE, controller, syncClientFactory);
    String readerGroup = "rg";
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(streamObj).startingStreamCuts(start).endingStreamCuts(end).build());
    // create reader on the stream
    @Cleanup final EventStreamReader<Long> reader = clientFactory.createReader("myreader", readerGroup, javaSerializer, ReaderConfig.builder().build());
    // read events from the reader.
    // verify that events read belong to the bound
    EventRead<Long> event = reader.readNextEvent(10000L);
    AtomicReference<TimeWindow> currentTimeWindow = new AtomicReference<>();
    AssertExtensions.assertEventuallyEquals(true, () -> {
        currentTimeWindow.set(reader.getCurrentTimeWindow(streamObj));
        return currentTimeWindow.get() != null && currentTimeWindow.get().getLowerTimeBound() != null && currentTimeWindow.get().getUpperTimeBound() != null;
    }, 100000);
    log.info("current time window = {}", currentTimeWindow.get());
    while (event.getEvent() != null) {
        Long time = event.getEvent();
        log.info("event read = {}", time);
        event.getPosition();
        assertTrue(time >= currentTimeWindow.get().getLowerTimeBound());
        event = reader.readNextEvent(10000L);
        if (event.isCheckpoint()) {
            event = reader.readNextEvent(10000L);
        }
    }
}

Example 5 with EventStreamWriter

use of io.pravega.client.stream.EventStreamWriter in project pravega by pravega.

the class WatermarkingTest method writeEvents.

private void writeEvents(EventStreamWriter<Long> writer, AtomicBoolean stopFlag) {
    AtomicInteger count = new AtomicInteger(0);
    AtomicLong currentTime = new AtomicLong();
    Futures.loop(() -> !stopFlag.get(), () -> Futures.delayedFuture(() -> {
        currentTime.set(System.currentTimeMillis());
        return writer.writeEvent(count.toString(), currentTime.get()).thenAccept(v -> {
            if (count.incrementAndGet() % 10 == 0) {
                writer.noteTime(currentTime.get());
            }
        });
    }, 1000L, executorService), executorService);
}