Examples with EventStreamClientFactory io.pravega.client.EventStreamClientFactory used on opensource projects

Example 21 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class WatermarkingTest method watermarkingTests.

@Test
public void watermarkingTests() throws Exception {
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
    // create 2 writers
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, clientConfig);
    JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<Long> writer1 = clientFactory.createEventWriter(STREAM, javaSerializer, EventWriterConfig.builder().build());
    @Cleanup EventStreamWriter<Long> writer2 = clientFactory.createEventWriter(STREAM, javaSerializer, EventWriterConfig.builder().build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    writeEvents(writer1, stopFlag);
    writeEvents(writer2, stopFlag);
    // scale the stream several times so that we get complex positions
    Stream streamObj = Stream.of(SCOPE, STREAM);
    scale(controller, streamObj);
    @Cleanup ClientFactoryImpl syncClientFactory = new ClientFactoryImpl(SCOPE, new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor()), connectionFactory);
    String markStream = NameUtils.getMarkStreamForStream(STREAM);
    RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
    // scale down one controller instance.
    Futures.getAndHandleExceptions(controllerInstance.scaleService(1), ExecutionException::new);
    // wait until at least 2 more watermarks are emitted
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 4, 100000);
    stopFlag.set(true);
    Watermark watermark0 = watermarks.take();
    Watermark watermark1 = watermarks.take();
    Watermark watermark2 = watermarks.take();
    Watermark watermark3 = watermarks.take();
    assertTrue(watermark0.getLowerTimeBound() <= watermark0.getUpperTimeBound());
    assertTrue(watermark1.getLowerTimeBound() <= watermark1.getUpperTimeBound());
    assertTrue(watermark2.getLowerTimeBound() <= watermark2.getUpperTimeBound());
    assertTrue(watermark3.getLowerTimeBound() <= watermark3.getUpperTimeBound());
    // verify that watermarks are increasing in time.
    assertTrue(watermark0.getLowerTimeBound() < watermark1.getLowerTimeBound());
    assertTrue(watermark1.getLowerTimeBound() < watermark2.getLowerTimeBound());
    assertTrue(watermark2.getLowerTimeBound() < watermark3.getLowerTimeBound());
    // use watermark as lower and upper bounds.
    Map<Segment, Long> positionMap0 = watermark0.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(SCOPE, STREAM, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutStart = new StreamCutImpl(streamObj, positionMap0);
    Map<Stream, StreamCut> start = Collections.singletonMap(streamObj, streamCutStart);
    Map<Segment, Long> positionMap2 = watermark2.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(SCOPE, STREAM, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutEnd = new StreamCutImpl(streamObj, positionMap2);
    Map<Stream, StreamCut> end = Collections.singletonMap(streamObj, streamCutEnd);
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(SCOPE, controller, syncClientFactory);
    String readerGroup = "rg";
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(streamObj).startingStreamCuts(start).endingStreamCuts(end).build());
    // create reader on the stream
    @Cleanup final EventStreamReader<Long> reader = clientFactory.createReader("myreader", readerGroup, javaSerializer, ReaderConfig.builder().build());
    // read events from the reader.
    // verify that events read belong to the bound
    EventRead<Long> event = reader.readNextEvent(10000L);
    AtomicReference<TimeWindow> currentTimeWindow = new AtomicReference<>();
    AssertExtensions.assertEventuallyEquals(true, () -> {
        currentTimeWindow.set(reader.getCurrentTimeWindow(streamObj));
        return currentTimeWindow.get() != null && currentTimeWindow.get().getLowerTimeBound() != null && currentTimeWindow.get().getUpperTimeBound() != null;
    }, 100000);
    log.info("current time window = {}", currentTimeWindow.get());
    while (event.getEvent() != null) {
        Long time = event.getEvent();
        log.info("event read = {}", time);
        event.getPosition();
        assertTrue(time >= currentTimeWindow.get().getLowerTimeBound());
        event = reader.readNextEvent(10000L);
        if (event.isCheckpoint()) {
            event = reader.readNextEvent(10000L);
        }
    }
}

Example 22 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class StreamsAndScopesManagementTest method testCreateSealAndDeleteStreams.

private void testCreateSealAndDeleteStreams(String scope) {
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    for (int j = 1; j <= NUM_STREAMS; j++) {
        final String stream = String.valueOf(j);
        StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(j)).build();
        // Create Stream with nonexistent scope, which should not be successful.
        log.info("Creating a stream in a deliberately nonexistent scope nonexistentScope/{}.", stream);
        assertThrows(RuntimeException.class, () -> streamManager.createStream("nonexistentScope", stream, StreamConfiguration.builder().build()));
        long iniTime = System.nanoTime();
        log.info("Creating stream {}/{}.", scope, stream);
        assertTrue("Creating stream", streamManager.createStream(scope, stream, config));
        controllerPerfStats.get("createStreamMs").add(timeDiffInMs(iniTime));
        // Update the configuration of the stream by doubling the number of segments.
        config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(j * 2)).build();
        iniTime = System.nanoTime();
        assertTrue(streamManager.updateStream(scope, stream, config));
        controllerPerfStats.get("updateStreamMs").add(timeDiffInMs(iniTime));
        // Perform tests on empty and non-empty streams.
        if (j % 2 == 0) {
            log.info("Writing events in stream {}/{}.", scope, stream);
            @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
            writeEvents(clientFactory, stream, NUM_EVENTS);
        }
        // Update the configuration of the stream.
        config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(j * 2)).build();
        assertTrue(streamManager.updateStream(scope, stream, config));
        // Attempting to delete non-empty scope and non-sealed stream.
        assertThrows(RuntimeException.class, () -> streamManager.deleteScope(scope));
        assertThrows(RuntimeException.class, () -> streamManager.deleteStream(scope, stream));
        // Seal and delete stream.
        log.info("Attempting to seal and delete stream {}/{}.", scope, stream);
        iniTime = System.nanoTime();
        assertTrue(streamManager.sealStream(scope, stream));
        controllerPerfStats.get("sealStreamMs").add(timeDiffInMs(iniTime));
        iniTime = System.nanoTime();
        assertTrue(streamManager.deleteStream(scope, stream));
        controllerPerfStats.get("deleteStreamMs").add(timeDiffInMs(iniTime));
        // Seal and delete already sealed/deleted streams.
        log.info("Sealing and deleting an already deleted stream {}/{}.", scope, stream);
        assertThrows(RuntimeException.class, () -> streamManager.sealStream(scope, stream));
        assertFalse(streamManager.deleteStream(scope, stream));
    }
}

Example 23 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class MultiSegmentStoreTest method testReadWrite.

private void testReadWrite() {
    List<URI> ctlURIs = this.controllerInstance.getServiceDetails();
    URI controllerUri = ctlURIs.get(0);
    String scope = "testscope" + RandomStringUtils.randomAlphanumeric(10);
    String stream = "teststream" + RandomStringUtils.randomAlphanumeric(10);
    ClientConfig clientConfig = Utils.buildClientConfig(controllerUri);
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    Assert.assertTrue(streamManager.createScope(scope));
    // Create stream with large number of segments so that most segment containers are used.
    Assert.assertTrue(streamManager.createStream(scope, stream, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(10)).build()));
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
    log.info("Invoking writer with controller URI: {}", controllerUri);
    @Cleanup EventStreamWriter<Serializable> writer = clientFactory.createEventWriter(stream, new JavaSerializer<>(), EventWriterConfig.builder().build());
    final int numEvents = 1000;
    final String fixedEvent = "testevent";
    for (int i = 0; i < numEvents; i++) {
        log.debug("Producing event: {} ", fixedEvent);
        writer.writeEvent(String.valueOf(i), fixedEvent);
    }
    writer.flush();
    log.info("Invoking reader with controller URI: {}", controllerUri);
    final String readerGroup = "testreadergroup" + RandomStringUtils.randomAlphanumeric(10);
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(scope, clientConfig);
    groupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(scope, stream)).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(UUID.randomUUID().toString(), readerGroup, new JavaSerializer<>(), ReaderConfig.builder().build());
    for (int i = 0; i < numEvents; i++) {
        try {
            String event = reader.readNextEvent(60000).getEvent();
            Assert.assertEquals(fixedEvent, event);
        } catch (ReinitializationRequiredException e) {
            log.error("Unexpected request to reinitialize {}", e);
            throw new IllegalStateException("Unexpected request to reinitialize");
        }
    }
}

Example 24 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class EndToEndTruncationTest method testSimpleOffsetTruncation.

/**
 * This test checks the basic operation of truncation with offsets. The test first writes two events on a Stream
 * (1 segment) and then truncates the Stream after the first event. We verify that a new reader first gets a
 * TruncatedDataException and then it reads only the second event written, as the first has been truncated.
 *
 * @throws ReinitializationRequiredException If a checkpoint or reset is performed on the reader group.
 */
@Test(timeout = 30000)
public void testSimpleOffsetTruncation() throws ReinitializationRequiredException {
    final String scope = "truncationTests";
    final String streamName = "testSimpleOffsetTruncation";
    final String readerGroupName = "RGTestSimpleOffsetTruncation";
    StreamConfiguration streamConfiguration = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    @Cleanup StreamManager streamManager = StreamManager.create(PRAVEGA.getControllerURI());
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, streamConfiguration);
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(scope, PRAVEGA.getControllerURI());
    groupManager.createReaderGroup(readerGroupName, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(scope + "/" + streamName).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup(readerGroupName);
    // Write two events to the Stream.
    writeEvents(clientFactory, streamName, 2);
    // Read only the first one.
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerGroupName + "1", readerGroupName, new UTF8StringSerializer(), ReaderConfig.builder().build());
    assertEquals(reader.readNextEvent(5000).getEvent(), "0");
    reader.close();
    // Create a Checkpoint, get StreamCut and truncate the Stream at that point.
    Checkpoint cp = readerGroup.initiateCheckpoint("myCheckpoint", executorService()).join();
    StreamCut streamCut = cp.asImpl().getPositions().values().iterator().next();
    assertTrue(streamManager.truncateStream(scope, streamName, streamCut));
    // Verify that a new reader reads from event 1 onwards.
    final String newReaderGroupName = readerGroupName + "new";
    groupManager.createReaderGroup(newReaderGroupName, ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).build());
    @Cleanup final EventStreamReader<String> newReader = clientFactory.createReader(newReaderGroupName + "2", newReaderGroupName, new UTF8StringSerializer(), ReaderConfig.builder().build());
    assertEquals("Expected read event: ", "1", newReader.readNextEvent(5000).getEvent());
    assertNull(newReader.readNextEvent(5000).getEvent());
}

Example 25 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class EndToEndTruncationTest method testParallelSegmentOffsetTruncation.

/**
 * This test verifies that truncation works specifying an offset that applies to multiple segments. To this end,
 * the test first writes a set of events on a Stream (with multiple segments) and truncates it at a specified offset
 * (truncatedEvents). The tests asserts that readers gets a TruncatedDataException after truncation and then it
 * (only) reads the remaining events that have not been truncated.
 */
@Test(timeout = 600000)
public void testParallelSegmentOffsetTruncation() {
    final String scope = "truncationTests";
    final String streamName = "testParallelSegmentOffsetTruncation";
    final int parallelism = 2;
    final int totalEvents = 100;
    final int truncatedEvents = 25;
    StreamConfiguration streamConf = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(parallelism)).build();
    @Cleanup StreamManager streamManager = StreamManager.create(PRAVEGA.getControllerURI());
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(scope, PRAVEGA.getControllerURI());
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    streamManager.createScope(scope);
    // Test truncation in new and re-created tests.
    for (int i = 0; i < 2; i++) {
        final String readerGroupName = "RGTestParallelSegmentOffsetTruncation" + i;
        streamManager.createStream(scope, streamName, streamConf);
        groupManager.createReaderGroup(readerGroupName, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(scope, streamName)).build());
        @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup(readerGroupName);
        // Write events to the Stream.
        writeEvents(clientFactory, streamName, totalEvents);
        // Instantiate readers to consume from Stream up to truncatedEvents.
        List<CompletableFuture<Integer>> futures = ReadWriteUtils.readEvents(clientFactory, readerGroupName, parallelism, truncatedEvents);
        Futures.allOf(futures).join();
        int eventsReadBeforeTruncation = futures.stream().map(CompletableFuture::join).reduce(Integer::sum).get();
        // Perform truncation on stream segment
        Checkpoint cp = readerGroup.initiateCheckpoint("myCheckpoint" + i, executorService()).join();
        StreamCut streamCut = cp.asImpl().getPositions().values().iterator().next();
        assertTrue(streamManager.truncateStream(scope, streamName, streamCut));
        // Just after the truncation, trying to read the whole stream should raise a TruncatedDataException.
        final String newGroupName = readerGroupName + "new";
        groupManager.createReaderGroup(newGroupName, ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).build());
        futures = readEvents(clientFactory, newGroupName, parallelism);
        Futures.allOf(futures).join();
        assertEquals("Expected read events: ", totalEvents - eventsReadBeforeTruncation, (int) futures.stream().map(CompletableFuture::join).reduce((a, b) -> a + b).get());
        assertTrue(streamManager.sealStream(scope, streamName));
        assertTrue(streamManager.deleteStream(scope, streamName));
    }
}