Examples with ConnectionFactory io.pravega.client.connection.impl.ConnectionFactory used on opensource projects

Example 51 with ConnectionFactory

use of io.pravega.client.connection.impl.ConnectionFactory in project pravega by pravega.

the class StreamCutsTest method testReaderGroupCuts.

@Test(timeout = 40000)
public void testReaderGroupCuts() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope("test", 0L).get();
    controller.createStream("test", "test", config).get();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter("test", new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "fpj was here").get();
    writer.writeEvent("0", "fpj was here again").get();
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
    groupManager.createReaderGroup("cuts", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/test").groupRefreshTimeMillis(0).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup("cuts");
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", "cuts", new JavaSerializer<>(), ReaderConfig.builder().initialAllocationDelay(0).build());
    EventRead<String> firstEvent = reader.readNextEvent(5000);
    assertNotNull(firstEvent.getEvent());
    assertEquals("fpj was here", firstEvent.getEvent());
    readerGroup.initiateCheckpoint("cp1", executor);
    EventRead<String> cpEvent = reader.readNextEvent(5000);
    assertEquals("cp1", cpEvent.getCheckpointName());
    EventRead<String> secondEvent = reader.readNextEvent(5000);
    assertNotNull(secondEvent.getEvent());
    assertEquals("fpj was here again", secondEvent.getEvent());
    Map<Stream, StreamCut> cuts = readerGroup.getStreamCuts();
    validateCuts(readerGroup, cuts, Collections.singleton(getQualifiedStreamSegmentName("test", "test", 0L)));
    // Scale the stream to verify that we get more segments in the cut.
    Stream stream = Stream.of("test", "test");
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0L), map, executor).getFuture().get();
    assertTrue(result);
    log.info("Finished 1st scaling");
    writer.writeEvent("0", "fpj was here again0").get();
    writer.writeEvent("1", "fpj was here again1").get();
    EventRead<String> eosEvent = reader.readNextEvent(100);
    // Reader does not yet see the data becasue there has been no CP
    assertNull(eosEvent.getEvent());
    CompletableFuture<Checkpoint> checkpoint = readerGroup.initiateCheckpoint("cp2", executor);
    cpEvent = reader.readNextEvent(100);
    EventRead<String> event0 = reader.readNextEvent(100);
    EventRead<String> event1 = reader.readNextEvent(100);
    cuts = checkpoint.get(5, TimeUnit.SECONDS).asImpl().getPositions();
    // Validate the reader did not release the segments before the checkpoint.
    // This is important because it means that once the checkpoint is initiated no segments change readers.
    Set<String> segmentNames = ImmutableSet.of(getQualifiedStreamSegmentName("test", "test", computeSegmentId(0, 0)));
    validateCuts(readerGroup, cuts, segmentNames);
    CompletableFuture<Map<Stream, StreamCut>> futureCuts = readerGroup.generateStreamCuts(executor);
    EventRead<String> emptyEvent = reader.readNextEvent(100);
    cuts = futureCuts.get();
    segmentNames = ImmutableSet.of(getQualifiedStreamSegmentName("test", "test", computeSegmentId(1, 1)), getQualifiedStreamSegmentName("test", "test", computeSegmentId(2, 1)));
    validateCuts(readerGroup, cuts, segmentNames);
    // Scale down to verify that the number drops back.
    map = new HashMap<>();
    map.put(0.0, 1.0);
    ArrayList<Long> toSeal = new ArrayList<>();
    toSeal.add(computeSegmentId(1, 1));
    toSeal.add(computeSegmentId(2, 1));
    result = controller.scaleStream(stream, Collections.unmodifiableList(toSeal), map, executor).getFuture().get();
    assertTrue(result);
    log.info("Finished 2nd scaling");
    writer.writeEvent("0", "fpj was here again2").get();
    // Reader sees the segment is empty
    emptyEvent = reader.readNextEvent(100);
    assertNull(emptyEvent.getEvent());
    checkpoint = readerGroup.initiateCheckpoint("cp3", executor);
    cpEvent = reader.readNextEvent(100);
    assertEquals("cp3", cpEvent.getCheckpointName());
    // Reader releases segments here
    event0 = reader.readNextEvent(5000);
    assertTrue(event0.getEvent().endsWith("2"));
    cuts = readerGroup.getStreamCuts();
    long three = computeSegmentId(3, 2);
    validateCuts(readerGroup, cuts, Collections.singleton(getQualifiedStreamSegmentName("test", "test", three)));
    // Scale up to 4 segments again.
    map = new HashMap<>();
    map.put(0.0, 0.25);
    map.put(0.25, 0.5);
    map.put(0.5, 0.75);
    map.put(0.75, 1.0);
    result = controller.scaleStream(stream, Collections.singletonList(three), map, executor).getFuture().get();
    assertTrue(result);
    log.info("Finished 3rd scaling");
    writer.writeEvent("0", "fpj was here again3").get();
    // Reader sees the segment is empty
    emptyEvent = reader.readNextEvent(100);
    assertNull(emptyEvent.getEvent());
    readerGroup.initiateCheckpoint("cp4", executor);
    cpEvent = reader.readNextEvent(1000);
    assertEquals("cp4", cpEvent.getCheckpointName());
    // Reader releases segments here
    event0 = reader.readNextEvent(5000);
    assertNotNull(event0.getEvent());
    cuts = readerGroup.getStreamCuts();
    segmentNames = new HashSet<>();
    long four = computeSegmentId(4, 3);
    long five = computeSegmentId(5, 3);
    long six = computeSegmentId(6, 3);
    long seven = computeSegmentId(7, 3);
    segmentNames.add(getQualifiedStreamSegmentName("test", "test", four));
    segmentNames.add(getQualifiedStreamSegmentName("test", "test", five));
    segmentNames.add(getQualifiedStreamSegmentName("test", "test", six));
    segmentNames.add(getQualifiedStreamSegmentName("test", "test", seven));
    validateCuts(readerGroup, cuts, Collections.unmodifiableSet(segmentNames));
}

Example 52 with ConnectionFactory

use of io.pravega.client.connection.impl.ConnectionFactory in project pravega by pravega.

the class ControllerWatermarkingTest method watermarkTest.

@Test(timeout = 60000)
public void watermarkTest() throws Exception {
    Controller controller = controllerWrapper.getController();
    String scope = "scope";
    String stream = "stream";
    controller.createScope(scope).join();
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    controller.createStream(scope, stream, config).join();
    String markStream = NameUtils.getMarkStreamForStream(stream);
    Stream streamObj = new StreamImpl(scope, stream);
    WriterPosition pos1 = WriterPosition.builder().segments(Collections.singletonMap(new Segment(scope, stream, 0L), 10L)).build();
    WriterPosition pos2 = WriterPosition.builder().segments(Collections.singletonMap(new Segment(scope, stream, 0L), 20L)).build();
    controller.noteTimestampFromWriter("1", streamObj, 1L, pos1).join();
    controller.noteTimestampFromWriter("2", streamObj, 2L, pos2).join();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    @Cleanup RevisionedStreamClient<Watermark> reader = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    AssertExtensions.assertEventuallyEquals(true, () -> {
        Iterator<Entry<Revision, Watermark>> watermarks = reader.readFrom(reader.fetchOldestRevision());
        return watermarks.hasNext();
    }, 30000);
    Iterator<Entry<Revision, Watermark>> watermarks = reader.readFrom(reader.fetchOldestRevision());
    Watermark watermark = watermarks.next().getValue();
    assertEquals(watermark.getLowerTimeBound(), 1L);
    assertTrue(watermark.getStreamCut().entrySet().stream().anyMatch(x -> x.getKey().getSegmentId() == 0L && x.getValue() == 20L));
    controller.sealStream(scope, stream).join();
    controller.deleteStream(scope, stream).join();
    AssertExtensions.assertFutureThrows("Mark Stream should not exist", controller.getCurrentSegments(scope, markStream), e -> Exceptions.unwrap(e) instanceof StoreException.DataNotFoundException);
}