Search in sources :

Example 16 with SocketConnectionFactoryImpl

use of io.pravega.client.connection.impl.SocketConnectionFactoryImpl in project pravega by pravega.

the class WatermarkingTest method watermarkTxnTest.

@Test(timeout = 120000)
public void watermarkTxnTest() throws Exception {
    Controller controller = PRAVEGA.getLocalController();
    String scope = "scopeTx";
    String stream = "watermarkTxnTest";
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    streamManager.createScope(scope);
    streamManager.createStream(scope, stream, config);
    Stream streamObj = Stream.of(scope, stream);
    // create 2 writers
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
    JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
    @Cleanup TransactionalEventStreamWriter<Long> writer1 = clientFactory.createTransactionalEventWriter("writer1", stream, new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutTime(10000).build());
    @Cleanup TransactionalEventStreamWriter<Long> writer2 = clientFactory.createTransactionalEventWriter("writer2", stream, new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutTime(10000).build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    CompletableFuture<Void> writer1Future = writeTxEvents(writer1, stopFlag);
    CompletableFuture<Void> writer2Future = writeTxEvents(writer2, stopFlag);
    // scale the stream several times so that we get complex positions
    scale(controller, streamObj, config);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    @Cleanup ClientFactoryImpl syncClientFactory = new ClientFactoryImpl(scope, new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor()), connectionFactory);
    String markStream = NameUtils.getMarkStreamForStream(stream);
    @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
    stopFlag.set(true);
    writer1Future.join();
    writer2Future.join();
    // read events from the stream
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, controller, syncClientFactory);
    Watermark watermark0 = watermarks.take();
    Watermark watermark1 = watermarks.take();
    assertTrue(watermark0.getLowerTimeBound() <= watermark0.getUpperTimeBound());
    assertTrue(watermark1.getLowerTimeBound() <= watermark1.getUpperTimeBound());
    assertTrue(watermark0.getLowerTimeBound() < watermark1.getLowerTimeBound());
    Map<Segment, Long> positionMap0 = watermark0.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    Map<Segment, Long> positionMap1 = watermark1.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutFirst = new StreamCutImpl(streamObj, positionMap0);
    StreamCut streamCutSecond = new StreamCutImpl(streamObj, positionMap1);
    Map<Stream, StreamCut> firstMarkStreamCut = Collections.singletonMap(streamObj, streamCutFirst);
    Map<Stream, StreamCut> secondMarkStreamCut = Collections.singletonMap(streamObj, streamCutSecond);
    // read from stream cut of first watermark
    String readerGroup = "watermarkTxnTest-group";
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(streamObj).startingStreamCuts(firstMarkStreamCut).endingStreamCuts(secondMarkStreamCut).disableAutomaticCheckpoints().build());
    @Cleanup final EventStreamReader<Long> reader = clientFactory.createReader("myreaderTx", readerGroup, javaSerializer, ReaderConfig.builder().build());
    EventRead<Long> event = reader.readNextEvent(10000L);
    TimeWindow currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    while (event.getEvent() != null && currentTimeWindow.getLowerTimeBound() == null && currentTimeWindow.getUpperTimeBound() == null) {
        event = reader.readNextEvent(10000L);
        currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    }
    assertNotNull(currentTimeWindow.getUpperTimeBound());
    // read all events and verify that all events are below the bounds
    while (event.getEvent() != null) {
        Long time = event.getEvent();
        log.info("timewindow = {} event = {}", currentTimeWindow, time);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || time >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || time <= currentTimeWindow.getUpperTimeBound());
        TimeWindow nextTimeWindow = reader.getCurrentTimeWindow(streamObj);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || nextTimeWindow.getLowerTimeBound() >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || nextTimeWindow.getUpperTimeBound() >= currentTimeWindow.getUpperTimeBound());
        currentTimeWindow = nextTimeWindow;
        event = reader.readNextEvent(10000L);
        if (event.isCheckpoint()) {
            event = reader.readNextEvent(10000L);
        }
    }
    assertNotNull(currentTimeWindow.getLowerTimeBound());
}
Also used : StreamCut(io.pravega.client.stream.StreamCut) EventStreamWriter(io.pravega.client.stream.EventStreamWriter) AssertExtensions(io.pravega.test.common.AssertExtensions) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) Cleanup(lombok.Cleanup) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) EventRead(io.pravega.client.stream.EventRead) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) Stream(io.pravega.client.stream.Stream) Duration(java.time.Duration) Map(java.util.Map) TimeWindow(io.pravega.client.stream.TimeWindow) ClassRule(org.junit.ClassRule) Transaction(io.pravega.client.stream.Transaction) ReaderGroupConfig(io.pravega.client.stream.ReaderGroupConfig) WatermarkSerializer(io.pravega.client.watermark.WatermarkSerializer) CompletionException(java.util.concurrent.CompletionException) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) ControllerImplConfig(io.pravega.client.control.impl.ControllerImplConfig) Slf4j(lombok.extern.slf4j.Slf4j) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) Controller(io.pravega.client.control.impl.Controller) Futures(io.pravega.common.concurrent.Futures) Segment(io.pravega.client.segment.impl.Segment) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) StreamManager(io.pravega.client.admin.StreamManager) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) StreamCutImpl(io.pravega.client.stream.impl.StreamCutImpl) EventStreamClientFactory(io.pravega.client.EventStreamClientFactory) RevisionedStreamClient(io.pravega.client.state.RevisionedStreamClient) SynchronizerConfig(io.pravega.client.state.SynchronizerConfig) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) TransactionalEventStreamWriter(io.pravega.client.stream.TransactionalEventStreamWriter) EventWriterConfig(io.pravega.client.stream.EventWriterConfig) NameUtils(io.pravega.shared.NameUtils) Iterator(java.util.Iterator) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) EventStreamReader(io.pravega.client.stream.EventStreamReader) Test(org.junit.Test) Watermark(io.pravega.shared.watermarks.Watermark) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) Assert.assertNull(org.junit.Assert.assertNull) SynchronizerClientFactory(io.pravega.client.SynchronizerClientFactory) ControllerImpl(io.pravega.client.control.impl.ControllerImpl) Revision(io.pravega.client.state.Revision) ReaderConfig(io.pravega.client.stream.ReaderConfig) TxnFailedException(io.pravega.client.stream.TxnFailedException) Collections(java.util.Collections) ScalingPolicy(io.pravega.client.stream.ScalingPolicy) Assert.assertEquals(org.junit.Assert.assertEquals) ClientConfig(io.pravega.client.ClientConfig) StreamCutImpl(io.pravega.client.stream.impl.StreamCutImpl) EventStreamClientFactory(io.pravega.client.EventStreamClientFactory) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Cleanup(lombok.Cleanup) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) Segment(io.pravega.client.segment.impl.Segment) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) Stream(io.pravega.client.stream.Stream) ClientConfig(io.pravega.client.ClientConfig) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) StreamCut(io.pravega.client.stream.StreamCut) ControllerImpl(io.pravega.client.control.impl.ControllerImpl) WatermarkSerializer(io.pravega.client.watermark.WatermarkSerializer) Controller(io.pravega.client.control.impl.Controller) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) TimeWindow(io.pravega.client.stream.TimeWindow) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) StreamManager(io.pravega.client.admin.StreamManager) AtomicLong(java.util.concurrent.atomic.AtomicLong) Watermark(io.pravega.shared.watermarks.Watermark) Map(java.util.Map) HashMap(java.util.HashMap) Test(org.junit.Test)

Example 17 with SocketConnectionFactoryImpl

use of io.pravega.client.connection.impl.SocketConnectionFactoryImpl in project pravega by pravega.

the class ReadTest method readThroughSegmentClient.

@Test(timeout = 10000)
public void readThroughSegmentClient() throws SegmentSealedException, EndOfSegmentException, SegmentTruncatedException {
    String endpoint = "localhost";
    String scope = "scope";
    String stream = "readThroughSegmentClient";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world\n";
    StreamSegmentStore store = SERVICE_BUILDER.createStreamSegmentService();
    TableStore tableStore = SERVICE_BUILDER.createTableStoreService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store, tableStore, SERVICE_BUILDER.getLowPriorityExecutor());
    server.startListening();
    @Cleanup SocketConnectionFactoryImpl clientCF = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ConnectionPoolImpl connectionPool = new ConnectionPoolImpl(ClientConfig.builder().build(), clientCF);
    @Cleanup Controller controller = new MockController(endpoint, port, connectionPool, true);
    controller.createScope(scope);
    controller.createStream(scope, stream, StreamConfiguration.builder().build());
    SegmentOutputStreamFactoryImpl segmentproducerClient = new SegmentOutputStreamFactoryImpl(controller, connectionPool);
    SegmentInputStreamFactoryImpl segmentConsumerClient = new SegmentInputStreamFactoryImpl(controller, connectionPool);
    Segment segment = Futures.getAndHandleExceptions(controller.getCurrentSegments(scope, stream), RuntimeException::new).getSegments().iterator().next();
    @Cleanup SegmentOutputStream out = segmentproducerClient.createOutputStreamForSegment(segment, segmentSealedCallback, EventWriterConfig.builder().build(), DelegationTokenProviderFactory.createWithEmptyToken());
    out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(testString.getBytes()), new CompletableFuture<>()));
    out.flush();
    @Cleanup EventSegmentReader in = segmentConsumerClient.createEventReaderForSegment(segment);
    ByteBuffer result = in.read();
    assertEquals(ByteBuffer.wrap(testString.getBytes()), result);
    // Test large write followed by read
    out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(new byte[15]), new CompletableFuture<>()));
    out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(new byte[15]), new CompletableFuture<>()));
    out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(new byte[150000]), new CompletableFuture<>()));
    assertEquals(in.read().capacity(), 15);
    assertEquals(in.read().capacity(), 15);
    assertEquals(in.read().capacity(), 150000);
}
Also used : ConnectionPoolImpl(io.pravega.client.connection.impl.ConnectionPoolImpl) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) MockController(io.pravega.client.stream.mock.MockController) Controller(io.pravega.client.control.impl.Controller) Cleanup(lombok.Cleanup) PravegaConnectionListener(io.pravega.segmentstore.server.host.handler.PravegaConnectionListener) ByteBuffer(java.nio.ByteBuffer) Segment(io.pravega.client.segment.impl.Segment) ReadSegment(io.pravega.shared.protocol.netty.WireCommands.ReadSegment) TableStore(io.pravega.segmentstore.contracts.tables.TableStore) StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) CompletableFuture(java.util.concurrent.CompletableFuture) SegmentOutputStreamFactoryImpl(io.pravega.client.segment.impl.SegmentOutputStreamFactoryImpl) SegmentOutputStream(io.pravega.client.segment.impl.SegmentOutputStream) EventSegmentReader(io.pravega.client.segment.impl.EventSegmentReader) MockController(io.pravega.client.stream.mock.MockController) SegmentInputStreamFactoryImpl(io.pravega.client.segment.impl.SegmentInputStreamFactoryImpl) Test(org.junit.Test)

Example 18 with SocketConnectionFactoryImpl

use of io.pravega.client.connection.impl.SocketConnectionFactoryImpl in project pravega by pravega.

the class ReadWriteTest method readWriteTest.

@Test(timeout = 60000)
public void readWriteTest() throws InterruptedException, ExecutionException {
    String scope = "testMultiReaderWriterScope";
    String readerGroupName = "testMultiReaderWriterReaderGroup";
    // 20  readers -> 20 stream segments ( to have max read parallelism)
    ScalingPolicy scalingPolicy = ScalingPolicy.fixed(20);
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(scalingPolicy).build();
    eventsReadFromPravega = new ConcurrentLinkedQueue<>();
    // data used by each of the writers.
    eventData = new AtomicLong();
    // used by readers to maintain a count of events.
    eventReadCount = new AtomicLong();
    stopReadFlag = new AtomicBoolean(false);
    ClientConfig clientConfig = ClientConfig.builder().build();
    try (ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
        StreamManager streamManager = new StreamManagerImpl(controller, cp)) {
        // create a scope
        Boolean createScopeStatus = streamManager.createScope(scope);
        log.info("Create scope status {}", createScopeStatus);
        // create a stream
        Boolean createStreamStatus = streamManager.createStream(scope, STREAM_NAME, config);
        log.info("Create stream status {}", createStreamStatus);
    }
    try (ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
        ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
        ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, controller, clientFactory)) {
        // start writing events to the stream
        log.info("Creating {} writers", NUM_WRITERS);
        List<CompletableFuture<Void>> writerList = new ArrayList<>();
        for (int i = 0; i < NUM_WRITERS; i++) {
            log.info("Starting writer{}", i);
            writerList.add(startNewWriter(eventData, clientFactory));
        }
        // create a reader group
        log.info("Creating Reader group : {}", readerGroupName);
        readerGroupManager.createReaderGroup(readerGroupName, ReaderGroupConfig.builder().stream(Stream.of(scope, STREAM_NAME)).build());
        log.info("Reader group name {} ", readerGroupManager.getReaderGroup(readerGroupName).getGroupName());
        log.info("Reader group scope {}", readerGroupManager.getReaderGroup(readerGroupName).getScope());
        // create readers
        log.info("Creating {} readers", NUM_READERS);
        List<CompletableFuture<Void>> readerList = new ArrayList<>();
        String readerName = "reader" + RandomFactory.create().nextInt(Integer.MAX_VALUE);
        // start reading events
        for (int i = 0; i < NUM_READERS; i++) {
            log.info("Starting reader{}", i);
            readerList.add(startNewReader(readerName + i, clientFactory, readerGroupName, eventsReadFromPravega, eventData, eventReadCount, stopReadFlag));
        }
        // wait for writers completion
        Futures.allOf(writerList).get();
        ExecutorServiceHelpers.shutdown(writerPool);
        // set stop read flag to true
        stopReadFlag.set(true);
        // wait for readers completion
        Futures.allOf(readerList).get();
        ExecutorServiceHelpers.shutdown(readerPool);
        // delete readergroup
        log.info("Deleting readergroup {}", readerGroupName);
        readerGroupManager.deleteReaderGroup(readerGroupName);
    }
    log.info("All writers have stopped. Setting Stop_Read_Flag. Event Written Count:{}, Event Read " + "Count: {}", eventData.get(), eventsReadFromPravega.size());
    assertEquals(TOTAL_NUM_EVENTS, eventsReadFromPravega.size());
    // check unique events.
    assertEquals(TOTAL_NUM_EVENTS, new TreeSet<>(eventsReadFromPravega).size());
    // seal the stream
    CompletableFuture<Boolean> sealStreamStatus = controller.sealStream(scope, STREAM_NAME);
    log.info("Sealing stream {}", STREAM_NAME);
    assertTrue(sealStreamStatus.get());
    // delete the stream
    CompletableFuture<Boolean> deleteStreamStatus = controller.deleteStream(scope, STREAM_NAME);
    log.info("Deleting stream {}", STREAM_NAME);
    assertTrue(deleteStreamStatus.get());
    // delete the  scope
    CompletableFuture<Boolean> deleteScopeStatus = controller.deleteScope(scope);
    log.info("Deleting scope {}", scope);
    assertTrue(deleteScopeStatus.get());
    log.info("Read write test succeeds");
}
Also used : ConnectionPool(io.pravega.client.connection.impl.ConnectionPool) ScalingPolicy(io.pravega.client.stream.ScalingPolicy) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) ConnectionPoolImpl(io.pravega.client.connection.impl.ConnectionPoolImpl) ArrayList(java.util.ArrayList) StreamManagerImpl(io.pravega.client.admin.impl.StreamManagerImpl) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AtomicLong(java.util.concurrent.atomic.AtomicLong) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) CompletableFuture(java.util.concurrent.CompletableFuture) StreamManager(io.pravega.client.admin.StreamManager) TreeSet(java.util.TreeSet) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) ClientConfig(io.pravega.client.ClientConfig) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) Test(org.junit.Test)

Example 19 with SocketConnectionFactoryImpl

use of io.pravega.client.connection.impl.SocketConnectionFactoryImpl in project pravega by pravega.

the class ReaderGroupNotificationTest method testEndOfStreamNotifications.

@Test(timeout = 40000)
public void testEndOfStreamNotifications() throws Exception {
    final String streamName = "stream2";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE, 0L).get();
    controller.createStream(SCOPE, streamName, config).get();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "data1").get();
    // scale
    Stream stream = new StreamImpl(SCOPE, streamName);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0L), map, executorService()).getFuture().get();
    assertTrue(result);
    writer.writeEvent("0", "data2").get();
    // seal stream
    assertTrue(controller.sealStream(SCOPE, streamName).get());
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory);
    groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, streamName)).groupRefreshTimeMillis(0).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup("reader");
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().initialAllocationDelay(0).build());
    // Add segment event listener
    Listener<EndOfDataNotification> l1 = notification -> {
        listenerInvoked.set(true);
        listenerLatch.release();
    };
    EndOfDataNotifier endOfDataNotifier = (EndOfDataNotifier) readerGroup.getEndOfDataNotifier(executorService());
    endOfDataNotifier.registerListener(l1);
    EventRead<String> event1 = reader1.readNextEvent(10000);
    assertEquals("data1", event1.getEvent());
    EventRead<String> emptyEvent = reader1.readNextEvent(0);
    assertNull(emptyEvent.getEvent());
    assertFalse(emptyEvent.isCheckpoint());
    readerGroup.initiateCheckpoint("cp", executorService());
    EventRead<String> cpEvent = reader1.readNextEvent(10000);
    assertTrue(cpEvent.isCheckpoint());
    EventRead<String> event2 = reader1.readNextEvent(10000);
    assertEquals("data2", event2.getEvent());
    emptyEvent = reader1.readNextEvent(0);
    assertNull(emptyEvent.getEvent());
    assertFalse(emptyEvent.isCheckpoint());
    emptyEvent = reader1.readNextEvent(0);
    assertNull(emptyEvent.getEvent());
    assertFalse(emptyEvent.isCheckpoint());
    readerGroup.initiateCheckpoint("cp2", executorService());
    cpEvent = reader1.readNextEvent(10000);
    assertTrue(cpEvent.isCheckpoint());
    emptyEvent = reader1.readNextEvent(0);
    assertNull(emptyEvent.getEvent());
    assertFalse(emptyEvent.isCheckpoint());
    endOfDataNotifier.pollNow();
    listenerLatch.await();
    assertTrue("Listener invoked", listenerInvoked.get());
}
Also used : EventStreamWriter(io.pravega.client.stream.EventStreamWriter) TableStore(io.pravega.segmentstore.contracts.tables.TableStore) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) Cleanup(lombok.Cleanup) ReaderGroup(io.pravega.client.stream.ReaderGroup) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) ServiceBuilderConfig(io.pravega.segmentstore.server.store.ServiceBuilderConfig) EventRead(io.pravega.client.stream.EventRead) EndOfDataNotification(io.pravega.client.stream.notifications.EndOfDataNotification) ServiceBuilder(io.pravega.segmentstore.server.store.ServiceBuilder) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) Stream(io.pravega.client.stream.Stream) Map(java.util.Map) After(org.junit.After) URI(java.net.URI) PravegaConnectionListener(io.pravega.segmentstore.server.host.handler.PravegaConnectionListener) ReaderGroupConfig(io.pravega.client.stream.ReaderGroupConfig) StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) EndOfDataNotifier(io.pravega.client.stream.notifications.notifier.EndOfDataNotifier) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) Slf4j(lombok.extern.slf4j.Slf4j) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) Assert.assertFalse(org.junit.Assert.assertFalse) TestUtils(io.pravega.test.common.TestUtils) ControllerWrapper(io.pravega.test.integration.demo.ControllerWrapper) Controller(io.pravega.client.control.impl.Controller) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) StreamImpl(io.pravega.client.stream.impl.StreamImpl) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) SegmentNotification(io.pravega.client.stream.notifications.SegmentNotification) TestingServerStarter(io.pravega.test.common.TestingServerStarter) TestingServer(org.apache.curator.test.TestingServer) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) ReusableLatch(io.pravega.common.util.ReusableLatch) Before(org.junit.Before) EventWriterConfig(io.pravega.client.stream.EventWriterConfig) Listener(io.pravega.client.stream.notifications.Listener) Assert.assertNotNull(org.junit.Assert.assertNotNull) lombok.val(lombok.val) SegmentNotifier(io.pravega.client.stream.notifications.notifier.SegmentNotifier) Assert.assertTrue(org.junit.Assert.assertTrue) EventStreamReader(io.pravega.client.stream.EventStreamReader) Test(org.junit.Test) Assert.assertNull(org.junit.Assert.assertNull) ReaderConfig(io.pravega.client.stream.ReaderConfig) Collections(java.util.Collections) ScalingPolicy(io.pravega.client.stream.ScalingPolicy) Assert.assertEquals(org.junit.Assert.assertEquals) ClientConfig(io.pravega.client.ClientConfig) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) HashMap(java.util.HashMap) ReaderGroup(io.pravega.client.stream.ReaderGroup) Controller(io.pravega.client.control.impl.Controller) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) Cleanup(lombok.Cleanup) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) EndOfDataNotification(io.pravega.client.stream.notifications.EndOfDataNotification) EndOfDataNotifier(io.pravega.client.stream.notifications.notifier.EndOfDataNotifier) StreamImpl(io.pravega.client.stream.impl.StreamImpl) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) Stream(io.pravega.client.stream.Stream) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) Test(org.junit.Test)

Example 20 with SocketConnectionFactoryImpl

use of io.pravega.client.connection.impl.SocketConnectionFactoryImpl in project pravega by pravega.

the class ReaderGroupNotificationTest method testSegmentNotifications.

@Test(timeout = 40000)
public void testSegmentNotifications() throws Exception {
    final String streamName = "stream1";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE, 0L).get();
    controller.createStream(SCOPE, streamName, config).get();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "data1").get();
    // scale
    Stream stream = new StreamImpl(SCOPE, streamName);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0L), map, executorService()).getFuture().get();
    assertTrue(result);
    writer.writeEvent("0", "data2").get();
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory);
    groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, streamName)).groupRefreshTimeMillis(0).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup("reader");
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().initialAllocationDelay(0).build());
    val notificationResults = new ArrayBlockingQueue<SegmentNotification>(2);
    // Add segment event listener
    Listener<SegmentNotification> l1 = notification -> {
        log.info("Number of Segments: {}, Number of Readers: {}", notification.getNumOfSegments(), notification.getNumOfReaders());
        notificationResults.add(notification);
    };
    SegmentNotifier segmentNotifier = (SegmentNotifier) readerGroup.getSegmentNotifier(executorService());
    segmentNotifier.registerListener(l1);
    // Read first event and validate notification.
    EventRead<String> event1 = reader1.readNextEvent(5000);
    assertEquals("data1", event1.getEvent());
    segmentNotifier.pollNow();
    SegmentNotification initialSegmentNotification = notificationResults.take();
    assertNotNull(initialSegmentNotification);
    assertEquals(1, initialSegmentNotification.getNumOfReaders());
    assertEquals(1, initialSegmentNotification.getNumOfSegments());
    EventRead<String> emptyEvent = reader1.readNextEvent(0);
    assertNull(emptyEvent.getEvent());
    assertFalse(emptyEvent.isCheckpoint());
    readerGroup.initiateCheckpoint("cp", executorService());
    EventRead<String> cpEvent = reader1.readNextEvent(1000);
    assertTrue(cpEvent.isCheckpoint());
    // Read second event and validate notification.
    EventRead<String> event2 = reader1.readNextEvent(10000);
    assertEquals("data2", event2.getEvent());
    segmentNotifier.pollNow();
    SegmentNotification segmentNotificationPostScale = notificationResults.take();
    assertEquals(1, segmentNotificationPostScale.getNumOfReaders());
    assertEquals(2, segmentNotificationPostScale.getNumOfSegments());
}
Also used : EventStreamWriter(io.pravega.client.stream.EventStreamWriter) TableStore(io.pravega.segmentstore.contracts.tables.TableStore) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) Cleanup(lombok.Cleanup) ReaderGroup(io.pravega.client.stream.ReaderGroup) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) ServiceBuilderConfig(io.pravega.segmentstore.server.store.ServiceBuilderConfig) EventRead(io.pravega.client.stream.EventRead) EndOfDataNotification(io.pravega.client.stream.notifications.EndOfDataNotification) ServiceBuilder(io.pravega.segmentstore.server.store.ServiceBuilder) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) Stream(io.pravega.client.stream.Stream) Map(java.util.Map) After(org.junit.After) URI(java.net.URI) PravegaConnectionListener(io.pravega.segmentstore.server.host.handler.PravegaConnectionListener) ReaderGroupConfig(io.pravega.client.stream.ReaderGroupConfig) StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) EndOfDataNotifier(io.pravega.client.stream.notifications.notifier.EndOfDataNotifier) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) Slf4j(lombok.extern.slf4j.Slf4j) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) Assert.assertFalse(org.junit.Assert.assertFalse) TestUtils(io.pravega.test.common.TestUtils) ControllerWrapper(io.pravega.test.integration.demo.ControllerWrapper) Controller(io.pravega.client.control.impl.Controller) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) StreamImpl(io.pravega.client.stream.impl.StreamImpl) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) SegmentNotification(io.pravega.client.stream.notifications.SegmentNotification) TestingServerStarter(io.pravega.test.common.TestingServerStarter) TestingServer(org.apache.curator.test.TestingServer) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) ReusableLatch(io.pravega.common.util.ReusableLatch) Before(org.junit.Before) EventWriterConfig(io.pravega.client.stream.EventWriterConfig) Listener(io.pravega.client.stream.notifications.Listener) Assert.assertNotNull(org.junit.Assert.assertNotNull) lombok.val(lombok.val) SegmentNotifier(io.pravega.client.stream.notifications.notifier.SegmentNotifier) Assert.assertTrue(org.junit.Assert.assertTrue) EventStreamReader(io.pravega.client.stream.EventStreamReader) Test(org.junit.Test) Assert.assertNull(org.junit.Assert.assertNull) ReaderConfig(io.pravega.client.stream.ReaderConfig) Collections(java.util.Collections) ScalingPolicy(io.pravega.client.stream.ScalingPolicy) Assert.assertEquals(org.junit.Assert.assertEquals) ClientConfig(io.pravega.client.ClientConfig) HashMap(java.util.HashMap) ReaderGroup(io.pravega.client.stream.ReaderGroup) Cleanup(lombok.Cleanup) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) Stream(io.pravega.client.stream.Stream) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) SegmentNotification(io.pravega.client.stream.notifications.SegmentNotification) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) lombok.val(lombok.val) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) Controller(io.pravega.client.control.impl.Controller) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) StreamImpl(io.pravega.client.stream.impl.StreamImpl) SegmentNotifier(io.pravega.client.stream.notifications.notifier.SegmentNotifier) Test(org.junit.Test)

Aggregations

SocketConnectionFactoryImpl (io.pravega.client.connection.impl.SocketConnectionFactoryImpl)90 Cleanup (lombok.Cleanup)63 Test (org.junit.Test)60 ClientFactoryImpl (io.pravega.client.stream.impl.ClientFactoryImpl)54 ConnectionFactory (io.pravega.client.connection.impl.ConnectionFactory)51 StreamConfiguration (io.pravega.client.stream.StreamConfiguration)49 ClientConfig (io.pravega.client.ClientConfig)48 ReaderGroupManager (io.pravega.client.admin.ReaderGroupManager)43 ReaderGroupManagerImpl (io.pravega.client.admin.impl.ReaderGroupManagerImpl)38 ConnectionPoolImpl (io.pravega.client.connection.impl.ConnectionPoolImpl)34 Controller (io.pravega.client.control.impl.Controller)31 Stream (io.pravega.client.stream.Stream)27 HashMap (java.util.HashMap)23 Before (org.junit.Before)23 ScalingPolicy (io.pravega.client.stream.ScalingPolicy)21 StreamImpl (io.pravega.client.stream.impl.StreamImpl)21 LocalController (io.pravega.controller.server.eventProcessor.LocalController)20 StreamManager (io.pravega.client.admin.StreamManager)19 ConnectionPool (io.pravega.client.connection.impl.ConnectionPool)19 PravegaConnectionListener (io.pravega.segmentstore.server.host.handler.PravegaConnectionListener)19