Search in sources :

Example 11 with ClientFactoryImpl

use of io.pravega.client.stream.impl.ClientFactoryImpl in project pravega by pravega.

the class LargeEventTest method testReadWriteScaleStreamSeal.

/**
 * Tests that if a Stream is sealed no data will continue to be produced by the {@link io.pravega.client.stream.impl.LargeEventWriter}
 */
@Test(timeout = 60000)
public void testReadWriteScaleStreamSeal() throws ExecutionException, InterruptedException, TimeoutException {
    String readerGroupName = "testLargeEventScaleStreamSealReaderGroup";
    String streamName = "ScaleStreamSeal";
    StreamConfiguration config = StreamConfiguration.builder().retentionPolicy(RetentionPolicy.bySizeBytes(Long.MAX_VALUE)).build();
    createScopeStream(SCOPE_NAME, streamName, config);
    AtomicInteger generation = new AtomicInteger(0);
    // Creates some data to write.
    int events = 1;
    Map<Integer, List<ByteBuffer>> data = generateEventData(NUM_WRITERS, events * generation.getAndIncrement(), events, LARGE_EVENT_SIZE);
    merge(eventsWrittenToPravega, data);
    // Perform a basic read-write cycle.
    Queue<ByteBuffer> reads = readWriteCycle(streamName, readerGroupName, eventsWrittenToPravega);
    validateEventReads(reads, eventsWrittenToPravega);
    // Define the scale operation.
    Runnable scale = () -> {
        Stream stream = new StreamImpl(SCOPE_NAME, streamName);
        try {
            StreamSegments segments = controller.getCurrentSegments(SCOPE_NAME, streamName).get();
            List<Long> ids = segments.getSegments().stream().map(Segment::getSegmentId).collect(Collectors.toList());
            controller.startScale(stream, ids, Map.of(0.0, 0.5, 0.5, 1.0)).get();
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            throw new RuntimeException(e);
        }
    };
    // Reset reads.
    eventReadCount.set(0);
    // Try to scale the segment after two `send` calls.
    AtomicInteger sendCount = new AtomicInteger(0);
    Supplier<Boolean> predicate = () -> sendCount.getAndIncrement() == CLOSE_WRITE_COUNT;
    AtomicReference<Boolean> latch = new AtomicReference<>(true);
    try (ConnectionExporter connectionFactory = new ConnectionExporter(ClientConfig.builder().build(), latch, scale, predicate);
        ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE_NAME, controller, connectionFactory);
        ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(SCOPE_NAME, controller, clientFactory)) {
        // Next set of writes.
        data = generateEventData(NUM_WRITERS, events * generation.getAndIncrement(), events, LARGE_EVENT_SIZE);
        merge(eventsWrittenToPravega, data);
        // Start writing events to the stream.
        val writers = createEventWriters(streamName, NUM_WRITERS, clientFactory, data);
        Futures.allOf(writers).get();
        // Wait for the scale event.
        TestUtils.await(() -> !latch.get(), 200, 2000);
        // Create a ReaderGroup.
        createReaderGroup(readerGroupName, readerGroupManager, streamName);
        // Create Readers.
        val readers = createEventReaders(NUM_READERS, clientFactory, readerGroupName, eventsReadFromPravega);
        stopReadFlag.set(true);
        Futures.allOf(readers).get();
        log.info("Deleting ReaderGroup: {}", readerGroupName);
        readerGroupManager.deleteReaderGroup(readerGroupName);
    }
    StreamSegments segments = controller.getCurrentSegments(SCOPE_NAME, streamName).get();
    // Make sure that the scale event has happened.
    Assert.assertEquals("Expected 2 StreamSegments.", 2, segments.getSegments().size());
    // This time there are successor segments, so the data should have been accepted.
    validateEventReads(eventsReadFromPravega, eventsWrittenToPravega);
}
Also used : Segment(io.pravega.client.segment.impl.Segment) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) List(java.util.List) ArrayList(java.util.ArrayList) Stream(io.pravega.client.stream.Stream) ExecutionException(java.util.concurrent.ExecutionException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) lombok.val(lombok.val) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) AtomicReference(java.util.concurrent.atomic.AtomicReference) ByteBuffer(java.nio.ByteBuffer) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) StreamImpl(io.pravega.client.stream.impl.StreamImpl) StreamSegments(io.pravega.client.stream.impl.StreamSegments) Test(org.junit.Test)

Example 12 with ClientFactoryImpl

use of io.pravega.client.stream.impl.ClientFactoryImpl in project pravega by pravega.

the class WatermarkingTest method watermarkTest.

@Test(timeout = 120000)
public void watermarkTest() throws Exception {
    Controller controller = PRAVEGA.getLocalController();
    String scope = "scope";
    String stream = "watermarkTest";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    streamManager.createScope(scope);
    streamManager.createStream(scope, stream, config);
    Stream streamObj = Stream.of(scope, stream);
    // create 2 writers
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
    JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<Long> writer1 = clientFactory.createEventWriter(stream, javaSerializer, EventWriterConfig.builder().build());
    @Cleanup EventStreamWriter<Long> writer2 = clientFactory.createEventWriter(stream, javaSerializer, EventWriterConfig.builder().build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    CompletableFuture<Void> writer1Future = writeEvents(writer1, stopFlag);
    CompletableFuture<Void> writer2Future = writeEvents(writer2, stopFlag);
    // scale the stream several times so that we get complex positions
    scale(controller, streamObj, config);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    @Cleanup ClientFactoryImpl syncClientFactory = new ClientFactoryImpl(scope, new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor()), connectionFactory);
    String markStream = NameUtils.getMarkStreamForStream(stream);
    @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
    stopFlag.set(true);
    writer1Future.join();
    writer2Future.join();
    // read events from the stream
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, controller, syncClientFactory);
    Watermark watermark0 = watermarks.take();
    Watermark watermark1 = watermarks.take();
    assertTrue(watermark0.getLowerTimeBound() <= watermark0.getUpperTimeBound());
    assertTrue(watermark1.getLowerTimeBound() <= watermark1.getUpperTimeBound());
    assertTrue(watermark0.getLowerTimeBound() < watermark1.getLowerTimeBound());
    Map<Segment, Long> positionMap0 = watermark0.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    Map<Segment, Long> positionMap1 = watermark1.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutFirst = new StreamCutImpl(streamObj, positionMap0);
    StreamCut streamCutSecond = new StreamCutImpl(streamObj, positionMap1);
    Map<Stream, StreamCut> firstMarkStreamCut = Collections.singletonMap(streamObj, streamCutFirst);
    Map<Stream, StreamCut> secondMarkStreamCut = Collections.singletonMap(streamObj, streamCutSecond);
    // read from stream cut of first watermark
    String readerGroup = "watermarkTest-group";
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(streamObj).startingStreamCuts(firstMarkStreamCut).endingStreamCuts(secondMarkStreamCut).disableAutomaticCheckpoints().build());
    @Cleanup final EventStreamReader<Long> reader = clientFactory.createReader("myreader", readerGroup, javaSerializer, ReaderConfig.builder().build());
    EventRead<Long> event = reader.readNextEvent(10000L);
    TimeWindow currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    while (event.getEvent() != null && currentTimeWindow.getLowerTimeBound() == null && currentTimeWindow.getUpperTimeBound() == null) {
        event = reader.readNextEvent(10000L);
        currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    }
    assertNotNull(currentTimeWindow.getUpperTimeBound());
    // read all events and verify that all events are below the bounds
    while (event.getEvent() != null) {
        Long time = event.getEvent();
        log.info("timewindow = {} event = {}", currentTimeWindow, time);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || time >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || time <= currentTimeWindow.getUpperTimeBound());
        TimeWindow nextTimeWindow = reader.getCurrentTimeWindow(streamObj);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || nextTimeWindow.getLowerTimeBound() >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || nextTimeWindow.getUpperTimeBound() >= currentTimeWindow.getUpperTimeBound());
        currentTimeWindow = nextTimeWindow;
        event = reader.readNextEvent(10000L);
        if (event.isCheckpoint()) {
            event = reader.readNextEvent(10000L);
        }
    }
    assertNotNull(currentTimeWindow.getLowerTimeBound());
}
Also used : StreamCut(io.pravega.client.stream.StreamCut) EventStreamWriter(io.pravega.client.stream.EventStreamWriter) AssertExtensions(io.pravega.test.common.AssertExtensions) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) Cleanup(lombok.Cleanup) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) EventRead(io.pravega.client.stream.EventRead) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) Stream(io.pravega.client.stream.Stream) Duration(java.time.Duration) Map(java.util.Map) TimeWindow(io.pravega.client.stream.TimeWindow) ClassRule(org.junit.ClassRule) Transaction(io.pravega.client.stream.Transaction) ReaderGroupConfig(io.pravega.client.stream.ReaderGroupConfig) WatermarkSerializer(io.pravega.client.watermark.WatermarkSerializer) CompletionException(java.util.concurrent.CompletionException) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) ControllerImplConfig(io.pravega.client.control.impl.ControllerImplConfig) Slf4j(lombok.extern.slf4j.Slf4j) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) Controller(io.pravega.client.control.impl.Controller) Futures(io.pravega.common.concurrent.Futures) Segment(io.pravega.client.segment.impl.Segment) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) StreamManager(io.pravega.client.admin.StreamManager) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) StreamCutImpl(io.pravega.client.stream.impl.StreamCutImpl) EventStreamClientFactory(io.pravega.client.EventStreamClientFactory) RevisionedStreamClient(io.pravega.client.state.RevisionedStreamClient) SynchronizerConfig(io.pravega.client.state.SynchronizerConfig) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) TransactionalEventStreamWriter(io.pravega.client.stream.TransactionalEventStreamWriter) EventWriterConfig(io.pravega.client.stream.EventWriterConfig) NameUtils(io.pravega.shared.NameUtils) Iterator(java.util.Iterator) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) EventStreamReader(io.pravega.client.stream.EventStreamReader) Test(org.junit.Test) Watermark(io.pravega.shared.watermarks.Watermark) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) Assert.assertNull(org.junit.Assert.assertNull) SynchronizerClientFactory(io.pravega.client.SynchronizerClientFactory) ControllerImpl(io.pravega.client.control.impl.ControllerImpl) Revision(io.pravega.client.state.Revision) ReaderConfig(io.pravega.client.stream.ReaderConfig) TxnFailedException(io.pravega.client.stream.TxnFailedException) Collections(java.util.Collections) ScalingPolicy(io.pravega.client.stream.ScalingPolicy) Assert.assertEquals(org.junit.Assert.assertEquals) ClientConfig(io.pravega.client.ClientConfig) StreamCutImpl(io.pravega.client.stream.impl.StreamCutImpl) EventStreamClientFactory(io.pravega.client.EventStreamClientFactory) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Cleanup(lombok.Cleanup) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) Segment(io.pravega.client.segment.impl.Segment) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) Stream(io.pravega.client.stream.Stream) ClientConfig(io.pravega.client.ClientConfig) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) StreamCut(io.pravega.client.stream.StreamCut) ControllerImpl(io.pravega.client.control.impl.ControllerImpl) WatermarkSerializer(io.pravega.client.watermark.WatermarkSerializer) Controller(io.pravega.client.control.impl.Controller) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) TimeWindow(io.pravega.client.stream.TimeWindow) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) StreamManager(io.pravega.client.admin.StreamManager) AtomicLong(java.util.concurrent.atomic.AtomicLong) Watermark(io.pravega.shared.watermarks.Watermark) Map(java.util.Map) HashMap(java.util.HashMap) Test(org.junit.Test)

Example 13 with ClientFactoryImpl

use of io.pravega.client.stream.impl.ClientFactoryImpl in project pravega by pravega.

the class WatermarkingTest method watermarkTxnTest.

@Test(timeout = 120000)
public void watermarkTxnTest() throws Exception {
    Controller controller = PRAVEGA.getLocalController();
    String scope = "scopeTx";
    String stream = "watermarkTxnTest";
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    streamManager.createScope(scope);
    streamManager.createStream(scope, stream, config);
    Stream streamObj = Stream.of(scope, stream);
    // create 2 writers
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
    JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
    @Cleanup TransactionalEventStreamWriter<Long> writer1 = clientFactory.createTransactionalEventWriter("writer1", stream, new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutTime(10000).build());
    @Cleanup TransactionalEventStreamWriter<Long> writer2 = clientFactory.createTransactionalEventWriter("writer2", stream, new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutTime(10000).build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    CompletableFuture<Void> writer1Future = writeTxEvents(writer1, stopFlag);
    CompletableFuture<Void> writer2Future = writeTxEvents(writer2, stopFlag);
    // scale the stream several times so that we get complex positions
    scale(controller, streamObj, config);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    @Cleanup ClientFactoryImpl syncClientFactory = new ClientFactoryImpl(scope, new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor()), connectionFactory);
    String markStream = NameUtils.getMarkStreamForStream(stream);
    @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
    stopFlag.set(true);
    writer1Future.join();
    writer2Future.join();
    // read events from the stream
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, controller, syncClientFactory);
    Watermark watermark0 = watermarks.take();
    Watermark watermark1 = watermarks.take();
    assertTrue(watermark0.getLowerTimeBound() <= watermark0.getUpperTimeBound());
    assertTrue(watermark1.getLowerTimeBound() <= watermark1.getUpperTimeBound());
    assertTrue(watermark0.getLowerTimeBound() < watermark1.getLowerTimeBound());
    Map<Segment, Long> positionMap0 = watermark0.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    Map<Segment, Long> positionMap1 = watermark1.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutFirst = new StreamCutImpl(streamObj, positionMap0);
    StreamCut streamCutSecond = new StreamCutImpl(streamObj, positionMap1);
    Map<Stream, StreamCut> firstMarkStreamCut = Collections.singletonMap(streamObj, streamCutFirst);
    Map<Stream, StreamCut> secondMarkStreamCut = Collections.singletonMap(streamObj, streamCutSecond);
    // read from stream cut of first watermark
    String readerGroup = "watermarkTxnTest-group";
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(streamObj).startingStreamCuts(firstMarkStreamCut).endingStreamCuts(secondMarkStreamCut).disableAutomaticCheckpoints().build());
    @Cleanup final EventStreamReader<Long> reader = clientFactory.createReader("myreaderTx", readerGroup, javaSerializer, ReaderConfig.builder().build());
    EventRead<Long> event = reader.readNextEvent(10000L);
    TimeWindow currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    while (event.getEvent() != null && currentTimeWindow.getLowerTimeBound() == null && currentTimeWindow.getUpperTimeBound() == null) {
        event = reader.readNextEvent(10000L);
        currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    }
    assertNotNull(currentTimeWindow.getUpperTimeBound());
    // read all events and verify that all events are below the bounds
    while (event.getEvent() != null) {
        Long time = event.getEvent();
        log.info("timewindow = {} event = {}", currentTimeWindow, time);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || time >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || time <= currentTimeWindow.getUpperTimeBound());
        TimeWindow nextTimeWindow = reader.getCurrentTimeWindow(streamObj);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || nextTimeWindow.getLowerTimeBound() >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || nextTimeWindow.getUpperTimeBound() >= currentTimeWindow.getUpperTimeBound());
        currentTimeWindow = nextTimeWindow;
        event = reader.readNextEvent(10000L);
        if (event.isCheckpoint()) {
            event = reader.readNextEvent(10000L);
        }
    }
    assertNotNull(currentTimeWindow.getLowerTimeBound());
}
Also used : StreamCut(io.pravega.client.stream.StreamCut) EventStreamWriter(io.pravega.client.stream.EventStreamWriter) AssertExtensions(io.pravega.test.common.AssertExtensions) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) Cleanup(lombok.Cleanup) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) EventRead(io.pravega.client.stream.EventRead) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) Stream(io.pravega.client.stream.Stream) Duration(java.time.Duration) Map(java.util.Map) TimeWindow(io.pravega.client.stream.TimeWindow) ClassRule(org.junit.ClassRule) Transaction(io.pravega.client.stream.Transaction) ReaderGroupConfig(io.pravega.client.stream.ReaderGroupConfig) WatermarkSerializer(io.pravega.client.watermark.WatermarkSerializer) CompletionException(java.util.concurrent.CompletionException) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) ControllerImplConfig(io.pravega.client.control.impl.ControllerImplConfig) Slf4j(lombok.extern.slf4j.Slf4j) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) Controller(io.pravega.client.control.impl.Controller) Futures(io.pravega.common.concurrent.Futures) Segment(io.pravega.client.segment.impl.Segment) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) StreamManager(io.pravega.client.admin.StreamManager) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) StreamCutImpl(io.pravega.client.stream.impl.StreamCutImpl) EventStreamClientFactory(io.pravega.client.EventStreamClientFactory) RevisionedStreamClient(io.pravega.client.state.RevisionedStreamClient) SynchronizerConfig(io.pravega.client.state.SynchronizerConfig) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) TransactionalEventStreamWriter(io.pravega.client.stream.TransactionalEventStreamWriter) EventWriterConfig(io.pravega.client.stream.EventWriterConfig) NameUtils(io.pravega.shared.NameUtils) Iterator(java.util.Iterator) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) EventStreamReader(io.pravega.client.stream.EventStreamReader) Test(org.junit.Test) Watermark(io.pravega.shared.watermarks.Watermark) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) Assert.assertNull(org.junit.Assert.assertNull) SynchronizerClientFactory(io.pravega.client.SynchronizerClientFactory) ControllerImpl(io.pravega.client.control.impl.ControllerImpl) Revision(io.pravega.client.state.Revision) ReaderConfig(io.pravega.client.stream.ReaderConfig) TxnFailedException(io.pravega.client.stream.TxnFailedException) Collections(java.util.Collections) ScalingPolicy(io.pravega.client.stream.ScalingPolicy) Assert.assertEquals(org.junit.Assert.assertEquals) ClientConfig(io.pravega.client.ClientConfig) StreamCutImpl(io.pravega.client.stream.impl.StreamCutImpl) EventStreamClientFactory(io.pravega.client.EventStreamClientFactory) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Cleanup(lombok.Cleanup) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) Segment(io.pravega.client.segment.impl.Segment) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) Stream(io.pravega.client.stream.Stream) ClientConfig(io.pravega.client.ClientConfig) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) StreamCut(io.pravega.client.stream.StreamCut) ControllerImpl(io.pravega.client.control.impl.ControllerImpl) WatermarkSerializer(io.pravega.client.watermark.WatermarkSerializer) Controller(io.pravega.client.control.impl.Controller) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) TimeWindow(io.pravega.client.stream.TimeWindow) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) StreamManager(io.pravega.client.admin.StreamManager) AtomicLong(java.util.concurrent.atomic.AtomicLong) Watermark(io.pravega.shared.watermarks.Watermark) Map(java.util.Map) HashMap(java.util.HashMap) Test(org.junit.Test)

Example 14 with ClientFactoryImpl

use of io.pravega.client.stream.impl.ClientFactoryImpl in project pravega by pravega.

the class MetricsTest method metricsTimeBasedCacheEvictionTest.

@Test(timeout = 120000)
public void metricsTimeBasedCacheEvictionTest() throws Exception {
    ClientConfig clientConfig = ClientConfig.builder().build();
    try (ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
        StreamManager streamManager = new StreamManagerImpl(controller, cp)) {
        boolean createScopeStatus = streamManager.createScope(scope);
        log.info("Create scope status {}", createScopeStatus);
        boolean createStreamStatus = streamManager.createStream(scope, STREAM_NAME, config);
        log.info("Create stream status {}", createStreamStatus);
    }
    try (ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
        ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
        ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, controller, clientFactory)) {
        @Cleanup EventStreamWriter<String> writer1 = clientFactory.createEventWriter(STREAM_NAME, new UTF8StringSerializer(), EventWriterConfig.builder().build());
        String event = "12345";
        long bytesWritten = TOTAL_NUM_EVENTS * (8 + event.length());
        writeEvents(event, writer1);
        String readerGroupName1 = readerGroupName + "1";
        log.info("Creating Reader group : {}", readerGroupName1);
        readerGroupManager.createReaderGroup(readerGroupName1, ReaderGroupConfig.builder().stream(Stream.of(scope, STREAM_NAME)).automaticCheckpointIntervalMillis(2000).build());
        EventStreamReader<String> reader1 = clientFactory.createReader(readerName, readerGroupName1, new UTF8StringSerializer(), ReaderConfig.builder().build());
        readAllEvents(reader1);
        final String[] streamTags = segmentTags(scope + "/" + STREAM_NAME + "/0.#epoch.0");
        assertEquals(bytesWritten, (long) MetricRegistryUtils.getCounter(SEGMENT_READ_BYTES, streamTags).count());
        // Wait for cache eviction to happen
        Thread.sleep(5000);
        String readerGroupName2 = readerGroupName + "2";
        log.info("Creating Reader group : {}", readerGroupName2);
        readerGroupManager.createReaderGroup(readerGroupName2, ReaderGroupConfig.builder().stream(Stream.of(scope, STREAM_NAME)).automaticCheckpointIntervalMillis(2000).build());
        EventStreamReader<String> reader2 = clientFactory.createReader(readerName, readerGroupName2, new UTF8StringSerializer(), ReaderConfig.builder().build());
        readAllEvents(reader2);
        // Metric is evicted from Cache, after cache eviction duration
        // Count starts from 0, rather than adding up to previously ready bytes, as cache is evicted.
        assertEquals(bytesWritten, (long) MetricRegistryUtils.getCounter(SEGMENT_READ_BYTES, streamTags).count());
        Map<Double, Double> map = new HashMap<>();
        map.put(0.0, 1.0);
        // Seal segment 0, create segment 1
        CompletableFuture<Boolean> scaleStatus = controller.scaleStream(new StreamImpl(scope, STREAM_NAME), Collections.singletonList(0L), map, executorService()).getFuture();
        Assert.assertTrue(scaleStatus.get());
        @Cleanup EventStreamWriter<String> writer2 = clientFactory.createEventWriter(STREAM_NAME, new UTF8StringSerializer(), EventWriterConfig.builder().build());
        writeEvents(event, writer2);
        readAllEvents(reader1);
        final String[] streamTags2nd = segmentTags(scope + "/" + STREAM_NAME + "/1.#epoch.1");
        assertEquals(bytesWritten, (long) MetricRegistryUtils.getCounter(SEGMENT_READ_BYTES, streamTags2nd).count());
        readerGroupManager.deleteReaderGroup(readerGroupName1);
        readerGroupManager.deleteReaderGroup(readerGroupName2);
        CompletableFuture<Boolean> sealStreamStatus = controller.sealStream(scope, STREAM_NAME);
        log.info("Sealing stream {}", STREAM_NAME);
        assertTrue(sealStreamStatus.get());
        CompletableFuture<Boolean> deleteStreamStatus = controller.deleteStream(scope, STREAM_NAME);
        log.info("Deleting stream {}", STREAM_NAME);
        assertTrue(deleteStreamStatus.get());
        CompletableFuture<Boolean> deleteScopeStatus = controller.deleteScope(scope);
        log.info("Deleting scope {}", scope);
        assertTrue(deleteScopeStatus.get());
    }
    log.info("Metrics Time based Cache Eviction test succeeds");
}
Also used : ConnectionPool(io.pravega.client.connection.impl.ConnectionPool) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) HashMap(java.util.HashMap) ConnectionPoolImpl(io.pravega.client.connection.impl.ConnectionPoolImpl) StreamManagerImpl(io.pravega.client.admin.impl.StreamManagerImpl) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) Cleanup(lombok.Cleanup) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) StreamManager(io.pravega.client.admin.StreamManager) StreamImpl(io.pravega.client.stream.impl.StreamImpl) UTF8StringSerializer(io.pravega.client.stream.impl.UTF8StringSerializer) ClientConfig(io.pravega.client.ClientConfig) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) Test(org.junit.Test)

Example 15 with ClientFactoryImpl

use of io.pravega.client.stream.impl.ClientFactoryImpl in project pravega by pravega.

the class WatermarkingTest method watermarkingTests.

@Test
public void watermarkingTests() throws Exception {
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
    // create 2 writers
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, clientConfig);
    JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<Long> writer1 = clientFactory.createEventWriter(STREAM, javaSerializer, EventWriterConfig.builder().build());
    @Cleanup EventStreamWriter<Long> writer2 = clientFactory.createEventWriter(STREAM, javaSerializer, EventWriterConfig.builder().build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    writeEvents(writer1, stopFlag);
    writeEvents(writer2, stopFlag);
    // scale the stream several times so that we get complex positions
    Stream streamObj = Stream.of(SCOPE, STREAM);
    scale(controller, streamObj);
    @Cleanup ClientFactoryImpl syncClientFactory = new ClientFactoryImpl(SCOPE, new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor()), connectionFactory);
    String markStream = NameUtils.getMarkStreamForStream(STREAM);
    RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
    // scale down one controller instance.
    Futures.getAndHandleExceptions(controllerInstance.scaleService(1), ExecutionException::new);
    // wait until at least 2 more watermarks are emitted
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 4, 100000);
    stopFlag.set(true);
    Watermark watermark0 = watermarks.take();
    Watermark watermark1 = watermarks.take();
    Watermark watermark2 = watermarks.take();
    Watermark watermark3 = watermarks.take();
    assertTrue(watermark0.getLowerTimeBound() <= watermark0.getUpperTimeBound());
    assertTrue(watermark1.getLowerTimeBound() <= watermark1.getUpperTimeBound());
    assertTrue(watermark2.getLowerTimeBound() <= watermark2.getUpperTimeBound());
    assertTrue(watermark3.getLowerTimeBound() <= watermark3.getUpperTimeBound());
    // verify that watermarks are increasing in time.
    assertTrue(watermark0.getLowerTimeBound() < watermark1.getLowerTimeBound());
    assertTrue(watermark1.getLowerTimeBound() < watermark2.getLowerTimeBound());
    assertTrue(watermark2.getLowerTimeBound() < watermark3.getLowerTimeBound());
    // use watermark as lower and upper bounds.
    Map<Segment, Long> positionMap0 = watermark0.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(SCOPE, STREAM, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutStart = new StreamCutImpl(streamObj, positionMap0);
    Map<Stream, StreamCut> start = Collections.singletonMap(streamObj, streamCutStart);
    Map<Segment, Long> positionMap2 = watermark2.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(SCOPE, STREAM, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutEnd = new StreamCutImpl(streamObj, positionMap2);
    Map<Stream, StreamCut> end = Collections.singletonMap(streamObj, streamCutEnd);
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(SCOPE, controller, syncClientFactory);
    String readerGroup = "rg";
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(streamObj).startingStreamCuts(start).endingStreamCuts(end).build());
    // create reader on the stream
    @Cleanup final EventStreamReader<Long> reader = clientFactory.createReader("myreader", readerGroup, javaSerializer, ReaderConfig.builder().build());
    // read events from the reader.
    // verify that events read belong to the bound
    EventRead<Long> event = reader.readNextEvent(10000L);
    AtomicReference<TimeWindow> currentTimeWindow = new AtomicReference<>();
    AssertExtensions.assertEventuallyEquals(true, () -> {
        currentTimeWindow.set(reader.getCurrentTimeWindow(streamObj));
        return currentTimeWindow.get() != null && currentTimeWindow.get().getLowerTimeBound() != null && currentTimeWindow.get().getUpperTimeBound() != null;
    }, 100000);
    log.info("current time window = {}", currentTimeWindow.get());
    while (event.getEvent() != null) {
        Long time = event.getEvent();
        log.info("event read = {}", time);
        event.getPosition();
        assertTrue(time >= currentTimeWindow.get().getLowerTimeBound());
        event = reader.readNextEvent(10000L);
        if (event.isCheckpoint()) {
            event = reader.readNextEvent(10000L);
        }
    }
}
Also used : StreamCut(io.pravega.client.stream.StreamCut) EventStreamWriter(io.pravega.client.stream.EventStreamWriter) MarathonException(mesosphere.marathon.client.MarathonException) AssertExtensions(io.pravega.test.common.AssertExtensions) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) Cleanup(lombok.Cleanup) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) EventRead(io.pravega.client.stream.EventRead) Service(io.pravega.test.system.framework.services.Service) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) Stream(io.pravega.client.stream.Stream) Duration(java.time.Duration) Map(java.util.Map) After(org.junit.After) TimeWindow(io.pravega.client.stream.TimeWindow) URI(java.net.URI) ReaderGroupConfig(io.pravega.client.stream.ReaderGroupConfig) Environment(io.pravega.test.system.framework.Environment) WatermarkSerializer(io.pravega.client.watermark.WatermarkSerializer) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) ControllerImplConfig(io.pravega.client.control.impl.ControllerImplConfig) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Controller(io.pravega.client.control.impl.Controller) Futures(io.pravega.common.concurrent.Futures) Segment(io.pravega.client.segment.impl.Segment) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) StreamManager(io.pravega.client.admin.StreamManager) RunWith(org.junit.runner.RunWith) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) StreamCutImpl(io.pravega.client.stream.impl.StreamCutImpl) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Timeout(org.junit.rules.Timeout) EventStreamClientFactory(io.pravega.client.EventStreamClientFactory) RevisionedStreamClient(io.pravega.client.state.RevisionedStreamClient) SynchronizerConfig(io.pravega.client.state.SynchronizerConfig) Utils(io.pravega.test.system.framework.Utils) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) Before(org.junit.Before) EventWriterConfig(io.pravega.client.stream.EventWriterConfig) NameUtils(io.pravega.shared.NameUtils) Iterator(java.util.Iterator) Assert.assertTrue(org.junit.Assert.assertTrue) EventStreamReader(io.pravega.client.stream.EventStreamReader) Test(org.junit.Test) Watermark(io.pravega.shared.watermarks.Watermark) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Rule(org.junit.Rule) ControllerImpl(io.pravega.client.control.impl.ControllerImpl) Revision(io.pravega.client.state.Revision) ReaderConfig(io.pravega.client.stream.ReaderConfig) ExecutorServiceHelpers(io.pravega.common.concurrent.ExecutorServiceHelpers) RandomFactory(io.pravega.common.hash.RandomFactory) Collections(java.util.Collections) ScalingPolicy(io.pravega.client.stream.ScalingPolicy) SystemTestRunner(io.pravega.test.system.framework.SystemTestRunner) ClientConfig(io.pravega.client.ClientConfig) StreamCutImpl(io.pravega.client.stream.impl.StreamCutImpl) EventStreamClientFactory(io.pravega.client.EventStreamClientFactory) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Cleanup(lombok.Cleanup) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) Segment(io.pravega.client.segment.impl.Segment) ConnectionFactory(io.pravega.client.connection.impl.ConnectionFactory) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) Stream(io.pravega.client.stream.Stream) ClientConfig(io.pravega.client.ClientConfig) ExecutionException(java.util.concurrent.ExecutionException) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) StreamCut(io.pravega.client.stream.StreamCut) ControllerImpl(io.pravega.client.control.impl.ControllerImpl) WatermarkSerializer(io.pravega.client.watermark.WatermarkSerializer) AtomicReference(java.util.concurrent.atomic.AtomicReference) SocketConnectionFactoryImpl(io.pravega.client.connection.impl.SocketConnectionFactoryImpl) TimeWindow(io.pravega.client.stream.TimeWindow) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AtomicLong(java.util.concurrent.atomic.AtomicLong) Watermark(io.pravega.shared.watermarks.Watermark) Map(java.util.Map) HashMap(java.util.HashMap) Test(org.junit.Test)

Aggregations

ClientFactoryImpl (io.pravega.client.stream.impl.ClientFactoryImpl)80 Test (org.junit.Test)63 Cleanup (lombok.Cleanup)59 SocketConnectionFactoryImpl (io.pravega.client.connection.impl.SocketConnectionFactoryImpl)54 StreamConfiguration (io.pravega.client.stream.StreamConfiguration)50 ReaderGroupManager (io.pravega.client.admin.ReaderGroupManager)44 ConnectionFactory (io.pravega.client.connection.impl.ConnectionFactory)40 ReaderGroupManagerImpl (io.pravega.client.admin.impl.ReaderGroupManagerImpl)39 ClientConfig (io.pravega.client.ClientConfig)37 Stream (io.pravega.client.stream.Stream)27 Controller (io.pravega.client.control.impl.Controller)25 StreamImpl (io.pravega.client.stream.impl.StreamImpl)24 HashMap (java.util.HashMap)23 ReaderGroup (io.pravega.client.stream.ReaderGroup)19 ScalingPolicy (io.pravega.client.stream.ScalingPolicy)19 JavaSerializer (io.pravega.client.stream.impl.JavaSerializer)19 EventWriterConfig (io.pravega.client.stream.EventWriterConfig)18 URI (java.net.URI)18 CompletableFuture (java.util.concurrent.CompletableFuture)18 ControllerImpl (io.pravega.client.control.impl.ControllerImpl)17