Examples with StreamManager io.pravega.client.admin.StreamManager used on opensource projects

Example 21 with StreamManager

use of io.pravega.client.admin.StreamManager in project pravega by pravega.

the class ByteStreamTest method testRecreateStream.

@Test(timeout = 30000)
public void testRecreateStream() {
    String scope = "ByteStreamTest";
    String stream = "testRecreateStream";
    StreamConfiguration config = StreamConfiguration.builder().build();
    @Cleanup StreamManager streamManager = new StreamManagerImpl(PRAVEGA.getLocalController(), Mockito.mock(ConnectionPool.class));
    // create a scope
    streamManager.createScope(scope);
    // create a stream
    assertTrue("Create stream failed", streamManager.createStream(scope, stream, config));
    // verify read and write.
    verifyByteClientReadWrite(scope, stream);
    // delete the stream and recreate
    assertTrue("Seal stream operation failed", streamManager.sealStream(scope, stream));
    assertTrue("Delete Stream operation failed", streamManager.deleteStream(scope, stream));
    assertTrue("Recreate stream failed", streamManager.createStream(scope, stream, config));
    // verify read and write.
    verifyByteClientReadWrite(scope, stream);
}

Example 22 with StreamManager

use of io.pravega.client.admin.StreamManager in project pravega by pravega.

the class WatermarkingTest method progressingWatermarkWithWriterTimeouts.

@Test(timeout = 60000)
public void progressingWatermarkWithWriterTimeouts() throws Exception {
    String scope = "Timeout";
    String streamName = "progressingWatermarkWithWriterTimeouts";
    int numSegments = 1;
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    assertNotNull(streamManager);
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(numSegments)).build());
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
    @Cleanup SynchronizerClientFactory syncClientFactory = SynchronizerClientFactory.withScope(scope, clientConfig);
    String markStream = NameUtils.getMarkStreamForStream(streamName);
    @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    // create two writers and write two sevent and call note time for each writer.
    @Cleanup EventStreamWriter<String> writer1 = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer1.writeEvent("1").get();
    writer1.noteTime(100L);
    @Cleanup EventStreamWriter<String> writer2 = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer2.writeEvent("2").get();
    writer2.noteTime(102L);
    // writer0 should timeout. writer1 and writer2 should result in two more watermarks with following times:
    // 1: 100L-101L 2: 101-101
    // then first writer should timeout and be discarded. But second writer should continue to be active as its time
    // is higher than first watermark. This should result in a second watermark to be emitted.
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() == 2, 100000);
    Watermark watermark1 = watermarks.poll();
    Watermark watermark2 = watermarks.poll();
    assertEquals(100L, watermark1.getLowerTimeBound());
    assertEquals(102L, watermark1.getUpperTimeBound());
    assertEquals(102L, watermark2.getLowerTimeBound());
    assertEquals(102L, watermark2.getUpperTimeBound());
    // stream cut should be same
    assertTrue(watermark2.getStreamCut().entrySet().stream().allMatch(x -> watermark1.getStreamCut().get(x.getKey()).equals(x.getValue())));
    // bring back writer1 and post an event with note time smaller than current watermark
    writer1.writeEvent("3").get();
    writer1.noteTime(101L);
    // no watermark should be emitted.
    Watermark nullMark = watermarks.poll(10, TimeUnit.SECONDS);
    assertNull(nullMark);
}

Example 23 with StreamManager

use of io.pravega.client.admin.StreamManager in project pravega by pravega.

the class DelegationTokenTest method createScopeStream.

private void createScopeStream(String scope, String streamName, int numSegments, ClientConfig clientConfig) {
    @Cleanup final StreamManager streamManager = StreamManager.create(clientConfig);
    assertNotNull(streamManager);
    log.debug("Done creating stream manager.");
    boolean isScopeCreated = streamManager.createScope(scope);
    assertTrue("Failed to create scope", isScopeCreated);
    log.debug("Done creating stream manager.");
    boolean isStreamCreated = streamManager.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(numSegments)).build());
    Assert.assertTrue("Failed to create the stream ", isStreamCreated);
}

Example 24 with StreamManager

use of io.pravega.client.admin.StreamManager in project pravega by pravega.

the class MetricsTest method metricsTimeBasedCacheEvictionTest.

@Test(timeout = 120000)
public void metricsTimeBasedCacheEvictionTest() throws Exception {
    ClientConfig clientConfig = ClientConfig.builder().build();
    try (ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
        StreamManager streamManager = new StreamManagerImpl(controller, cp)) {
        boolean createScopeStatus = streamManager.createScope(scope);
        log.info("Create scope status {}", createScopeStatus);
        boolean createStreamStatus = streamManager.createStream(scope, STREAM_NAME, config);
        log.info("Create stream status {}", createStreamStatus);
    }
    try (ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
        ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
        ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, controller, clientFactory)) {
        @Cleanup EventStreamWriter<String> writer1 = clientFactory.createEventWriter(STREAM_NAME, new UTF8StringSerializer(), EventWriterConfig.builder().build());
        String event = "12345";
        long bytesWritten = TOTAL_NUM_EVENTS * (8 + event.length());
        writeEvents(event, writer1);
        String readerGroupName1 = readerGroupName + "1";
        log.info("Creating Reader group : {}", readerGroupName1);
        readerGroupManager.createReaderGroup(readerGroupName1, ReaderGroupConfig.builder().stream(Stream.of(scope, STREAM_NAME)).automaticCheckpointIntervalMillis(2000).build());
        EventStreamReader<String> reader1 = clientFactory.createReader(readerName, readerGroupName1, new UTF8StringSerializer(), ReaderConfig.builder().build());
        readAllEvents(reader1);
        final String[] streamTags = segmentTags(scope + "/" + STREAM_NAME + "/0.#epoch.0");
        assertEquals(bytesWritten, (long) MetricRegistryUtils.getCounter(SEGMENT_READ_BYTES, streamTags).count());
        // Wait for cache eviction to happen
        Thread.sleep(5000);
        String readerGroupName2 = readerGroupName + "2";
        log.info("Creating Reader group : {}", readerGroupName2);
        readerGroupManager.createReaderGroup(readerGroupName2, ReaderGroupConfig.builder().stream(Stream.of(scope, STREAM_NAME)).automaticCheckpointIntervalMillis(2000).build());
        EventStreamReader<String> reader2 = clientFactory.createReader(readerName, readerGroupName2, new UTF8StringSerializer(), ReaderConfig.builder().build());
        readAllEvents(reader2);
        // Metric is evicted from Cache, after cache eviction duration
        // Count starts from 0, rather than adding up to previously ready bytes, as cache is evicted.
        assertEquals(bytesWritten, (long) MetricRegistryUtils.getCounter(SEGMENT_READ_BYTES, streamTags).count());
        Map<Double, Double> map = new HashMap<>();
        map.put(0.0, 1.0);
        // Seal segment 0, create segment 1
        CompletableFuture<Boolean> scaleStatus = controller.scaleStream(new StreamImpl(scope, STREAM_NAME), Collections.singletonList(0L), map, executorService()).getFuture();
        Assert.assertTrue(scaleStatus.get());
        @Cleanup EventStreamWriter<String> writer2 = clientFactory.createEventWriter(STREAM_NAME, new UTF8StringSerializer(), EventWriterConfig.builder().build());
        writeEvents(event, writer2);
        readAllEvents(reader1);
        final String[] streamTags2nd = segmentTags(scope + "/" + STREAM_NAME + "/1.#epoch.1");
        assertEquals(bytesWritten, (long) MetricRegistryUtils.getCounter(SEGMENT_READ_BYTES, streamTags2nd).count());
        readerGroupManager.deleteReaderGroup(readerGroupName1);
        readerGroupManager.deleteReaderGroup(readerGroupName2);
        CompletableFuture<Boolean> sealStreamStatus = controller.sealStream(scope, STREAM_NAME);
        log.info("Sealing stream {}", STREAM_NAME);
        assertTrue(sealStreamStatus.get());
        CompletableFuture<Boolean> deleteStreamStatus = controller.deleteStream(scope, STREAM_NAME);
        log.info("Deleting stream {}", STREAM_NAME);
        assertTrue(deleteStreamStatus.get());
        CompletableFuture<Boolean> deleteScopeStatus = controller.deleteScope(scope);
        log.info("Deleting scope {}", scope);
        assertTrue(deleteScopeStatus.get());
    }
    log.info("Metrics Time based Cache Eviction test succeeds");
}

Example 25 with StreamManager

use of io.pravega.client.admin.StreamManager in project pravega by pravega.

the class ControllerGrpcListStreamsTest method listStreams.

private Set<Stream> listStreams(ClientConfig clientConfig, String scopeName) {
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    assertNotNull(streamManager);
    Iterator<Stream> streamsIter = streamManager.listStreams(scopeName);
    Set<Stream> streams = new HashSet<>();
    streamsIter.forEachRemaining(s -> streams.add(s));
    return streams;
}