Examples with StreamConfiguration io.pravega.client.stream.StreamConfiguration used on opensource projects

Example 96 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class BoundedStreamReaderTest method createStream.

private void createStream(String streamName) throws Exception {
    Controller controller = controllerWrapper.getController();
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    val f1 = controller.createStream(SCOPE, streamName, config);
    f1.get();
}

Example 97 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class ControllerMetricsTest method streamMetricsTest.

/**
 * This test verifies that the appropriate metrics for Stream operations are updated correctly (counters, latency
 * histograms). Note that this test performs "at least" assertions on metrics as in an environment with concurrent
 * tests running, it might be possible that metrics get updated by other tests.
 */
@Test(timeout = 300000)
public void streamMetricsTest() throws Exception {
    // make unique scope to improve the test isolation.
    final String scope = "controllerMetricsTestScope" + RandomFactory.getSeed();
    final String streamName = "controllerMetricsTestStream";
    final String readerGroupName = "RGControllerMetricsTestStream";
    final int parallelism = 4;
    final int eventsWritten = 10;
    int iterations = 3;
    // At this point, we have at least 6 internal streams.
    StreamConfiguration streamConfiguration = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(parallelism)).build();
    @Cleanup StreamManager streamManager = StreamManager.create(controllerURI);
    streamManager.createScope(scope);
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, ClientConfig.builder().controllerURI(controllerURI).build());
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(scope, controllerURI);
    for (int i = 0; i < iterations; i++) {
        final String iterationStreamName = streamName + i;
        final String iterationReaderGroupName = readerGroupName + RandomFactory.getSeed();
        // Check that the number of streams in metrics has been incremented.
        streamManager.createStream(scope, iterationStreamName, streamConfiguration);
        Counter createdStreamsCounter = MetricRegistryUtils.getCounter(CREATE_STREAM);
        AssertExtensions.assertGreaterThanOrEqual("The counter of created streams", i, (long) createdStreamsCounter.count());
        groupManager.createReaderGroup(iterationReaderGroupName, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(scope + "/" + iterationStreamName).build());
        for (long j = 1; j < iterations + 1; j++) {
            @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup(iterationReaderGroupName);
            // Update the Stream and check that the number of updated streams and per-stream updates is incremented.
            streamManager.updateStream(scope, iterationStreamName, streamConfiguration);
            Counter updatedStreamsCounter = MetricRegistryUtils.getCounter(globalMetricName(UPDATE_STREAM));
            Counter streamUpdatesCounter = MetricRegistryUtils.getCounter(UPDATE_STREAM, streamTags(scope, iterationStreamName));
            Assert.assertTrue(iterations * i + j <= updatedStreamsCounter.count());
            Assert.assertTrue(j == streamUpdatesCounter.count());
            // Read and write some events.
            writeEvents(clientFactory, iterationStreamName, eventsWritten);
            Futures.allOf(readEvents(clientFactory, iterationReaderGroupName, parallelism));
            // Get a StreamCut for truncating the Stream.
            StreamCut streamCut = readerGroup.generateStreamCuts(executorService()).join().get(Stream.of(scope, iterationStreamName));
            // Truncate the Stream and check that the number of truncated Streams and per-Stream truncations is incremented.
            streamManager.truncateStream(scope, iterationStreamName, streamCut);
            Counter streamTruncationCounter = MetricRegistryUtils.getCounter(globalMetricName(TRUNCATE_STREAM));
            Counter perStreamTruncationCounter = MetricRegistryUtils.getCounter(TRUNCATE_STREAM, streamTags(scope, iterationStreamName));
            Assert.assertTrue(iterations * i + j <= streamTruncationCounter.count());
            Assert.assertTrue(j == perStreamTruncationCounter.count());
        }
        // Check metrics accounting for sealed and deleted streams.
        streamManager.sealStream(scope, iterationStreamName);
        Counter streamSealCounter = MetricRegistryUtils.getCounter(SEAL_STREAM);
        Assert.assertTrue(i + 1 <= streamSealCounter.count());
        streamManager.deleteStream(scope, iterationStreamName);
        Counter streamDeleteCounter = MetricRegistryUtils.getCounter(DELETE_STREAM);
        Assert.assertTrue(i + 1 <= streamDeleteCounter.count());
    }
    // Put assertion on different lines so it can tell more information in case of failure.
    Timer latencyValues1 = MetricRegistryUtils.getTimer(CREATE_STREAM_LATENCY);
    Assert.assertNotNull(latencyValues1);
    AssertExtensions.assertGreaterThanOrEqual("Number of iterations and latency count do not match.", iterations, latencyValues1.count());
    Timer latencyValues2 = MetricRegistryUtils.getTimer(SEAL_STREAM_LATENCY);
    Assert.assertNotNull(latencyValues2);
    Assert.assertEquals(iterations, latencyValues2.count());
    Timer latencyValues3 = MetricRegistryUtils.getTimer(DELETE_STREAM_LATENCY);
    Assert.assertNotNull(latencyValues3);
    Assert.assertEquals(iterations, latencyValues3.count());
    Timer latencyValues4 = MetricRegistryUtils.getTimer(UPDATE_STREAM_LATENCY);
    Assert.assertNotNull(latencyValues4);
    Assert.assertEquals(iterations * iterations, latencyValues4.count());
    Timer latencyValues5 = MetricRegistryUtils.getTimer(TRUNCATE_STREAM_LATENCY);
    Assert.assertNotNull(latencyValues5);
    Assert.assertEquals(iterations * iterations, latencyValues5.count());
}

Example 98 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class ByteStreamTest method testBlockingRead.

@Test(timeout = 30000)
public void testBlockingRead() throws IOException {
    String scope = "ByteStreamTest";
    String stream = "testBlockingRead";
    StreamConfiguration config = StreamConfiguration.builder().build();
    @Cleanup StreamManager streamManager = new StreamManagerImpl(PRAVEGA.getLocalController(), Mockito.mock(ConnectionPool.class));
    // create a scope
    Boolean createScopeStatus = streamManager.createScope(scope);
    log.info("Create scope status {}", createScopeStatus);
    // create a stream
    Boolean createStreamStatus = streamManager.createStream(scope, stream, config);
    log.info("Create stream status {}", createStreamStatus);
    @Cleanup ByteStreamClientFactory client = createClientFactory(scope);
    byte[] payload = new byte[100];
    Arrays.fill(payload, (byte) 1);
    byte[] readBuffer = new byte[200];
    Arrays.fill(readBuffer, (byte) 0);
    @Cleanup ByteStreamWriter writer = client.createByteStreamWriter(stream);
    @Cleanup ByteStreamReader reader = client.createByteStreamReader(stream);
    AssertExtensions.assertBlocks(() -> {
        assertEquals(100, reader.read(readBuffer));
    }, () -> writer.write(payload));
    assertEquals(1, readBuffer[99]);
    assertEquals(0, readBuffer[100]);
    Arrays.fill(readBuffer, (byte) 0);
    writer.write(payload);
    assertEquals(100, reader.read(readBuffer));
    assertEquals(1, readBuffer[99]);
    assertEquals(0, readBuffer[100]);
    writer.write(payload);
    writer.write(payload);
    assertEquals(200, StreamHelpers.readAll(reader, readBuffer, 0, readBuffer.length));
    AssertExtensions.assertBlocks(() -> {
        assertEquals(100, reader.read(readBuffer));
    }, () -> writer.write(payload));
    writer.closeAndSeal();
    assertEquals(-1, reader.read());
}

Example 99 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class ByteStreamTest method testRecreateStream.

@Test(timeout = 30000)
public void testRecreateStream() {
    String scope = "ByteStreamTest";
    String stream = "testRecreateStream";
    StreamConfiguration config = StreamConfiguration.builder().build();
    @Cleanup StreamManager streamManager = new StreamManagerImpl(PRAVEGA.getLocalController(), Mockito.mock(ConnectionPool.class));
    // create a scope
    streamManager.createScope(scope);
    // create a stream
    assertTrue("Create stream failed", streamManager.createStream(scope, stream, config));
    // verify read and write.
    verifyByteClientReadWrite(scope, stream);
    // delete the stream and recreate
    assertTrue("Seal stream operation failed", streamManager.sealStream(scope, stream));
    assertTrue("Delete Stream operation failed", streamManager.deleteStream(scope, stream));
    assertTrue("Recreate stream failed", streamManager.createStream(scope, stream, config));
    // verify read and write.
    verifyByteClientReadWrite(scope, stream);
}

Example 100 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class LargeEventTest method testNormalThenLargeEvent.

@Test(timeout = 60000)
public void testNormalThenLargeEvent() throws ExecutionException, InterruptedException {
    String streamName = "NormalEventLargeEvent";
    String readerGroupName = "testNormalThenLargeEvent";
    StreamConfiguration config = getStreamConfiguration(NUM_READERS);
    createScopeStream(SCOPE_NAME, streamName, config);
    int events = 1;
    AtomicInteger generation = new AtomicInteger(0);
    // Normal Event Write/Read.
    merge(eventsWrittenToPravega, generateEventData(NUM_WRITERS, events * generation.getAndIncrement(), events, LARGE_EVENT_SIZE));
    log.info("Writing {} new events.", eventsWrittenToPravega.size());
    eventsReadFromPravega = readWriteCycle(streamName, readerGroupName, eventsWrittenToPravega);
    log.info("Read back {} events.", eventsReadFromPravega.size());
    validateEventReads(eventsReadFromPravega, eventsWrittenToPravega);
    // Large Event Write/Read.
    Map<Integer, List<ByteBuffer>> data = generateEventData(NUM_WRITERS, events * generation.getAndIncrement(), events, TINY_EVENT_SIZE);
    merge(eventsWrittenToPravega, data);
    log.info("Writing {} new events.", eventsWrittenToPravega.size());
    eventsReadFromPravega = readWriteCycle(streamName, readerGroupName, data);
    log.info("Read back {} events.", eventsReadFromPravega.size());
    validateEventReads(eventsReadFromPravega, eventsWrittenToPravega);
    validateCleanUp(streamName);
}