Examples with StreamCut io.pravega.client.stream.StreamCut used on opensource projects

Example 71 with StreamCut

use of io.pravega.client.stream.StreamCut in project pravega by pravega.

the class StreamCutsTest method streamCutsTest.

/**
 * This test verifies the correct operation of readers using StreamCuts. Concretely, the test creates two streams
 * with different number of segments and it writes some events (TOTAL_EVENTS / 2) in them. Then, the test creates a
 * list of StreamCuts that encompasses both streams every CUT_SIZE events. The test asserts that new groups of
 * readers can be initialized at these sequential StreamCut intervals and that only CUT_SIZE events are read. Also,
 * the test checks the correctness of different combinations of StreamCuts that have not been sequentially created.
 * After creating StreamCuts and tests the correctness of reads, the test also checks resetting a reader group to a
 * specific initial read point. The previous process is repeated twice: before and after scaling streams, to test if
 * StreamCuts work correctly under scaling events (thus writing TOTAL_EVENTS). Finally, this test checks reading
 * different StreamCut combinations in both streams for all events (encompassing events before and after scaling).
 */
@Test
public void streamCutsTest() {
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, clientConfig);
    @Cleanup ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(SCOPE, clientConfig);
    readerGroupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().stream(Stream.of(SCOPE, STREAM_ONE)).stream(Stream.of(SCOPE, STREAM_TWO)).build());
    @Cleanup ReaderGroup readerGroup = readerGroupManager.getReaderGroup(READER_GROUP);
    // Perform write of events, slice by slice StreamCuts test and combinations StreamCuts test.
    log.info("Write, slice by slice and combinations test before scaling.");
    final int parallelismBeforeScale = RG_PARALLELISM_ONE + RG_PARALLELISM_TWO;
    List<Map<Stream, StreamCut>> slicesBeforeScale = writeEventsAndCheckSlices(clientFactory, readerGroup, readerGroupManager, parallelismBeforeScale);
    // Now, we perform a manual scale on both streams and wait until it occurs.
    CompletableFuture<Boolean> scaleStreamOne = scaleStream(SCOPE, STREAM_ONE, RG_PARALLELISM_ONE * 2, executor);
    checkScaleStatus(scaleStreamOne);
    // Perform again the same test on the stream segments after scaling.
    final int parallelSegmentsAfterScale = RG_PARALLELISM_ONE * 2 + RG_PARALLELISM_TWO;
    final String newReaderGroupName = READER_GROUP + "new";
    final Map<Stream, StreamCut> streamCutBeforeScale = slicesBeforeScale.get(slicesBeforeScale.size() - 1);
    readerGroupManager.createReaderGroup(newReaderGroupName, ReaderGroupConfig.builder().stream(Stream.of(SCOPE, STREAM_ONE)).stream(Stream.of(SCOPE, STREAM_TWO)).startingStreamCuts(streamCutBeforeScale).build());
    @Cleanup ReaderGroup newReaderGroup = readerGroupManager.getReaderGroup(newReaderGroupName);
    log.info("Checking slices again starting from {}.", streamCutBeforeScale);
    List<Map<Stream, StreamCut>> slicesAfterScale = writeEventsAndCheckSlices(clientFactory, newReaderGroup, readerGroupManager, parallelSegmentsAfterScale);
    // Perform combinations including StreamCuts before and after the scale event.
    slicesAfterScale.remove(0);
    slicesBeforeScale.addAll(slicesAfterScale);
    log.info("Performing combinations in the whole stream.");
    combineSlicesAndVerify(readerGroupManager, clientFactory, parallelSegmentsAfterScale, slicesBeforeScale);
    log.info("All events correctly read from StreamCut slices on multiple Streams. StreamCuts test passed.");
}

Example 72 with StreamCut

use of io.pravega.client.stream.StreamCut in project pravega by pravega.

the class StreamCutsTest method writeEventsAndCheckSlices.

// Start utils region
private List<Map<Stream, StreamCut>> writeEventsAndCheckSlices(EventStreamClientFactory clientFactory, ReaderGroup readerGroup, ReaderGroupManager readerGroupManager, int parallelSegments) {
    // First, write half of total events before scaling (1/4 in each Stream).
    writeEvents(clientFactory, STREAM_ONE, TOTAL_EVENTS / 4);
    writeEvents(clientFactory, STREAM_TWO, TOTAL_EVENTS / 4);
    log.info("Finished writing events to streams.");
    Map<Stream, StreamCut> initialPosition = new HashMap<>(readerGroup.getStreamCuts());
    log.info("Creating StreamCuts from: {}.", initialPosition);
    // Get StreamCuts for each slice from both Streams at the same time (may be different in each execution).
    List<Map<Stream, StreamCut>> streamSlices = getStreamCutSlices(clientFactory, readerGroup, TOTAL_EVENTS / 2);
    streamSlices.add(0, initialPosition);
    log.info("Finished creating StreamCuts {}.", streamSlices);
    // Ensure that reader groups can correctly read slice by slice from different Streams.
    readSliceBySliceAndVerify(readerGroupManager, clientFactory, parallelSegments, streamSlices);
    log.info("Finished checking sequentially slice by slice.");
    // Perform different combinations of StreamCuts and verify that read event boundaries are still correct.
    combineSlicesAndVerify(readerGroupManager, clientFactory, parallelSegments, streamSlices);
    log.info("Finished checking StreamCut combinations.");
    // Test that a reader group can be reset correctly.
    ReaderGroupConfig firstSliceConfig = ReaderGroupConfig.builder().stream(Stream.of(SCOPE, STREAM_ONE)).stream(Stream.of(SCOPE, STREAM_TWO)).startingStreamCuts(initialPosition).endingStreamCuts(streamSlices.get(streamSlices.size() - 1)).build();
    readerGroup.resetReaderGroup(firstSliceConfig);
    log.info("Resetting existing reader group {} to stream cut {}.", READER_GROUP, initialPosition);
    final int readEvents = readAllEvents(readerGroupManager, clientFactory, readerGroup.getGroupName(), parallelSegments).stream().map(CompletableFuture::join).reduce(Integer::sum).get();
    assertEquals("Expected read events: ", TOTAL_EVENTS / 2, readEvents);
    return streamSlices;
}

Example 73 with StreamCut

use of io.pravega.client.stream.StreamCut in project pravega by pravega.

the class ControllerServiceWithStreamTest method createReaderGroupTest.

@Test(timeout = 5000)
public void createReaderGroupTest() throws Exception {
    String stream = "stream1";
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scalingPolicy(policy1).build();
    // Start time  when stream is created.
    long start = System.currentTimeMillis();
    // Create scope
    Controller.CreateScopeStatus scopeStatus = consumer.createScope(SCOPE, 0L).join();
    assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, scopeStatus.getStatus());
    // create stream
    Controller.CreateStreamStatus streamStatus = consumer.createStream(SCOPE, stream, configuration1, start, 0L).get();
    assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamStatus.getStatus());
    final Segment seg0 = new Segment(SCOPE, "stream1", 0L);
    final Segment seg1 = new Segment(SCOPE, "stream1", 1L);
    ImmutableMap<Segment, Long> startStreamCut = ImmutableMap.of(seg0, 10L, seg1, 10L);
    Map<Stream, StreamCut> startSC = ImmutableMap.of(Stream.of(SCOPE, "stream1"), new StreamCutImpl(Stream.of(SCOPE, "stream1"), startStreamCut));
    ImmutableMap<Segment, Long> endStreamCut = ImmutableMap.of(seg0, 200L, seg1, 300L);
    Map<Stream, StreamCut> endSC = ImmutableMap.of(Stream.of(SCOPE, "stream1"), new StreamCutImpl(Stream.of(SCOPE, "stream1"), endStreamCut));
    ReaderGroupConfig rgConfig = ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(30000L).groupRefreshTimeMillis(20000L).maxOutstandingCheckpointRequest(2).retentionType(ReaderGroupConfig.StreamDataRetention.AUTOMATIC_RELEASE_AT_LAST_CHECKPOINT).startingStreamCuts(startSC).endingStreamCuts(endSC).build();
    Controller.CreateReaderGroupResponse rgStatus = consumer.createReaderGroup(SCOPE, "rg1", rgConfig, System.currentTimeMillis(), 0L).get();
    assertEquals(Controller.CreateReaderGroupResponse.Status.SUCCESS, rgStatus.getStatus());
    // there should be 1 invocation
    verify(streamStore, times(1)).createReaderGroup(anyString(), anyString(), any(), anyLong(), any(), any());
    rgStatus = consumer.createReaderGroup(SCOPE, "rg1", rgConfig, System.currentTimeMillis(), 0L).get();
    assertEquals(Controller.CreateReaderGroupResponse.Status.SUCCESS, rgStatus.getStatus());
    // verify that create readergroup is not called again
    verify(streamStore, times(1)).createReaderGroup(anyString(), anyString(), any(), anyLong(), any(), any());
}

Example 74 with StreamCut

use of io.pravega.client.stream.StreamCut in project pravega by pravega.

the class SerializationTest method testStreamCut.

@Test
public void testStreamCut() {
    StreamCutImpl cut = new StreamCutImpl(Stream.of("Foo/Bar"), ImmutableMap.of(Segment.fromScopedName("Foo/Bar/1"), 3L));
    ByteBuffer bytes = cut.toBytes();
    StreamCut cut2 = StreamCut.fromBytes(bytes);
    assertEquals(cut, cut2);
    bytes = StreamCut.UNBOUNDED.toBytes();
    assertEquals(StreamCut.UNBOUNDED, StreamCut.fromBytes(bytes));
    assertNotEquals(cut, StreamCut.UNBOUNDED);
}

Example 75 with StreamCut

use of io.pravega.client.stream.StreamCut in project pravega by pravega.

the class ReaderGroupImplTest method getUnreadBytesBasedOnLastCheckpointPosition.

@Test
public void getUnreadBytesBasedOnLastCheckpointPosition() {
    final String stream = "s1";
    final StreamCut startStreamCut = getStreamCut(stream, 10L, 1, 2);
    final StreamCut endStreamCut = getStreamCut(stream, 25L, 1, 2);
    // setup mocks
    when(state.getPositionsForLastCompletedCheckpoint()).thenReturn(Optional.of(ImmutableMap.of(Stream.of(SCOPE, stream), startStreamCut.asImpl().getPositions())));
    when(state.getEndSegments()).thenReturn(endStreamCut.asImpl().getPositions());
    when(synchronizer.getState()).thenReturn(state);
    ImmutableSet<Segment> segmentSet = ImmutableSet.<Segment>builder().addAll(startStreamCut.asImpl().getPositions().keySet()).addAll(endStreamCut.asImpl().getPositions().keySet()).build();
    when(controller.getSegments(startStreamCut, endStreamCut)).thenReturn(CompletableFuture.completedFuture(new StreamSegmentSuccessors(segmentSet, "")));
    assertEquals(30L, readerGroup.unreadBytes());
}