Examples with SegmentRange io.pravega.client.batch.SegmentRange used on opensource projects

Example 11 with SegmentRange

use of io.pravega.client.batch.SegmentRange in project pravega by pravega.

the class BatchClientSimpleTest method batchClientSimpleTest.

/**
 * This test verifies the basic functionality of {@link BatchClientFactory}, including stream metadata checks, segment
 * counts, parallel segment reads and reads with offsets using stream cuts.
 */
@Test
@SuppressWarnings("deprecation")
public void batchClientSimpleTest() {
    final int totalEvents = RG_PARALLELISM * 100;
    final int offsetEvents = RG_PARALLELISM * 20;
    final int batchIterations = 4;
    final Stream stream = Stream.of(SCOPE, STREAM);
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    @Cleanup BatchClientFactory batchClient = BatchClientFactory.withScope(SCOPE, clientConfig);
    log.info("Invoking batchClientSimpleTest test with Controller URI: {}", controllerURI);
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, clientConfig);
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(SCOPE + "/" + STREAM).build());
    ReaderGroup readerGroup = groupManager.getReaderGroup(READER_GROUP);
    log.info("Writing events to stream");
    // Write events to the Stream.
    writeEvents(clientFactory, STREAM, totalEvents);
    // Instantiate readers to consume from Stream up to truncatedEvents.
    List<CompletableFuture<Integer>> futures = readEventFutures(clientFactory, READER_GROUP, RG_PARALLELISM, offsetEvents);
    Futures.allOf(futures).join();
    // Create a stream cut on the specified offset position.
    Checkpoint cp = readerGroup.initiateCheckpoint("batchClientCheckpoint", executor).join();
    StreamCut streamCut = cp.asImpl().getPositions().values().iterator().next();
    // Instantiate the batch client and assert it provides correct stream info.
    log.debug("Creating batch client.");
    StreamInfo streamInfo = streamManager.getStreamInfo(SCOPE, stream.getStreamName());
    log.debug("Validating stream metadata fields.");
    assertEquals("Expected Stream name: ", STREAM, streamInfo.getStreamName());
    assertEquals("Expected Scope name: ", SCOPE, streamInfo.getScope());
    // Test that we can read events from parallel segments from an offset onwards.
    log.debug("Reading events from stream cut onwards in parallel.");
    List<SegmentRange> ranges = Lists.newArrayList(batchClient.getSegments(stream, streamCut, StreamCut.UNBOUNDED).getIterator());
    assertEquals("Expected events read: ", totalEvents - offsetEvents, readFromRanges(ranges, batchClient));
    // Emulate the behavior of Hadoop client: i) Get tail of Stream, ii) Read from current point until tail, iii) repeat.
    log.debug("Reading in batch iterations.");
    StreamCut currentTailStreamCut = streamManager.getStreamInfo(SCOPE, stream.getStreamName()).getTailStreamCut();
    int readEvents = 0;
    for (int i = 0; i < batchIterations; i++) {
        writeEvents(clientFactory, STREAM, totalEvents);
        // Read all the existing events in parallel segments from the previous tail to the current one.
        ranges = Lists.newArrayList(batchClient.getSegments(stream, currentTailStreamCut, StreamCut.UNBOUNDED).getIterator());
        assertEquals("Expected number of segments: ", RG_PARALLELISM, ranges.size());
        readEvents += readFromRanges(ranges, batchClient);
        log.debug("Events read in parallel so far: {}.", readEvents);
        currentTailStreamCut = streamManager.getStreamInfo(SCOPE, stream.getStreamName()).getTailStreamCut();
    }
    assertEquals("Expected events read: .", totalEvents * batchIterations, readEvents);
    // Truncate the stream in first place.
    log.debug("Truncating stream at event {}.", offsetEvents);
    assertTrue(controller.truncateStream(SCOPE, STREAM, streamCut).join());
    // Test the batch client when we select to start reading a Stream from a truncation point.
    StreamCut initialPosition = streamManager.getStreamInfo(SCOPE, stream.getStreamName()).getHeadStreamCut();
    List<SegmentRange> newRanges = Lists.newArrayList(batchClient.getSegments(stream, initialPosition, StreamCut.UNBOUNDED).getIterator());
    assertEquals("Expected events read: ", (totalEvents - offsetEvents) + totalEvents * batchIterations, readFromRanges(newRanges, batchClient));
    log.debug("Events correctly read from Stream: simple batch client test passed.");
}

Example 12 with SegmentRange

use of io.pravega.client.batch.SegmentRange in project pravega by pravega.

the class BatchClientSimpleTest method readFromRanges.

// Start utils region
private int readFromRanges(List<SegmentRange> ranges, BatchClientFactory batchClient) {
    List<CompletableFuture<Integer>> eventCounts = ranges.parallelStream().map(range -> CompletableFuture.supplyAsync(() -> batchClient.readSegment(range, new JavaSerializer<>())).thenApplyAsync(segmentIterator -> {
        log.debug("Thread " + Thread.currentThread().getId() + " reading events.");
        int numEvents = Lists.newArrayList(segmentIterator).size();
        segmentIterator.close();
        return numEvents;
    })).collect(Collectors.toList());
    return eventCounts.stream().map(CompletableFuture::join).mapToInt(Integer::intValue).sum();
}

Example 13 with SegmentRange

use of io.pravega.client.batch.SegmentRange in project pravega by pravega.

the class BatchClientImplTest method testGetSegmentsWithStreamCut.

@Test(timeout = 5000)
public void testGetSegmentsWithStreamCut() throws Exception {
    PravegaNodeUri location = new PravegaNodeUri("localhost", 0);
    MockConnectionFactoryImpl connectionFactory = getMockConnectionFactory(location);
    MockController mockController = new MockController(location.getEndpoint(), location.getPort(), connectionFactory, false);
    Stream stream = createStream(SCOPE, STREAM, 3, mockController);
    @Cleanup BatchClientFactoryImpl client = new BatchClientFactoryImpl(mockController, ClientConfig.builder().maxConnectionsPerSegmentStore(1).build(), connectionFactory);
    Iterator<SegmentRange> boundedSegments = client.getSegments(stream, getStreamCut(5L, 0, 1, 2), getStreamCut(15L, 0, 1, 2)).getIterator();
    assertTrue(boundedSegments.hasNext());
    assertEquals(0L, boundedSegments.next().asImpl().getSegment().getSegmentId());
    assertTrue(boundedSegments.hasNext());
    assertEquals(1L, boundedSegments.next().asImpl().getSegment().getSegmentId());
    assertTrue(boundedSegments.hasNext());
    assertEquals(2L, boundedSegments.next().asImpl().getSegment().getSegmentId());
    assertFalse(boundedSegments.hasNext());
}

Example 14 with SegmentRange

use of io.pravega.client.batch.SegmentRange in project pravega by pravega.

the class BatchClientImplTest method testGetSegmentsWithMultipleSegments.

@Test(timeout = 5000)
public void testGetSegmentsWithMultipleSegments() throws Exception {
    PravegaNodeUri location = new PravegaNodeUri("localhost", 0);
    @Cleanup MockConnectionFactoryImpl connectionFactory = getMockConnectionFactory(location);
    MockController mockController = new MockController(location.getEndpoint(), location.getPort(), connectionFactory, false);
    MockController stubbedController = spy(mockController);
    Stream stream = createStream(SCOPE, STREAM, 2, stubbedController);
    Set<Segment> segments = ImmutableSet.<Segment>builder().add(new Segment(SCOPE, STREAM, 0L), new Segment(SCOPE, STREAM, 1L), new Segment(SCOPE, STREAM, 2L)).build();
    // Setup mock.
    doReturn(CompletableFuture.completedFuture(new StreamSegmentSuccessors(segments, ""))).when(stubbedController).getSegments(any(StreamCut.class), any(StreamCut.class));
    @Cleanup BatchClientFactoryImpl client = new BatchClientFactoryImpl(stubbedController, ClientConfig.builder().maxConnectionsPerSegmentStore(1).build(), connectionFactory);
    Iterator<SegmentRange> segmentIterator = client.getSegments(stream, null, null).getIterator();
    assertTrue(segmentIterator.hasNext());
    assertEquals(0L, segmentIterator.next().asImpl().getSegment().getSegmentId());
    assertTrue(segmentIterator.hasNext());
    assertEquals(1L, segmentIterator.next().asImpl().getSegment().getSegmentId());
    assertTrue(segmentIterator.hasNext());
    assertEquals(2L, segmentIterator.next().asImpl().getSegment().getSegmentId());
    assertFalse(segmentIterator.hasNext());
}

Example 15 with SegmentRange

use of io.pravega.client.batch.SegmentRange in project pravega by pravega.

the class BatchClientImplTest method testGetSegmentsWithNullStreamCut.

@Test(timeout = 5000)
public void testGetSegmentsWithNullStreamCut() throws Exception {
    PravegaNodeUri location = new PravegaNodeUri("localhost", 0);
    @Cleanup MockConnectionFactoryImpl connectionFactory = getMockConnectionFactory(location);
    MockController mockController = new MockController(location.getEndpoint(), location.getPort(), connectionFactory, false);
    Stream stream = createStream(SCOPE, STREAM, 3, mockController);
    @Cleanup BatchClientFactoryImpl client = new BatchClientFactoryImpl(mockController, ClientConfig.builder().maxConnectionsPerSegmentStore(1).build(), connectionFactory);
    Iterator<SegmentRange> segments = client.getSegments(stream, null, null).getIterator();
    assertTrue(segments.hasNext());
    assertEquals(0L, segments.next().asImpl().getSegment().getSegmentId());
    assertTrue(segments.hasNext());
    assertEquals(1L, segments.next().asImpl().getSegment().getSegmentId());
    assertTrue(segments.hasNext());
    assertEquals(2L, segments.next().asImpl().getSegment().getSegmentId());
    assertFalse(segments.hasNext());
}