Examples with SegmentRange io.pravega.client.batch.SegmentRange used on opensource projects

Example 6 with SegmentRange

use of io.pravega.client.batch.SegmentRange in project pravega by pravega.

the class BatchClientTest method testBatchClientWithStreamTruncationPostGetSegments.

@Test(expected = TruncatedDataException.class, timeout = 50000)
public void testBatchClientWithStreamTruncationPostGetSegments() throws InterruptedException, ExecutionException {
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, clientConfig);
    createTestStreamWithEvents(clientFactory);
    @Cleanup BatchClientFactory batchClient = BatchClientFactory.withScope(SCOPE, clientConfig);
    // 1. Fetch Segments.
    ArrayList<SegmentRange> segmentsPostTruncation = Lists.newArrayList(batchClient.getSegments(Stream.of(SCOPE, STREAM), StreamCut.UNBOUNDED, StreamCut.UNBOUNDED).getIterator());
    // 2. Create a StreamCut at the end of segment 0 ( offset = 3 * 30 = 90)
    StreamCut streamCut90L = new StreamCutImpl(Stream.of(SCOPE, STREAM), ImmutableMap.of(new Segment(SCOPE, STREAM, 0), 90L));
    // 3. Truncate stream.
    assertTrue("truncate stream", controllerWrapper.getController().truncateStream(SCOPE, STREAM, streamCut90L).join());
    // 4. Use SegmentRange obtained before truncation.
    SegmentRange s0 = segmentsPostTruncation.stream().filter(segmentRange -> segmentRange.getSegmentId() == 0L).findFirst().get();
    // 5. Read non existent segment.
    List<String> eventList = new ArrayList<>();
    @Cleanup SegmentIterator<String> segmentIterator = batchClient.readSegment(s0, serializer);
    eventList.addAll(Lists.newArrayList(segmentIterator));
}

Example 7 with SegmentRange

use of io.pravega.client.batch.SegmentRange in project pravega by pravega.

the class BatchClientTest method listAndReadSegmentsUsingBatchClient.

protected void listAndReadSegmentsUsingBatchClient(String scopeName, String streamName, ClientConfig config) throws InterruptedException, ExecutionException {
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scopeName, config);
    createTestStreamWithEvents(clientFactory);
    log.info("Done creating test event stream with test events");
    @Cleanup BatchClientFactory batchClient = BatchClientFactory.withScope(scopeName, config);
    // List out all the segments in the stream.
    ArrayList<SegmentRange> segments = Lists.newArrayList(batchClient.getSegments(Stream.of(scopeName, streamName), null, null).getIterator());
    assertEquals("Expected number of segments", 6, segments.size());
    // Batch read all events from stream.
    List<String> batchEventList = new ArrayList<>();
    segments.forEach(segInfo -> {
        @Cleanup SegmentIterator<String> segmentIterator = batchClient.readSegment(segInfo, serializer);
        batchEventList.addAll(Lists.newArrayList(segmentIterator));
    });
    assertEquals("Event count", 9, batchEventList.size());
    // Read from a given offset.
    Segment seg0 = new Segment(scopeName, streamName, 0);
    SegmentRange seg0Info = SegmentRangeImpl.builder().segment(seg0).startOffset(60).endOffset(90).build();
    @Cleanup SegmentIterator<String> seg0Iterator = batchClient.readSegment(seg0Info, serializer);
    ArrayList<String> dataAtOffset = Lists.newArrayList(seg0Iterator);
    assertEquals(1, dataAtOffset.size());
    assertEquals(DATA_OF_SIZE_30, dataAtOffset.get(0));
}

Example 8 with SegmentRange

use of io.pravega.client.batch.SegmentRange in project pravega by pravega.

the class DelegationTokenTest method testBatchClientDelegationTokenRenewal.

/**
 * This test verifies that a batch client continues to read events as a result of automatic delegation token
 * renewal, after the initial delegation token it uses expires.
 * <p>
 * We use an extraordinarily high test timeout and read timeouts to account for any inordinate delays that may be
 * encountered in testing environments.
 */
@Test(timeout = 50000)
public void testBatchClientDelegationTokenRenewal() throws InterruptedException {
    // Delegation token renewal threshold is 5 seconds, so we are using 6 seconds as Token TTL so that token doesn't
    // get renewed before each use.
    @Cleanup ClusterWrapper pravegaCluster = ClusterWrapper.builder().authEnabled(true).tokenTtlInSeconds(6).build();
    pravegaCluster.start();
    final String scope = "testscope";
    final String streamName = "teststream";
    final ClientConfig clientConfig = ClientConfig.builder().controllerURI(URI.create(pravegaCluster.controllerUri())).credentials(new DefaultCredentials("1111_aaaa", "admin")).build();
    log.debug("Done creating client config.");
    // Create Scope and Stream.
    createScopeStream(scope, streamName, 1, clientConfig);
    // write ten Events.
    writeTenEvents(scope, streamName, clientConfig);
    // Now, read the events from the stream using Batch client.
    @Cleanup BatchClientFactory batchClientFactory = BatchClientFactory.withScope(scope, clientConfig);
    List<SegmentRange> segmentRanges = Lists.newArrayList(batchClientFactory.getSegments(Stream.of(scope, streamName), StreamCut.UNBOUNDED, StreamCut.UNBOUNDED).getIterator());
    assertEquals("The number of segments in the stream is 1", 1, segmentRanges.size());
    SegmentIterator<String> segmentIterator = batchClientFactory.readSegment(segmentRanges.get(0), new JavaSerializer<>());
    int eventReadCount = 0;
    while (segmentIterator.hasNext()) {
        // We are keeping sleep time relatively large, just to make sure that the delegation token expires
        // midway.
        Thread.sleep(500);
        String event = segmentIterator.next();
        log.debug("Done reading event {}", event);
        eventReadCount++;
    }
    // Assert that we end up reading 10 events even though delegation token must have expired midway.
    // 
    // To look for evidence of delegation token renewal check the logs for the following message:
    // - "Token is nearing expiry, so refreshing it"
    assertEquals(10, eventReadCount);
}

Example 9 with SegmentRange

use of io.pravega.client.batch.SegmentRange in project pravega by pravega.

the class BatchClientImpl method getStreamSegmentInfo.

private StreamSegmentsIterator getStreamSegmentInfo(final StreamCut startStreamCut, final StreamCut endStreamCut) {
    log.debug("Start stream cut: {}, End stream cut: {}", startStreamCut, endStreamCut);
    StreamSegmentsInfoImpl.validateStreamCuts(startStreamCut, endStreamCut);
    final SortedSet<Segment> segmentSet = new TreeSet<>();
    StreamSegmentSuccessors segments = getAndHandleExceptions(controller.getSegments(startStreamCut, endStreamCut), RuntimeException::new);
    segmentSet.addAll(segments.getSegments());
    synchronized (this) {
        latestDelegationToken.set(segments.getDelegationToken());
    }
    log.debug("List of Segments between the start and end stream cuts : {}", segmentSet);
    Iterator<SegmentRange> iterator = Iterators.transform(segmentSet.iterator(), s -> getSegmentRange(s, startStreamCut, endStreamCut));
    return StreamSegmentsInfoImpl.builder().segmentRangeIterator(iterator).startStreamCut(startStreamCut).endStreamCut(endStreamCut).build();
}

Example 10 with SegmentRange

use of io.pravega.client.batch.SegmentRange in project pravega by pravega.

the class BatchClientImplTest method testSegmentIterator.

@Test(timeout = 5000)
public void testSegmentIterator() throws ConnectionFailedException {
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    ClientConnection connection = Mockito.mock(ClientConnection.class);
    PravegaNodeUri location = new PravegaNodeUri("localhost", 0);
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            CreateSegment request = (CreateSegment) invocation.getArgument(0);
            connectionFactory.getProcessor(location).process(new SegmentCreated(request.getRequestId(), request.getSegment()));
            return null;
        }
    }).when(connection).send(Mockito.any(CreateSegment.class));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            GetStreamSegmentInfo request = (GetStreamSegmentInfo) invocation.getArgument(0);
            connectionFactory.getProcessor(location).process(new StreamSegmentInfo(request.getRequestId(), request.getSegmentName(), true, false, false, 0, 0, 0));
            return null;
        }
    }).when(connection).send(Mockito.any(GetStreamSegmentInfo.class));
    connectionFactory.provideConnection(location, connection);
    MockController mockController = new MockController(location.getEndpoint(), location.getPort(), connectionFactory);
    BatchClientImpl client = new BatchClientImpl(mockController, connectionFactory);
    Stream stream = new StreamImpl("scope", "stream");
    mockController.createScope("scope");
    mockController.createStream(StreamConfiguration.builder().scope("scope").streamName("stream").scalingPolicy(ScalingPolicy.fixed(3)).build()).join();
    Iterator<SegmentRange> segments = client.getSegments(stream, null, null).getIterator();
    assertTrue(segments.hasNext());
    assertEquals(0, segments.next().asImpl().getSegment().getSegmentNumber());
    assertTrue(segments.hasNext());
    assertEquals(1, segments.next().asImpl().getSegment().getSegmentNumber());
    assertTrue(segments.hasNext());
    assertEquals(2, segments.next().asImpl().getSegment().getSegmentNumber());
    assertFalse(segments.hasNext());
}