Examples with BatchClientFactory io.pravega.client.BatchClientFactory used on opensource projects

Example 1 with BatchClientFactory

use of io.pravega.client.BatchClientFactory in project pravega by pravega.

the class BatchClientTest method testBatchClientWithStreamTruncation.

@Test(timeout = 50000)
@SuppressWarnings("deprecation")
public void testBatchClientWithStreamTruncation() throws InterruptedException, ExecutionException {
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, clientConfig);
    createTestStreamWithEvents(clientFactory);
    log.info("Done creating a test stream with test events");
    @Cleanup BatchClientFactory batchClient = BatchClientFactory.withScope(SCOPE, clientConfig);
    log.info("Done creating batch client factory");
    // 1. Create a StreamCut after 2 events(offset = 2 * 30 = 60).
    StreamCut streamCut60L = new StreamCutImpl(Stream.of(SCOPE, STREAM), ImmutableMap.of(new Segment(SCOPE, STREAM, 0), 60L));
    // 2. Truncate stream.
    assertTrue("truncate stream", controllerWrapper.getController().truncateStream(SCOPE, STREAM, streamCut60L).join());
    // 3a. Fetch Segments using StreamCut.UNBOUNDED>
    ArrayList<SegmentRange> segmentsPostTruncation1 = Lists.newArrayList(batchClient.getSegments(Stream.of(SCOPE, STREAM), StreamCut.UNBOUNDED, StreamCut.UNBOUNDED).getIterator());
    // 3b. Fetch Segments using getStreamInfo() api.
    StreamInfo streamInfo = streamManager.getStreamInfo(SCOPE, STREAM);
    ArrayList<SegmentRange> segmentsPostTruncation2 = Lists.newArrayList(batchClient.getSegments(Stream.of(SCOPE, STREAM), streamInfo.getHeadStreamCut(), streamInfo.getTailStreamCut()).getIterator());
    // Validate results.
    validateSegmentCountAndEventCount(batchClient, segmentsPostTruncation1);
    validateSegmentCountAndEventCount(batchClient, segmentsPostTruncation2);
}

Example 2 with BatchClientFactory

use of io.pravega.client.BatchClientFactory in project pravega by pravega.

the class BatchClientTest method testBatchClientWithStreamTruncationPostGetSegments.

@Test(expected = TruncatedDataException.class, timeout = 50000)
public void testBatchClientWithStreamTruncationPostGetSegments() throws InterruptedException, ExecutionException {
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, clientConfig);
    createTestStreamWithEvents(clientFactory);
    @Cleanup BatchClientFactory batchClient = BatchClientFactory.withScope(SCOPE, clientConfig);
    // 1. Fetch Segments.
    ArrayList<SegmentRange> segmentsPostTruncation = Lists.newArrayList(batchClient.getSegments(Stream.of(SCOPE, STREAM), StreamCut.UNBOUNDED, StreamCut.UNBOUNDED).getIterator());
    // 2. Create a StreamCut at the end of segment 0 ( offset = 3 * 30 = 90)
    StreamCut streamCut90L = new StreamCutImpl(Stream.of(SCOPE, STREAM), ImmutableMap.of(new Segment(SCOPE, STREAM, 0), 90L));
    // 3. Truncate stream.
    assertTrue("truncate stream", controllerWrapper.getController().truncateStream(SCOPE, STREAM, streamCut90L).join());
    // 4. Use SegmentRange obtained before truncation.
    SegmentRange s0 = segmentsPostTruncation.stream().filter(segmentRange -> segmentRange.getSegmentId() == 0L).findFirst().get();
    // 5. Read non existent segment.
    List<String> eventList = new ArrayList<>();
    @Cleanup SegmentIterator<String> segmentIterator = batchClient.readSegment(s0, serializer);
    eventList.addAll(Lists.newArrayList(segmentIterator));
}

Example 3 with BatchClientFactory

use of io.pravega.client.BatchClientFactory in project pravega by pravega.

the class BatchClientTest method listAndReadSegmentsUsingBatchClient.

protected void listAndReadSegmentsUsingBatchClient(String scopeName, String streamName, ClientConfig config) throws InterruptedException, ExecutionException {
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scopeName, config);
    createTestStreamWithEvents(clientFactory);
    log.info("Done creating test event stream with test events");
    @Cleanup BatchClientFactory batchClient = BatchClientFactory.withScope(scopeName, config);
    // List out all the segments in the stream.
    ArrayList<SegmentRange> segments = Lists.newArrayList(batchClient.getSegments(Stream.of(scopeName, streamName), null, null).getIterator());
    assertEquals("Expected number of segments", 6, segments.size());
    // Batch read all events from stream.
    List<String> batchEventList = new ArrayList<>();
    segments.forEach(segInfo -> {
        @Cleanup SegmentIterator<String> segmentIterator = batchClient.readSegment(segInfo, serializer);
        batchEventList.addAll(Lists.newArrayList(segmentIterator));
    });
    assertEquals("Event count", 9, batchEventList.size());
    // Read from a given offset.
    Segment seg0 = new Segment(scopeName, streamName, 0);
    SegmentRange seg0Info = SegmentRangeImpl.builder().segment(seg0).startOffset(60).endOffset(90).build();
    @Cleanup SegmentIterator<String> seg0Iterator = batchClient.readSegment(seg0Info, serializer);
    ArrayList<String> dataAtOffset = Lists.newArrayList(seg0Iterator);
    assertEquals(1, dataAtOffset.size());
    assertEquals(DATA_OF_SIZE_30, dataAtOffset.get(0));
}

Example 4 with BatchClientFactory

use of io.pravega.client.BatchClientFactory in project pravega by pravega.

the class DelegationTokenTest method testBatchClientDelegationTokenRenewal.

/**
 * This test verifies that a batch client continues to read events as a result of automatic delegation token
 * renewal, after the initial delegation token it uses expires.
 * <p>
 * We use an extraordinarily high test timeout and read timeouts to account for any inordinate delays that may be
 * encountered in testing environments.
 */
@Test(timeout = 50000)
public void testBatchClientDelegationTokenRenewal() throws InterruptedException {
    // Delegation token renewal threshold is 5 seconds, so we are using 6 seconds as Token TTL so that token doesn't
    // get renewed before each use.
    @Cleanup ClusterWrapper pravegaCluster = ClusterWrapper.builder().authEnabled(true).tokenTtlInSeconds(6).build();
    pravegaCluster.start();
    final String scope = "testscope";
    final String streamName = "teststream";
    final ClientConfig clientConfig = ClientConfig.builder().controllerURI(URI.create(pravegaCluster.controllerUri())).credentials(new DefaultCredentials("1111_aaaa", "admin")).build();
    log.debug("Done creating client config.");
    // Create Scope and Stream.
    createScopeStream(scope, streamName, 1, clientConfig);
    // write ten Events.
    writeTenEvents(scope, streamName, clientConfig);
    // Now, read the events from the stream using Batch client.
    @Cleanup BatchClientFactory batchClientFactory = BatchClientFactory.withScope(scope, clientConfig);
    List<SegmentRange> segmentRanges = Lists.newArrayList(batchClientFactory.getSegments(Stream.of(scope, streamName), StreamCut.UNBOUNDED, StreamCut.UNBOUNDED).getIterator());
    assertEquals("The number of segments in the stream is 1", 1, segmentRanges.size());
    SegmentIterator<String> segmentIterator = batchClientFactory.readSegment(segmentRanges.get(0), new JavaSerializer<>());
    int eventReadCount = 0;
    while (segmentIterator.hasNext()) {
        // We are keeping sleep time relatively large, just to make sure that the delegation token expires
        // midway.
        Thread.sleep(500);
        String event = segmentIterator.next();
        log.debug("Done reading event {}", event);
        eventReadCount++;
    }
    // Assert that we end up reading 10 events even though delegation token must have expired midway.
    // 
    // To look for evidence of delegation token renewal check the logs for the following message:
    // - "Token is nearing expiry, so refreshing it"
    assertEquals(10, eventReadCount);
}

Example 5 with BatchClientFactory

use of io.pravega.client.BatchClientFactory in project pravega by pravega.

the class BatchClientSimpleTest method batchClientSimpleTest.

/**
 * This test verifies the basic functionality of {@link BatchClientFactory}, including stream metadata checks, segment
 * counts, parallel segment reads and reads with offsets using stream cuts.
 */
@Test
@SuppressWarnings("deprecation")
public void batchClientSimpleTest() {
    final int totalEvents = RG_PARALLELISM * 100;
    final int offsetEvents = RG_PARALLELISM * 20;
    final int batchIterations = 4;
    final Stream stream = Stream.of(SCOPE, STREAM);
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    @Cleanup BatchClientFactory batchClient = BatchClientFactory.withScope(SCOPE, clientConfig);
    log.info("Invoking batchClientSimpleTest test with Controller URI: {}", controllerURI);
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, clientConfig);
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(SCOPE + "/" + STREAM).build());
    ReaderGroup readerGroup = groupManager.getReaderGroup(READER_GROUP);
    log.info("Writing events to stream");
    // Write events to the Stream.
    writeEvents(clientFactory, STREAM, totalEvents);
    // Instantiate readers to consume from Stream up to truncatedEvents.
    List<CompletableFuture<Integer>> futures = readEventFutures(clientFactory, READER_GROUP, RG_PARALLELISM, offsetEvents);
    Futures.allOf(futures).join();
    // Create a stream cut on the specified offset position.
    Checkpoint cp = readerGroup.initiateCheckpoint("batchClientCheckpoint", executor).join();
    StreamCut streamCut = cp.asImpl().getPositions().values().iterator().next();
    // Instantiate the batch client and assert it provides correct stream info.
    log.debug("Creating batch client.");
    StreamInfo streamInfo = streamManager.getStreamInfo(SCOPE, stream.getStreamName());
    log.debug("Validating stream metadata fields.");
    assertEquals("Expected Stream name: ", STREAM, streamInfo.getStreamName());
    assertEquals("Expected Scope name: ", SCOPE, streamInfo.getScope());
    // Test that we can read events from parallel segments from an offset onwards.
    log.debug("Reading events from stream cut onwards in parallel.");
    List<SegmentRange> ranges = Lists.newArrayList(batchClient.getSegments(stream, streamCut, StreamCut.UNBOUNDED).getIterator());
    assertEquals("Expected events read: ", totalEvents - offsetEvents, readFromRanges(ranges, batchClient));
    // Emulate the behavior of Hadoop client: i) Get tail of Stream, ii) Read from current point until tail, iii) repeat.
    log.debug("Reading in batch iterations.");
    StreamCut currentTailStreamCut = streamManager.getStreamInfo(SCOPE, stream.getStreamName()).getTailStreamCut();
    int readEvents = 0;
    for (int i = 0; i < batchIterations; i++) {
        writeEvents(clientFactory, STREAM, totalEvents);
        // Read all the existing events in parallel segments from the previous tail to the current one.
        ranges = Lists.newArrayList(batchClient.getSegments(stream, currentTailStreamCut, StreamCut.UNBOUNDED).getIterator());
        assertEquals("Expected number of segments: ", RG_PARALLELISM, ranges.size());
        readEvents += readFromRanges(ranges, batchClient);
        log.debug("Events read in parallel so far: {}.", readEvents);
        currentTailStreamCut = streamManager.getStreamInfo(SCOPE, stream.getStreamName()).getTailStreamCut();
    }
    assertEquals("Expected events read: .", totalEvents * batchIterations, readEvents);
    // Truncate the stream in first place.
    log.debug("Truncating stream at event {}.", offsetEvents);
    assertTrue(controller.truncateStream(SCOPE, STREAM, streamCut).join());
    // Test the batch client when we select to start reading a Stream from a truncation point.
    StreamCut initialPosition = streamManager.getStreamInfo(SCOPE, stream.getStreamName()).getHeadStreamCut();
    List<SegmentRange> newRanges = Lists.newArrayList(batchClient.getSegments(stream, initialPosition, StreamCut.UNBOUNDED).getIterator());
    assertEquals("Expected events read: ", (totalEvents - offsetEvents) + totalEvents * batchIterations, readFromRanges(newRanges, batchClient));
    log.debug("Events correctly read from Stream: simple batch client test passed.");
}