Examples with ClientFactoryImpl io.pravega.client.stream.impl.ClientFactoryImpl used on opensource projects

Example 16 with ClientFactoryImpl

use of io.pravega.client.stream.impl.ClientFactoryImpl in project pravega by pravega.

the class RetentionTest method retentionTest.

private CompletableFuture<Void> retentionTest(String streamName, boolean sizeBased) throws Exception {
    return CompletableFuture.runAsync(() -> {
        final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
        @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
        ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
        @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
        log.info("Invoking Writer test with Controller URI: {}", controllerURI);
        // create a writer
        @Cleanup EventStreamWriter<Serializable> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
        // write an event
        String writeEvent = "event";
        writer.writeEvent(writeEvent);
        if (sizeBased) {
            // since truncation always happens at an event boundary, for size based, we will write two events,
            // so that truncation can happen at the first event.
            writer.writeEvent(writeEvent);
        }
        writer.flush();
        log.debug("Writing event: {} ", writeEvent);
        // sleep for 5 mins -- retention frequency is set to 2 minutes. So in 5 minutes we should definitely have
        // 2 retention cycles, with a stream cut being computed in first cycle and truncation happening on the
        // previously computed streamcut in second cycle.
        // for time based retention, we wrote one event, which would get truncated.
        // for size based retention we wrote two events such that stream would retain at least 1 byte as prescribed by
        // the policy
        Exceptions.handleInterrupted(() -> Thread.sleep(5 * 60 * 1000));
        // create a reader
        ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, clientConfig);
        String groupName = READER_GROUP + streamName;
        groupManager.createReaderGroup(groupName, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, streamName)).build());
        EventStreamReader<String> reader = clientFactory.createReader(UUID.randomUUID().toString(), groupName, new JavaSerializer<>(), ReaderConfig.builder().build());
        if (sizeBased) {
            // we should read one write event back from the stream.
            String event = reader.readNextEvent(6000).getEvent();
            assertEquals(event, writeEvent);
        }
        // verify reader functionality is unaffected post truncation
        String event = "newEvent";
        writer.writeEvent(event);
        log.info("Writing event: {}", event);
        Assert.assertEquals(event, reader.readNextEvent(6000).getEvent());
        log.debug("The stream is already truncated.Simple retention test passed.");
    });
}

Example 17 with ClientFactoryImpl

use of io.pravega.client.stream.impl.ClientFactoryImpl in project pravega by pravega.

the class AutoScaleTest method scaleUpTest.

/**
 * Invoke the simple scale up Test, produce traffic from multiple writers in parallel.
 * The test will periodically check if a scale event has occurred by talking to controller via
 * controller client.
 *
 * @throws InterruptedException if interrupted
 * @throws URISyntaxException   If URI is invalid
 */
private CompletableFuture<Void> scaleUpTest() {
    ClientFactoryImpl clientFactory = getClientFactory();
    ControllerImpl controller = getController();
    final AtomicBoolean exit = new AtomicBoolean(false);
    createWriters(clientFactory, 6, SCOPE, SCALE_UP_STREAM_NAME);
    // overall wait for test to complete in 260 seconds (4.2 minutes) or scale up, whichever happens first.
    return Retry.withExpBackoff(10, 10, 30, Duration.ofSeconds(10).toMillis()).retryingOn(ScaleOperationNotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments(SCOPE, SCALE_UP_STREAM_NAME).thenAccept(x -> {
        log.debug("size ==" + x.getSegments().size());
        if (x.getSegments().size() == 1) {
            throw new ScaleOperationNotDoneException();
        } else {
            log.info("scale up done successfully");
            exit.set(true);
        }
    }), scaleExecutorService);
}

Example 18 with ClientFactoryImpl

use of io.pravega.client.stream.impl.ClientFactoryImpl in project pravega by pravega.

the class MultiReaderTxnWriterWithFailoverTest method setup.

@Before
public void setup() {
    // Get zk details to verify if controller, SSS are running
    Service zkService = Utils.createZookeeperService();
    List<URI> zkUris = zkService.getServiceDetails();
    log.debug("Zookeeper service details: {}", zkUris);
    // get the zk ip details and pass it to  host, controller
    URI zkUri = zkUris.get(0);
    // Verify controller is running.
    controllerInstance = Utils.createPravegaControllerService(zkUri);
    assertTrue(controllerInstance.isRunning());
    List<URI> conURIs = controllerInstance.getServiceDetails();
    log.info("Pravega Controller service instance details: {}", conURIs);
    // Fetch all the RPC endpoints and construct the client URIs.
    final List<String> uris = conURIs.stream().filter(ISGRPC).map(URI::getAuthority).collect(Collectors.toList());
    controllerURIDirect = URI.create((Utils.TLS_AND_AUTH_ENABLED ? TLS : TCP) + String.join(",", uris));
    log.info("Controller Service direct URI: {}", controllerURIDirect);
    // Verify segment store is running.
    segmentStoreInstance = Utils.createPravegaSegmentStoreService(zkUri, controllerURIDirect);
    assertTrue(segmentStoreInstance.isRunning());
    log.info("Pravega Segmentstore service instance details: {}", segmentStoreInstance.getServiceDetails());
    // executor service
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(NUM_READERS + NUM_WRITERS + 2, "MultiReaderTxnWriterWithFailoverTest-main");
    controllerExecutorService = ExecutorServiceHelpers.newScheduledThreadPool(2, "MultiReaderTxnWriterWithFailoverTest-controller");
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURIDirect);
    // get Controller Uri
    controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).maxBackoffMillis(5000).build(), controllerExecutorService);
    testState = new TestState(true);
    // read and write count variables
    streamManager = new StreamManagerImpl(clientConfig);
    createScopeAndStream(scope, STREAM_NAME, config, streamManager);
    log.info("Scope passed to client factory {}", scope);
    clientFactory = new ClientFactoryImpl(scope, controller, new SocketConnectionFactoryImpl(clientConfig));
    readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
}

Example 19 with ClientFactoryImpl

use of io.pravega.client.stream.impl.ClientFactoryImpl in project pravega by pravega.

the class OffsetTruncationTest method offsetTruncationTest.

/**
 * This test verifies that truncation works specifying an offset that applies to multiple segments. To this end,
 * the test first writes a set of events on a Stream (with multiple segments) and truncates it at a specified offset
 * (truncatedEvents). The tests asserts that readers first get a TruncatedDataException as they are attempting to
 * read a truncated segment, and then they only read the remaining events that have not been truncated.
 */
@Test
public void offsetTruncationTest() {
    final int totalEvents = 200;
    final int truncatedEvents = 50;
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    log.info("Invoking offsetTruncationTest test with Controller URI: {}", controllerURI);
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, clientConfig);
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().stream(Stream.of(SCOPE, STREAM)).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup(READER_GROUP);
    // Write events to the Stream.
    writeEvents(clientFactory, STREAM, totalEvents);
    // Instantiate readers to consume from Stream up to truncatedEvents.
    List<CompletableFuture<Integer>> futures = readEventFutures(clientFactory, READER_GROUP, PARALLELISM, truncatedEvents);
    Futures.allOf(futures).join();
    // Ensure that we have read all the events required before initiating the checkpoint.
    assertEquals("Number of events read is not the expected one.", (Integer) truncatedEvents, futures.stream().map(f -> Futures.getAndHandleExceptions(f, RuntimeException::new)).reduce(Integer::sum).get());
    // Perform truncation on stream segment.
    Checkpoint cp = readerGroup.initiateCheckpoint("truncationCheckpoint", executor).join();
    StreamCut streamCut = cp.asImpl().getPositions().values().iterator().next();
    StreamCut alternativeStreamCut = readerGroup.generateStreamCuts(executor).join().get(Stream.of(SCOPE, STREAM));
    assertEquals("StreamCuts for reader group differ depending on how they are generated.", streamCut, alternativeStreamCut);
    assertTrue(streamManager.truncateStream(SCOPE, STREAM, streamCut));
    // Just after the truncation, read events from the offset defined in truncate call onwards.
    final String newGroupName = READER_GROUP + "new";
    groupManager.createReaderGroup(newGroupName, ReaderGroupConfig.builder().stream(Stream.of(SCOPE, STREAM)).build());
    futures = readEventFutures(clientFactory, newGroupName, PARALLELISM);
    Futures.allOf(futures).join();
    assertEquals("Expected read events: ", totalEvents - truncatedEvents, (int) futures.stream().map(CompletableFuture::join).reduce(Integer::sum).get());
    log.debug("The stream has been successfully truncated at event {}. Offset truncation test passed.", truncatedEvents);
}

Example 20 with ClientFactoryImpl

use of io.pravega.client.stream.impl.ClientFactoryImpl in project pravega by pravega.

the class ReadTxnWriteScaleWithFailoverTest method setup.

@Before
public void setup() {
    // Get zk details to verify if controller, segmentstore are running
    Service zkService = Utils.createZookeeperService();
    List<URI> zkUris = zkService.getServiceDetails();
    log.debug("Zookeeper service details: {}", zkUris);
    // get the zk ip details and pass it to  host, controller
    URI zkUri = zkUris.get(0);
    // Verify controller is running.
    controllerInstance = Utils.createPravegaControllerService(zkUri);
    assertTrue(controllerInstance.isRunning());
    List<URI> conURIs = controllerInstance.getServiceDetails();
    log.info("Pravega Controller service instance details: {}", conURIs);
    // Fetch all the RPC endpoints and construct the client URIs.
    final List<String> uris = conURIs.stream().filter(ISGRPC).map(URI::getAuthority).collect(Collectors.toList());
    controllerURIDirect = URI.create((Utils.TLS_AND_AUTH_ENABLED ? TLS : TCP) + String.join(",", uris));
    log.info("Controller Service direct URI: {}", controllerURIDirect);
    // Verify segment store is running.
    segmentStoreInstance = Utils.createPravegaSegmentStoreService(zkUri, controllerURIDirect);
    assertTrue(segmentStoreInstance.isRunning());
    log.info("Pravega Segmentstore service instance details: {}", segmentStoreInstance.getServiceDetails());
    // num. of readers + num. of writers + 1 to run checkScale operation
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(NUM_READERS + NUM_WRITERS + 1, "ReadTxnWriteScaleWithFailoverTest-main");
    controllerExecutorService = ExecutorServiceHelpers.newScheduledThreadPool(2, "ReadTxnWriteScaleWithFailoverTest-controller");
    ClientConfig clientConfig = Utils.buildClientConfig(controllerURIDirect);
    // get Controller Uri
    controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).maxBackoffMillis(5000).build(), controllerExecutorService);
    testState = new TestState(true);
    streamManager = new StreamManagerImpl(clientConfig);
    createScopeAndStream(scope, stream, config, streamManager);
    log.info("Scope passed to client factory {}", scope);
    clientFactory = new ClientFactoryImpl(scope, controller, clientConfig);
    readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
}