Examples with ControllerImpl io.pravega.client.control.impl.ControllerImpl used on opensource projects

Example 11 with ControllerImpl

use of io.pravega.client.control.impl.ControllerImpl in project pravega by pravega.

the class MultiReaderTxnWriterWithFailoverTest method setup.

@Before
public void setup() {
    // Get zk details to verify if controller, SSS are running
    Service zkService = Utils.createZookeeperService();
    List<URI> zkUris = zkService.getServiceDetails();
    log.debug("Zookeeper service details: {}", zkUris);
    // get the zk ip details and pass it to  host, controller
    URI zkUri = zkUris.get(0);
    // Verify controller is running.
    controllerInstance = Utils.createPravegaControllerService(zkUri);
    assertTrue(controllerInstance.isRunning());
    List<URI> conURIs = controllerInstance.getServiceDetails();
    log.info("Pravega Controller service instance details: {}", conURIs);
    // Fetch all the RPC endpoints and construct the client URIs.
    final List<String> uris = conURIs.stream().filter(ISGRPC).map(URI::getAuthority).collect(Collectors.toList());
    controllerURIDirect = URI.create((Utils.TLS_AND_AUTH_ENABLED ? TLS : TCP) + String.join(",", uris));
    log.info("Controller Service direct URI: {}", controllerURIDirect);
    // Verify segment store is running.
    segmentStoreInstance = Utils.createPravegaSegmentStoreService(zkUri, controllerURIDirect);
    assertTrue(segmentStoreInstance.isRunning());
    log.info("Pravega Segmentstore service instance details: {}", segmentStoreInstance.getServiceDetails());
    // executor service
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(NUM_READERS + NUM_WRITERS + 2, "MultiReaderTxnWriterWithFailoverTest-main");
    controllerExecutorService = ExecutorServiceHelpers.newScheduledThreadPool(2, "MultiReaderTxnWriterWithFailoverTest-controller");
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURIDirect);
    // get Controller Uri
    controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).maxBackoffMillis(5000).build(), controllerExecutorService);
    testState = new TestState(true);
    // read and write count variables
    streamManager = new StreamManagerImpl(clientConfig);
    createScopeAndStream(scope, STREAM_NAME, config, streamManager);
    log.info("Scope passed to client factory {}", scope);
    clientFactory = new ClientFactoryImpl(scope, controller, new SocketConnectionFactoryImpl(clientConfig));
    readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
}

Example 12 with ControllerImpl

use of io.pravega.client.control.impl.ControllerImpl in project pravega by pravega.

the class MetadataScalabilityLargeNumSegmentsTest method largeNumSegmentsScalability.

@Test
public void largeNumSegmentsScalability() {
    testState = new TestState(false);
    ControllerImpl controller = getController();
    List<List<Segment>> listOfEpochs = scale(controller);
    truncation(controller, listOfEpochs);
    sealAndDeleteStream(controller);
}

Example 13 with ControllerImpl

use of io.pravega.client.control.impl.ControllerImpl in project pravega by pravega.

the class OffsetTruncationTest method offsetTruncationTest.

/**
 * This test verifies that truncation works specifying an offset that applies to multiple segments. To this end,
 * the test first writes a set of events on a Stream (with multiple segments) and truncates it at a specified offset
 * (truncatedEvents). The tests asserts that readers first get a TruncatedDataException as they are attempting to
 * read a truncated segment, and then they only read the remaining events that have not been truncated.
 */
@Test
public void offsetTruncationTest() {
    final int totalEvents = 200;
    final int truncatedEvents = 50;
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    log.info("Invoking offsetTruncationTest test with Controller URI: {}", controllerURI);
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, clientConfig);
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().stream(Stream.of(SCOPE, STREAM)).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup(READER_GROUP);
    // Write events to the Stream.
    writeEvents(clientFactory, STREAM, totalEvents);
    // Instantiate readers to consume from Stream up to truncatedEvents.
    List<CompletableFuture<Integer>> futures = readEventFutures(clientFactory, READER_GROUP, PARALLELISM, truncatedEvents);
    Futures.allOf(futures).join();
    // Ensure that we have read all the events required before initiating the checkpoint.
    assertEquals("Number of events read is not the expected one.", (Integer) truncatedEvents, futures.stream().map(f -> Futures.getAndHandleExceptions(f, RuntimeException::new)).reduce(Integer::sum).get());
    // Perform truncation on stream segment.
    Checkpoint cp = readerGroup.initiateCheckpoint("truncationCheckpoint", executor).join();
    StreamCut streamCut = cp.asImpl().getPositions().values().iterator().next();
    StreamCut alternativeStreamCut = readerGroup.generateStreamCuts(executor).join().get(Stream.of(SCOPE, STREAM));
    assertEquals("StreamCuts for reader group differ depending on how they are generated.", streamCut, alternativeStreamCut);
    assertTrue(streamManager.truncateStream(SCOPE, STREAM, streamCut));
    // Just after the truncation, read events from the offset defined in truncate call onwards.
    final String newGroupName = READER_GROUP + "new";
    groupManager.createReaderGroup(newGroupName, ReaderGroupConfig.builder().stream(Stream.of(SCOPE, STREAM)).build());
    futures = readEventFutures(clientFactory, newGroupName, PARALLELISM);
    Futures.allOf(futures).join();
    assertEquals("Expected read events: ", totalEvents - truncatedEvents, (int) futures.stream().map(CompletableFuture::join).reduce(Integer::sum).get());
    log.debug("The stream has been successfully truncated at event {}. Offset truncation test passed.", truncatedEvents);
}

Example 14 with ControllerImpl

use of io.pravega.client.control.impl.ControllerImpl in project pravega by pravega.

the class ReadTxnWriteScaleWithFailoverTest method setup.

@Before
public void setup() {
    // Get zk details to verify if controller, segmentstore are running
    Service zkService = Utils.createZookeeperService();
    List<URI> zkUris = zkService.getServiceDetails();
    log.debug("Zookeeper service details: {}", zkUris);
    // get the zk ip details and pass it to  host, controller
    URI zkUri = zkUris.get(0);
    // Verify controller is running.
    controllerInstance = Utils.createPravegaControllerService(zkUri);
    assertTrue(controllerInstance.isRunning());
    List<URI> conURIs = controllerInstance.getServiceDetails();
    log.info("Pravega Controller service instance details: {}", conURIs);
    // Fetch all the RPC endpoints and construct the client URIs.
    final List<String> uris = conURIs.stream().filter(ISGRPC).map(URI::getAuthority).collect(Collectors.toList());
    controllerURIDirect = URI.create((Utils.TLS_AND_AUTH_ENABLED ? TLS : TCP) + String.join(",", uris));
    log.info("Controller Service direct URI: {}", controllerURIDirect);
    // Verify segment store is running.
    segmentStoreInstance = Utils.createPravegaSegmentStoreService(zkUri, controllerURIDirect);
    assertTrue(segmentStoreInstance.isRunning());
    log.info("Pravega Segmentstore service instance details: {}", segmentStoreInstance.getServiceDetails());
    // num. of readers + num. of writers + 1 to run checkScale operation
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(NUM_READERS + NUM_WRITERS + 1, "ReadTxnWriteScaleWithFailoverTest-main");
    controllerExecutorService = ExecutorServiceHelpers.newScheduledThreadPool(2, "ReadTxnWriteScaleWithFailoverTest-controller");
    ClientConfig clientConfig = Utils.buildClientConfig(controllerURIDirect);
    // get Controller Uri
    controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).maxBackoffMillis(5000).build(), controllerExecutorService);
    testState = new TestState(true);
    streamManager = new StreamManagerImpl(clientConfig);
    createScopeAndStream(scope, stream, config, streamManager);
    log.info("Scope passed to client factory {}", scope);
    clientFactory = new ClientFactoryImpl(scope, controller, clientConfig);
    readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
}

Example 15 with ControllerImpl

use of io.pravega.client.control.impl.ControllerImpl in project pravega by pravega.

the class ReadWriteAndAutoScaleWithFailoverTest method setup.

@Before
public void setup() {
    // Get zk details to verify if controller, SSS are running
    Service zkService = Utils.createZookeeperService();
    List<URI> zkUris = zkService.getServiceDetails();
    log.debug("Zookeeper service details: {}", zkUris);
    // get the zk ip details and pass it to  host, controller
    URI zkUri = zkUris.get(0);
    // Verify controller is running.
    controllerInstance = Utils.createPravegaControllerService(zkUri);
    assertTrue(controllerInstance.isRunning());
    List<URI> conURIs = controllerInstance.getServiceDetails();
    log.info("Pravega Controller service instance details: {}", conURIs);
    // Fetch all the RPC endpoints and construct the client URIs.
    final List<String> uris = conURIs.stream().filter(ISGRPC).map(URI::getAuthority).collect(Collectors.toList());
    controllerURIDirect = URI.create("tcp://" + String.join(",", uris));
    log.info("Controller Service direct URI: {}", controllerURIDirect);
    // Verify segment store is running.
    segmentStoreInstance = Utils.createPravegaSegmentStoreService(zkUri, controllerURIDirect);
    assertTrue(segmentStoreInstance.isRunning());
    log.info("Pravega Segmentstore service instance details: {}", segmentStoreInstance.getServiceDetails());
    // executor service
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(NUM_READERS + TOTAL_NUM_WRITERS + 1, "ReadWriteAndAutoScaleWithFailoverTest-main");
    controllerExecutorService = ExecutorServiceHelpers.newScheduledThreadPool(2, "ReadWriteAndAutoScaleWithFailoverTest-controller");
    // get Controller Uri
    ClientConfig clientConfig = Utils.buildClientConfig(controllerURIDirect);
    controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).maxBackoffMillis(5000).build(), controllerExecutorService);
    testState = new TestState(false);
    streamManager = new StreamManagerImpl(clientConfig);
    createScopeAndStream(scope, AUTO_SCALE_STREAM, config, streamManager);
    log.info("Scope passed to client factory {}", scope);
    clientFactory = new ClientFactoryImpl(scope, controller, clientConfig);
    readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
}