Examples with ControllerImpl io.pravega.client.stream.impl.ControllerImpl used on opensource projects

Example 6 with ControllerImpl

use of io.pravega.client.stream.impl.ControllerImpl in project pravega by pravega.

the class ReadWriteAndAutoScaleWithFailoverTest method setup.

@Before
public void setup() {
    // Get zk details to verify if controller, SSS are running
    Service zkService = Utils.createZookeeperService();
    List<URI> zkUris = zkService.getServiceDetails();
    log.debug("Zookeeper service details: {}", zkUris);
    // get the zk ip details and pass it to  host, controller
    URI zkUri = zkUris.get(0);
    // Verify controller is running.
    controllerInstance = Utils.createPravegaControllerService(zkUri);
    assertTrue(controllerInstance.isRunning());
    List<URI> conURIs = controllerInstance.getServiceDetails();
    log.info("Pravega Controller service instance details: {}", conURIs);
    // Fetch all the RPC endpoints and construct the client URIs.
    final List<String> uris = conURIs.stream().filter(uri -> Utils.DOCKER_BASED ? uri.getPort() == Utils.DOCKER_CONTROLLER_PORT : uri.getPort() == Utils.MARATHON_CONTROLLER_PORT).map(URI::getAuthority).collect(Collectors.toList());
    controllerURIDirect = URI.create("tcp://" + String.join(",", uris));
    log.info("Controller Service direct URI: {}", controllerURIDirect);
    // Verify segment store is running.
    segmentStoreInstance = Utils.createPravegaSegmentStoreService(zkUri, controllerURIDirect);
    assertTrue(segmentStoreInstance.isRunning());
    log.info("Pravega Segmentstore service instance details: {}", segmentStoreInstance.getServiceDetails());
    // executor service
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(NUM_READERS + TOTAL_NUM_WRITERS + 1, "ReadWriteAndAutoScaleWithFailoverTest-main");
    controllerExecutorService = ExecutorServiceHelpers.newScheduledThreadPool(2, "ReadWriteAndAutoScaleWithFailoverTest-controller");
    // get Controller Uri
    controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(ClientConfig.builder().controllerURI(controllerURIDirect).build()).maxBackoffMillis(5000).build(), controllerExecutorService);
    testState = new TestState(false);
    testState.writersListComplete.add(0, testState.writersComplete);
    testState.writersListComplete.add(1, testState.newWritersComplete);
    streamManager = new StreamManagerImpl(ClientConfig.builder().controllerURI(controllerURIDirect).build());
    createScopeAndStream(scope, AUTO_SCALE_STREAM, config, streamManager);
    log.info("Scope passed to client factory {}", scope);
    clientFactory = new ClientFactoryImpl(scope, controller);
    readerGroupManager = ReaderGroupManager.withScope(scope, ClientConfig.builder().controllerURI(controllerURIDirect).build());
}

Example 7 with ControllerImpl

use of io.pravega.client.stream.impl.ControllerImpl in project pravega by pravega.

the class SetupUtils method startAllServices.

/**
 * Start all pravega related services required for the test deployment.
 *
 * @param numThreads the number of threads for the internal client threadpool.
 * @throws Exception on any errors.
 */
public void startAllServices(Integer numThreads) throws Exception {
    if (!this.started.compareAndSet(false, true)) {
        log.warn("Services already started, not attempting to start again");
        return;
    }
    this.connectionFactory = new ConnectionFactoryImpl(clientConfig, numThreads);
    this.controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
    this.clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    // Start zookeeper.
    this.zkTestServer = new TestingServerStarter().start();
    this.zkTestServer.start();
    // Start Pravega Service.
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    this.server = new PravegaConnectionListener(false, servicePort, store);
    this.server.startListening();
    log.info("Started Pravega Service");
    // Start Controller.
    this.controllerWrapper = new ControllerWrapper(this.zkTestServer.getConnectString(), false, true, controllerRPCPort, "localhost", servicePort, Config.HOST_STORE_CONTAINER_COUNT, controllerRESTPort);
    this.controllerWrapper.awaitRunning();
    this.controllerWrapper.getController().createScope(scope).get();
    log.info("Initialized Pravega Controller");
}

Example 8 with ControllerImpl

use of io.pravega.client.stream.impl.ControllerImpl in project pravega by pravega.

the class AutoScaleTest method scaleDownTest.

/**
 * Invoke the simple scale down Test, produce no into a stream.
 * The test will periodically check if a scale event has occured by talking to controller via
 * controller client.
 *
 * @throws InterruptedException if interrupted
 * @throws URISyntaxException   If URI is invalid
 */
private CompletableFuture<Void> scaleDownTest() {
    final ControllerImpl controller = getController();
    final AtomicBoolean exit = new AtomicBoolean(false);
    // overall wait for test to complete in 260 seconds (4.2 minutes) or scale down, whichever happens first.
    return Retry.withExpBackoff(10, 10, 30, Duration.ofSeconds(10).toMillis()).retryingOn(ScaleOperationNotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments(SCOPE, SCALE_DOWN_STREAM_NAME).thenAccept(x -> {
        if (x.getSegments().size() == 2) {
            throw new ScaleOperationNotDoneException();
        } else {
            log.info("scale down done successfully");
            exit.set(true);
        }
    }), EXECUTOR_SERVICE);
}

Example 9 with ControllerImpl

use of io.pravega.client.stream.impl.ControllerImpl in project pravega by pravega.

the class AutoScaleTest method scaleUpTxnTest.

/**
 * Invoke the scale up Test with transactional writes. Produce traffic from multiple writers in parallel.
 * Each writer writes using transactions.
 * Transactions are committed quickly to give
 * The test will periodically check if a scale event has occured by talking to controller via
 * controller client.
 *
 * @throws InterruptedException if interrupted
 * @throws URISyntaxException   If URI is invalid
 */
private CompletableFuture<Void> scaleUpTxnTest() {
    ControllerImpl controller = getController();
    final AtomicBoolean exit = new AtomicBoolean(false);
    ClientFactory clientFactory = getClientFactory();
    startNewTxnWriter(clientFactory, exit);
    // overall wait for test to complete in 260 seconds (4.2 minutes) or scale up, whichever happens first.
    return Retry.withExpBackoff(10, 10, 30, Duration.ofSeconds(10).toMillis()).retryingOn(ScaleOperationNotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments(SCOPE, SCALE_UP_TXN_STREAM_NAME).thenAccept(x -> {
        if (x.getSegments().size() == 1) {
            throw new ScaleOperationNotDoneException();
        } else {
            log.info("txn test scale up done successfully");
            exit.set(true);
        }
    }), EXECUTOR_SERVICE);
}

Example 10 with ControllerImpl

use of io.pravega.client.stream.impl.ControllerImpl in project pravega by pravega.

the class ControllerFailoverTest method failoverTest.

@Test(timeout = 180000)
public void failoverTest() throws InterruptedException, ExecutionException {
    String scope = "testFailoverScope" + RandomStringUtils.randomAlphabetic(5);
    String stream = "testFailoverStream" + RandomStringUtils.randomAlphabetic(5);
    int initialSegments = 2;
    List<Integer> segmentsToSeal = Collections.singletonList(0);
    Map<Double, Double> newRangesToCreate = new HashMap<>();
    newRangesToCreate.put(0.0, 0.25);
    newRangesToCreate.put(0.25, 0.5);
    long lease = 29000;
    long maxExecutionTime = 60000;
    long scaleGracePeriod = 30000;
    // Connect with first controller instance.
    final Controller controller1 = new ControllerImpl(ControllerImplConfig.builder().clientConfig(ClientConfig.builder().controllerURI(controllerURIDirect).build()).build(), EXECUTOR_SERVICE);
    // Create scope, stream, and a transaction with high timeout value.
    controller1.createScope(scope).join();
    log.info("Scope {} created successfully", scope);
    createStream(controller1, scope, stream, ScalingPolicy.fixed(initialSegments));
    log.info("Stream {}/{} created successfully", scope, stream);
    long txnCreationTimestamp = System.nanoTime();
    StreamImpl stream1 = new StreamImpl(scope, stream);
    TxnSegments txnSegments = controller1.createTransaction(stream1, lease, scaleGracePeriod).join();
    log.info("Transaction {} created successfully, beginTime={}", txnSegments.getTxnId(), txnCreationTimestamp);
    // Initiate scale operation. It will block until ongoing transaction is complete.
    controller1.startScale(stream1, segmentsToSeal, newRangesToCreate).join();
    // Ensure that scale is not yet done.
    boolean scaleStatus = controller1.checkScaleStatus(stream1, 0).join();
    log.info("Status of scale operation isDone={}", scaleStatus);
    Assert.assertTrue(!scaleStatus);
    // Now stop the controller instance executing scale operation.
    Futures.getAndHandleExceptions(controllerService1.scaleService(1), ExecutionException::new);
    log.info("Successfully stopped one instance of controller service");
    List<URI> conUris = controllerService1.getServiceDetails();
    // Fetch all the RPC endpoints and construct the client URIs.
    final List<String> uris = conUris.stream().filter(uri -> Utils.DOCKER_BASED ? uri.getPort() == Utils.DOCKER_CONTROLLER_PORT : uri.getPort() == Utils.MARATHON_CONTROLLER_PORT).map(URI::getAuthority).collect(Collectors.toList());
    controllerURIDirect = URI.create("tcp://" + String.join(",", uris));
    log.info("Controller Service direct URI: {}", controllerURIDirect);
    // Connect to another controller instance.
    final Controller controller2 = new ControllerImpl(ControllerImplConfig.builder().clientConfig(ClientConfig.builder().controllerURI(controllerURIDirect).build()).build(), EXECUTOR_SERVICE);
    // Fetch status of transaction.
    log.info("Fetching status of transaction {}, time elapsed since its creation={}", txnSegments.getTxnId(), System.nanoTime() - txnCreationTimestamp);
    Transaction.Status status = controller2.checkTransactionStatus(stream1, txnSegments.getTxnId()).join();
    log.info("Transaction {} status={}", txnSegments.getTxnId(), status);
    if (status == Transaction.Status.OPEN) {
        // Abort the ongoing transaction.
        log.info("Trying to abort transaction {}, by sending request to controller at {}", txnSegments.getTxnId(), controllerURIDirect);
        controller2.abortTransaction(stream1, txnSegments.getTxnId()).join();
    }
    // Note: if scale does not complete within desired time, test will timeout.
    while (!scaleStatus) {
        scaleStatus = controller2.checkScaleStatus(stream1, 0).join();
        Thread.sleep(30000);
    }
    // Ensure that the stream has 3 segments now.
    log.info("Checking whether scale operation succeeded by fetching current segments");
    StreamSegments streamSegments = controller2.getCurrentSegments(scope, stream).join();
    log.info("Current segment count=", streamSegments.getSegments().size());
    Assert.assertEquals(initialSegments - segmentsToSeal.size() + newRangesToCreate.size(), streamSegments.getSegments().size());
}