Examples with CuratorFramework org.apache.curator.framework.CuratorFramework used on opensource projects

Example 76 with CuratorFramework

use of org.apache.curator.framework.CuratorFramework in project hive by apache.

the class ZooKeeperTokenStore method addToken.

@Override
public boolean addToken(DelegationTokenIdentifier tokenIdentifier, DelegationTokenInformation token) {
    byte[] tokenBytes = MetastoreDelegationTokenSupport.encodeDelegationTokenInformation(token);
    String tokenPath = getTokenPath(tokenIdentifier);
    CuratorFramework zk = getSession();
    String newNode;
    try {
        newNode = zk.create().withMode(CreateMode.PERSISTENT).withACL(newNodeAcl).forPath(tokenPath, tokenBytes);
    } catch (Exception e) {
        throw new TokenStoreException("Error creating new node with path " + tokenPath, e);
    }
    LOGGER.info("Added token: {}", newNode);
    return true;
}

Example 77 with CuratorFramework

use of org.apache.curator.framework.CuratorFramework in project hive by apache.

the class ZooKeeperTokenStore method updateMasterKey.

@Override
public void updateMasterKey(int keySeq, String s) {
    CuratorFramework zk = getSession();
    String keyPath = rootNode + NODE_KEYS + "/" + String.format(ZK_SEQ_FORMAT, keySeq);
    try {
        zk.setData().forPath(keyPath, s.getBytes());
    } catch (Exception e) {
        throw new TokenStoreException("Error setting data in " + keyPath, e);
    }
}

Example 78 with CuratorFramework

use of org.apache.curator.framework.CuratorFramework in project pravega by pravega.

the class ControllerServiceMain method run.

@Override
protected void run() throws Exception {
    long traceId = LoggerHelpers.traceEnter(log, this.objectId, "run");
    try {
        while (isRunning()) {
            // Create store client.
            log.info("Creating store client");
            storeClient = StoreClientFactory.createStoreClient(serviceConfig.getStoreClientConfig());
            boolean hasZkConnection = serviceConfig.getStoreClientConfig().getStoreType().equals(StoreType.Zookeeper) || serviceConfig.isControllerClusterListenerEnabled();
            CompletableFuture<Void> sessionExpiryFuture = new CompletableFuture<>();
            if (hasZkConnection) {
                CuratorFramework client = (CuratorFramework) storeClient.getClient();
                log.info("Awaiting ZK client connection to ZK server");
                client.blockUntilConnected();
                // Await ZK session expiry.
                log.info("Awaiting ZK session expiry or termination trigger for ControllerServiceMain");
                client.getZookeeperClient().getZooKeeper().register(new ZKWatcher(sessionExpiryFuture));
            }
            // Start controller services.
            starter = starterFactory.apply(serviceConfig, storeClient);
            log.info("Starting controller services");
            notifyServiceStateChange(ServiceState.STARTING);
            starter.startAsync();
            log.info("Awaiting controller services start");
            starter.awaitRunning();
            if (hasZkConnection) {
                // At this point, wait until either of the two things happen
                // 1. ZK session expires, i.e., sessionExpiryFuture completes, or
                // 2. This ControllerServiceMain instance is stopped by invoking stopAsync() method,
                // i.e., serviceStopFuture completes.
                CompletableFuture.anyOf(sessionExpiryFuture, this.serviceStopFuture).join();
                // stop ControllerServiceStarter.
                if (sessionExpiryFuture.isDone()) {
                    log.info("ZK session expired");
                    storeClient.close();
                }
            } else {
                this.serviceStopFuture.join();
            }
            log.info("Stopping ControllerServiceStarter");
            notifyServiceStateChange(ServiceState.PAUSING);
            starter.stopAsync();
            log.info("Awaiting termination of ControllerServiceStarter");
            starter.awaitTerminated();
        }
    } catch (Exception e) {
        log.error("Controller Service Main thread exited exceptionally", e);
        throw e;
    } finally {
        if (storeClient != null) {
            storeClient.close();
        }
        LoggerHelpers.traceLeave(log, this.objectId, "run", traceId);
    }
}

Example 79 with CuratorFramework

use of org.apache.curator.framework.CuratorFramework in project pravega by pravega.

the class ControllerServiceStarter method startUp.

@Override
protected void startUp() {
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.objectId, "startUp");
    log.info("Initiating controller service startUp");
    log.info("Event processors enabled = {}", serviceConfig.getEventProcessorConfig().isPresent());
    log.info("Cluster listener enabled = {}", serviceConfig.isControllerClusterListenerEnabled());
    log.info("    Host monitor enabled = {}", serviceConfig.getHostMonitorConfig().isHostMonitorEnabled());
    log.info("     gRPC server enabled = {}", serviceConfig.getGRPCServerConfig().isPresent());
    log.info("     REST server enabled = {}", serviceConfig.getRestServerConfig().isPresent());
    final StreamMetadataStore streamStore;
    final TaskMetadataStore taskMetadataStore;
    final HostControllerStore hostStore;
    final CheckpointStore checkpointStore;
    try {
        // Initialize the executor service.
        controllerExecutor = ExecutorServiceHelpers.newScheduledThreadPool(serviceConfig.getThreadPoolSize(), "controllerpool");
        retentionExecutor = ExecutorServiceHelpers.newScheduledThreadPool(Config.RETENTION_THREAD_POOL_SIZE, "retentionpool");
        log.info("Creating the stream store");
        streamStore = StreamStoreFactory.createStore(storeClient, controllerExecutor);
        log.info("Creating the task store");
        taskMetadataStore = TaskStoreFactory.createStore(storeClient, controllerExecutor);
        log.info("Creating the host store");
        hostStore = HostStoreFactory.createStore(serviceConfig.getHostMonitorConfig(), storeClient);
        log.info("Creating the checkpoint store");
        checkpointStore = CheckpointStoreFactory.create(storeClient);
        // On each controller process restart, we use a fresh hostId,
        // which is a combination of hostname and random GUID.
        String hostName = getHostName();
        Host host = new Host(hostName, getPort(), UUID.randomUUID().toString());
        if (serviceConfig.getHostMonitorConfig().isHostMonitorEnabled()) {
            // Start the Segment Container Monitor.
            monitor = new SegmentContainerMonitor(hostStore, (CuratorFramework) storeClient.getClient(), new UniformContainerBalancer(), serviceConfig.getHostMonitorConfig().getHostMonitorMinRebalanceInterval());
            log.info("Starting segment container monitor");
            monitor.startAsync();
        }
        ClientConfig clientConfig = ClientConfig.builder().controllerURI(URI.create((serviceConfig.getGRPCServerConfig().get().isTlsEnabled() ? "tls://" : "tcp://") + "localhost")).trustStore(serviceConfig.getGRPCServerConfig().get().getTlsTrustStore()).validateHostName(false).build();
        connectionFactory = new ConnectionFactoryImpl(clientConfig);
        SegmentHelper segmentHelper = new SegmentHelper();
        streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelper, controllerExecutor, host.getHostId(), connectionFactory, serviceConfig.getGRPCServerConfig().get().isAuthorizationEnabled(), serviceConfig.getGRPCServerConfig().get().getTokenSigningKey());
        streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelper, controllerExecutor, host.getHostId(), serviceConfig.getTimeoutServiceConfig(), connectionFactory, serviceConfig.getGRPCServerConfig().get().isAuthorizationEnabled(), serviceConfig.getGRPCServerConfig().get().getTokenSigningKey());
        streamCutService = new StreamCutService(Config.BUCKET_COUNT, host.getHostId(), streamStore, streamMetadataTasks, retentionExecutor);
        log.info("starting auto retention service asynchronously");
        streamCutService.startAsync();
        streamCutService.awaitRunning();
        // Controller has a mechanism to track the currently active controller host instances. On detecting a failure of
        // any controller instance, the failure detector stores the failed HostId in a failed hosts directory (FH), and
        // invokes the taskSweeper.sweepOrphanedTasks for each failed host. When all resources under the failed hostId
        // are processed and deleted, that failed HostId is removed from FH folder.
        // Moreover, on controller process startup, it detects any hostIds not in the currently active set of
        // controllers and starts sweeping tasks orphaned by those hostIds.
        TaskSweeper taskSweeper = new TaskSweeper(taskMetadataStore, host.getHostId(), controllerExecutor, streamMetadataTasks);
        TxnSweeper txnSweeper = new TxnSweeper(streamStore, streamTransactionMetadataTasks, serviceConfig.getTimeoutServiceConfig().getMaxLeaseValue(), controllerExecutor);
        if (serviceConfig.isControllerClusterListenerEnabled()) {
            cluster = new ClusterZKImpl((CuratorFramework) storeClient.getClient(), ClusterType.CONTROLLER);
        }
        controllerService = new ControllerService(streamStore, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, new SegmentHelper(), controllerExecutor, cluster);
        // Setup event processors.
        setController(new LocalController(controllerService, serviceConfig.getGRPCServerConfig().get().isAuthorizationEnabled(), serviceConfig.getGRPCServerConfig().get().getTokenSigningKey()));
        if (serviceConfig.getEventProcessorConfig().isPresent()) {
            // Create ControllerEventProcessor object.
            controllerEventProcessors = new ControllerEventProcessors(host.getHostId(), serviceConfig.getEventProcessorConfig().get(), localController, checkpointStore, streamStore, hostStore, segmentHelper, connectionFactory, streamMetadataTasks, controllerExecutor);
            // Bootstrap and start it asynchronously.
            log.info("Starting event processors");
            controllerEventProcessors.bootstrap(streamTransactionMetadataTasks, streamMetadataTasks).thenAcceptAsync(x -> controllerEventProcessors.startAsync(), controllerExecutor);
        }
        // Setup and start controller cluster listener after all sweepers have been initialized.
        if (serviceConfig.isControllerClusterListenerEnabled()) {
            List<FailoverSweeper> failoverSweepers = new ArrayList<>();
            failoverSweepers.add(taskSweeper);
            failoverSweepers.add(txnSweeper);
            if (serviceConfig.getEventProcessorConfig().isPresent()) {
                assert controllerEventProcessors != null;
                failoverSweepers.add(controllerEventProcessors);
            }
            controllerClusterListener = new ControllerClusterListener(host, cluster, controllerExecutor, failoverSweepers);
            log.info("Starting controller cluster listener");
            controllerClusterListener.startAsync();
        }
        // Start RPC server.
        if (serviceConfig.getGRPCServerConfig().isPresent()) {
            grpcServer = new GRPCServer(controllerService, serviceConfig.getGRPCServerConfig().get());
            grpcServer.startAsync();
            log.info("Awaiting start of rpc server");
            grpcServer.awaitRunning();
        }
        // Start REST server.
        if (serviceConfig.getRestServerConfig().isPresent()) {
            restServer = new RESTServer(this.localController, controllerService, grpcServer.getPravegaAuthManager(), serviceConfig.getRestServerConfig().get(), connectionFactory);
            restServer.startAsync();
            log.info("Awaiting start of REST server");
            restServer.awaitRunning();
        }
        // Wait for controller event processors to start.
        if (serviceConfig.getEventProcessorConfig().isPresent()) {
            log.info("Awaiting start of controller event processors");
            controllerEventProcessors.awaitRunning();
        }
        // Wait for controller cluster listeners to start.
        if (serviceConfig.isControllerClusterListenerEnabled()) {
            log.info("Awaiting start of controller cluster listener");
            controllerClusterListener.awaitRunning();
        }
    } finally {
        LoggerHelpers.traceLeave(log, this.objectId, "startUp", traceId);
    }
}

Example 80 with CuratorFramework

use of org.apache.curator.framework.CuratorFramework in project pravega by pravega.

the class ZKSegmentContainerManagerTest method testInitializeSucceeds.

/**
 * Test if initialization completes.
 *
 * @throws Exception if an error occurred.
 */
@Test
public void testInitializeSucceeds() throws Exception {
    @Cleanup CuratorFramework zkClient = startClient();
    @Cleanup ZKSegmentContainerManager segManager = createContainerManager(createMockContainerRegistry(), zkClient);
    segManager.initialize();
}