Examples with HelixConfigScopeBuilder org.apache.helix.model.builder.HelixConfigScopeBuilder used on opensource projects

Example 16 with HelixConfigScopeBuilder

use of org.apache.helix.model.builder.HelixConfigScopeBuilder in project ambry by linkedin.

the class VcrTestUtil method populateZkInfoAndStartController.

/**
 * Populate info on ZooKeeper server and start {@link HelixControllerManager}.
 * @param zkConnectString zk connect string to zk server.
 * @param vcrClusterName the vcr cluster name.
 * @param clusterMap the {@link ClusterMap} to use.
 * @param vcrHelixStateModel the state model to use for helix cluster events.
 * @return the created {@link HelixControllerManager}.
 */
public static HelixControllerManager populateZkInfoAndStartController(String zkConnectString, String vcrClusterName, ClusterMap clusterMap, String vcrHelixStateModel) {
    HelixZkClient zkClient = DedicatedZkClientFactory.getInstance().buildZkClient(new HelixZkClient.ZkConnectionConfig(zkConnectString), new HelixZkClient.ZkClientConfig());
    try {
        zkClient.setZkSerializer(new ZNRecordSerializer());
        ClusterSetup clusterSetup = new ClusterSetup(zkClient);
        clusterSetup.addCluster(vcrClusterName, true);
        HelixAdmin admin = new ZKHelixAdmin(zkClient);
        // set ALLOW_PARTICIPANT_AUTO_JOIN
        HelixConfigScope configScope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER).forCluster(vcrClusterName).build();
        Map<String, String> helixClusterProperties = new HashMap<>();
        helixClusterProperties.put(ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, String.valueOf(true));
        admin.setConfig(configScope, helixClusterProperties);
        // set PersistBestPossibleAssignment
        ConfigAccessor configAccessor = new ConfigAccessor(zkClient);
        ClusterConfig clusterConfig = configAccessor.getClusterConfig(vcrClusterName);
        clusterConfig.setPersistBestPossibleAssignment(true);
        configAccessor.setClusterConfig(vcrClusterName, clusterConfig);
        FullAutoModeISBuilder builder = new FullAutoModeISBuilder(helixResource);
        builder.setStateModel(vcrHelixStateModel);
        for (PartitionId partitionId : clusterMap.getAllPartitionIds(null)) {
            builder.add(partitionId.toPathString());
        }
        builder.setMinActiveReplica(MIN_ACTIVE_REPLICAS);
        builder.setRebalanceDelay((int) REBALANCE_DELAY_MS);
        builder.setRebalancerClass(DelayedAutoRebalancer.class.getName());
        builder.setRebalanceStrategy(CrushEdRebalanceStrategy.class.getName());
        IdealState idealState = builder.build();
        admin.addResource(vcrClusterName, helixResource, idealState);
        admin.rebalance(vcrClusterName, helixResource, NUM_REPLICAS, "", "");
        HelixControllerManager helixControllerManager = new HelixControllerManager(zkConnectString, vcrClusterName);
        helixControllerManager.syncStart();
        return helixControllerManager;
    } finally {
        zkClient.close();
    }
}

Example 17 with HelixConfigScopeBuilder

use of org.apache.helix.model.builder.HelixConfigScopeBuilder in project pinot by linkedin.

the class ControllerRequestBuilderUtil method addFakeDataInstancesToAutoJoinHelixCluster.

public static void addFakeDataInstancesToAutoJoinHelixCluster(String helixClusterName, String zkServer, int numInstances, boolean isSingleTenant, int adminPort) throws Exception {
    for (int i = 0; i < numInstances; ++i) {
        final String instanceId = "Server_localhost_" + i;
        final HelixManager helixZkManager = HelixManagerFactory.getZKHelixManager(helixClusterName, instanceId, InstanceType.PARTICIPANT, zkServer);
        final StateMachineEngine stateMachineEngine = helixZkManager.getStateMachineEngine();
        final StateModelFactory<?> stateModelFactory = new EmptySegmentOnlineOfflineStateModelFactory();
        stateMachineEngine.registerStateModelFactory(EmptySegmentOnlineOfflineStateModelFactory.getStateModelDef(), stateModelFactory);
        helixZkManager.connect();
        if (isSingleTenant) {
            helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
            helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
        } else {
            helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, UNTAGGED_SERVER_INSTANCE);
        }
        HelixConfigScope scope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.PARTICIPANT, helixClusterName).forParticipant(instanceId).build();
        Map<String, String> props = new HashMap<>();
        props.put(CommonConstants.Helix.Instance.ADMIN_PORT_KEY, String.valueOf(adminPort + i));
        helixZkManager.getClusterManagmentTool().setConfig(scope, props);
    }
}

Example 18 with HelixConfigScopeBuilder

use of org.apache.helix.model.builder.HelixConfigScopeBuilder in project pinot by linkedin.

the class HelixServerStarter method updateInstanceConfigInHelix.

private void updateInstanceConfigInHelix(Map<String, String> props) {
    HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.PARTICIPANT, _helixClusterName).forParticipant(_instanceId).build();
    _helixAdmin.setConfig(scope, props);
}

Example 19 with HelixConfigScopeBuilder

use of org.apache.helix.model.builder.HelixConfigScopeBuilder in project ambry by linkedin.

the class HelixVcrUtil method createCluster.

/**
 * Create a helix cluster with given information.
 * @param destZkString the cluster's zk string
 * @param destClusterName the cluster's name
 */
public static void createCluster(String destZkString, String destClusterName, VcrHelixConfig config) {
    HelixZkClient destZkClient = getHelixZkClient(destZkString);
    HelixAdmin destAdmin = new ZKHelixAdmin(destZkClient);
    if (ZKUtil.isClusterSetup(destClusterName, destZkClient)) {
        errorAndExit("Failed to create cluster because " + destClusterName + " already exist.");
    }
    ClusterSetup clusterSetup = new ClusterSetup.Builder().setZkAddress(destZkString).build();
    clusterSetup.addCluster(destClusterName, true);
    // set ALLOW_PARTICIPANT_AUTO_JOIN
    HelixConfigScope configScope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER).forCluster(destClusterName).build();
    Map<String, String> helixClusterProperties = new HashMap<>();
    helixClusterProperties.put(ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, String.valueOf(config.getClusterConfigFields().isAllowAutoJoin()));
    destAdmin.setConfig(configScope, helixClusterProperties);
    setClusterConfig(destZkClient, destClusterName, config, false);
    logger.info("Cluster {} is created successfully!", destClusterName);
}

Example 20 with HelixConfigScopeBuilder

use of org.apache.helix.model.builder.HelixConfigScopeBuilder in project helix by apache.

the class HelixTaskExecutor method updateStateTransitionMessageThreadPool.

/**
 * Dedicated Thread pool can be provided in configuration or by client.
 *  This method is to check it and update the thread pool if necessary.
 */
private void updateStateTransitionMessageThreadPool(Message message, HelixManager manager) {
    if (!message.getMsgType().equals(MessageType.STATE_TRANSITION.name())) {
        return;
    }
    String resourceName = message.getResourceName();
    String factoryName = message.getStateModelFactoryName();
    String stateModelName = message.getStateModelDef();
    if (factoryName == null) {
        factoryName = HelixConstants.DEFAULT_STATE_MODEL_FACTORY;
    }
    StateModelFactory<? extends StateModel> stateModelFactory = manager.getStateMachineEngine().getStateModelFactory(stateModelName, factoryName);
    String perStateTransitionTypeKey = getStateTransitionType(getPerResourceStateTransitionPoolName(resourceName), message.getFromState(), message.getToState());
    if (perStateTransitionTypeKey != null && stateModelFactory != null && !_transitionTypeThreadpoolChecked.contains(perStateTransitionTypeKey)) {
        ExecutorService perStateTransitionTypeExecutor = stateModelFactory.getExecutorService(resourceName, message.getFromState(), message.getToState());
        _transitionTypeThreadpoolChecked.add(perStateTransitionTypeKey);
        if (perStateTransitionTypeExecutor != null) {
            _executorMap.put(perStateTransitionTypeKey, perStateTransitionTypeExecutor);
            LOG.info(String.format("Added client specified dedicate threadpool for resource %s from %s to %s", getPerResourceStateTransitionPoolName(resourceName), message.getFromState(), message.getToState()));
            return;
        }
    }
    if (!_resourcesThreadpoolChecked.contains(resourceName)) {
        int threadpoolSize = -1;
        ConfigAccessor configAccessor = manager.getConfigAccessor();
        // Changes to this configuration on thread pool size will only take effect after the participant get restarted.
        if (configAccessor != null) {
            HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.RESOURCE).forCluster(manager.getClusterName()).forResource(resourceName).build();
            String threadpoolSizeStr = configAccessor.get(scope, MAX_THREADS);
            try {
                if (threadpoolSizeStr != null) {
                    threadpoolSize = Integer.parseInt(threadpoolSizeStr);
                }
            } catch (Exception e) {
                LOG.error("Failed to parse ThreadPoolSize from resourceConfig for resource" + resourceName, e);
            }
        }
        final String key = getPerResourceStateTransitionPoolName(resourceName);
        if (threadpoolSize > 0) {
            _executorMap.put(key, Executors.newFixedThreadPool(threadpoolSize, new ThreadFactory() {

                @Override
                public Thread newThread(Runnable r) {
                    return new Thread(r, "GerenricHelixController-message_handle_" + key);
                }
            }));
            LOG.info("Added dedicate threadpool for resource: " + resourceName + " with size: " + threadpoolSize);
        } else {
            // check whether client specifies customized threadpool.
            if (stateModelFactory != null) {
                ExecutorService executor = stateModelFactory.getExecutorService(resourceName);
                if (executor != null) {
                    _executorMap.put(key, executor);
                    LOG.info("Added client specified dedicate threadpool for resource: " + key);
                }
            } else {
                LOG.error(String.format("Fail to get dedicate threadpool defined in stateModelFactory %s: using factoryName: %s for resource %s. No stateModelFactory was found!", stateModelName, factoryName, resourceName));
            }
        }
        _resourcesThreadpoolChecked.add(resourceName);
    }
}