Examples with StateModelDefinition org.apache.helix.model.StateModelDefinition used on opensource projects

Example 1 with StateModelDefinition

use of org.apache.helix.model.StateModelDefinition in project pinot by linkedin.

the class HelixSetupUtils method createHelixClusterIfNeeded.

public static void createHelixClusterIfNeeded(String helixClusterName, String zkPath, boolean isUpdateStateModel) {
    final HelixAdmin admin = new ZKHelixAdmin(zkPath);
    final String segmentStateModelName = PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL;
    if (admin.getClusters().contains(helixClusterName)) {
        LOGGER.info("cluster already exists ********************************************* ");
        if (isUpdateStateModel) {
            final StateModelDefinition curStateModelDef = admin.getStateModelDef(helixClusterName, segmentStateModelName);
            List<String> states = curStateModelDef.getStatesPriorityList();
            if (states.contains(PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE)) {
                LOGGER.info("State model {} already updated to contain CONSUMING state", segmentStateModelName);
                return;
            } else {
                LOGGER.info("Updating {} to add states for low level kafka consumers", segmentStateModelName);
                StateModelDefinition newStateModelDef = PinotHelixSegmentOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition();
                ZkClient zkClient = new ZkClient(zkPath);
                zkClient.waitUntilConnected(20, TimeUnit.SECONDS);
                zkClient.setZkSerializer(new ZNRecordSerializer());
                HelixDataAccessor accessor = new ZKHelixDataAccessor(helixClusterName, new ZkBaseDataAccessor<ZNRecord>(zkClient));
                PropertyKey.Builder keyBuilder = accessor.keyBuilder();
                accessor.setProperty(keyBuilder.stateModelDef(segmentStateModelName), newStateModelDef);
                LOGGER.info("Completed updating statemodel {}", segmentStateModelName);
                zkClient.close();
            }
        }
        return;
    }
    LOGGER.info("Creating a new cluster, as the helix cluster : " + helixClusterName + " was not found ********************************************* ");
    admin.addCluster(helixClusterName, false);
    LOGGER.info("Enable auto join.");
    final HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER).forCluster(helixClusterName).build();
    final Map<String, String> props = new HashMap<String, String>();
    props.put(ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, String.valueOf(true));
    //we need only one segment to be loaded at a time
    props.put(MessageType.STATE_TRANSITION + "." + HelixTaskExecutor.MAX_THREADS, String.valueOf(1));
    admin.setConfig(scope, props);
    LOGGER.info("Adding state model {} (with CONSUMED state) generated using {} **********************************************", segmentStateModelName, PinotHelixSegmentOnlineOfflineStateModelGenerator.class.toString());
    // If this is a fresh cluster we are creating, then the cluster will see the CONSUMING state in the
    // state model. But then the servers will never be asked to go to that STATE (whether they have the code
    // to handle it or not) unil we complete the feature using low-level kafka consumers and turn the feature on.
    admin.addStateModelDef(helixClusterName, segmentStateModelName, PinotHelixSegmentOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition());
    LOGGER.info("Adding state model definition named : " + PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL + " generated using : " + PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.class.toString() + " ********************************************** ");
    admin.addStateModelDef(helixClusterName, PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL, PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition());
    LOGGER.info("Adding empty ideal state for Broker!");
    HelixHelper.updateResourceConfigsFor(new HashMap<String, String>(), CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, helixClusterName, admin);
    IdealState idealState = PinotTableIdealStateBuilder.buildEmptyIdealStateForBrokerResource(admin, helixClusterName);
    admin.setResourceIdealState(helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, idealState);
    initPropertyStorePath(helixClusterName, zkPath);
    LOGGER.info("New Cluster setup completed... ********************************************** ");
}

Example 2 with StateModelDefinition

use of org.apache.helix.model.StateModelDefinition in project pinot by linkedin.

the class PinotHelixSegmentOnlineOfflineStateModelGenerator method generatePinotStateModelDefinition.

public static StateModelDefinition generatePinotStateModelDefinition() {
    StateModelDefinition.Builder builder = new StateModelDefinition.Builder(PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL);
    builder.initialState(OFFLINE_STATE);
    builder.addState(ONLINE_STATE);
    builder.addState(CONSUMING_STATE);
    builder.addState(OFFLINE_STATE);
    builder.addState(DROPPED_STATE);
    // Set the initial state when the node starts
    // Add transitions between the states.
    builder.addTransition(CONSUMING_STATE, ONLINE_STATE);
    builder.addTransition(OFFLINE_STATE, CONSUMING_STATE);
    builder.addTransition(OFFLINE_STATE, ONLINE_STATE);
    builder.addTransition(CONSUMING_STATE, OFFLINE_STATE);
    builder.addTransition(ONLINE_STATE, OFFLINE_STATE);
    builder.addTransition(OFFLINE_STATE, DROPPED_STATE);
    // set constraints on states.
    // static constraint
    builder.dynamicUpperBound(ONLINE_STATE, "R");
    // dynamic constraint, R means it should be derived based on the replication
    // factor.
    builder.dynamicUpperBound(CONSUMING_STATE, "R");
    StateModelDefinition statemodelDefinition = builder.build();
    return statemodelDefinition;
}

Example 3 with StateModelDefinition

use of org.apache.helix.model.StateModelDefinition in project databus by linkedin.

the class DatabusCluster method create.

/**
     * create a cluster with a partitioned resource .
     * If successful, a non-zero number indicating the number of partitions created in the cluster is returned ;
     * This is meant to be atomic. If more than one thread/instance attempts to create the same cluster with diff number of partitions
     * only one of them will win . If a cluster exists, the number returned however will be the same in both those instances.
     * If the cluster could not be reached 0 is returned (retry possible) , and if there are other errors -1 is returned (retry not possible)
     */
public static int create(ZKHelixAdmin admin, ZkClient zkClient, String clusterName, int numPartitions) {
    boolean clusterAdded = true;
    // add cluster
    try {
        /**
           *  TODO : HACK !! : Copying this logic from OLD Helix library to mimic similar old "behavior".
           *
           *  Please see DDSDBUS-2579/HELIX-137
           *  The helix addCluster() ( in 0.6.2.3) API  has a new problem  where callers could not differentiate
           *  between the case when new cluster is created and the case where it was created by some other client. This was needed
           *  so that the follow-up steps of adding state-model (non-idempotent operation) can be done only by the client creating the cluster.
           *  Both old (0.6.1.3) and new Helix (0.6.2.3 ) library has the following issue:
           *   (a)  "No Atomicity in the face of the ZK client disconnects which results in cluster only partly
           *       initialized and unusable. This is noticed in PCL/PCS environment"
           *
           *  In order to workaround the backwards incompatibility issue between the 2 helix versions, we are reproducing part
           *  of the old addCluster() implementation below to get the same behavior as that of using 0.6.1.3 . The problem referred
           *  as (a) still exists.
           *
           */
        if (zkClient.exists("/" + clusterName)) {
            throw new Exception("Cluster already exists !!");
        }
        clusterAdded = admin.addCluster(clusterName, false);
        if (!clusterAdded) {
            LOG.error("Problem creating cluster (" + clusterName + ")");
        }
    } catch (Exception e) {
        LOG.warn("Warn! Cluster might already exist! " + clusterName + " Exception=" + e.getMessage());
        clusterAdded = false;
    }
    if (clusterAdded) {
        // add state model definition
        try {
            admin.addStateModelDef(clusterName, DEFAULT_STATE_MODEL, new StateModelDefinition(StateModelConfigGenerator.generateConfigForOnlineOffline()));
            admin.addResource(clusterName, DEFAULT_RESOURCE_NAME, numPartitions, DEFAULT_STATE_MODEL, IdealStateModeProperty.AUTO_REBALANCE.toString());
            admin.rebalance(clusterName, DEFAULT_RESOURCE_NAME, 1);
        } catch (Exception e) {
            LOG.warn("Resource addition incomplete. May have been completed by another instance: " + e.getMessage());
            clusterAdded = false;
        }
    }
    //Ensure that cluster is setup fully
    int part = getNumPartitionsInResource(admin, clusterName, DEFAULT_RESOURCE_NAME);
    if (part == 0) {
        long startTimeMs = System.currentTimeMillis();
        try {
            do {
                Thread.sleep(100);
                part = getNumPartitionsInResource(admin, clusterName, DEFAULT_RESOURCE_NAME);
            } while (part == 0 && ((System.currentTimeMillis() - startTimeMs) < DEFAULT_CLUSTER_CREATE_WAIT_MS));
        } catch (InterruptedException e) {
            LOG.warn("Cluster create wait interrupted for cluster=" + clusterName + " exception= " + e.getMessage());
        }
    }
    return part;
}

Example 4 with StateModelDefinition

use of org.apache.helix.model.StateModelDefinition in project pinot by linkedin.

the class PinotHelixBrokerResourceOnlineOfflineStateModelGenerator method generatePinotStateModelDefinition.

public static StateModelDefinition generatePinotStateModelDefinition() {
    ZNRecord record = new ZNRecord(PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL);
    /*
     * initial state in always offline for an instance.
     *
     */
    record.setSimpleField(StateModelDefinitionProperty.INITIAL_STATE.toString(), OFFLINE_STATE);
    /*
     * this is a ondered list of states in which we want the instances to be in. the first entry is
     * given the top most priority.
     *
     */
    List<String> statePriorityList = new ArrayList<String>();
    statePriorityList.add(ONLINE_STATE);
    statePriorityList.add(OFFLINE_STATE);
    statePriorityList.add(DROPPED_STATE);
    record.setListField(StateModelDefinitionProperty.STATE_PRIORITY_LIST.toString(), statePriorityList);
    /**
     *
     * If you are wondering what R and -1 signify, here is an explanation -1 means that don't even
     * try to keep any instances in this state. R says that all instances in the preference list
     * should be in this state.
     *
     */
    for (String state : statePriorityList) {
        String key = state + ".meta";
        Map<String, String> metadata = new HashMap<String, String>();
        if (state.equals(ONLINE_STATE)) {
            metadata.put("count", "R");
            record.setMapField(key, metadata);
        }
        if (state.equals(OFFLINE_STATE)) {
            metadata.put("count", "-1");
            record.setMapField(key, metadata);
        }
        if (state.equals(DROPPED_STATE)) {
            metadata.put("count", "-1");
            record.setMapField(key, metadata);
        }
    }
    /*
     * construction a state transition table, this tells the controller the next state given initial
     * and final states.
     *
     */
    for (String state : statePriorityList) {
        String key = state + ".next";
        if (state.equals(ONLINE_STATE)) {
            Map<String, String> metadata = new HashMap<String, String>();
            metadata.put(OFFLINE_STATE, OFFLINE_STATE);
            metadata.put(DROPPED_STATE, DROPPED_STATE);
            record.setMapField(key, metadata);
        }
        if (state.equals("OFFLINE")) {
            Map<String, String> metadata = new HashMap<String, String>();
            metadata.put(ONLINE_STATE, ONLINE_STATE);
            metadata.put(DROPPED_STATE, DROPPED_STATE);
            record.setMapField(key, metadata);
        }
    }
    /*
     * This is the transition priority list, again the first inserted gets the top most priority.
     *
     */
    List<String> stateTransitionPriorityList = new ArrayList<String>();
    stateTransitionPriorityList.add("ONLINE-OFFLINE");
    stateTransitionPriorityList.add("ONLINE-DROPPED");
    stateTransitionPriorityList.add("OFFLINE-ONLINE");
    stateTransitionPriorityList.add("OFFLINE-DROPPED");
    record.setListField(StateModelDefinitionProperty.STATE_TRANSITION_PRIORITYLIST.toString(), stateTransitionPriorityList);
    return new StateModelDefinition(record);
}