Examples with ZKHelixAdmin org.apache.helix.manager.zk.ZKHelixAdmin used on opensource projects

Example 1 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project pinot by linkedin.

the class HelixSetupUtils method createHelixClusterIfNeeded.

public static void createHelixClusterIfNeeded(String helixClusterName, String zkPath, boolean isUpdateStateModel) {
    final HelixAdmin admin = new ZKHelixAdmin(zkPath);
    final String segmentStateModelName = PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL;
    if (admin.getClusters().contains(helixClusterName)) {
        LOGGER.info("cluster already exists ********************************************* ");
        if (isUpdateStateModel) {
            final StateModelDefinition curStateModelDef = admin.getStateModelDef(helixClusterName, segmentStateModelName);
            List<String> states = curStateModelDef.getStatesPriorityList();
            if (states.contains(PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE)) {
                LOGGER.info("State model {} already updated to contain CONSUMING state", segmentStateModelName);
                return;
            } else {
                LOGGER.info("Updating {} to add states for low level kafka consumers", segmentStateModelName);
                StateModelDefinition newStateModelDef = PinotHelixSegmentOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition();
                ZkClient zkClient = new ZkClient(zkPath);
                zkClient.waitUntilConnected(20, TimeUnit.SECONDS);
                zkClient.setZkSerializer(new ZNRecordSerializer());
                HelixDataAccessor accessor = new ZKHelixDataAccessor(helixClusterName, new ZkBaseDataAccessor<ZNRecord>(zkClient));
                PropertyKey.Builder keyBuilder = accessor.keyBuilder();
                accessor.setProperty(keyBuilder.stateModelDef(segmentStateModelName), newStateModelDef);
                LOGGER.info("Completed updating statemodel {}", segmentStateModelName);
                zkClient.close();
            }
        }
        return;
    }
    LOGGER.info("Creating a new cluster, as the helix cluster : " + helixClusterName + " was not found ********************************************* ");
    admin.addCluster(helixClusterName, false);
    LOGGER.info("Enable auto join.");
    final HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER).forCluster(helixClusterName).build();
    final Map<String, String> props = new HashMap<String, String>();
    props.put(ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, String.valueOf(true));
    //we need only one segment to be loaded at a time
    props.put(MessageType.STATE_TRANSITION + "." + HelixTaskExecutor.MAX_THREADS, String.valueOf(1));
    admin.setConfig(scope, props);
    LOGGER.info("Adding state model {} (with CONSUMED state) generated using {} **********************************************", segmentStateModelName, PinotHelixSegmentOnlineOfflineStateModelGenerator.class.toString());
    // If this is a fresh cluster we are creating, then the cluster will see the CONSUMING state in the
    // state model. But then the servers will never be asked to go to that STATE (whether they have the code
    // to handle it or not) unil we complete the feature using low-level kafka consumers and turn the feature on.
    admin.addStateModelDef(helixClusterName, segmentStateModelName, PinotHelixSegmentOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition());
    LOGGER.info("Adding state model definition named : " + PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL + " generated using : " + PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.class.toString() + " ********************************************** ");
    admin.addStateModelDef(helixClusterName, PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL, PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition());
    LOGGER.info("Adding empty ideal state for Broker!");
    HelixHelper.updateResourceConfigsFor(new HashMap<String, String>(), CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, helixClusterName, admin);
    IdealState idealState = PinotTableIdealStateBuilder.buildEmptyIdealStateForBrokerResource(admin, helixClusterName);
    admin.setResourceIdealState(helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, idealState);
    initPropertyStorePath(helixClusterName, zkPath);
    LOGGER.info("New Cluster setup completed... ********************************************** ");
}

Example 2 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project pinot by linkedin.

the class StarTreeClusterIntegrationTest method waitForExternalViewUpdate.

/**
   * Wait for External View to be in sync with Ideal State.
   * @return
   */
private boolean waitForExternalViewUpdate() {
    final ZKHelixAdmin helixAdmin = new ZKHelixAdmin(ZkStarter.DEFAULT_ZK_STR);
    ClusterStateVerifier.Verifier customVerifier = new ClusterStateVerifier.Verifier() {

        @Override
        public boolean verify() {
            String clusterName = getHelixClusterName();
            List<String> resourcesInCluster = helixAdmin.getResourcesInCluster(clusterName);
            LOGGER.info("Waiting for external view to update for resources: {} startTime: {}", resourcesInCluster, new Timestamp(System.currentTimeMillis()));
            for (String resourceName : resourcesInCluster) {
                IdealState idealState = helixAdmin.getResourceIdealState(clusterName, resourceName);
                ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, resourceName);
                LOGGER.info("HERE for {},\n IS:{} \n EV:{}", resourceName, idealState, externalView);
                if (idealState == null || externalView == null) {
                    return false;
                }
                Set<String> partitionSet = idealState.getPartitionSet();
                for (String partition : partitionSet) {
                    Map<String, String> instanceStateMapIS = idealState.getInstanceStateMap(partition);
                    Map<String, String> instanceStateMapEV = externalView.getStateMap(partition);
                    if (instanceStateMapIS == null || instanceStateMapEV == null) {
                        return false;
                    }
                    if (!instanceStateMapIS.equals(instanceStateMapEV)) {
                        return false;
                    }
                }
                LOGGER.info("External View updated successfully for {},\n IS:{} \n EV:{}", resourceName, idealState, externalView);
            }
            LOGGER.info("External View updated successfully for {}", resourcesInCluster);
            return true;
        }
    };
    return ClusterStateVerifier.verifyByPolling(customVerifier, TIMEOUT_IN_MILLISECONDS);
}

Example 3 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project pinot by linkedin.

the class UpdateSegmentState method init.

private void init() {
    LOGGER.info("Trying to connect to " + _zkAddress + " cluster " + _clusterName);
    _helixAdmin = new ZKHelixAdmin(_zkAddress);
    ZNRecordSerializer serializer = new ZNRecordSerializer();
    String path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, _clusterName);
    _propertyStore = new ZkHelixPropertyStore<>(_zkAddress, serializer, path);
}

Example 4 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project pinot by linkedin.

the class VerifySegmentState method execute.

@Override
public boolean execute() throws Exception {
    ZKHelixAdmin helixAdmin = new ZKHelixAdmin(zkAddress);
    List<String> resourcesInCluster = helixAdmin.getResourcesInCluster(clusterName);
    for (String resourceName : resourcesInCluster) {
        if (resourceName.startsWith(tablePrefix)) {
            IdealState resourceIdealState = helixAdmin.getResourceIdealState(clusterName, resourceName);
            ExternalView resourceExternalView = helixAdmin.getResourceExternalView(clusterName, resourceName);
            Map<String, Map<String, String>> mapFieldsFromIS = resourceIdealState.getRecord().getMapFields();
            Map<String, Map<String, String>> mapFieldsFromEV = resourceExternalView.getRecord().getMapFields();
            boolean error = false;
            if (mapFieldsFromIS.size() != mapFieldsFromEV.size()) {
                LOGGER.info("Table: {}, idealState size: {} does NOT match external view size: {}", resourceName, mapFieldsFromIS.size(), mapFieldsFromEV.size());
                error = true;
            }
            if (!mapFieldsFromIS.keySet().equals(mapFieldsFromEV.keySet())) {
                Set<String> idealStateKeys = mapFieldsFromIS.keySet();
                Set<String> externalViewKeys = mapFieldsFromEV.keySet();
                Sets.SetView<String> isToEVDiff = Sets.difference(idealStateKeys, externalViewKeys);
                for (String segmentName : isToEVDiff) {
                    LOGGER.info("Segment: {} is missing in external view, ideal state: {}", segmentName, mapFieldsFromIS.get(segmentName));
                }
                Sets.SetView<String> evToISDiff = Sets.difference(externalViewKeys, idealStateKeys);
                for (String segmentName : evToISDiff) {
                    LOGGER.error("Segment: {} is missing in ideal state, external view: {}", segmentName, mapFieldsFromEV.get(segmentName));
                }
                error = true;
            }
            for (Map.Entry<String, Map<String, String>> idealEntry : mapFieldsFromIS.entrySet()) {
                String segmentName = idealEntry.getKey();
                Map<String, String> segmentIdealState = idealEntry.getValue();
                // try to format consistently for tool based parsing
                if (!mapFieldsFromEV.containsKey(segmentName)) {
                    LOGGER.info("Segment: {} idealstate: {} is MISSING in external view: {}", segmentName, segmentIdealState, "");
                }
                Map<String, String> segmentExternalView = mapFieldsFromEV.get(segmentName);
                if (!segmentIdealState.equals(segmentExternalView)) {
                    LOGGER.info("Segment: {} idealstate: {} does NOT match external view: {}", segmentName, segmentIdealState, segmentExternalView);
                    error = true;
                }
            }
            LOGGER.info(resourceName + " = " + (error ? "ERROR" : "OK"));
        }
    }
    return true;
}

Example 5 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project helix by apache.

the class IdealStateExample method main.

public static void main(String[] args) throws Exception {
    if (args.length < 3) {
        System.err.println("USAGE: IdealStateExample zkAddress clusterName idealStateMode (FULL_AUTO, SEMI_AUTO, or CUSTOMIZED) idealStateJsonFile (required for CUSTOMIZED mode)");
        System.exit(1);
    }
    final String zkAddr = args[0];
    final String clusterName = args[1];
    final String idealStateRebalancerModeStr = args[2].toUpperCase();
    String idealStateJsonFile = null;
    RebalanceMode idealStateRebalancerMode = RebalanceMode.valueOf(idealStateRebalancerModeStr);
    if (idealStateRebalancerMode == RebalanceMode.CUSTOMIZED) {
        if (args.length < 4) {
            System.err.println("Missng idealStateJsonFile for CUSTOMIZED ideal state mode");
            System.exit(1);
        }
        idealStateJsonFile = args[3];
    }
    // add cluster {clusterName}
    ZkClient zkclient = new ZkClient(zkAddr, ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer());
    ZKHelixAdmin admin = new ZKHelixAdmin(zkclient);
    admin.addCluster(clusterName, true);
    // add MasterSlave state mode definition
    StateModelConfigGenerator generator = new StateModelConfigGenerator();
    admin.addStateModelDef(clusterName, "MasterSlave", new StateModelDefinition(generator.generateConfigForMasterSlave()));
    // add 3 participants: "localhost:{12918, 12919, 12920}"
    for (int i = 0; i < 3; i++) {
        int port = 12918 + i;
        InstanceConfig config = new InstanceConfig("localhost_" + port);
        config.setHostName("localhost");
        config.setPort(Integer.toString(port));
        config.setInstanceEnabled(true);
        admin.addInstance(clusterName, config);
    }
    // add resource "TestDB" which has 4 partitions and uses MasterSlave state model
    String resourceName = "TestDB";
    if (idealStateRebalancerMode == RebalanceMode.SEMI_AUTO || idealStateRebalancerMode == RebalanceMode.FULL_AUTO) {
        admin.addResource(clusterName, resourceName, 4, "MasterSlave", idealStateRebalancerModeStr);
        // rebalance resource "TestDB" using 3 replicas
        admin.rebalance(clusterName, resourceName, 3);
    } else if (idealStateRebalancerMode == RebalanceMode.CUSTOMIZED) {
        admin.addIdealState(clusterName, resourceName, idealStateJsonFile);
    }
    // start helix controller
    new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                HelixControllerMain.main(new String[] { "--zkSvr", zkAddr, "--cluster", clusterName });
            } catch (Exception e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    }).start();
    // start 3 dummy participants
    for (int i = 0; i < 3; i++) {
        int port = 12918 + i;
        final String instanceName = "localhost_" + port;
        new Thread(new Runnable() {

            @Override
            public void run() {
                DummyParticipant.main(new String[] { zkAddr, clusterName, instanceName });
            }
        }).start();
    }
}