Examples with IdealState org.apache.helix.model.IdealState used on opensource projects

Example 61 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class BalanceNumSegmentAssignmentStrategyIntegrationTest method getSegmentsPerInstance.

private Map<String, Integer> getSegmentsPerInstance(String tableName) {
    Map<String, Integer> segmentsPerInstance = new HashMap<String, Integer>();
    IdealState idealState = _helixAdmin.getResourceIdealState(getHelixClusterName(), tableName + "_OFFLINE");
    for (String partitionName : idealState.getPartitionSet()) {
        for (String instanceName : idealState.getInstanceSet(partitionName)) {
            if (!segmentsPerInstance.containsKey(instanceName)) {
                segmentsPerInstance.put(instanceName, 1);
            } else {
                segmentsPerInstance.put(instanceName, segmentsPerInstance.get(instanceName) + 1);
            }
        }
    }
    return segmentsPerInstance;
}

Example 62 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotNumReplicaChanger method changeNumReplicas.

public void changeNumReplicas(final String tableName) throws Exception {
    // Get the number of replicas in the tableconfig.
    final String offlineTableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(tableName);
    final AbstractTableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(propertyStore, offlineTableName);
    final int newNumReplicas = Integer.parseInt(offlineTableConfig.getValidationConfig().getReplication());
    // Now get the idealstate, and get the number of replicas in it.
    IdealState currentIdealState = helixAdmin.getResourceIdealState(clusterName, offlineTableName);
    int currentNumReplicas = Integer.parseInt(currentIdealState.getReplicas());
    if (newNumReplicas > currentNumReplicas) {
        LOGGER.info("Increasing replicas not yet supported");
    } else if (newNumReplicas == currentNumReplicas) {
        LOGGER.info("Number of replicas ({}) match in table definition and Idealstate. Nothing to do for {}", newNumReplicas, offlineTableName);
    } else if (newNumReplicas < currentNumReplicas) {
        if (dryRun) {
            IdealState newIdealState = updateIdealState(currentIdealState, newNumReplicas);
            LOGGER.info("Final segment Assignment:");
            printSegmentAssignment(newIdealState.getRecord().getMapFields());
        } else {
            HelixHelper.updateIdealState(helixManager, offlineTableName, new Function<IdealState, IdealState>() {

                @Nullable
                @Override
                public IdealState apply(IdealState idealState) {
                    return updateIdealState(idealState, newNumReplicas);
                }
            }, RetryPolicies.exponentialBackoffRetryPolicy(5, 500L, 2.0f));
            waitForStable(offlineTableName);
            LOGGER.info("Successfully changed numReplicas to {} for table {}", newNumReplicas, offlineTableName);
            LOGGER.warn("*** You need to rebalance table {} ***", offlineTableName);
        }
    }
}

Example 63 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotZKChanger method isStable.

/**
   * return true if IdealState = ExternalView
   * @return
   */
public int isStable(String tableName) {
    IdealState idealState = helixAdmin.getResourceIdealState(clusterName, tableName);
    ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, tableName);
    Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields();
    Map<String, Map<String, String>> mapFieldsEV = externalView.getRecord().getMapFields();
    int numDiff = 0;
    for (String segment : mapFieldsIS.keySet()) {
        Map<String, String> mapIS = mapFieldsIS.get(segment);
        Map<String, String> mapEV = mapFieldsEV.get(segment);
        for (String server : mapIS.keySet()) {
            String state = mapIS.get(server);
            if (mapEV == null || mapEV.get(server) == null || !mapEV.get(server).equals(state)) {
                LOGGER.info("Mismatch: segment" + segment + " server:" + server + " state:" + state);
                numDiff = numDiff + 1;
            }
        }
    }
    return numDiff;
}

Example 64 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class ShowClusterInfoCommand method execute.

@Override
public boolean execute() throws Exception {
    Set<String> includeTableSet = new HashSet<>();
    String[] includeTables = _tables.split(",");
    for (String includeTable : includeTables) {
        String name = stripTypeFromName(includeTable.trim());
        if (name.length() > 0) {
            includeTableSet.add(name);
        }
    }
    Set<String> includeTagSet = new HashSet<>();
    String[] includeTags = _tags.split(",");
    for (String includeTag : includeTags) {
        String name = stripTypeFromName(includeTag.trim());
        if (name.length() > 0) {
            includeTagSet.add(name);
        }
    }
    ClusterInfo clusterInfo = new ClusterInfo();
    clusterInfo.clusterName = _clusterName;
    ZKHelixAdmin zkHelixAdmin = new ZKHelixAdmin(_zkAddress);
    if (!zkHelixAdmin.getClusters().contains(_clusterName)) {
        LOGGER.error("Cluster {} not found in {}.", _clusterName, _zkAddress);
        return false;
    }
    List<String> instancesInCluster = zkHelixAdmin.getInstancesInCluster(_clusterName);
    List<String> tables = zkHelixAdmin.getResourcesInCluster(_clusterName);
    ZkClient zkClient = new ZkClient(_zkAddress);
    zkClient.setZkSerializer(new ZNRecordStreamingSerializer());
    LOGGER.info("Connecting to Zookeeper at: {}", _zkAddress);
    zkClient.waitUntilConnected();
    ZkBaseDataAccessor<ZNRecord> baseDataAccessor = new ZkBaseDataAccessor<>(zkClient);
    ZKHelixDataAccessor zkHelixDataAccessor = new ZKHelixDataAccessor(_clusterName, baseDataAccessor);
    PropertyKey property = zkHelixDataAccessor.keyBuilder().liveInstances();
    List<String> liveInstances = zkHelixDataAccessor.getChildNames(property);
    PropertyKey controllerLeaderKey = zkHelixDataAccessor.keyBuilder().controllerLeader();
    LiveInstance controllerLeaderLiveInstance = zkHelixDataAccessor.getProperty(controllerLeaderKey);
    ControllerInfo controllerInfo = new ControllerInfo();
    controllerInfo.leaderName = controllerLeaderLiveInstance.getId();
    clusterInfo.controllerInfo = controllerInfo;
    for (String server : instancesInCluster) {
        if (server.startsWith("Server")) {
            ServerInfo serverInfo = new ServerInfo();
            serverInfo.name = server;
            serverInfo.state = (liveInstances.contains(server)) ? "ONLINE" : "OFFLINE";
            InstanceConfig config = zkHelixAdmin.getInstanceConfig(_clusterName, server);
            serverInfo.tags = config.getRecord().getListField("TAG_LIST");
            clusterInfo.addServerInfo(serverInfo);
        }
        if (server.startsWith("Broker")) {
            BrokerInfo brokerInfo = new BrokerInfo();
            brokerInfo.name = server;
            brokerInfo.state = (liveInstances.contains(server)) ? "ONLINE" : "OFFLINE";
            InstanceConfig config = zkHelixAdmin.getInstanceConfig(_clusterName, server);
            brokerInfo.tags = config.getRecord().getListField("TAG_LIST");
            clusterInfo.addBrokerInfo(brokerInfo);
        }
    }
    for (String table : tables) {
        if ("brokerResource".equalsIgnoreCase(table)) {
            continue;
        }
        TableInfo tableInfo = new TableInfo();
        IdealState idealState = zkHelixAdmin.getResourceIdealState(_clusterName, table);
        ExternalView externalView = zkHelixAdmin.getResourceExternalView(_clusterName, table);
        Set<String> segmentsFromIdealState = idealState.getPartitionSet();
        tableInfo.tableName = table;
        tableInfo.tag = idealState.getRecord().getSimpleField("INSTANCE_GROUP_TAG");
        String rawTableName = stripTypeFromName(tableInfo.tableName);
        String rawTagName = stripTypeFromName(tableInfo.tag);
        if (!includeTableSet.isEmpty() && !includeTableSet.contains(rawTableName)) {
            continue;
        }
        if (!includeTagSet.isEmpty() && !includeTagSet.contains(rawTagName)) {
            continue;
        }
        for (String segment : segmentsFromIdealState) {
            SegmentInfo segmentInfo = new SegmentInfo();
            segmentInfo.name = segment;
            Map<String, String> serverStateMapFromIS = idealState.getInstanceStateMap(segment);
            if (serverStateMapFromIS == null) {
                LOGGER.info("Unassigned segment {} in ideal state", segment);
                serverStateMapFromIS = Collections.emptyMap();
            }
            Map<String, String> serverStateMapFromEV = externalView.getStateMap(segment);
            if (serverStateMapFromEV == null) {
                LOGGER.info("Unassigned segment {} in external view", segment);
                serverStateMapFromEV = Collections.emptyMap();
            }
            for (String serverName : serverStateMapFromIS.keySet()) {
                segmentInfo.segmentStateMap.put(serverName, serverStateMapFromEV.get(serverName));
            }
            tableInfo.addSegmentInfo(segmentInfo);
        }
        clusterInfo.addTableInfo(tableInfo);
    }
    Yaml yaml = new Yaml();
    StringWriter sw = new StringWriter();
    yaml.dump(clusterInfo, sw);
    LOGGER.info(sw.toString());
    return true;
}

Example 65 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PerfBenchmarkDriver method waitForExternalViewUpdate.

public static void waitForExternalViewUpdate(String zkAddress, final String clusterName, long timeoutInMilliseconds) {
    final ZKHelixAdmin helixAdmin = new ZKHelixAdmin(zkAddress);
    Verifier customVerifier = new Verifier() {

        @Override
        public boolean verify() {
            List<String> resourcesInCluster = helixAdmin.getResourcesInCluster(clusterName);
            LOGGER.info("Waiting for the cluster to be set up and indexes to be loaded on the servers" + new Timestamp(System.currentTimeMillis()));
            for (String resourceName : resourcesInCluster) {
                IdealState idealState = helixAdmin.getResourceIdealState(clusterName, resourceName);
                ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, resourceName);
                if (idealState == null || externalView == null) {
                    return false;
                }
                Set<String> partitionSet = idealState.getPartitionSet();
                for (String partition : partitionSet) {
                    Map<String, String> instanceStateMapIS = idealState.getInstanceStateMap(partition);
                    Map<String, String> instanceStateMapEV = externalView.getStateMap(partition);
                    if (instanceStateMapIS == null || instanceStateMapEV == null) {
                        return false;
                    }
                    if (!instanceStateMapIS.equals(instanceStateMapEV)) {
                        return false;
                    }
                }
            }
            LOGGER.info("Cluster is ready to serve queries");
            return true;
        }
    };
    ClusterStateVerifier.verifyByPolling(customVerifier, timeoutInMilliseconds);
}