Examples with IdealState org.apache.helix.model.IdealState used on opensource projects

Example 36 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class UpdateSegmentState method fixTableIdealState.

public void fixTableIdealState(String tableName) throws Exception {
    IdealState idealState = _helixAdmin.getResourceIdealState(_clusterName, tableName);
    if (idealState == null) {
        LOGGER.info("No IDEALSTATE found for table " + tableName);
        return;
    }
    Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields();
    int nChanges = 0;
    for (String segment : mapFieldsIS.keySet()) {
        Map<String, String> mapIS = mapFieldsIS.get(segment);
        for (String server : mapIS.keySet()) {
            String state = mapIS.get(server);
            if (state.equals(fromState)) {
                if (_fix) {
                    mapIS.put(server, toState);
                } else {
                    LOGGER.info("Table:" + tableName + ",Segment:" + segment + ",Server:" + server + ":" + fromState);
                }
                nChanges++;
            }
        }
    }
    if (nChanges == 0) {
        LOGGER.info("No segments detected in " + fromState + " state for table " + tableName);
    } else {
        if (_fix) {
            LOGGER.info("Replacing IDEALSTATE for table " + tableName + " with " + nChanges + " changes");
            _helixAdmin.setResourceIdealState(_clusterName, tableName, idealState);
        } else {
            LOGGER.info("Detected " + nChanges + " instances in " + fromState + " in table " + tableName);
        }
    }
}

Example 37 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotSegmentRebalancer method isStable.

/**
   * return true if IdealState = ExternalView
   * @return
   */
public int isStable(String tableName) {
    IdealState idealState = helixAdmin.getResourceIdealState(clusterName, tableName);
    ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, tableName);
    Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields();
    Map<String, Map<String, String>> mapFieldsEV = externalView.getRecord().getMapFields();
    int numDiff = 0;
    for (String segment : mapFieldsIS.keySet()) {
        Map<String, String> mapIS = mapFieldsIS.get(segment);
        Map<String, String> mapEV = mapFieldsEV.get(segment);
        for (String server : mapIS.keySet()) {
            String state = mapIS.get(server);
            if (mapEV == null || mapEV.get(server) == null || !mapEV.get(server).equals(state)) {
                LOGGER.info("Mismatch: segment" + segment + " server:" + server + " state:" + state);
                numDiff = numDiff + 1;
            }
        }
    }
    return numDiff;
}

Example 38 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class VerifySegmentState method execute.

@Override
public boolean execute() throws Exception {
    ZKHelixAdmin helixAdmin = new ZKHelixAdmin(zkAddress);
    List<String> resourcesInCluster = helixAdmin.getResourcesInCluster(clusterName);
    for (String resourceName : resourcesInCluster) {
        if (resourceName.startsWith(tablePrefix)) {
            IdealState resourceIdealState = helixAdmin.getResourceIdealState(clusterName, resourceName);
            ExternalView resourceExternalView = helixAdmin.getResourceExternalView(clusterName, resourceName);
            Map<String, Map<String, String>> mapFieldsFromIS = resourceIdealState.getRecord().getMapFields();
            Map<String, Map<String, String>> mapFieldsFromEV = resourceExternalView.getRecord().getMapFields();
            boolean error = false;
            if (mapFieldsFromIS.size() != mapFieldsFromEV.size()) {
                LOGGER.info("Table: {}, idealState size: {} does NOT match external view size: {}", resourceName, mapFieldsFromIS.size(), mapFieldsFromEV.size());
                error = true;
            }
            if (!mapFieldsFromIS.keySet().equals(mapFieldsFromEV.keySet())) {
                Set<String> idealStateKeys = mapFieldsFromIS.keySet();
                Set<String> externalViewKeys = mapFieldsFromEV.keySet();
                Sets.SetView<String> isToEVDiff = Sets.difference(idealStateKeys, externalViewKeys);
                for (String segmentName : isToEVDiff) {
                    LOGGER.info("Segment: {} is missing in external view, ideal state: {}", segmentName, mapFieldsFromIS.get(segmentName));
                }
                Sets.SetView<String> evToISDiff = Sets.difference(externalViewKeys, idealStateKeys);
                for (String segmentName : evToISDiff) {
                    LOGGER.error("Segment: {} is missing in ideal state, external view: {}", segmentName, mapFieldsFromEV.get(segmentName));
                }
                error = true;
            }
            for (Map.Entry<String, Map<String, String>> idealEntry : mapFieldsFromIS.entrySet()) {
                String segmentName = idealEntry.getKey();
                Map<String, String> segmentIdealState = idealEntry.getValue();
                // try to format consistently for tool based parsing
                if (!mapFieldsFromEV.containsKey(segmentName)) {
                    LOGGER.info("Segment: {} idealstate: {} is MISSING in external view: {}", segmentName, segmentIdealState, "");
                }
                Map<String, String> segmentExternalView = mapFieldsFromEV.get(segmentName);
                if (!segmentIdealState.equals(segmentExternalView)) {
                    LOGGER.info("Segment: {} idealstate: {} does NOT match external view: {}", segmentName, segmentIdealState, segmentExternalView);
                    error = true;
                }
            }
            LOGGER.info(resourceName + " = " + (error ? "ERROR" : "OK"));
        }
    }
    return true;
}

Example 39 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class DeleteOverlappingSegmentsInPinot method deleteOverlappingSegments.

public static boolean deleteOverlappingSegments(String zkUrl, String zkCluster, String tableName) {
    boolean updateSuccessful = false;
    if (!tableName.endsWith("_OFFLINE")) {
        tableName = tableName + "_OFFLINE";
    }
    ZkClient zkClient = new ZkClient(zkUrl);
    ZNRecordSerializer zkSerializer = new ZNRecordSerializer();
    zkClient.setZkSerializer(zkSerializer);
    BaseDataAccessor<ZNRecord> baseDataAccessor = new ZkBaseDataAccessor<>(zkClient);
    HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(zkCluster, baseDataAccessor);
    Builder keyBuilder = helixDataAccessor.keyBuilder();
    PropertyKey idealStateKey = keyBuilder.idealStates(tableName);
    PropertyKey externalViewKey = keyBuilder.externalView(tableName);
    IdealState currentIdealState = helixDataAccessor.getProperty(idealStateKey);
    byte[] serializeIS = zkSerializer.serialize(currentIdealState.getRecord());
    String name = tableName + ".idealstate." + System.currentTimeMillis();
    File outputFile = new File("/tmp", name);
    try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
        IOUtils.write(serializeIS, fileOutputStream);
    } catch (IOException e) {
        LOG.error("Exception in delete overlapping segments", e);
        return updateSuccessful;
    }
    LOG.info("Saved current idealstate to {}", outputFile);
    IdealState newIdealState;
    do {
        newIdealState = computeNewIdealStateAfterDeletingOverlappingSegments(helixDataAccessor, idealStateKey);
        LOG.info("Updating IdealState");
        updateSuccessful = helixDataAccessor.getBaseDataAccessor().set(idealStateKey.getPath(), newIdealState.getRecord(), newIdealState.getRecord().getVersion(), AccessOption.PERSISTENT);
        if (updateSuccessful) {
            int numSegmentsDeleted = currentIdealState.getPartitionSet().size() - newIdealState.getPartitionSet().size();
            LOG.info("Successfully updated IdealState: Removed segments: {}", (numSegmentsDeleted));
        }
    } while (!updateSuccessful);
    try {
        while (true) {
            Thread.sleep(10000);
            ExternalView externalView = helixDataAccessor.getProperty(externalViewKey);
            IdealState idealState = helixDataAccessor.getProperty(idealStateKey);
            Set<String> evPartitionSet = externalView.getPartitionSet();
            Set<String> isPartitionSet = idealState.getPartitionSet();
            if (evPartitionSet.equals(isPartitionSet)) {
                LOG.info("Table {} has reached stable state. i.e segments in external view match idealstates", tableName);
                break;
            }
        }
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    return updateSuccessful;
}

Example 40 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class DeleteOverlappingSegmentsInPinot method computeNewIdealStateAfterDeletingOverlappingSegments.

private static IdealState computeNewIdealStateAfterDeletingOverlappingSegments(HelixDataAccessor helixDataAccessor, PropertyKey idealStatesKey) {
    IdealState is = helixDataAccessor.getProperty(idealStatesKey);
    // compute existing DAILY segments
    Set<String> daysWithDailySegments = new HashSet<>();
    for (String segmentName : is.getPartitionSet()) {
        LOG.info("Segment Name : {}", segmentName);
        if (segmentName.indexOf("DAILY") > -1) {
            String[] splits = segmentName.split("_");
            String endDay = splits[splits.length - 2].substring(0, "yyyy-mm-dd".length());
            String startDay = splits[splits.length - 3].substring(0, "yyyy-mm-dd".length());
            LOG.info("Start : {} End : {}", startDay, endDay);
            daysWithDailySegments.add(startDay);
        }
    }
    // compute list of HOURLY segments to be deleted
    Set<String> hourlySegmentsToDelete = new TreeSet<>();
    for (String segmentName : is.getPartitionSet()) {
        LOG.info("Segment name {}", segmentName);
        if (segmentName.indexOf("HOURLY") > -1) {
            String[] splits = segmentName.split("_");
            String endDay = splits[splits.length - 2].substring(0, "yyyy-mm-dd".length());
            String startDay = splits[splits.length - 3].substring(0, "yyyy-mm-dd".length());
            LOG.info("Start : {} End : {}", startDay, endDay);
            if (daysWithDailySegments.contains(startDay)) {
                hourlySegmentsToDelete.add(segmentName);
            }
        }
    }
    LOG.info("HOURLY segments that can be deleted: {}", hourlySegmentsToDelete.size());
    LOG.info("Hourly segments to delete {}", hourlySegmentsToDelete.toString().replaceAll(",", "\n"));
    IdealState newIdealState = new IdealState(is.getRecord());
    for (String hourlySegmentToDelete : hourlySegmentsToDelete) {
        newIdealState.getRecord().getMapFields().remove(hourlySegmentToDelete);
    }
    return newIdealState;
}