Examples with IdealState org.apache.helix.model.IdealState used on opensource projects

Example 56 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotLLCRealtimeSegmentManager method completeCommittingSegmentsInternal.

private void completeCommittingSegmentsInternal(String realtimeTableName, Map<Integer, MinMaxPriorityQueue<LLCSegmentName>> partitionToLatestSegments) {
    IdealState idealState = getTableIdealState(realtimeTableName);
    Set<String> segmentNamesIS = idealState.getPartitionSet();
    final ZNRecord partitionAssignment = getKafkaPartitionAssignment(realtimeTableName);
    for (Map.Entry<Integer, MinMaxPriorityQueue<LLCSegmentName>> entry : partitionToLatestSegments.entrySet()) {
        final LLCSegmentName segmentName = entry.getValue().pollFirst();
        final String segmentId = segmentName.getSegmentName();
        final int partitionId = entry.getKey();
        if (!segmentNamesIS.contains(segmentId)) {
            LOGGER.info("{}:Repairing segment for partition {}. Segment {} not found in idealstate", realtimeTableName, partitionId, segmentId);
            List<String> newInstances = partitionAssignment.getListField(Integer.toString(partitionId));
            LOGGER.info("{}: Assigning segment {} to {}", realtimeTableName, segmentId, newInstances);
            // TODO Re-write num-partitions in metadata if needed.
            // If there was a prev segment in the same partition, then we need to fix it to be ONLINE.
            LLCSegmentName prevSegmentName = entry.getValue().pollLast();
            String prevSegmentNameStr = null;
            if (prevSegmentName != null) {
                prevSegmentNameStr = prevSegmentName.getSegmentName();
            }
            updateIdealState(realtimeTableName, newInstances, prevSegmentNameStr, segmentId);
        }
    }
}

Example 57 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class ValidationManager method validateLLCSegments.

// For LLC segments, validate that there is at least one segment in CONSUMING state for every partition.
void validateLLCSegments(final String realtimeTableName, AbstractTableConfig tableConfig) {
    LOGGER.info("Validating LLC Segments for {}", realtimeTableName);
    Map<String, String> streamConfigs = tableConfig.getIndexingConfig().getStreamConfigs();
    ZNRecord partitionAssignment = _llcRealtimeSegmentManager.getKafkaPartitionAssignment(realtimeTableName);
    if (partitionAssignment == null) {
        LOGGER.warn("No partition assignment found for table {}", realtimeTableName);
        return;
    }
    Map<String, List<String>> partitionToHostsMap = partitionAssignment.getListFields();
    // Keep a set of kafka partitions, and remove the partition when we find a segment in CONSUMING state in
    // that partition.
    Set<Integer> nonConsumingKafkaPartitions = new HashSet<>(partitionToHostsMap.size());
    for (String partitionStr : partitionToHostsMap.keySet()) {
        nonConsumingKafkaPartitions.add(Integer.valueOf(partitionStr));
    }
    IdealState idealState = HelixHelper.getTableIdealState(_pinotHelixResourceManager.getHelixZkManager(), realtimeTableName);
    if (!idealState.isEnabled()) {
        // No validation to be done.
        LOGGER.info("Skipping validation for {} since it is disabled", realtimeTableName);
        return;
    }
    // Walk through all segments in the idealState, looking for one instance that is in CONSUMING state. If we find one
    // remove the kafka partition that the segment belongs to, from the kafka partition set.
    // Make sure that there are at least some LLC segments in place. If there are no LLC segments, it is possible
    // that this table is in the process of being disabled for LLC
    Set<String> segmentIds = idealState.getPartitionSet();
    List<String> llcSegments = new ArrayList<>(segmentIds.size());
    for (String segmentId : segmentIds) {
        if (SegmentName.isLowLevelConsumerSegmentName(segmentId)) {
            llcSegments.add(segmentId);
            Map<String, String> stateMap = idealState.getInstanceStateMap(segmentId);
            Iterator<String> iterator = stateMap.values().iterator();
            // If there is at least one instance in CONSUMING state, we are good.
            boolean foundConsuming = false;
            while (iterator.hasNext() && !foundConsuming) {
                String stateString = iterator.next();
                if (stateString.equals(PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE)) {
                    LOGGER.info("Found CONSUMING segment {}", segmentId);
                    foundConsuming = true;
                }
            }
            if (foundConsuming) {
                LLCSegmentName llcSegmentName = new LLCSegmentName(segmentId);
                nonConsumingKafkaPartitions.remove(llcSegmentName.getPartitionId());
            }
        }
    }
    // Kafka partition set now has all the partitions that do not have any segments in CONSUMING state.
    if (!llcSegments.isEmpty()) {
        // Raise the metric only if there is at least one llc segment in the idealstate.
        _validationMetrics.updateNumNonConsumingPartitionsMetric(realtimeTableName, nonConsumingKafkaPartitions.size());
        // Recreate a segment for the partitions that are missing one.
        for (Integer kafkaPartition : nonConsumingKafkaPartitions) {
            LOGGER.warn("Table {}, kafka partition {} has no segments in CONSUMING state (out of {} llc segments)", realtimeTableName, kafkaPartition, llcSegments.size());
        }
        if (_autoCreateOnError) {
            _llcRealtimeSegmentManager.createConsumingSegment(realtimeTableName, nonConsumingKafkaPartitions, llcSegments, tableConfig);
            _llcRealtimeSegmentManager.completeCommittingSegments(realtimeTableName, llcSegments);
        }
    }
    // Make this call after other validations (so that we verify that we are consistent against the existing partition
    // assignment). This call may end up changing the kafka partition assignment for the table.
    _llcRealtimeSegmentManager.updateKafkaPartitionsIfNecessary(realtimeTableName, tableConfig);
}

Example 58 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotTableIdealStateBuilder method dropSegmentFromIdealStateFor.

/**
   * Remove a segment is also required to recompute the ideal state.
   *
   * @param tableName
   * @param segmentId
   * @param helixAdmin
   * @param helixClusterName
   * @return
   */
public static synchronized IdealState dropSegmentFromIdealStateFor(String tableName, String segmentId, HelixAdmin helixAdmin, String helixClusterName) {
    final IdealState currentIdealState = helixAdmin.getResourceIdealState(helixClusterName, tableName);
    final Set<String> currentInstanceSet = currentIdealState.getInstanceSet(segmentId);
    if (!currentInstanceSet.isEmpty() && currentIdealState.getPartitionSet().contains(segmentId)) {
        for (String instanceName : currentIdealState.getInstanceSet(segmentId)) {
            currentIdealState.setPartitionState(segmentId, instanceName, "DROPPED");
        }
    } else {
        throw new RuntimeException("Cannot found segmentId - " + segmentId + " in table - " + tableName);
    }
    return currentIdealState;
}

Example 59 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotHelixResourceManager method getAllInstancesForTable.

private Set<String> getAllInstancesForTable(String tableName) {
    Set<String> instanceSet = new HashSet<String>();
    IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
    for (String partition : tableIdealState.getPartitionSet()) {
        instanceSet.addAll(tableIdealState.getInstanceSet(partition));
    }
    return instanceSet;
}

Example 60 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotHelixResourceManager method toggleSegmentState.

/**
   * Toggle the status of segment between ONLINE (enable = true) and OFFLINE (enable = FALSE).
   *
   * @param tableName: Name of table to which the segment belongs.
   * @param segments: List of segment for which to toggle the status.
   * @param enable: True for ONLINE, False for OFFLINE.
   * @param timeoutInSeconds Time out for toggling segment state.
   * @return
   */
public PinotResourceManagerResponse toggleSegmentState(String tableName, List<String> segments, boolean enable, long timeoutInSeconds) {
    String status = (enable) ? "ONLINE" : "OFFLINE";
    HelixDataAccessor helixDataAccessor = _helixZkManager.getHelixDataAccessor();
    PropertyKey idealStatePropertyKey = _keyBuilder.idealStates(tableName);
    boolean updateSuccessful;
    boolean externalViewUpdateSuccessful = true;
    long deadline = System.currentTimeMillis() + 1000 * timeoutInSeconds;
    // Set all partitions to offline to unload them from the servers
    do {
        final IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
        for (String segmentName : segments) {
            final Set<String> instanceSet = idealState.getInstanceSet(segmentName);
            if (instanceSet == null || instanceSet.isEmpty()) {
                return new PinotResourceManagerResponse("Segment " + segmentName + " not found.", false);
            }
            for (final String instance : instanceSet) {
                idealState.setPartitionState(segmentName, instance, status);
            }
        }
        updateSuccessful = helixDataAccessor.updateProperty(idealStatePropertyKey, idealState);
    } while (!updateSuccessful && (System.currentTimeMillis() <= deadline));
    // Check that the ideal state has been updated.
    LOGGER.info("Ideal state successfully updated, waiting to update external view");
    IdealState updatedIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
    for (String segmentName : segments) {
        Map<String, String> instanceStateMap = updatedIdealState.getInstanceStateMap(segmentName);
        for (String state : instanceStateMap.values()) {
            if (!status.equals(state)) {
                return new PinotResourceManagerResponse("Error: Failed to update Ideal state when setting status " + status + " for segment " + segmentName, false);
            }
        }
        // Wait until the partitions are offline in the external view
        if (!ifExternalViewChangeReflectedForState(tableName, segmentName, status, (timeoutInSeconds * 1000), true)) {
            externalViewUpdateSuccessful = false;
        }
    }
    return (externalViewUpdateSuccessful) ? new PinotResourceManagerResponse(("Success: Segment(s) " + " now " + status), true) : new PinotResourceManagerResponse("Error: Timed out. External view not completely updated", false);
}