Examples with HelixDataAccessor org.apache.helix.HelixDataAccessor used on opensource projects

Example 1 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project pinot by linkedin.

the class HelixHelper method updateIdealState.

/**
   * Updates the ideal state, retrying if necessary in case of concurrent updates to the ideal state.
   *
   * @param helixManager The HelixManager used to interact with the Helix cluster
   * @param resourceName The resource for which to update the ideal state
   * @param updater A function that returns an updated ideal state given an input ideal state
   */
public static void updateIdealState(final HelixManager helixManager, final String resourceName, final Function<IdealState, IdealState> updater, RetryPolicy policy) {
    boolean successful = policy.attempt(new Callable<Boolean>() {

        @Override
        public Boolean call() {
            HelixDataAccessor dataAccessor = helixManager.getHelixDataAccessor();
            PropertyKey propertyKey = dataAccessor.keyBuilder().idealStates(resourceName);
            // Create an updated version of the ideal state
            IdealState idealState = dataAccessor.getProperty(propertyKey);
            PropertyKey key = dataAccessor.keyBuilder().idealStates(resourceName);
            String path = key.getPath();
            // Make a copy of the the idealState above to pass it to the updater, instead of querying again,
            // as the state my change between the queries.
            ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer();
            IdealState idealStateCopy = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord())));
            IdealState updatedIdealState;
            try {
                updatedIdealState = updater.apply(idealStateCopy);
            } catch (Exception e) {
                LOGGER.error("Caught exception while updating ideal state", e);
                return false;
            }
            // If there are changes to apply, apply them
            if (!EqualityUtils.isEqual(idealState, updatedIdealState) && updatedIdealState != null) {
                BaseDataAccessor<ZNRecord> baseDataAccessor = dataAccessor.getBaseDataAccessor();
                boolean success;
                // If the ideal state is large enough, enable compression
                if (MAX_PARTITION_COUNT_IN_UNCOMPRESSED_IDEAL_STATE < updatedIdealState.getPartitionSet().size()) {
                    updatedIdealState.getRecord().setBooleanField("enableCompression", true);
                }
                try {
                    success = baseDataAccessor.set(path, updatedIdealState.getRecord(), idealState.getRecord().getVersion(), AccessOption.PERSISTENT);
                } catch (Exception e) {
                    boolean idealStateIsCompressed = updatedIdealState.getRecord().getBooleanField("enableCompression", false);
                    LOGGER.warn("Caught exception while updating ideal state for resource {} (compressed={}), retrying.", resourceName, idealStateIsCompressed, e);
                    return false;
                }
                if (success) {
                    return true;
                } else {
                    LOGGER.warn("Failed to update ideal state for resource {}, retrying.", resourceName);
                    return false;
                }
            } else {
                LOGGER.warn("Idempotent or null ideal state update for resource {}, skipping update.", resourceName);
                return true;
            }
        }
    });
    if (!successful) {
        throw new RuntimeException("Failed to update ideal state for resource " + resourceName);
    }
}

Example 2 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project pinot by linkedin.

the class HelixHelper method getTableIdealState.

public static IdealState getTableIdealState(HelixManager manager, String resourceName) {
    final HelixDataAccessor accessor = manager.getHelixDataAccessor();
    final Builder builder = accessor.keyBuilder();
    return accessor.getProperty(builder.idealStates(resourceName));
}

Example 3 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project pinot by linkedin.

the class PinotHelixResourceManager method updateExistedSegment.

private boolean updateExistedSegment(SegmentZKMetadata segmentZKMetadata) {
    final String tableName;
    if (segmentZKMetadata instanceof RealtimeSegmentZKMetadata) {
        tableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(segmentZKMetadata.getTableName());
    } else {
        tableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(segmentZKMetadata.getTableName());
    }
    final String segmentName = segmentZKMetadata.getSegmentName();
    HelixDataAccessor helixDataAccessor = _helixZkManager.getHelixDataAccessor();
    PropertyKey idealStatePropertyKey = _keyBuilder.idealStates(tableName);
    // Set all partitions to offline to unload them from the servers
    boolean updateSuccessful;
    do {
        final IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
        final Set<String> instanceSet = idealState.getInstanceSet(segmentName);
        if (instanceSet == null || instanceSet.size() == 0) {
            // We are trying to refresh a segment, but there are no instances currently assigned for fielding this segment.
            // When those instances do come up, the segment will be uploaded correctly, so return success but log a warning.
            LOGGER.warn("No instances as yet for segment {}, table {}", segmentName, tableName);
            return true;
        }
        for (final String instance : instanceSet) {
            idealState.setPartitionState(segmentName, instance, "OFFLINE");
        }
        updateSuccessful = helixDataAccessor.updateProperty(idealStatePropertyKey, idealState);
    } while (!updateSuccessful);
    // Check that the ideal state has been written to ZK
    IdealState updatedIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
    Map<String, String> instanceStateMap = updatedIdealState.getInstanceStateMap(segmentName);
    for (String state : instanceStateMap.values()) {
        if (!"OFFLINE".equals(state)) {
            LOGGER.error("Failed to write OFFLINE ideal state!");
            return false;
        }
    }
    // Wait until the partitions are offline in the external view
    LOGGER.info("Wait until segment - " + segmentName + " to be OFFLINE in ExternalView");
    if (!ifExternalViewChangeReflectedForState(tableName, segmentName, "OFFLINE", _externalViewOnlineToOfflineTimeoutMillis, false)) {
        LOGGER.error("External view for segment {} did not reflect the ideal state of OFFLINE within the {} ms time limit", segmentName, _externalViewOnlineToOfflineTimeoutMillis);
        return false;
    }
    // Set all partitions to online so that they load the new segment data
    do {
        final IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
        final Set<String> instanceSet = idealState.getInstanceSet(segmentName);
        LOGGER.info("Found {} instances for segment '{}', in ideal state", instanceSet.size(), segmentName);
        for (final String instance : instanceSet) {
            idealState.setPartitionState(segmentName, instance, "ONLINE");
            LOGGER.info("Setting Ideal State for segment '{}' to ONLINE for instance '{}'", segmentName, instance);
        }
        updateSuccessful = helixDataAccessor.updateProperty(idealStatePropertyKey, idealState);
    } while (!updateSuccessful);
    // Check that the ideal state has been written to ZK
    updatedIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
    instanceStateMap = updatedIdealState.getInstanceStateMap(segmentName);
    LOGGER.info("Found {} instances for segment '{}', after updating ideal state", instanceStateMap.size(), segmentName);
    for (String state : instanceStateMap.values()) {
        if (!"ONLINE".equals(state)) {
            LOGGER.error("Failed to write ONLINE ideal state!");
            return false;
        }
    }
    LOGGER.info("Refresh is done for segment - " + segmentName);
    return true;
}

Example 4 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project pinot by linkedin.

the class PinotHelixResourceManager method dropInstance.

/**
   * Drop the instance from helix cluster. Instance will not be dropped if:
   * - It is a live instance.
   * - Has at least one ONLINE segment.
   *
   * @param instanceName: Name of the instance to be dropped.
   * @return
   */
public PinotResourceManagerResponse dropInstance(String instanceName) {
    if (!instanceExists(instanceName)) {
        return new PinotResourceManagerResponse("Instance " + instanceName + " does not exist.", false);
    }
    HelixDataAccessor helixDataAccessor = _helixZkManager.getHelixDataAccessor();
    LiveInstance liveInstance = helixDataAccessor.getProperty(_keyBuilder.liveInstance(instanceName));
    if (liveInstance != null) {
        PropertyKey currentStatesKey = _keyBuilder.currentStates(instanceName, liveInstance.getSessionId());
        List<CurrentState> currentStates = _helixDataAccessor.getChildValues(currentStatesKey);
        if (currentStates != null) {
            for (CurrentState currentState : currentStates) {
                for (String state : currentState.getPartitionStateMap().values()) {
                    if (state.equalsIgnoreCase(SegmentOnlineOfflineStateModel.ONLINE)) {
                        return new PinotResourceManagerResponse(("Instance " + instanceName + " has online partitions"), false);
                    }
                }
            }
        } else {
            return new PinotResourceManagerResponse("Cannot drop live instance " + instanceName + " please stop the instance first.", false);
        }
    }
    // Disable the instance first.
    toogleInstance(instanceName, false, 10);
    _helixAdmin.dropInstance(_helixClusterName, getHelixInstanceConfig(instanceName));
    return new PinotResourceManagerResponse("Instance " + instanceName + " dropped.", true);
}

Example 5 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project pinot by linkedin.

the class PinotHelixResourceManager method instanceExists.

/**
   * Check if an Instance exists in the Helix cluster.
   *
   * @param instanceName: Name of instance to check.
   * @return True if instance exists in the Helix cluster, False otherwise.
   */
public boolean instanceExists(String instanceName) {
    HelixDataAccessor helixDataAccessor = _helixZkManager.getHelixDataAccessor();
    InstanceConfig config = helixDataAccessor.getProperty(_keyBuilder.instanceConfig(instanceName));
    return (config != null);
}