Examples with IdealState org.apache.helix.model.IdealState used on opensource projects

Example 6 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotHelixResourceManager method updateExistedSegment.

private boolean updateExistedSegment(SegmentZKMetadata segmentZKMetadata) {
    final String tableName;
    if (segmentZKMetadata instanceof RealtimeSegmentZKMetadata) {
        tableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(segmentZKMetadata.getTableName());
    } else {
        tableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(segmentZKMetadata.getTableName());
    }
    final String segmentName = segmentZKMetadata.getSegmentName();
    HelixDataAccessor helixDataAccessor = _helixZkManager.getHelixDataAccessor();
    PropertyKey idealStatePropertyKey = _keyBuilder.idealStates(tableName);
    // Set all partitions to offline to unload them from the servers
    boolean updateSuccessful;
    do {
        final IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
        final Set<String> instanceSet = idealState.getInstanceSet(segmentName);
        if (instanceSet == null || instanceSet.size() == 0) {
            // We are trying to refresh a segment, but there are no instances currently assigned for fielding this segment.
            // When those instances do come up, the segment will be uploaded correctly, so return success but log a warning.
            LOGGER.warn("No instances as yet for segment {}, table {}", segmentName, tableName);
            return true;
        }
        for (final String instance : instanceSet) {
            idealState.setPartitionState(segmentName, instance, "OFFLINE");
        }
        updateSuccessful = helixDataAccessor.updateProperty(idealStatePropertyKey, idealState);
    } while (!updateSuccessful);
    // Check that the ideal state has been written to ZK
    IdealState updatedIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
    Map<String, String> instanceStateMap = updatedIdealState.getInstanceStateMap(segmentName);
    for (String state : instanceStateMap.values()) {
        if (!"OFFLINE".equals(state)) {
            LOGGER.error("Failed to write OFFLINE ideal state!");
            return false;
        }
    }
    // Wait until the partitions are offline in the external view
    LOGGER.info("Wait until segment - " + segmentName + " to be OFFLINE in ExternalView");
    if (!ifExternalViewChangeReflectedForState(tableName, segmentName, "OFFLINE", _externalViewOnlineToOfflineTimeoutMillis, false)) {
        LOGGER.error("External view for segment {} did not reflect the ideal state of OFFLINE within the {} ms time limit", segmentName, _externalViewOnlineToOfflineTimeoutMillis);
        return false;
    }
    // Set all partitions to online so that they load the new segment data
    do {
        final IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
        final Set<String> instanceSet = idealState.getInstanceSet(segmentName);
        LOGGER.info("Found {} instances for segment '{}', in ideal state", instanceSet.size(), segmentName);
        for (final String instance : instanceSet) {
            idealState.setPartitionState(segmentName, instance, "ONLINE");
            LOGGER.info("Setting Ideal State for segment '{}' to ONLINE for instance '{}'", segmentName, instance);
        }
        updateSuccessful = helixDataAccessor.updateProperty(idealStatePropertyKey, idealState);
    } while (!updateSuccessful);
    // Check that the ideal state has been written to ZK
    updatedIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
    instanceStateMap = updatedIdealState.getInstanceStateMap(segmentName);
    LOGGER.info("Found {} instances for segment '{}', after updating ideal state", instanceStateMap.size(), segmentName);
    for (String state : instanceStateMap.values()) {
        if (!"ONLINE".equals(state)) {
            LOGGER.error("Failed to write ONLINE ideal state!");
            return false;
        }
    }
    LOGGER.info("Refresh is done for segment - " + segmentName);
    return true;
}

Example 7 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotHelixResourceManager method ensureRealtimeClusterIsSetUp.

private void ensureRealtimeClusterIsSetUp(AbstractTableConfig config, String realtimeTableName, IndexingConfig indexingConfig) {
    KafkaStreamMetadata kafkaStreamMetadata = new KafkaStreamMetadata(indexingConfig.getStreamConfigs());
    IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, realtimeTableName);
    if (kafkaStreamMetadata.hasHighLevelKafkaConsumerType()) {
        if (kafkaStreamMetadata.hasSimpleKafkaConsumerType()) {
            // We may be adding on low-level, or creating both.
            if (idealState == null) {
                // Need to create both. Create high-level consumer first.
                createHelixEntriesForHighLevelConsumer(config, realtimeTableName, idealState);
                idealState = _helixAdmin.getResourceIdealState(_helixClusterName, realtimeTableName);
                LOGGER.info("Configured new HLC for table {}", realtimeTableName);
            }
        // Fall through to create low-level consumers
        } else {
            // Only high-level consumer specified in the config.
            createHelixEntriesForHighLevelConsumer(config, realtimeTableName, idealState);
            // Clean up any LLC table if they are present
            PinotLLCRealtimeSegmentManager.getInstance().cleanupLLC(realtimeTableName);
        }
    }
    // Either we have only low-level consumer, or both.
    if (kafkaStreamMetadata.hasSimpleKafkaConsumerType()) {
        // Will either create idealstate entry, or update the IS entry with new segments
        // (unless there are low-level segments already present)
        final String llcKafkaPartitionAssignmentPath = ZKMetadataProvider.constructPropertyStorePathForKafkaPartitions(realtimeTableName);
        if (!_propertyStore.exists(llcKafkaPartitionAssignmentPath, AccessOption.PERSISTENT)) {
            PinotTableIdealStateBuilder.buildLowLevelRealtimeIdealStateFor(realtimeTableName, config, _helixAdmin, _helixClusterName, idealState);
            LOGGER.info("Successfully added Helix entries for low-level consumers for {} ", realtimeTableName);
        } else {
            LOGGER.info("LLC is already set up for table {}, not configuring again", realtimeTableName);
        }
    }
}

Example 8 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class TableViews method getIdealState.

@Nullable
public Map<String, Map<String, String>> getIdealState(@Nonnull String tableNameOptType, @Parameter(name = "tableType", in = "query", description = "Table type", required = false) @Nullable TableType tableType) {
    String tableName = getTableName(tableNameOptType, tableType);
    IdealState resourceIdealState = _pinotHelixResourceManager.getHelixAdmin().getResourceIdealState(_pinotHelixResourceManager.getHelixClusterName(), tableName);
    return resourceIdealState == null ? null : resourceIdealState.getRecord().getMapFields();
}

Example 9 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotSegmentUploadRestletResource method getSegments.

private JSONArray getSegments(String tableName, String tableType) {
    final JSONArray ret = new JSONArray();
    String realtimeTableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(tableName);
    String offlineTableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(tableName);
    String tableNameWithType;
    if (CommonConstants.Helix.TableType.valueOf(tableType).toString().equals("REALTIME")) {
        tableNameWithType = realtimeTableName;
    } else {
        tableNameWithType = offlineTableName;
    }
    List<String> segmentList = _pinotHelixResourceManager.getAllSegmentsForResource(tableNameWithType);
    IdealState idealState = HelixHelper.getTableIdealState(_pinotHelixResourceManager.getHelixZkManager(), tableNameWithType);
    for (String segmentName : segmentList) {
        Map<String, String> map = idealState.getInstanceStateMap(segmentName);
        if (map == null) {
            continue;
        }
        if (!map.containsValue(PinotHelixSegmentOnlineOfflineStateModelGenerator.OFFLINE_STATE)) {
            ret.put(segmentName);
        }
    }
    return ret;
}

Example 10 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotHelixResourceManager method addInstanceToBrokerIdealState.

private void addInstanceToBrokerIdealState(String brokerTenantTag, String instanceName) {
    IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    for (String tableName : tableIdealState.getPartitionSet()) {
        if (TableNameBuilder.getTableTypeFromTableName(tableName) == TableType.OFFLINE) {
            String brokerTag = ControllerTenantNameBuilder.getBrokerTenantNameForTenant(ZKMetadataProvider.getOfflineTableConfig(getPropertyStore(), tableName).getTenantConfig().getBroker());
            if (brokerTag.equals(brokerTenantTag)) {
                tableIdealState.setPartitionState(tableName, instanceName, BrokerOnlineOfflineStateModel.ONLINE);
            }
        } else if (TableNameBuilder.getTableTypeFromTableName(tableName) == TableType.REALTIME) {
            String brokerTag = ControllerTenantNameBuilder.getBrokerTenantNameForTenant(ZKMetadataProvider.getRealtimeTableConfig(getPropertyStore(), tableName).getTenantConfig().getBroker());
            if (brokerTag.equals(brokerTenantTag)) {
                tableIdealState.setPartitionState(tableName, instanceName, BrokerOnlineOfflineStateModel.ONLINE);
            }
        }
    }
    _helixAdmin.setResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, tableIdealState);
}