Examples with IdealState org.apache.helix.model.IdealState used on opensource projects

Example 21 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotTableIdealStateBuilder method buildEmptyIdealStateForBrokerResource.

/**
   *
   * Building an empty idealState for a given table.
   * Used when creating a new table.
   *
   * @param helixAdmin
   * @param helixClusterName
   * @return
   */
public static IdealState buildEmptyIdealStateForBrokerResource(HelixAdmin helixAdmin, String helixClusterName) {
    final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    customModeIdealStateBuilder.setStateModel(PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL).setMaxPartitionsPerNode(Integer.MAX_VALUE).setNumReplica(Integer.MAX_VALUE).setNumPartitions(Integer.MAX_VALUE);
    final IdealState idealState = customModeIdealStateBuilder.build();
    return idealState;
}

Example 22 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotTableIdealStateBuilder method buildEmptyKafkaConsumerRealtimeIdealStateFor.

public static IdealState buildEmptyKafkaConsumerRealtimeIdealStateFor(String realtimeTableName, int replicaCount) {
    final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(realtimeTableName);
    customModeIdealStateBuilder.setStateModel(PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL).setNumPartitions(0).setNumReplica(replicaCount).setMaxPartitionsPerNode(1);
    final IdealState idealState = customModeIdealStateBuilder.build();
    idealState.setInstanceGroupTag(realtimeTableName);
    return idealState;
}

Example 23 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class SegmentDeletionManager method deleteSegmentFromPropertyStoreAndLocal.

protected synchronized void deleteSegmentFromPropertyStoreAndLocal(String tableName, Collection<String> segmentIds, long deletionDelay) {
    // Check if segment got removed from ExternalView and IdealStates
    if (_helixAdmin.getResourceExternalView(_helixClusterName, tableName) == null || _helixAdmin.getResourceIdealState(_helixClusterName, tableName) == null) {
        LOGGER.warn("Resource: {} is not set up in idealState or ExternalView, won't do anything", tableName);
        return;
    }
    // Has the segments that will be deleted
    List<String> segmentsToDelete = new ArrayList<>(segmentIds.size());
    // List of segments that we need to retry
    Set<String> segmentsToRetryLater = new HashSet<>(segmentIds.size());
    try {
        ExternalView externalView = _helixAdmin.getResourceExternalView(_helixClusterName, tableName);
        IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
        for (String segmentId : segmentIds) {
            Map<String, String> segmentToInstancesMapFromExternalView = externalView.getStateMap(segmentId);
            Map<String, String> segmentToInstancesMapFromIdealStates = idealState.getInstanceStateMap(segmentId);
            if ((segmentToInstancesMapFromExternalView == null || segmentToInstancesMapFromExternalView.isEmpty()) && (segmentToInstancesMapFromIdealStates == null || segmentToInstancesMapFromIdealStates.isEmpty())) {
                segmentsToDelete.add(segmentId);
            } else {
                segmentsToRetryLater.add(segmentId);
            }
        }
    } catch (Exception e) {
        LOGGER.warn("Caught exception while checking helix states for table {} " + tableName, e);
        segmentsToDelete.clear();
        segmentsToDelete.addAll(segmentIds);
        segmentsToRetryLater.clear();
    }
    if (!segmentsToDelete.isEmpty()) {
        List<String> propStorePathList = new ArrayList<>(segmentsToDelete.size());
        for (String segmentId : segmentsToDelete) {
            String segmentPropertyStorePath = ZKMetadataProvider.constructPropertyStorePathForSegment(tableName, segmentId);
            propStorePathList.add(segmentPropertyStorePath);
        }
        boolean[] deleteSuccessful = _propertyStore.remove(propStorePathList, AccessOption.PERSISTENT);
        List<String> propStoreFailedSegs = new ArrayList<>(segmentsToDelete.size());
        for (int i = 0; i < deleteSuccessful.length; i++) {
            final String segmentId = segmentsToDelete.get(i);
            if (!deleteSuccessful[i]) {
                // remove API can fail because the prop store entry did not exist, so check first.
                if (_propertyStore.exists(propStorePathList.get(i), AccessOption.PERSISTENT)) {
                    LOGGER.info("Could not delete {} from propertystore", propStorePathList.get(i));
                    segmentsToRetryLater.add(segmentId);
                    propStoreFailedSegs.add(segmentId);
                }
            }
        }
        segmentsToDelete.removeAll(propStoreFailedSegs);
        for (String segmentId : segmentsToDelete) {
            removeSegmentFromStore(tableName, segmentId);
        }
    }
    LOGGER.info("Deleted {} segments from table {}:{}", segmentsToDelete.size(), tableName, segmentsToDelete.size() <= 5 ? segmentsToDelete : "");
    if (segmentsToRetryLater.size() > 0) {
        long effectiveDeletionDelay = Math.min(deletionDelay * 2, MAX_DELETION_DELAY_SECONDS);
        LOGGER.info("Postponing deletion of {} segments from table {}", segmentsToRetryLater.size(), tableName);
        deleteSegmentsWithDelay(tableName, segmentsToRetryLater, effectiveDeletionDelay);
        return;
    }
}

Example 24 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class SegmentStatusCheckerTest method noReplicas.

@Test
public void noReplicas() throws Exception {
    final String tableName = "myTable_REALTIME";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = new IdealState(tableName);
    idealState.setPartitionState("myTable_0", "pinot1", "OFFLINE");
    idealState.setPartitionState("myTable_0", "pinot2", "OFFLINE");
    idealState.setPartitionState("myTable_0", "pinot3", "OFFLINE");
    idealState.setReplicas("0");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(null);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 1);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS), 100);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
    segmentStatusChecker.stop();
}

Example 25 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class SegmentStatusCheckerTest method noIdealState.

@Test
public void noIdealState() throws Exception {
    final String tableName = "myTable_REALTIME";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = null;
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(null);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 1);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS), 100);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
    segmentStatusChecker.stop();
}