Examples with IdealState org.apache.helix.model.IdealState used on opensource projects

Example 41 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class HelixHelperTest method testWriteLargeIdealState.

/**
   * Regression test for large ideal state updates failing silently
   */
@Test
public void testWriteLargeIdealState() {
    final int segmentCount = 20000;
    PinotHelixResourceManager resourceManager = _controller.getHelixResourceManager();
    final IdealState idealState = new IdealState(RESOURCE_NAME);
    idealState.setStateModelDefRef("OnlineOffline");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    idealState.setReplicas("0");
    resourceManager.getHelixAdmin().addResource(CLUSTER_NAME, RESOURCE_NAME, idealState);
    HelixHelper.updateIdealState(resourceManager.getHelixZkManager(), RESOURCE_NAME, new Function<IdealState, IdealState>() {

        @Override
        public IdealState apply(@Nullable IdealState idealState) {
            for (int i = 0; i < segmentCount; ++i) {
                idealState.setPartitionState("segment_" + i, INSTANCE_NAME, "ONLINE");
            }
            return idealState;
        }
    }, RetryPolicies.noDelayRetryPolicy(1));
    Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
    final IdealState resourceIdealState = resourceManager.getHelixAdmin().getResourceIdealState(CLUSTER_NAME, RESOURCE_NAME);
    for (int i = 0; i < segmentCount; ++i) {
        assertEquals(resourceIdealState.getInstanceStateMap("segment_" + i).get(INSTANCE_NAME), "ONLINE");
    }
}

Example 42 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class PinotResourceManagerTest method testConcurrentAddingAndDeletingSegments.

/**
   * Creates 5 threads that concurrently try to add 20 segments each, and asserts that we have
   * 100 segments in the end. Then launches 5 threads again that concurrently try to delete all segments,
   * and makes sure that we have zero segments left in the end.
   * @throws Exception
   */
@Test
public void testConcurrentAddingAndDeletingSegments() throws Exception {
    ExecutorService addSegmentExecutor = Executors.newFixedThreadPool(5);
    for (int i = 0; i < 5; ++i) {
        addSegmentExecutor.execute(new Runnable() {

            @Override
            public void run() {
                for (int i = 0; i < 20; ++i) {
                    addOneSegment(TABLE_NAME);
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException e) {
                        Assert.assertFalse(true, "Exception caught during sleep.");
                    }
                }
            }
        });
    }
    addSegmentExecutor.shutdown();
    while (!addSegmentExecutor.isTerminated()) {
    }
    final String offlineTableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(TABLE_NAME);
    IdealState idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, offlineTableName);
    Assert.assertEquals(idealState.getPartitionSet().size(), 100);
    ExecutorService deleteSegmentExecutor = Executors.newFixedThreadPool(5);
    for (final String segment : idealState.getPartitionSet()) {
        deleteSegmentExecutor.execute(new Runnable() {

            @Override
            public void run() {
                deleteOneSegment(offlineTableName, segment);
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    Assert.assertFalse(true, "Exception caught during sleep.");
                }
            }
        });
    }
    deleteSegmentExecutor.shutdown();
    while (!deleteSegmentExecutor.isTerminated()) {
    }
    idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, offlineTableName);
    Assert.assertEquals(idealState.getPartitionSet().size(), 0);
}

Example 43 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class SegmentStatusCheckerTest method basicTest.

@Test
public void basicTest() throws Exception {
    final String tableName = "myTable_OFFLINE";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = new IdealState(tableName);
    idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_0", "pinot2", "ONLINE");
    idealState.setPartitionState("myTable_0", "pinot3", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot3", "ONLINE");
    idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE");
    idealState.setReplicas("2");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    ExternalView externalView = new ExternalView(tableName);
    externalView.setState("myTable_0", "pinot1", "ONLINE");
    externalView.setState("myTable_0", "pinot2", "ONLINE");
    externalView.setState("myTable_1", "pinot1", "ERROR");
    externalView.setState("myTable_1", "pinot2", "ONLINE");
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(externalView);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 1);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS), 1);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_OF_REPLICAS), 33);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
    segmentStatusChecker.stop();
}

Example 44 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class SegmentStatusCheckerTest method missingEVTest.

@Test
public void missingEVTest() throws Exception {
    final String tableName = "myTable_REALTIME";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = new IdealState(tableName);
    idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_0", "pinot2", "ONLINE");
    idealState.setPartitionState("myTable_0", "pinot3", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot3", "ONLINE");
    idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE");
    idealState.setReplicas("2");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(null);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 0);
    segmentStatusChecker.stop();
}

Example 45 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class SegmentStatusCheckerTest method noSegments.

@Test
public void noSegments() throws Exception {
    final String tableName = "myTable_REALTIME";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = new IdealState(tableName);
    idealState.setReplicas("0");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(null);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 1);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS), 100);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
    segmentStatusChecker.stop();
}