Search in sources :

Example 51 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class SegmentStatusCheckerTest method missingEVTest.

@Test
public void missingEVTest() throws Exception {
    final String tableName = "myTable_REALTIME";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = new IdealState(tableName);
    idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_0", "pinot2", "ONLINE");
    idealState.setPartitionState("myTable_0", "pinot3", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot3", "ONLINE");
    idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE");
    idealState.setReplicas("2");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(null);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 0);
    segmentStatusChecker.stop();
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ArrayList(java.util.ArrayList) HelixAdmin(org.apache.helix.HelixAdmin) IdealState(org.apache.helix.model.IdealState) ControllerMetrics(com.linkedin.pinot.common.metrics.ControllerMetrics) Test(org.testng.annotations.Test)

Example 52 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class SegmentStatusCheckerTest method noSegments.

@Test
public void noSegments() throws Exception {
    final String tableName = "myTable_REALTIME";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = new IdealState(tableName);
    idealState.setReplicas("0");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(null);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 1);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS), 100);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
    segmentStatusChecker.stop();
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ArrayList(java.util.ArrayList) HelixAdmin(org.apache.helix.HelixAdmin) IdealState(org.apache.helix.model.IdealState) ControllerMetrics(com.linkedin.pinot.common.metrics.ControllerMetrics) Test(org.testng.annotations.Test)

Example 53 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class SegmentStatusCheckerTest method missingEVPartitionTest.

@Test
public void missingEVPartitionTest() throws Exception {
    final String tableName = "myTable_OFFLINE";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = new IdealState(tableName);
    idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_0", "pinot2", "ONLINE");
    idealState.setPartitionState("myTable_0", "pinot3", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot3", "ONLINE");
    idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE");
    idealState.setPartitionState("myTable_3", "pinot3", "ONLINE");
    idealState.setReplicas("2");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    ExternalView externalView = new ExternalView(tableName);
    externalView.setState("myTable_0", "pinot1", "ONLINE");
    externalView.setState("myTable_0", "pinot2", "ONLINE");
    externalView.setState("myTable_1", "pinot1", "ERROR");
    externalView.setState("myTable_1", "pinot2", "ONLINE");
    ZNRecord znrecord = new ZNRecord("myTable_0");
    znrecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, "myTable_0");
    znrecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, "myTable_OFFLINE");
    znrecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
    znrecord.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.OFFLINE);
    znrecord.setLongField(CommonConstants.Segment.START_TIME, 1000);
    znrecord.setLongField(CommonConstants.Segment.END_TIME, 2000);
    znrecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString());
    znrecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, 10000);
    znrecord.setLongField(CommonConstants.Segment.CRC, 1234);
    znrecord.setLongField(CommonConstants.Segment.CREATION_TIME, 3000);
    znrecord.setSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL, "http://localhost:8000/myTable_0");
    znrecord.setLongField(CommonConstants.Segment.Offline.PUSH_TIME, System.currentTimeMillis());
    znrecord.setLongField(CommonConstants.Segment.Offline.REFRESH_TIME, System.currentTimeMillis());
    ZkHelixPropertyStore<ZNRecord> propertyStore;
    {
        propertyStore = (ZkHelixPropertyStore<ZNRecord>) mock(ZkHelixPropertyStore.class);
        when(propertyStore.get("/SEGMENTS/myTable_OFFLINE/myTable_3", null, AccessOption.PERSISTENT)).thenReturn(znrecord);
    }
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(externalView);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
        when(helixResourceManager.getPropertyStore()).thenReturn(propertyStore);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(0);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 1);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 75);
    segmentStatusChecker.stop();
}
Also used : ExternalView(org.apache.helix.model.ExternalView) MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ZkHelixPropertyStore(org.apache.helix.store.zk.ZkHelixPropertyStore) ArrayList(java.util.ArrayList) HelixAdmin(org.apache.helix.HelixAdmin) IdealState(org.apache.helix.model.IdealState) ZNRecord(org.apache.helix.ZNRecord) ControllerMetrics(com.linkedin.pinot.common.metrics.ControllerMetrics) Test(org.testng.annotations.Test)

Example 54 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class SegmentStatusCheckerTest method missingIdealTest.

@Test
public void missingIdealTest() throws Exception {
    final String tableName = "myTable_REALTIME";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = null;
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(null);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(null);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 1);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS), 100);
    segmentStatusChecker.stop();
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ArrayList(java.util.ArrayList) HelixAdmin(org.apache.helix.HelixAdmin) IdealState(org.apache.helix.model.IdealState) ControllerMetrics(com.linkedin.pinot.common.metrics.ControllerMetrics) Test(org.testng.annotations.Test)

Example 55 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class SegmentStatusCheckerTest method missingEVPartitionPushTest.

@Test
public void missingEVPartitionPushTest() throws Exception {
    final String tableName = "myTable_OFFLINE";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = new IdealState(tableName);
    idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
    idealState.setReplicas("2");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    ExternalView externalView = new ExternalView(tableName);
    externalView.setState("myTable_1", "pinot1", "ONLINE");
    externalView.setState("myTable_1", "pinot2", "ONLINE");
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", "myTable_OFFLINE")).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", "myTable_OFFLINE")).thenReturn(externalView);
    }
    ZNRecord znrecord = new ZNRecord("myTable_0");
    znrecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, "myTable_0");
    znrecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, "myTable_OFFLINE");
    znrecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
    znrecord.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.OFFLINE);
    znrecord.setLongField(CommonConstants.Segment.START_TIME, 1000);
    znrecord.setLongField(CommonConstants.Segment.END_TIME, 2000);
    znrecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString());
    znrecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, 10000);
    znrecord.setLongField(CommonConstants.Segment.CRC, 1234);
    znrecord.setLongField(CommonConstants.Segment.CREATION_TIME, 3000);
    znrecord.setSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL, "http://localhost:8000/myTable_0");
    znrecord.setLongField(CommonConstants.Segment.Offline.PUSH_TIME, System.currentTimeMillis());
    znrecord.setLongField(CommonConstants.Segment.Offline.REFRESH_TIME, System.currentTimeMillis());
    ZkHelixPropertyStore<ZNRecord> propertyStore;
    {
        propertyStore = (ZkHelixPropertyStore<ZNRecord>) mock(ZkHelixPropertyStore.class);
        when(propertyStore.get("/SEGMENTS/myTable_OFFLINE/myTable_0", null, AccessOption.PERSISTENT)).thenReturn(znrecord);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
        when(helixResourceManager.getPropertyStore()).thenReturn(propertyStore);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS), 2);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_OF_REPLICAS), 100);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
    segmentStatusChecker.stop();
}
Also used : ExternalView(org.apache.helix.model.ExternalView) MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ZkHelixPropertyStore(org.apache.helix.store.zk.ZkHelixPropertyStore) ArrayList(java.util.ArrayList) HelixAdmin(org.apache.helix.HelixAdmin) IdealState(org.apache.helix.model.IdealState) ZNRecord(org.apache.helix.ZNRecord) ControllerMetrics(com.linkedin.pinot.common.metrics.ControllerMetrics) Test(org.testng.annotations.Test)

Aggregations

HelixAdmin (org.apache.helix.HelixAdmin)104 IdealState (org.apache.helix.model.IdealState)44 ZKHelixAdmin (org.apache.helix.manager.zk.ZKHelixAdmin)43 Test (org.testng.annotations.Test)40 HashMap (java.util.HashMap)30 ZNRecord (org.apache.helix.ZNRecord)28 Date (java.util.Date)22 InstanceConfig (org.apache.helix.model.InstanceConfig)22 ArrayList (java.util.ArrayList)18 Map (java.util.Map)17 HashSet (java.util.HashSet)16 ExternalView (org.apache.helix.model.ExternalView)16 StateModelDefinition (org.apache.helix.model.StateModelDefinition)16 IOException (java.io.IOException)13 ClusterControllerManager (org.apache.helix.integration.manager.ClusterControllerManager)13 HelixDataAccessor (org.apache.helix.HelixDataAccessor)12 MockParticipantManager (org.apache.helix.integration.manager.MockParticipantManager)12 TreeMap (java.util.TreeMap)11 PropertyKey (org.apache.helix.PropertyKey)11 HelixConfigScopeBuilder (org.apache.helix.model.builder.HelixConfigScopeBuilder)11