Search in sources :

Example 41 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class SegmentStatusCheckerTest method missingEVTest.

@Test
public void missingEVTest() throws Exception {
    final String tableName = "myTable_REALTIME";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = new IdealState(tableName);
    idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_0", "pinot2", "ONLINE");
    idealState.setPartitionState("myTable_0", "pinot3", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot3", "ONLINE");
    idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE");
    idealState.setReplicas("2");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(null);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 0);
    segmentStatusChecker.stop();
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ArrayList(java.util.ArrayList) HelixAdmin(org.apache.helix.HelixAdmin) IdealState(org.apache.helix.model.IdealState) ControllerMetrics(com.linkedin.pinot.common.metrics.ControllerMetrics) Test(org.testng.annotations.Test)

Example 42 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class SegmentStatusCheckerTest method noSegments.

@Test
public void noSegments() throws Exception {
    final String tableName = "myTable_REALTIME";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = new IdealState(tableName);
    idealState.setReplicas("0");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(null);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 1);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS), 100);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
    segmentStatusChecker.stop();
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ArrayList(java.util.ArrayList) HelixAdmin(org.apache.helix.HelixAdmin) IdealState(org.apache.helix.model.IdealState) ControllerMetrics(com.linkedin.pinot.common.metrics.ControllerMetrics) Test(org.testng.annotations.Test)

Example 43 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class SegmentStatusCheckerTest method missingEVPartitionTest.

@Test
public void missingEVPartitionTest() throws Exception {
    final String tableName = "myTable_OFFLINE";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = new IdealState(tableName);
    idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_0", "pinot2", "ONLINE");
    idealState.setPartitionState("myTable_0", "pinot3", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot3", "ONLINE");
    idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE");
    idealState.setPartitionState("myTable_3", "pinot3", "ONLINE");
    idealState.setReplicas("2");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    ExternalView externalView = new ExternalView(tableName);
    externalView.setState("myTable_0", "pinot1", "ONLINE");
    externalView.setState("myTable_0", "pinot2", "ONLINE");
    externalView.setState("myTable_1", "pinot1", "ERROR");
    externalView.setState("myTable_1", "pinot2", "ONLINE");
    ZNRecord znrecord = new ZNRecord("myTable_0");
    znrecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, "myTable_0");
    znrecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, "myTable_OFFLINE");
    znrecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
    znrecord.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.OFFLINE);
    znrecord.setLongField(CommonConstants.Segment.START_TIME, 1000);
    znrecord.setLongField(CommonConstants.Segment.END_TIME, 2000);
    znrecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString());
    znrecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, 10000);
    znrecord.setLongField(CommonConstants.Segment.CRC, 1234);
    znrecord.setLongField(CommonConstants.Segment.CREATION_TIME, 3000);
    znrecord.setSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL, "http://localhost:8000/myTable_0");
    znrecord.setLongField(CommonConstants.Segment.Offline.PUSH_TIME, System.currentTimeMillis());
    znrecord.setLongField(CommonConstants.Segment.Offline.REFRESH_TIME, System.currentTimeMillis());
    ZkHelixPropertyStore<ZNRecord> propertyStore;
    {
        propertyStore = (ZkHelixPropertyStore<ZNRecord>) mock(ZkHelixPropertyStore.class);
        when(propertyStore.get("/SEGMENTS/myTable_OFFLINE/myTable_3", null, AccessOption.PERSISTENT)).thenReturn(znrecord);
    }
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(externalView);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
        when(helixResourceManager.getPropertyStore()).thenReturn(propertyStore);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(0);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 1);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 75);
    segmentStatusChecker.stop();
}
Also used : ExternalView(org.apache.helix.model.ExternalView) MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ZkHelixPropertyStore(org.apache.helix.store.zk.ZkHelixPropertyStore) ArrayList(java.util.ArrayList) HelixAdmin(org.apache.helix.HelixAdmin) IdealState(org.apache.helix.model.IdealState) ZNRecord(org.apache.helix.ZNRecord) ControllerMetrics(com.linkedin.pinot.common.metrics.ControllerMetrics) Test(org.testng.annotations.Test)

Example 44 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class SegmentStatusCheckerTest method missingIdealTest.

@Test
public void missingIdealTest() throws Exception {
    final String tableName = "myTable_REALTIME";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = null;
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(null);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(null);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 1);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS), 100);
    segmentStatusChecker.stop();
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ArrayList(java.util.ArrayList) HelixAdmin(org.apache.helix.HelixAdmin) IdealState(org.apache.helix.model.IdealState) ControllerMetrics(com.linkedin.pinot.common.metrics.ControllerMetrics) Test(org.testng.annotations.Test)

Example 45 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class SegmentStatusCheckerTest method missingEVPartitionPushTest.

@Test
public void missingEVPartitionPushTest() throws Exception {
    final String tableName = "myTable_OFFLINE";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = new IdealState(tableName);
    idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
    idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
    idealState.setReplicas("2");
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    ExternalView externalView = new ExternalView(tableName);
    externalView.setState("myTable_1", "pinot1", "ONLINE");
    externalView.setState("myTable_1", "pinot2", "ONLINE");
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", "myTable_OFFLINE")).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", "myTable_OFFLINE")).thenReturn(externalView);
    }
    ZNRecord znrecord = new ZNRecord("myTable_0");
    znrecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, "myTable_0");
    znrecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, "myTable_OFFLINE");
    znrecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
    znrecord.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.OFFLINE);
    znrecord.setLongField(CommonConstants.Segment.START_TIME, 1000);
    znrecord.setLongField(CommonConstants.Segment.END_TIME, 2000);
    znrecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString());
    znrecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, 10000);
    znrecord.setLongField(CommonConstants.Segment.CRC, 1234);
    znrecord.setLongField(CommonConstants.Segment.CREATION_TIME, 3000);
    znrecord.setSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL, "http://localhost:8000/myTable_0");
    znrecord.setLongField(CommonConstants.Segment.Offline.PUSH_TIME, System.currentTimeMillis());
    znrecord.setLongField(CommonConstants.Segment.Offline.REFRESH_TIME, System.currentTimeMillis());
    ZkHelixPropertyStore<ZNRecord> propertyStore;
    {
        propertyStore = (ZkHelixPropertyStore<ZNRecord>) mock(ZkHelixPropertyStore.class);
        when(propertyStore.get("/SEGMENTS/myTable_OFFLINE/myTable_0", null, AccessOption.PERSISTENT)).thenReturn(znrecord);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
        when(helixResourceManager.getPropertyStore()).thenReturn(propertyStore);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS), 2);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_OF_REPLICAS), 100);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
    segmentStatusChecker.stop();
}
Also used : ExternalView(org.apache.helix.model.ExternalView) MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ZkHelixPropertyStore(org.apache.helix.store.zk.ZkHelixPropertyStore) ArrayList(java.util.ArrayList) HelixAdmin(org.apache.helix.HelixAdmin) IdealState(org.apache.helix.model.IdealState) ZNRecord(org.apache.helix.ZNRecord) ControllerMetrics(com.linkedin.pinot.common.metrics.ControllerMetrics) Test(org.testng.annotations.Test)

Aggregations

HelixAdmin (org.apache.helix.HelixAdmin)83 Test (org.testng.annotations.Test)40 IdealState (org.apache.helix.model.IdealState)36 ZNRecord (org.apache.helix.ZNRecord)28 ZKHelixAdmin (org.apache.helix.manager.zk.ZKHelixAdmin)23 Date (java.util.Date)22 HashMap (java.util.HashMap)19 ExternalView (org.apache.helix.model.ExternalView)16 InstanceConfig (org.apache.helix.model.InstanceConfig)15 ClusterControllerManager (org.apache.helix.integration.manager.ClusterControllerManager)13 ArrayList (java.util.ArrayList)12 HelixDataAccessor (org.apache.helix.HelixDataAccessor)12 MockParticipantManager (org.apache.helix.integration.manager.MockParticipantManager)12 PropertyKey (org.apache.helix.PropertyKey)11 StateModelDefinition (org.apache.helix.model.StateModelDefinition)11 HelixException (org.apache.helix.HelixException)10 ControllerMetrics (com.linkedin.pinot.common.metrics.ControllerMetrics)9 MetricsRegistry (com.yammer.metrics.core.MetricsRegistry)9 Path (javax.ws.rs.Path)9 HelixConfigScopeBuilder (org.apache.helix.model.builder.HelixConfigScopeBuilder)9