Examples with ExternalView org.apache.helix.model.ExternalView used on opensource projects

Example 36 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class SegmentDeletionManagerTest method makeHelixAdmin.

HelixAdmin makeHelixAdmin() {
    HelixAdmin admin = mock(HelixAdmin.class);
    ExternalView ev = mock(ExternalView.class);
    IdealState is = mock(IdealState.class);
    when(admin.getResourceExternalView(clusterName, tableName)).thenReturn(ev);
    when(admin.getResourceIdealState(clusterName, tableName)).thenReturn(is);
    List<String> segmentsInIs = segmentsInIdealStateOrExtView();
    Map<String, String> dummy = new HashMap<>(1);
    dummy.put("someHost", "ONLINE");
    for (String segment : segmentsInIs) {
        when(is.getInstanceStateMap(segment)).thenReturn(dummy);
    }
    when(ev.getStateMap(anyString())).thenReturn(null);
    return admin;
}

Example 37 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class TimeBoundaryServiceTest method testExternalViewBasedTimeBoundaryService.

@Test
public void testExternalViewBasedTimeBoundaryService() throws Exception {
    addingTableToPropertyStore("testResource0");
    addingTableToPropertyStore("testResource1");
    HelixExternalViewBasedTimeBoundaryService tbs = new HelixExternalViewBasedTimeBoundaryService(_propertyStore);
    addingSegmentsToPropertyStore(5, _propertyStore, "testResource0");
    ExternalView externalView = constructExternalView("testResource0");
    tbs.updateTimeBoundaryService(externalView);
    TimeBoundaryInfo tbi = tbs.getTimeBoundaryInfoFor("testResource0");
    Assert.assertEquals(tbi.getTimeColumn(), "timestamp");
    Assert.assertEquals(tbi.getTimeValue(), "4");
    addingSegmentsToPropertyStore(50, _propertyStore, "testResource1");
    externalView = constructExternalView("testResource1");
    tbs.updateTimeBoundaryService(externalView);
    tbi = tbs.getTimeBoundaryInfoFor("testResource1");
    Assert.assertEquals(tbi.getTimeColumn(), "timestamp");
    Assert.assertEquals(tbi.getTimeValue(), "49");
    addingSegmentsToPropertyStore(50, _propertyStore, "testResource0");
    externalView = constructExternalView("testResource0");
    tbs.updateTimeBoundaryService(externalView);
    tbi = tbs.getTimeBoundaryInfoFor("testResource0");
    Assert.assertEquals(tbi.getTimeColumn(), "timestamp");
    Assert.assertEquals(tbi.getTimeValue(), "49");
}

Example 38 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class HelixBrokerStarterTest method setUp.

@BeforeTest
public void setUp() throws Exception {
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    _zkClient = new ZkClient(ZkStarter.DEFAULT_ZK_STR);
    final String instanceId = "localhost_helixController";
    _pinotResourceManager = new PinotHelixResourceManager(ZkStarter.DEFAULT_ZK_STR, HELIX_CLUSTER_NAME, instanceId, null, 10000L, true, /*isUpdateStateModel=*/
    false);
    _pinotResourceManager.start();
    final String helixZkURL = HelixConfig.getAbsoluteZkPathForHelix(ZkStarter.DEFAULT_ZK_STR);
    _helixZkManager = HelixSetupUtils.setup(HELIX_CLUSTER_NAME, helixZkURL, instanceId, /*isUpdateStateModel=*/
    false);
    _helixAdmin = _helixZkManager.getClusterManagmentTool();
    Thread.sleep(3000);
    final Configuration pinotHelixBrokerProperties = DefaultHelixBrokerConfig.getDefaultBrokerConf();
    pinotHelixBrokerProperties.addProperty(CommonConstants.Helix.KEY_OF_BROKER_QUERY_PORT, 8943);
    _helixBrokerStarter = new HelixBrokerStarter(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, pinotHelixBrokerProperties);
    Thread.sleep(1000);
    ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, 5, true);
    ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, 1, true);
    final String tableName = "dining";
    JSONObject buildCreateOfflineTableV2JSON = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(tableName, null, null, 1);
    AbstractTableConfig config = AbstractTableConfig.init(buildCreateOfflineTableV2JSON.toString());
    _pinotResourceManager.addTable(config);
    for (int i = 1; i <= 5; i++) {
        addOneSegment(tableName);
        Thread.sleep(2000);
        final ExternalView externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(tableName));
        Assert.assertEquals(externalView.getPartitionSet().size(), i);
    }
}

Example 39 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class BaseClusterIntegrationTest method setupSegmentCountCountDownLatch.

protected CountDownLatch setupSegmentCountCountDownLatch(final String tableName, final int expectedSegmentCount) throws Exception {
    final CountDownLatch latch = new CountDownLatch(1);
    HelixManager manager = HelixManagerFactory.getZKHelixManager(getHelixClusterName(), "test_instance", InstanceType.SPECTATOR, ZkStarter.DEFAULT_ZK_STR);
    manager.connect();
    manager.addExternalViewChangeListener(new ExternalViewChangeListener() {

        private boolean _hasBeenTriggered = false;

        @Override
        public void onExternalViewChange(List<ExternalView> externalViewList, NotificationContext changeContext) {
            // Nothing to do?
            if (_hasBeenTriggered) {
                return;
            }
            for (ExternalView externalView : externalViewList) {
                if (externalView.getId().contains(tableName)) {
                    Set<String> partitionSet = externalView.getPartitionSet();
                    if (partitionSet.size() == expectedSegmentCount) {
                        int onlinePartitionCount = 0;
                        for (String partitionId : partitionSet) {
                            Map<String, String> partitionStateMap = externalView.getStateMap(partitionId);
                            if (partitionStateMap.containsValue("ONLINE")) {
                                onlinePartitionCount++;
                            }
                        }
                        if (onlinePartitionCount == expectedSegmentCount) {
                            //                System.out.println("Got " + expectedSegmentCount + " online tables, unlatching the main thread");
                            latch.countDown();
                            _hasBeenTriggered = true;
                        }
                    }
                }
            }
        }
    });
    return latch;
}

Example 40 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class PinotZKChanger method isStable.

/**
   * return true if IdealState = ExternalView
   * @return
   */
public int isStable(String tableName) {
    IdealState idealState = helixAdmin.getResourceIdealState(clusterName, tableName);
    ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, tableName);
    Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields();
    Map<String, Map<String, String>> mapFieldsEV = externalView.getRecord().getMapFields();
    int numDiff = 0;
    for (String segment : mapFieldsIS.keySet()) {
        Map<String, String> mapIS = mapFieldsIS.get(segment);
        Map<String, String> mapEV = mapFieldsEV.get(segment);
        for (String server : mapIS.keySet()) {
            String state = mapIS.get(server);
            if (mapEV == null || mapEV.get(server) == null || !mapEV.get(server).equals(state)) {
                LOGGER.info("Mismatch: segment" + segment + " server:" + server + " state:" + state);
                numDiff = numDiff + 1;
            }
        }
    }
    return numDiff;
}