Examples with Resource org.apache.helix.model.Resource used on opensource projects

Example 31 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class TaskAssignmentStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
    MessageThrottleStageOutput messageOutput = event.getAttribute(AttributeName.MESSAGES_THROTTLE.name());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    Map<String, LiveInstance> liveInstanceMap = cache.getLiveInstances();
    if (manager == null || resourceMap == null || messageOutput == null || cache == null || liveInstanceMap == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires HelixManager|RESOURCES|MESSAGES_THROTTLE|DataCache|liveInstanceMap");
    }
    HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
    List<Message> messagesToSend = new ArrayList<Message>();
    for (String resourceName : resourceMap.keySet()) {
        Resource resource = resourceMap.get(resourceName);
        for (Partition partition : resource.getPartitions()) {
            List<Message> messages = messageOutput.getMessages(resourceName, partition);
            messagesToSend.addAll(messages);
        }
    }
    List<Message> outputMessages = batchMessage(dataAccessor.keyBuilder(), messagesToSend, resourceMap, liveInstanceMap, manager.getProperties());
    sendMessages(dataAccessor, outputMessages);
    // TODO: Need also count messages from task rebalancer
    if (!cache.isTaskCache()) {
        ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
        if (clusterStatusMonitor != null) {
            clusterStatusMonitor.increaseMessageReceived(outputMessages);
        }
    }
    long cacheStart = System.currentTimeMillis();
    cache.cacheMessages(outputMessages);
    long cacheEnd = System.currentTimeMillis();
    logger.debug("Caching messages took " + (cacheEnd - cacheStart) + " ms");
}

Example 32 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class TestP2PStateTransitionMessages method testP2PMessage.

private void testP2PMessage(ClusterConfig clusterConfig, Boolean p2pMessageEnabled) throws Exception {
    Map<String, Resource> resourceMap = getResourceMap(new String[] { db }, numPartition, BuiltInStateModelDefinitions.MasterSlave.name(), clusterConfig, null);
    event.addAttribute(AttributeName.RESOURCES.name(), resourceMap);
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resourceMap);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), new CurrentStateOutput());
    event.addAttribute(AttributeName.helixmanager.name(), manager);
    Pipeline pipeline = createPipeline();
    pipeline.handle(event);
    BestPossibleStateOutput bestPossibleStateOutput = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    CurrentStateOutput currentStateOutput = populateCurrentStateFromBestPossible(bestPossibleStateOutput);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    Partition p = new Partition(db + "_0");
    String masterInstance = getTopStateInstance(bestPossibleStateOutput.getInstanceStateMap(db, p), MasterSlaveSMD.States.MASTER.name());
    Assert.assertNotNull(masterInstance);
    admin.enableInstance(_clusterName, masterInstance, false);
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    cache.notifyDataChange(HelixConstants.ChangeType.INSTANCE_CONFIG);
    pipeline.handle(event);
    bestPossibleStateOutput = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    MessageSelectionStageOutput messageOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    List<Message> messages = messageOutput.getMessages(db, p);
    Assert.assertEquals(messages.size(), 1);
    Message message = messages.get(0);
    Assert.assertEquals(message.getTgtName(), masterInstance);
    Assert.assertEquals(message.getFromState(), MasterSlaveSMD.States.MASTER.name());
    Assert.assertEquals(message.getToState(), MasterSlaveSMD.States.SLAVE.name());
    if (p2pMessageEnabled) {
        Assert.assertEquals(message.getRelayMessages().entrySet().size(), 1);
        String newMasterInstance = getTopStateInstance(bestPossibleStateOutput.getInstanceStateMap(db, p), MasterSlaveSMD.States.MASTER.name());
        Message relayMessage = message.getRelayMessage(newMasterInstance);
        Assert.assertNotNull(relayMessage);
        Assert.assertEquals(relayMessage.getMsgSubType(), Message.MessageType.RELAYED_MESSAGE.name());
        Assert.assertEquals(relayMessage.getTgtName(), newMasterInstance);
        Assert.assertEquals(relayMessage.getRelaySrcHost(), masterInstance);
        Assert.assertEquals(relayMessage.getFromState(), MasterSlaveSMD.States.SLAVE.name());
        Assert.assertEquals(relayMessage.getToState(), MasterSlaveSMD.States.MASTER.name());
    } else {
        Assert.assertTrue(message.getRelayMessages().entrySet().isEmpty());
    }
}

Example 33 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class TestClusterStatusMonitor method testReportData.

@Test()
public void testReportData() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    int n = 5;
    String testDB = "TestDB";
    String testDB_0 = testDB + "_0";
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    ClusterStatusMonitor monitor = new ClusterStatusMonitor(clusterName);
    monitor.active();
    ObjectName clusterMonitorObjName = monitor.getObjectName(monitor.clusterBeanName());
    try {
        _server.getMBeanInfo(clusterMonitorObjName);
    } catch (Exception e) {
        Assert.fail("Fail to register ClusterStatusMonitor");
    }
    // Test #setPerInstanceResourceStatus()
    BestPossibleStateOutput bestPossibleStates = new BestPossibleStateOutput();
    bestPossibleStates.setState(testDB, new Partition(testDB_0), "localhost_12918", "MASTER");
    bestPossibleStates.setState(testDB, new Partition(testDB_0), "localhost_12919", "SLAVE");
    bestPossibleStates.setState(testDB, new Partition(testDB_0), "localhost_12920", "SLAVE");
    bestPossibleStates.setState(testDB, new Partition(testDB_0), "localhost_12921", "OFFLINE");
    bestPossibleStates.setState(testDB, new Partition(testDB_0), "localhost_12922", "DROPPED");
    Map<String, InstanceConfig> instanceConfigMap = Maps.newHashMap();
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        InstanceConfig config = new InstanceConfig(instanceName);
        instanceConfigMap.put(instanceName, config);
    }
    Map<String, Resource> resourceMap = Maps.newHashMap();
    Resource db = new Resource(testDB);
    db.setStateModelDefRef("MasterSlave");
    db.addPartition(testDB_0);
    resourceMap.put(testDB, db);
    Map<String, StateModelDefinition> stateModelDefMap = Maps.newHashMap();
    StateModelDefinition msStateModelDef = new StateModelDefinition(StateModelConfigGenerator.generateConfigForMasterSlave());
    stateModelDefMap.put("MasterSlave", msStateModelDef);
    monitor.setPerInstanceResourceStatus(bestPossibleStates, instanceConfigMap, resourceMap, stateModelDefMap);
    // localhost_12918 should have 1 partition because it's MASTER
    ObjectName objName = monitor.getObjectName(monitor.getPerInstanceResourceBeanName("localhost_12918", testDB));
    Object value = _server.getAttribute(objName, "PartitionGauge");
    Assert.assertTrue(value instanceof Long);
    Assert.assertEquals((Long) value, new Long(1));
    value = _server.getAttribute(objName, "SensorName");
    Assert.assertTrue(value instanceof String);
    Assert.assertEquals((String) value, String.format("%s.%s.%s.%s.%s", ClusterStatusMonitor.PARTICIPANT_STATUS_KEY, clusterName, ClusterStatusMonitor.DEFAULT_TAG, "localhost_12918", testDB));
    // localhost_12919 should have 1 partition because it's SLAVE
    objName = monitor.getObjectName(monitor.getPerInstanceResourceBeanName("localhost_12919", testDB));
    value = _server.getAttribute(objName, "PartitionGauge");
    Assert.assertTrue(value instanceof Long);
    Assert.assertEquals((Long) value, new Long(1));
    // localhost_12921 should have 0 partition because it's OFFLINE
    objName = monitor.getObjectName(monitor.getPerInstanceResourceBeanName("localhost_12921", testDB));
    value = _server.getAttribute(objName, "PartitionGauge");
    Assert.assertTrue(value instanceof Long);
    Assert.assertEquals((Long) value, new Long(0));
    // localhost_12922 should have 0 partition because it's DROPPED
    objName = monitor.getObjectName(monitor.getPerInstanceResourceBeanName("localhost_12922", testDB));
    value = _server.getAttribute(objName, "PartitionGauge");
    Assert.assertTrue(value instanceof Long);
    Assert.assertEquals((Long) value, new Long(0));
    // Missing localhost_12918 in best possible ideal-state should remove it from mbean
    bestPossibleStates.getInstanceStateMap(testDB, new Partition(testDB_0)).remove("localhost_12918");
    monitor.setPerInstanceResourceStatus(bestPossibleStates, instanceConfigMap, resourceMap, stateModelDefMap);
    try {
        objName = monitor.getObjectName(monitor.getPerInstanceResourceBeanName("localhost_12918", testDB));
        _server.getMBeanInfo(objName);
        Assert.fail("Fail to unregister PerInstanceResource mbean for localhost_12918");
    } catch (InstanceNotFoundException e) {
    // OK
    }
    // Clean up
    monitor.reset();
    try {
        objName = monitor.getObjectName(monitor.getPerInstanceResourceBeanName("localhost_12920", testDB));
        _server.getMBeanInfo(objName);
        Assert.fail("Fail to unregister PerInstanceResource mbean for localhost_12920");
    } catch (InstanceNotFoundException e) {
    // OK
    }
    try {
        _server.getMBeanInfo(clusterMonitorObjName);
        Assert.fail("Fail to unregister ClusterStatusMonitor");
    } catch (InstanceNotFoundException e) {
    // OK
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 34 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class TestRebalancerMetrics method testRecoveryRebalanceMetrics.

@Test
public void testRecoveryRebalanceMetrics() {
    System.out.println("START testRecoveryRebalanceMetrics at " + new Date(System.currentTimeMillis()));
    String resource = "testResourceName";
    int numPartition = 100;
    int numReplica = 3;
    int maxPending = 3;
    setupIdealState(5, new String[] { resource }, numPartition, numReplica, IdealState.RebalanceMode.FULL_AUTO, BuiltInStateModelDefinitions.MasterSlave.name());
    setupInstances(5);
    setupLiveInstances(5);
    setupStateModel();
    Map<String, Resource> resourceMap = getResourceMap(new String[] { resource }, numPartition, BuiltInStateModelDefinitions.MasterSlave.name());
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    event.addAttribute(AttributeName.RESOURCES.name(), resourceMap);
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resourceMap);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    ClusterStatusMonitor monitor = new ClusterStatusMonitor(_clusterName);
    monitor.active();
    event.addAttribute(AttributeName.clusterStatusMonitor.name(), monitor);
    runStage(event, new ReadClusterDataStage());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    setupThrottleConfig(cache.getClusterConfig(), StateTransitionThrottleConfig.RebalanceType.RECOVERY_BALANCE, maxPending);
    runStage(event, new BestPossibleStateCalcStage());
    runStage(event, new IntermediateStateCalcStage());
    ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
    ResourceMonitor resourceMonitor = clusterStatusMonitor.getResourceMonitor(resource);
    Assert.assertEquals(resourceMonitor.getPendingRecoveryRebalancePartitionGauge(), numPartition);
    Assert.assertEquals(resourceMonitor.getRecoveryRebalanceThrottledPartitionGauge(), numPartition - maxPending);
    System.out.println("END testRecoveryRebalanceMetrics at " + new Date(System.currentTimeMillis()));
}

Example 35 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class TestTopStateHandoffMetrics method preSetup.

public void preSetup() {
    setupLiveInstances(3);
    setupStateModel();
    Resource resource = new Resource(TEST_RESOURCE);
    resource.setStateModelDefRef("MasterSlave");
    resource.addPartition(PARTITION);
    event.addAttribute(AttributeName.RESOURCES.name(), Collections.singletonMap(TEST_RESOURCE, resource));
    ClusterStatusMonitor monitor = new ClusterStatusMonitor("TestCluster");
    monitor.active();
    event.addAttribute(AttributeName.clusterStatusMonitor.name(), monitor);
}