Examples with Partition org.apache.helix.model.Partition used on opensource projects

Example 46 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TaskAssignmentStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
    MessageThrottleStageOutput messageOutput = event.getAttribute(AttributeName.MESSAGES_THROTTLE.name());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    Map<String, LiveInstance> liveInstanceMap = cache.getLiveInstances();
    if (manager == null || resourceMap == null || messageOutput == null || cache == null || liveInstanceMap == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires HelixManager|RESOURCES|MESSAGES_THROTTLE|DataCache|liveInstanceMap");
    }
    HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
    List<Message> messagesToSend = new ArrayList<Message>();
    for (String resourceName : resourceMap.keySet()) {
        Resource resource = resourceMap.get(resourceName);
        for (Partition partition : resource.getPartitions()) {
            List<Message> messages = messageOutput.getMessages(resourceName, partition);
            messagesToSend.addAll(messages);
        }
    }
    List<Message> outputMessages = batchMessage(dataAccessor.keyBuilder(), messagesToSend, resourceMap, liveInstanceMap, manager.getProperties());
    sendMessages(dataAccessor, outputMessages);
    // TODO: Need also count messages from task rebalancer
    if (!cache.isTaskCache()) {
        ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
        if (clusterStatusMonitor != null) {
            clusterStatusMonitor.increaseMessageReceived(outputMessages);
        }
    }
    long cacheStart = System.currentTimeMillis();
    cache.cacheMessages(outputMessages);
    long cacheEnd = System.currentTimeMillis();
    logger.debug("Caching messages took " + (cacheEnd - cacheStart) + " ms");
}

Example 47 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class AbstractRebalancer method currentMapping.

protected Map<String, Map<String, String>> currentMapping(CurrentStateOutput currentStateOutput, String resourceName, List<String> partitions, Map<String, Integer> stateCountMap) {
    Map<String, Map<String, String>> map = new HashMap<>();
    for (String partition : partitions) {
        Map<String, String> curStateMap = currentStateOutput.getCurrentStateMap(resourceName, new Partition(partition));
        map.put(partition, new HashMap<String, String>());
        for (String node : curStateMap.keySet()) {
            String state = curStateMap.get(node);
            map.get(partition).put(node, state);
        }
        Map<String, String> pendingStateMap = currentStateOutput.getPendingStateMap(resourceName, new Partition(partition));
        for (String node : pendingStateMap.keySet()) {
            String state = pendingStateMap.get(node);
            map.get(partition).put(node, state);
        }
    }
    return map;
}

Example 48 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class MaintenanceRebalancer method computeNewIdealState.

@Override
public IdealState computeNewIdealState(String resourceName, IdealState currentIdealState, CurrentStateOutput currentStateOutput, ClusterDataCache clusterData) {
    LOG.info(String.format("Start computing ideal state for resource %s in maintenance mode.", resourceName));
    Map<Partition, Map<String, String>> currentStateMap = currentStateOutput.getCurrentStateMap(resourceName);
    if (currentStateMap == null || currentStateMap.size() == 0) {
        LOG.warn(String.format("No new partition will be assigned for %s in maintenance mode", resourceName));
        currentIdealState.setPreferenceLists(Collections.EMPTY_MAP);
        return currentIdealState;
    }
    // Derived preference list from current state with state priority
    for (Partition partition : currentStateMap.keySet()) {
        Map<String, String> stateMap = currentStateMap.get(partition);
        List<String> preferenceList = new ArrayList<>(stateMap.keySet());
        Collections.sort(preferenceList, new PreferenceListNodeComparator(stateMap, clusterData.getStateModelDef(currentIdealState.getStateModelDefRef())));
        currentIdealState.setPreferenceList(partition.getPartitionName(), preferenceList);
    }
    LOG.info("End computing ideal state for resource %s in maintenance mode.");
    return currentIdealState;
}

Example 49 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TestP2PStateTransitionMessages method testP2PMessage.

private void testP2PMessage(ClusterConfig clusterConfig, Boolean p2pMessageEnabled) throws Exception {
    Map<String, Resource> resourceMap = getResourceMap(new String[] { db }, numPartition, BuiltInStateModelDefinitions.MasterSlave.name(), clusterConfig, null);
    event.addAttribute(AttributeName.RESOURCES.name(), resourceMap);
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resourceMap);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), new CurrentStateOutput());
    event.addAttribute(AttributeName.helixmanager.name(), manager);
    Pipeline pipeline = createPipeline();
    pipeline.handle(event);
    BestPossibleStateOutput bestPossibleStateOutput = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    CurrentStateOutput currentStateOutput = populateCurrentStateFromBestPossible(bestPossibleStateOutput);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    Partition p = new Partition(db + "_0");
    String masterInstance = getTopStateInstance(bestPossibleStateOutput.getInstanceStateMap(db, p), MasterSlaveSMD.States.MASTER.name());
    Assert.assertNotNull(masterInstance);
    admin.enableInstance(_clusterName, masterInstance, false);
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    cache.notifyDataChange(HelixConstants.ChangeType.INSTANCE_CONFIG);
    pipeline.handle(event);
    bestPossibleStateOutput = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    MessageSelectionStageOutput messageOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    List<Message> messages = messageOutput.getMessages(db, p);
    Assert.assertEquals(messages.size(), 1);
    Message message = messages.get(0);
    Assert.assertEquals(message.getTgtName(), masterInstance);
    Assert.assertEquals(message.getFromState(), MasterSlaveSMD.States.MASTER.name());
    Assert.assertEquals(message.getToState(), MasterSlaveSMD.States.SLAVE.name());
    if (p2pMessageEnabled) {
        Assert.assertEquals(message.getRelayMessages().entrySet().size(), 1);
        String newMasterInstance = getTopStateInstance(bestPossibleStateOutput.getInstanceStateMap(db, p), MasterSlaveSMD.States.MASTER.name());
        Message relayMessage = message.getRelayMessage(newMasterInstance);
        Assert.assertNotNull(relayMessage);
        Assert.assertEquals(relayMessage.getMsgSubType(), Message.MessageType.RELAYED_MESSAGE.name());
        Assert.assertEquals(relayMessage.getTgtName(), newMasterInstance);
        Assert.assertEquals(relayMessage.getRelaySrcHost(), masterInstance);
        Assert.assertEquals(relayMessage.getFromState(), MasterSlaveSMD.States.SLAVE.name());
        Assert.assertEquals(relayMessage.getToState(), MasterSlaveSMD.States.MASTER.name());
    } else {
        Assert.assertTrue(message.getRelayMessages().entrySet().isEmpty());
    }
}

Example 50 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TestClusterStatusMonitor method testReportData.

@Test()
public void testReportData() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    int n = 5;
    String testDB = "TestDB";
    String testDB_0 = testDB + "_0";
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    ClusterStatusMonitor monitor = new ClusterStatusMonitor(clusterName);
    monitor.active();
    ObjectName clusterMonitorObjName = monitor.getObjectName(monitor.clusterBeanName());
    try {
        _server.getMBeanInfo(clusterMonitorObjName);
    } catch (Exception e) {
        Assert.fail("Fail to register ClusterStatusMonitor");
    }
    // Test #setPerInstanceResourceStatus()
    BestPossibleStateOutput bestPossibleStates = new BestPossibleStateOutput();
    bestPossibleStates.setState(testDB, new Partition(testDB_0), "localhost_12918", "MASTER");
    bestPossibleStates.setState(testDB, new Partition(testDB_0), "localhost_12919", "SLAVE");
    bestPossibleStates.setState(testDB, new Partition(testDB_0), "localhost_12920", "SLAVE");
    bestPossibleStates.setState(testDB, new Partition(testDB_0), "localhost_12921", "OFFLINE");
    bestPossibleStates.setState(testDB, new Partition(testDB_0), "localhost_12922", "DROPPED");
    Map<String, InstanceConfig> instanceConfigMap = Maps.newHashMap();
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        InstanceConfig config = new InstanceConfig(instanceName);
        instanceConfigMap.put(instanceName, config);
    }
    Map<String, Resource> resourceMap = Maps.newHashMap();
    Resource db = new Resource(testDB);
    db.setStateModelDefRef("MasterSlave");
    db.addPartition(testDB_0);
    resourceMap.put(testDB, db);
    Map<String, StateModelDefinition> stateModelDefMap = Maps.newHashMap();
    StateModelDefinition msStateModelDef = new StateModelDefinition(StateModelConfigGenerator.generateConfigForMasterSlave());
    stateModelDefMap.put("MasterSlave", msStateModelDef);
    monitor.setPerInstanceResourceStatus(bestPossibleStates, instanceConfigMap, resourceMap, stateModelDefMap);
    // localhost_12918 should have 1 partition because it's MASTER
    ObjectName objName = monitor.getObjectName(monitor.getPerInstanceResourceBeanName("localhost_12918", testDB));
    Object value = _server.getAttribute(objName, "PartitionGauge");
    Assert.assertTrue(value instanceof Long);
    Assert.assertEquals((Long) value, new Long(1));
    value = _server.getAttribute(objName, "SensorName");
    Assert.assertTrue(value instanceof String);
    Assert.assertEquals((String) value, String.format("%s.%s.%s.%s.%s", ClusterStatusMonitor.PARTICIPANT_STATUS_KEY, clusterName, ClusterStatusMonitor.DEFAULT_TAG, "localhost_12918", testDB));
    // localhost_12919 should have 1 partition because it's SLAVE
    objName = monitor.getObjectName(monitor.getPerInstanceResourceBeanName("localhost_12919", testDB));
    value = _server.getAttribute(objName, "PartitionGauge");
    Assert.assertTrue(value instanceof Long);
    Assert.assertEquals((Long) value, new Long(1));
    // localhost_12921 should have 0 partition because it's OFFLINE
    objName = monitor.getObjectName(monitor.getPerInstanceResourceBeanName("localhost_12921", testDB));
    value = _server.getAttribute(objName, "PartitionGauge");
    Assert.assertTrue(value instanceof Long);
    Assert.assertEquals((Long) value, new Long(0));
    // localhost_12922 should have 0 partition because it's DROPPED
    objName = monitor.getObjectName(monitor.getPerInstanceResourceBeanName("localhost_12922", testDB));
    value = _server.getAttribute(objName, "PartitionGauge");
    Assert.assertTrue(value instanceof Long);
    Assert.assertEquals((Long) value, new Long(0));
    // Missing localhost_12918 in best possible ideal-state should remove it from mbean
    bestPossibleStates.getInstanceStateMap(testDB, new Partition(testDB_0)).remove("localhost_12918");
    monitor.setPerInstanceResourceStatus(bestPossibleStates, instanceConfigMap, resourceMap, stateModelDefMap);
    try {
        objName = monitor.getObjectName(monitor.getPerInstanceResourceBeanName("localhost_12918", testDB));
        _server.getMBeanInfo(objName);
        Assert.fail("Fail to unregister PerInstanceResource mbean for localhost_12918");
    } catch (InstanceNotFoundException e) {
    // OK
    }
    // Clean up
    monitor.reset();
    try {
        objName = monitor.getObjectName(monitor.getPerInstanceResourceBeanName("localhost_12920", testDB));
        _server.getMBeanInfo(objName);
        Assert.fail("Fail to unregister PerInstanceResource mbean for localhost_12920");
    } catch (InstanceNotFoundException e) {
    // OK
    }
    try {
        _server.getMBeanInfo(clusterMonitorObjName);
        Assert.fail("Fail to unregister ClusterStatusMonitor");
    } catch (InstanceNotFoundException e) {
    // OK
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}