Examples with ConsistentHashRouter org.apache.rocketmq.common.consistenthash.ConsistentHashRouter used on opensource projects

Example 1 with ConsistentHashRouter

use of org.apache.rocketmq.common.consistenthash.ConsistentHashRouter in project rocketmq-externals by apache.

the class DivideTaskByConsistentHash method divide.

@Override
public List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicMap, TaskDivideConfig tdc) {
    List<KeyValue> config = new ArrayList<>();
    int parallelism = tdc.getTaskParallelism();
    Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<>();
    int id = -1;
    Collection<ClientNode> cidNodes = new ArrayList<>();
    for (int i = 0; i < parallelism; i++) {
        cidNodes.add(new ClientNode(i, Integer.toString(i)));
        queueTopicList.put(i, new ArrayList<>());
    }
    ConsistentHashRouter<ClientNode> router = new ConsistentHashRouter<>(cidNodes, cidNodes.size());
    for (String t : topicMap.keySet()) {
        for (TaskTopicInfo queue : topicMap.get(t)) {
            ClientNode clientNode = router.routeNode(queue.toString());
            if (clientNode != null) {
                queueTopicList.get(clientNode.index).add(queue);
            }
        }
    }
    for (int i = 0; i < parallelism; i++) {
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
        keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
        keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACL_ENABLE.getKey(), String.valueOf(tdc.isSrcAclEnable()));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACCESS_KEY.getKey(), tdc.getSrcAccessKey());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_SECRET_KEY.getKey(), tdc.getSrcSecretKey());
        config.add(keyValue);
    }
    return config;
}

Example 2 with ConsistentHashRouter

use of org.apache.rocketmq.common.consistenthash.ConsistentHashRouter in project rocketmq-externals by apache.

the class AllocateConnAndTaskStrategyByConsistentHash method allocate.

@Override
public ConnAndTaskConfigs allocate(List<String> allWorker, String curWorker, Map<String, ConnectKeyValue> connectorConfigs, Map<String, List<ConnectKeyValue>> taskConfigs) {
    ConnAndTaskConfigs allocateResult = new ConnAndTaskConfigs();
    if (null == allWorker || 0 == allWorker.size()) {
        return allocateResult;
    }
    Collection<ClientNode> cidNodes = allWorker.stream().map(ClientNode::new).collect(Collectors.toList());
    ConsistentHashRouter router = getRouter(cidNodes);
    connectorConfigs.entrySet().stream().filter(task -> curWorker.equals(router.routeNode(task.getKey()).getKey())).forEach(task -> allocateResult.getConnectorConfigs().put(task.getKey(), task.getValue()));
    for (Map.Entry<String, List<ConnectKeyValue>> connector : taskConfigs.entrySet()) {
        connector.getValue().stream().filter(kv -> curWorker.equals(router.routeNode(kv.toString()).getKey())).forEach(allocateResult.getTaskConfigs().computeIfAbsent(connector.getKey(), k -> new ArrayList<>())::add);
    }
    log.debug("allocate result: " + allocateResult);
    return allocateResult;
}