Examples with QueueInfo com.dtstack.taier.scheduler.server.queue.QueueInfo used on opensource projects

Example 1 with QueueInfo

use of com.dtstack.taier.scheduler.server.queue.QueueInfo in project Taier by DTStack.

the class JobSchedulerListener method getAllNodesJobQueueInfo.

/**
 * 同步所有节点的 type类型下的 job实例信息
 * key1: nodeAddress,
 * key2: scheduleType
 */
public Map<String, Map<Integer, QueueInfo>> getAllNodesJobQueueInfo() {
    List<String> allNodeAddress = zkService.getAliveBrokersChildren();
    Pair<String, String> cycTime = getCycTimeLimit();
    Map<String, Map<Integer, QueueInfo>> allNodeJobInfo = Maps.newHashMap();
    for (String nodeAddress : allNodeAddress) {
        if (StringUtils.isBlank(nodeAddress)) {
            continue;
        }
        allNodeJobInfo.computeIfAbsent(nodeAddress, na -> {
            Map<Integer, QueueInfo> nodeJobInfo = Maps.newHashMap();
            for (EScheduleType scheduleType : EScheduleType.values()) {
                executors.forEach(executor -> nodeJobInfo.computeIfAbsent(scheduleType.getType(), k -> {
                    int queueSize = scheduleJobMapper.countTasksByCycTimeTypeAndAddress(nodeAddress, scheduleType.getType(), cycTime.getLeft(), cycTime.getRight());
                    QueueInfo queueInfo = new QueueInfo();
                    queueInfo.setSize(queueSize);
                    return queueInfo;
                }));
            }
            return nodeJobInfo;
        });
    }
    return allNodeJobInfo;
}

Example 2 with QueueInfo

use of com.dtstack.taier.scheduler.server.queue.QueueInfo in project Taier by DTStack.

the class JobPartitioner method computeBatchJobSize.

/**
 * compute job number per node
 */
public Map<String, Integer> computeBatchJobSize(Integer type, int jobSize) {
    // 节点挂了就会迁移的
    List<String> aliveNodes = zkService.getAliveBrokersChildren();
    Map<Integer, Map<String, QueueInfo>> allNodesJobQueueInfo = queueListener.getAllNodesJobQueueInfo();
    if (allNodesJobQueueInfo.isEmpty()) {
        return getDefaultStrategy(aliveNodes, jobSize);
    }
    Map<String, QueueInfo> nodesJobQueue = allNodesJobQueueInfo.get(type);
    if (nodesJobQueue == null || nodesJobQueue.isEmpty()) {
        return getDefaultStrategy(aliveNodes, jobSize);
    }
    Map<String, Integer> nodeSort = Maps.newHashMap();
    int total = jobSize;
    for (Map.Entry<String, QueueInfo> queueInfoEntry : nodesJobQueue.entrySet()) {
        QueueInfo queueInfo = queueInfoEntry.getValue();
        total += queueInfo.getSize();
        // 排除宕机节点
        if (aliveNodes.contains(queueInfoEntry.getKey())) {
            nodeSort.put(queueInfoEntry.getKey(), queueInfo.getSize());
        }
    }
    if (nodeSort.isEmpty()) {
        return getDefaultStrategy(aliveNodes, jobSize);
    }
    int avg = (total / nodeSort.size()) + 1;
    for (Map.Entry<String, Integer> entry : nodeSort.entrySet()) {
        entry.setValue(avg - entry.getValue());
    }
    return nodeSort;
}