use of com.tencent.angel.master.task.AMTaskManager in project angel by Tencent.
the class TaskCalPerfChecker method check.
@Override
public List<Id> check(AMContext context) {
double slowestDiscount = context.getConf().getDouble(AngelConf.ANGEL_AM_TASK_SLOWEST_DISCOUNT, AngelConf.DEFAULT_ANGEL_AM_TASK_SLOWEST_DISCOUNT);
LOG.info("start to check slow workers use TaskCalPerfChecker policy, slowestDiscount = " + slowestDiscount);
Set<Id> slowWorkers = new HashSet<Id>();
AMTaskManager taskManage = context.getTaskManager();
WorkerManager workerManager = context.getWorkerManager();
Collection<AMTask> tasks = taskManage.getTasks();
long totalSamples = 0;
long totalCalTimeMs = 0;
double averageRate = 0.0;
Map<TaskId, Double> taskIdToRateMap = new HashMap<TaskId, Double>(tasks.size());
for (AMTask task : tasks) {
if (task.getMetrics().containsKey(TaskCounter.TOTAL_CALCULATE_SAMPLES) && task.getMetrics().containsKey(TaskCounter.TOTAL_CALCULATE_TIME_MS)) {
long sampleNum = Long.valueOf(task.getMetrics().get(TaskCounter.TOTAL_CALCULATE_SAMPLES));
double calTimeMs = Long.valueOf(task.getMetrics().get(TaskCounter.TOTAL_CALCULATE_TIME_MS));
LOG.info("for task " + task.getTaskId() + ", sampleNum = " + sampleNum + ", calTimeMs = " + calTimeMs);
totalSamples += sampleNum;
totalCalTimeMs += calTimeMs;
if (sampleNum > 5000000) {
LOG.info("task " + task.getTaskId() + " calculate rate = " + (calTimeMs * 10000 / sampleNum));
taskIdToRateMap.put(task.getTaskId(), calTimeMs * 10000 / sampleNum);
}
}
}
if (totalSamples != 0) {
averageRate = (double) totalCalTimeMs * 10000 / totalSamples;
}
LOG.info("totalSamples = " + totalSamples + ", totalCalTimeMs = " + totalCalTimeMs + ", average calulate time for 10000 samples = " + averageRate + ", the maximum calulate time for 10000 sample = " + averageRate / slowestDiscount);
for (Map.Entry<TaskId, Double> rateEntry : taskIdToRateMap.entrySet()) {
if (averageRate < rateEntry.getValue() * slowestDiscount) {
LOG.info("task " + rateEntry.getKey() + " rate = " + rateEntry.getValue() + " is < " + averageRate * slowestDiscount);
AMWorker worker = workerManager.getWorker(rateEntry.getKey());
if (worker != null) {
LOG.info("put worker " + worker.getId() + " to slow worker list");
slowWorkers.add(worker.getId());
}
}
}
List<Id> slowWorkerList = new ArrayList<>(slowWorkers.size());
slowWorkerList.addAll(slowWorkers);
return slowWorkerList;
}
Aggregations