Examples with AMTask com.tencent.angel.master.task.AMTask used on opensource projects

Example 1 with AMTask

use of com.tencent.angel.master.task.AMTask in project angel by Tencent.

the class ProgressBlock method render.

@Override
protected void render(Block html) {
    set(TITLE, join("Angel Progress"));
    Hamlet.TABLE<Hamlet.DIV<Hamlet>> table = html.div(_INFO_WRAP).table("#job");
    Hamlet.TR<Hamlet.THEAD<Hamlet.TABLE<Hamlet.DIV<Hamlet>>>> headTr = table.thead().tr();
    headTr.th(_TH, "taskid").th(_TH, "state").th(_TH, "current iteration").th(_TH, "workerlog");
    headTr._()._();
    float current_iteration_progress = (float) 0.0;
    float current_clock_progress = (float) 0.0;
    Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> tbody = table.tbody();
    List<AMTask> amTaskList = new ArrayList();
    Map<AMTask, WorkerAttempt> map = new HashMap<>();
    Collection<AMWorkerGroup> amWorkerGroupSet = amContext.getWorkerManager().getWorkerGroupMap().values();
    for (AMWorkerGroup amWorkerGroup : amWorkerGroupSet) {
        Collection<AMWorker> amWorkerSet = amWorkerGroup.getWorkerSet();
        for (AMWorker amWorker : amWorkerSet) {
            Collection<WorkerAttempt> workerAttempts = amWorker.getAttempts().values();
            for (WorkerAttempt workerAttempt : workerAttempts) {
                Collection<AMTask> amTasks = workerAttempt.getTaskMap().values();
                for (AMTask amTask : amTasks) {
                    map.put(amTask, workerAttempt);
                }
            }
        }
    }
    for (AMTask amTask : amContext.getTaskManager().getTasks()) {
        amTaskList.add(amTask);
    }
    Collections.sort(amTaskList, new Comparator<AMTask>() {

        @Override
        public int compare(AMTask task1, AMTask task2) {
            return task1.getTaskId().getIndex() - task2.getTaskId().getIndex();
        }
    });
    for (AMTask task : amTaskList) {
        WorkerAttempt workerAttempt = map.get(task);
        if (task.getProgress() >= 0 && task.getProgress() <= 1)
            current_iteration_progress = task.getProgress();
        current_clock_progress = ((float) task.getIteration()) / ((float) amContext.getTotalIterationNum());
        Hamlet.TR<Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>>> tr = tbody.tr();
        tr.td(task.getTaskId().toString()).td(task.getState().toString()).td(String.valueOf(task.getIteration()) + "/" + amContext.getTotalIterationNum()).td().a(url(MRWebAppUtil.getYARNWebappScheme(), workerAttempt.getNodeHttpAddr(), "node", "containerlogs", workerAttempt.getContainerIdStr(), amContext.getUser().toString()), workerAttempt.getId().toString())._();
        tr._();
    }
    tbody._()._()._();
}

Example 2 with AMTask

use of com.tencent.angel.master.task.AMTask in project angel by Tencent.

the class WorkerBlock method render.

@Override
protected void render(Block html) {
    set(TITLE, join("Angel Worker Attempt ", $(WORKER_ATTEMPT_ID)));
    String workerAttemptIdStr = $(WORKER_ATTEMPT_ID);
    if (workerAttemptIdStr == null || workerAttemptIdStr.isEmpty()) {
        html.p()._("Sorry, can't do anything without a WorkerId.")._();
        return;
    }
    WorkerAttemptId workerAttemptId = null;
    try {
        workerAttemptId = new WorkerAttemptId(workerAttemptIdStr);
    } catch (UnvalidIdStrException e) {
        LOG.error("unvalid id string, ", e);
        return;
    }
    AMWorker worker;
    worker = amContext.getWorkerManager().getWorker(workerAttemptId.getWorkerId());
    if (worker == null) {
        html.p()._("Sorry, can't find worker " + workerAttemptId.getWorkerId())._();
        return;
    }
    WorkerAttempt workerAttempt = worker.getWorkerAttempt(workerAttemptId);
    TABLE<DIV<Hamlet>> table = html.div(_INFO_WRAP).table("#job");
    TR<THEAD<TABLE<DIV<Hamlet>>>> headTr = table.thead().tr();
    headTr.th(_TH, "taskid").th(_TH, "state").th(_TH, "current iteration").th(_TH, "current iteration bar").th(_TH, "current progress").th(_TH, "current progress bar").th(_TH, "taskcounters");
    headTr._()._();
    float current_iteration_progress = (float) 0.0;
    float current_clock_progress = (float) 0.0;
    TBODY<TABLE<DIV<Hamlet>>> tbody = table.tbody();
    for (AMTask task : workerAttempt.getTaskMap().values()) {
        if (task.getProgress() >= 0 && task.getProgress() <= 1)
            current_iteration_progress = task.getProgress();
        current_clock_progress = ((float) task.getIteration()) / ((float) amContext.getTotalIterationNum());
        TR<TBODY<TABLE<DIV<Hamlet>>>> tr = tbody.tr();
        tr.td(task.getTaskId().toString()).td(task.getState().toString()).td(String.valueOf(task.getIteration()) + "/" + amContext.getTotalIterationNum()).td().div(_PROGRESSBAR).$title(// tooltip
        join(String.valueOf(current_clock_progress * 100), '%')).div(_PROGRESSBAR_VALUE).$style(join("width:", String.valueOf(current_clock_progress * 100), '%'))._()._()._().td(String.valueOf(current_iteration_progress)).td().div(_PROGRESSBAR).$title(join(String.valueOf(current_iteration_progress * 100), '%')).div(_PROGRESSBAR_VALUE).$style(join("width:", String.valueOf(current_iteration_progress * 100), '%'))._()._()._().td().a(url("angel/taskCountersPage/", task.getTaskId().toString()), "taskcounters")._();
        tr._();
    }
    tbody._()._()._();
}

Example 3 with AMTask

use of com.tencent.angel.master.task.AMTask in project angel by Tencent.

the class MasterService method taskIteration.

/**
 * task update iteration number
 * @param controller rpc controller of protobuf
 * @param request contains task id, iteration number
 * @throws ServiceException
 */
@Override
public TaskIterationResponse taskIteration(RpcController controller, TaskIterationRequest request) throws ServiceException {
    LOG.debug("task iteration, " + request);
    TaskId taskId = ProtobufUtil.convertToId(request.getTaskId());
    // get Task meta from task manager, if can not find, just new a AMTask object and put it to task manager
    // in ANGEL_PS mode, task id may can not know advance
    AMTask task = context.getTaskManager().getTask(taskId);
    if (task == null) {
        task = new AMTask(taskId, null);
        context.getTaskManager().putTask(taskId, task);
    }
    // update task iteration
    task.iteration(request.getIteration());
    context.getEventHandler().handle(new MetricsEvent(MetricsEventType.TASK_ITERATION_UPDATE));
    return TaskIterationResponse.newBuilder().build();
}

Example 4 with AMTask

use of com.tencent.angel.master.task.AMTask in project angel by Tencent.

the class AMWorker method addAndScheduleAttempt.

@SuppressWarnings("unchecked")
private void addAndScheduleAttempt() {
    WorkerAttempt attempt = null;
    writeLock.lock();
    try {
        // init a worker attempt for the worker
        attempt = createWorkerAttempt();
        for (TaskId taskId : taskIds) {
            AMTask task = context.getTaskManager().getTask(taskId);
            if (task != null) {
                task.resetCounters();
            }
        }
        attempts.put(attempt.getId(), attempt);
        LOG.info("scheduling " + attempt.getId());
        runningAttemptId = attempt.getId();
        lastAttemptId = attempt.getId();
    } finally {
        writeLock.unlock();
    }
    // schedule the worker attempt
    context.getEventHandler().handle(new WorkerAttemptEvent(WorkerAttemptEventType.SCHEDULE, attempt.getId()));
}

Example 5 with AMTask

use of com.tencent.angel.master.task.AMTask in project angel by Tencent.

the class ProtobufUtil method buildWorkerMetaProto.

private static WorkerMetaInfoProto buildWorkerMetaProto(AMWorker worker) {
    WorkerMetaInfoProto.Builder builder = WorkerMetaInfoProto.newBuilder();
    WorkerAttempt attempt = worker.getRunningAttempt();
    WorkerAttemptIdProto workerAttemptIdProto = convertToIdProto(attempt.getId());
    Location location = attempt.getLocation();
    WorkerLocationProto.Builder locBuilder = WorkerLocationProto.newBuilder();
    locBuilder.setWorkerAttemptId(workerAttemptIdProto);
    if (location != null) {
        locBuilder.setLocation(buildLocation(location));
    }
    builder.setWorkerLocation(locBuilder.build());
    TaskMetaInfoProto.Builder taskMetaBuilder = TaskMetaInfoProto.newBuilder();
    MatrixClock.Builder clockBuilder = MatrixClock.newBuilder();
    for (Entry<TaskId, AMTask> taskEntry : attempt.getTaskMap().entrySet()) {
        AMTask task = taskEntry.getValue();
        taskMetaBuilder.setTaskId(convertToIdProto(taskEntry.getKey()));
        taskMetaBuilder.setIteration(task.getIteration());
        Int2IntOpenHashMap matrixClocks = task.getMatrixClocks();
        for (it.unimi.dsi.fastutil.ints.Int2IntMap.Entry clockEntry : matrixClocks.int2IntEntrySet()) {
            taskMetaBuilder.addMatrixClock(clockBuilder.setMatrixId(clockEntry.getIntKey()).setClock(clockEntry.getIntValue()).build());
        }
        builder.addTasks(taskMetaBuilder.build());
        LOG.debug("task meta=" + taskMetaBuilder.build());
    }
    return builder.build();
}