Examples with TaskId com.tencent.angel.worker.task.TaskId used on opensource projects

Example 6 with TaskId

use of com.tencent.angel.worker.task.TaskId in project angel by Tencent.

the class ProtobufUtil method convertToId.

public static TaskId convertToId(TaskIdProto idProto) {
    TaskId id = protoToTaskIdMap.get(idProto);
    if (id == null) {
        id = new TaskId(idProto.getTaskIndex());
        protoToTaskIdMap.put(idProto, id);
    }
    return id;
}

Example 7 with TaskId

use of com.tencent.angel.worker.task.TaskId in project angel by Tencent.

the class AMWorker method addAndScheduleAttempt.

@SuppressWarnings("unchecked")
private void addAndScheduleAttempt() {
    WorkerAttempt attempt = null;
    writeLock.lock();
    try {
        // init a worker attempt for the worker
        attempt = createWorkerAttempt();
        for (TaskId taskId : taskIds) {
            AMTask task = context.getTaskManager().getTask(taskId);
            if (task != null) {
                task.resetCounters();
            }
        }
        attempts.put(attempt.getId(), attempt);
        LOG.info("scheduling " + attempt.getId());
        runningAttemptId = attempt.getId();
        lastAttemptId = attempt.getId();
    } finally {
        writeLock.unlock();
    }
    // schedule the worker attempt
    context.getEventHandler().handle(new WorkerAttemptEvent(WorkerAttemptEventType.SCHEDULE, attempt.getId()));
}

Example 8 with TaskId

use of com.tencent.angel.worker.task.TaskId in project angel by Tencent.

the class WorkerManager method initWorkers.

private void initWorkers() {
    int base = 0;
    // init all tasks , workers and worker groups and put them to the corresponding maps
    for (int i = 0; i < workergroupNumber; i++) {
        Map<WorkerId, AMWorker> workers = new HashMap<WorkerId, AMWorker>();
        WorkerId leader = null;
        WorkerGroupId groupId = new WorkerGroupId(i);
        for (int j = 0; j < workersInGroup; j++) {
            base = (i * workersInGroup + j) * taskNumberInEachWorker;
            List<TaskId> taskIds = new ArrayList<TaskId>(taskNumberInEachWorker);
            for (int k = 0; k < taskNumberInEachWorker && (base < totalTaskNumber); k++, base++) {
                taskIds.add(new TaskId(base));
            }
            WorkerId workerId = new WorkerId(groupId, i * workersInGroup + j);
            AMWorker worker = new AMWorker(workerId, context, taskIds);
            workersMap.put(workerId, worker);
            workers.put(workerId, worker);
            if (j == 0) {
                leader = workerId;
            }
        }
        AMWorkerGroup group = new AMWorkerGroup(groupId, context, workers, leader, i);
        for (WorkerId id : workers.keySet()) {
            findWorkerGroupMap.put(id, group);
            for (TaskId taskId : workers.get(id).getTaskIds()) {
                taskIdToWorkerMap.put(taskId, workers.get(id));
            }
        }
        workerGroupMap.put(groupId, group);
        group.handle(new AMWorkerGroupEvent(AMWorkerGroupEventType.INIT, groupId));
    }
    LOG.info("to init taskClockManager!");
}

Example 9 with TaskId

use of com.tencent.angel.worker.task.TaskId in project angel by Tencent.

the class PSAgentTest method setup.

@BeforeClass
public static void setup() throws Exception {
    try {
        // set basic configuration keys
        Configuration conf = new Configuration();
        conf.setBoolean("mapred.mapper.new-api", true);
        conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
        conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
        conf.set(AngelConf.ANGEL_PS_ROUTER_TYPE, "range");
        // use local deploy mode and dummy dataspliter
        conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
        conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
        // conf.setInt(AngelConf.ANGEL_PREPROCESS_VECTOR_MAXDIM, 10000);
        conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
        conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
        conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
        conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
        conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
        conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
        conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
        conf.setInt(AngelConf.ANGEL_PSAGENT_CACHE_SYNC_TIMEINTERVAL_MS, 10);
        conf.setInt(AngelConf.ANGEL_WORKER_HEARTBEAT_INTERVAL_MS, 1000);
        conf.setInt(AngelConf.ANGEL_PS_HEARTBEAT_INTERVAL_MS, 1000);
        conf.setInt(AngelConf.ANGEL_WORKER_MAX_ATTEMPTS, 1);
        conf.setInt(AngelConf.ANGEL_PS_MAX_ATTEMPTS, 1);
        // get a angel client
        angelClient = AngelClientFactory.get(conf);
        // add matrix
        MatrixContext mMatrix = new MatrixContext();
        mMatrix.setName("w1");
        mMatrix.setRowNum(1);
        mMatrix.setColNum(100000);
        mMatrix.setMaxRowNumInBlock(1);
        mMatrix.setMaxColNumInBlock(50000);
        mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
        mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
        mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
        mMatrix.setPartitionerClass(ColumnRangePartitioner.class);
        angelClient.addMatrix(mMatrix);
        MatrixContext mMatrix2 = new MatrixContext();
        mMatrix2.setName("w2");
        mMatrix2.setRowNum(1);
        mMatrix2.setColNum(100000);
        mMatrix2.setMaxRowNumInBlock(1);
        mMatrix2.setMaxColNumInBlock(50000);
        mMatrix2.setRowType(RowType.T_DOUBLE_DENSE);
        mMatrix2.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        mMatrix2.set(MatrixConf.MATRIX_HOGWILD, "true");
        mMatrix2.set(MatrixConf.MATRIX_AVERAGE, "false");
        mMatrix2.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
        mMatrix2.setPartitionerClass(ColumnRangePartitioner.class);
        angelClient.addMatrix(mMatrix2);
        angelClient.startPSServer();
        angelClient.run();
        Thread.sleep(5000);
        group0Id = new WorkerGroupId(0);
        worker0Id = new WorkerId(group0Id, 0);
        worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
        task0Id = new TaskId(0);
        task1Id = new TaskId(1);
        psId = new ParameterServerId(0);
        psAttempt0Id = new PSAttemptId(psId, 0);
    } catch (Exception x) {
        LOG.error("setup failed ", x);
        throw x;
    }
}

Example 10 with TaskId

use of com.tencent.angel.worker.task.TaskId in project angel by Tencent.

the class TaskCountersBlock method render.

@Override
protected void render(Block html) {
    set(TITLE, join("Angel TaskCountersBlock", $(TASK_ID)));
    String taskIdSr = $(TASK_ID);
    html.h1(taskIdSr);
    try {
        TaskId taskId = new TaskId(taskIdSr);
        String taskCounters = amContext.getTaskManager().getTask(taskId).getMetrics().get("taskCounters");
        html.pre()._(taskCounters)._();
    } catch (UnvalidIdStrException e) {
        LOG.error("unvalid id string ", e);
    }
}