Examples with PSAttemptId com.tencent.angel.ps.PSAttemptId used on opensource projects

Example 1 with PSAttemptId

use of com.tencent.angel.ps.PSAttemptId in project angel by Tencent.

the class ParameterServerServiceTest method setup.

@Before
public void setup() throws Exception {
    try {
        // set basic configuration keys
        conf = new Configuration();
        conf.setBoolean("mapred.mapper.new-api", true);
        conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
        conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
        // use local deploy mode and dummy dataspliter
        conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
        conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
        conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
        conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
        conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
        conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
        conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
        conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
        conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
        // get a angel client
        angelClient = AngelClientFactory.get(conf);
        // add matrix
        MatrixContext mMatrix = new MatrixContext();
        mMatrix.setName("w1");
        mMatrix.setRowNum(1);
        mMatrix.setColNum(100000);
        mMatrix.setMaxRowNumInBlock(1);
        mMatrix.setMaxColNumInBlock(50000);
        mMatrix.setRowType(RowType.T_INT_DENSE);
        mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
        mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
        angelClient.addMatrix(mMatrix);
        mMatrix = new MatrixContext();
        mMatrix.setName("w2");
        mMatrix.setRowNum(1);
        mMatrix.setColNum(100000);
        mMatrix.setMaxRowNumInBlock(1);
        mMatrix.setMaxColNumInBlock(50000);
        mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
        mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        mMatrix.set(MatrixConf.MATRIX_HOGWILD, "false");
        mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
        angelClient.addMatrix(mMatrix);
        angelClient.startPSServer();
        angelClient.run();
        Thread.sleep(5000);
        // group0Id = new WorkerGroupId(0);
        // worker0Id = new WorkerId(group0Id, 0);
        // worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
        // task0Id = new TaskId(0);
        // task1Id = new TaskId(1);
        psId = new ParameterServerId(0);
        psAttempt0Id = new PSAttemptId(psId, 0);
    } catch (Exception x) {
        LOG.error("setup failed ", x);
        throw x;
    }
}

Example 2 with PSAttemptId

use of com.tencent.angel.ps.PSAttemptId in project angel by Tencent.

the class PSAgentTest method setup.

@BeforeClass
public static void setup() throws Exception {
    try {
        // set basic configuration keys
        Configuration conf = new Configuration();
        conf.setBoolean("mapred.mapper.new-api", true);
        conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
        conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
        // use local deploy mode and dummy dataspliter
        conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
        conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
        // conf.setInt(AngelConf.ANGEL_PREPROCESS_VECTOR_MAXDIM, 10000);
        conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
        conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
        conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
        conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
        conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
        conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
        conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
        // get a angel client
        angelClient = AngelClientFactory.get(conf);
        // add matrix
        MatrixContext mMatrix = new MatrixContext();
        mMatrix.setName("w1");
        mMatrix.setRowNum(1);
        mMatrix.setColNum(100000);
        mMatrix.setMaxRowNumInBlock(1);
        mMatrix.setMaxColNumInBlock(50000);
        mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
        mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
        mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
        angelClient.addMatrix(mMatrix);
        MatrixContext mMatrix2 = new MatrixContext();
        mMatrix2.setName("w2");
        mMatrix2.setRowNum(1);
        mMatrix2.setColNum(100000);
        mMatrix2.setMaxRowNumInBlock(1);
        mMatrix2.setMaxColNumInBlock(50000);
        mMatrix2.setRowType(RowType.T_DOUBLE_DENSE);
        mMatrix2.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        mMatrix2.set(MatrixConf.MATRIX_HOGWILD, "true");
        mMatrix2.set(MatrixConf.MATRIX_AVERAGE, "false");
        mMatrix2.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
        angelClient.addMatrix(mMatrix2);
        angelClient.startPSServer();
        angelClient.run();
        Thread.sleep(10000);
        group0Id = new WorkerGroupId(0);
        worker0Id = new WorkerId(group0Id, 0);
        worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
        task0Id = new TaskId(0);
        task1Id = new TaskId(1);
        psId = new ParameterServerId(0);
        psAttempt0Id = new PSAttemptId(psId, 0);
    } catch (Exception x) {
        LOG.error("setup failed ", x);
        throw x;
    }
}

Example 3 with PSAttemptId

use of com.tencent.angel.ps.PSAttemptId in project angel by Tencent.

the class TransportTest method setup.

@BeforeClass
public static void setup() throws Exception {
    try {
        // Set basic configuration keys
        Configuration conf = new Configuration();
        conf.setBoolean("mapred.mapper.new-api", true);
        conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
        conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
        // Use local deploy mode and dummy data spliter
        conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
        conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
        conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
        conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
        conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
        conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
        conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
        conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
        conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
        // Create an Angel client
        angelClient = AngelClientFactory.get(conf);
        // Add different types of matrix
        MatrixContext matrix = new MatrixContext();
        matrix.setName("dense_double_mat");
        matrix.setRowNum(ddRow);
        matrix.setColNum(ddCol);
        matrix.setMaxRowNumInBlock(ddRow / 2);
        matrix.setMaxColNumInBlock(ddCol / 2);
        matrix.setRowType(RowType.T_DOUBLE_DENSE);
        matrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        matrix.set(MatrixConf.MATRIX_HOGWILD, "false");
        matrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        matrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
        angelClient.addMatrix(matrix);
        matrix = new MatrixContext();
        matrix.setName("dense_double_mat_1");
        matrix.setRowNum(ddRow);
        matrix.setColNum(ddCol);
        matrix.setMaxRowNumInBlock(ddRow / 2);
        matrix.setMaxColNumInBlock(ddCol / 2);
        matrix.setRowType(RowType.T_DOUBLE_DENSE);
        matrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        matrix.set(MatrixConf.MATRIX_HOGWILD, "false");
        matrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        matrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
        angelClient.addMatrix(matrix);
        matrix = new MatrixContext();
        matrix.setName("dense_int_mat");
        matrix.setRowNum(diRow);
        matrix.setColNum(diCol);
        matrix.setMaxRowNumInBlock(diRow / 2);
        matrix.setMaxColNumInBlock(diCol / 2);
        matrix.setRowType(RowType.T_INT_DENSE);
        matrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        matrix.set(MatrixConf.MATRIX_HOGWILD, "false");
        matrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        matrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
        angelClient.addMatrix(matrix);
        matrix = new MatrixContext();
        matrix.setName("dense_int_mat_1");
        matrix.setRowNum(diRow);
        matrix.setColNum(diCol);
        matrix.setMaxRowNumInBlock(diRow / 2);
        matrix.setMaxColNumInBlock(diCol / 2);
        matrix.setRowType(RowType.T_INT_DENSE);
        matrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        matrix.set(MatrixConf.MATRIX_HOGWILD, "false");
        matrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        matrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
        angelClient.addMatrix(matrix);
        matrix = new MatrixContext();
        matrix.setName("dense_float_mat");
        matrix.setRowNum(dfRow);
        matrix.setColNum(dfCol);
        matrix.setMaxRowNumInBlock(dfRow / 2);
        matrix.setMaxColNumInBlock(dfCol / 2);
        matrix.setRowType(RowType.T_FLOAT_DENSE);
        matrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        matrix.set(MatrixConf.MATRIX_HOGWILD, "false");
        matrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        matrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_FLOAT");
        angelClient.addMatrix(matrix);
        angelClient.startPSServer();
        angelClient.run();
        Thread.sleep(10000);
        group0Id = new WorkerGroupId(0);
        worker0Id = new WorkerId(group0Id, 0);
        worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
        task0Id = new TaskId(0);
        psId = new ParameterServerId(0);
        psAttempt0Id = new PSAttemptId(psId, 0);
    } catch (Exception x) {
        LOG.error("setup failed ", x);
        throw x;
    }
}

Example 4 with PSAttemptId

use of com.tencent.angel.ps.PSAttemptId in project angel by Tencent.

the class LocalClusterHelper method setup.

public static void setup() throws Exception {
    // set basic configuration keys
    Configuration conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
    // use local deploy mode and dummy data spliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
    // get a angel client
    angelClient = AngelClientFactory.get(conf);
    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(10);
    mMatrix.setColNum(1000);
    mMatrix.setMaxRowNumInBlock(10);
    mMatrix.setMaxColNumInBlock(500);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
    angelClient.addMatrix(mMatrix);
    MatrixContext mMatrix2 = new MatrixContext();
    mMatrix2.setName("w2");
    mMatrix2.setRowNum(10);
    mMatrix2.setColNum(100);
    mMatrix2.setMaxRowNumInBlock(5);
    mMatrix2.setMaxColNumInBlock(50);
    mMatrix2.setRowType(RowType.T_DOUBLE_DENSE);
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix2.set(MatrixConf.MATRIX_HOGWILD, "false");
    mMatrix2.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
    angelClient.addMatrix(mMatrix2);
    angelClient.startPSServer();
    angelClient.run();
    Thread.sleep(2 * 1000);
    group0Id = new WorkerGroupId(0);
    worker0Id = new WorkerId(group0Id, 0);
    worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
    task0Id = new TaskId(0);
    task1Id = new TaskId(1);
    psId = new ParameterServerId(0);
    psAttempt0Id = new PSAttemptId(psId, 0);
}

Example 5 with PSAttemptId

use of com.tencent.angel.ps.PSAttemptId in project angel by Tencent.

the class ParameterServersBlock method render.

@Override
protected void render(Block html) {
    set(TITLE, join("Angel ParameterServers"));
    TABLE<Hamlet> table = html.table("#job");
    TR<THEAD<TABLE<Hamlet>>> headTr = table.thead().tr();
    headTr.th(_TH, "id").th(_TH, "state").th(_TH, "node address").th(_TH, "start time").th(_TH, "end time").th(_TH, "elapsed time").th(_TH, "log").th(_TH, "threadstack");
    headTr._()._();
    Set<PSAttemptStateInternal> stateSet = transformToInternalState($(PARAMETERSERVER_STATE));
    TBODY<TABLE<Hamlet>> tbody = table.tbody();
    for (AMParameterServer ps : amContext.getParameterServerManager().getParameterServerMap().values()) {
        Map<PSAttemptId, PSAttempt> psAttempts = ps.getPSAttempts();
        for (PSAttempt psAttempt : psAttempts.values()) {
            if (stateSet.contains(psAttempt.getInternalState())) {
                TR<TBODY<TABLE<Hamlet>>> tr = tbody.tr();
                long elaspedTs = 0;
                if (psAttempt.getLaunchTime() != 0 && psAttempt.getFinishTime() != 0) {
                    elaspedTs = psAttempt.getFinishTime() - psAttempt.getLaunchTime();
                } else if (psAttempt.getLaunchTime() != 0 && psAttempt.getFinishTime() == 0) {
                    elaspedTs = System.currentTimeMillis() - psAttempt.getLaunchTime();
                }
                if (psAttempt.getNodeHttpAddr() == null) {
                    tr.td(psAttempt.getId().toString()).td($(PARAMETERSERVER_STATE)).td("N/A").td(psAttempt.getLaunchTime() == 0 ? "N/A" : new Date(psAttempt.getLaunchTime()).toString()).td(psAttempt.getFinishTime() == 0 ? "N/A" : new Date(psAttempt.getFinishTime()).toString()).td(elaspedTs == 0 ? "N/A" : new Date(elaspedTs).toString()).td("N/A").td("N/A");
                    tr._();
                } else {
                    tr.td(psAttempt.getId().toString()).td($(PARAMETERSERVER_STATE)).td().a(url(MRWebAppUtil.getYARNWebappScheme(), psAttempt.getNodeHttpAddr()), psAttempt.getNodeHttpAddr())._().td(psAttempt.getLaunchTime() == 0 ? "N/A" : new Date(psAttempt.getLaunchTime()).toString()).td(psAttempt.getFinishTime() == 0 ? "N/A" : new Date(psAttempt.getFinishTime()).toString()).td(elaspedTs == 0 ? "N/A" : StringUtils.formatTime(elaspedTs)).td().a(url(MRWebAppUtil.getYARNWebappScheme(), psAttempt.getNodeHttpAddr(), "node", "containerlogs", psAttempt.getContainerIdStr(), amContext.getUser().toString()), "log")._().td().a(url("/angel/parameterServerThreadStackPage/", psAttempt.getId().toString()), "psthreadstack")._();
                    tr._();
                }
            }
        }
    }
    tbody._()._();
}