use of com.tencent.angel.ml.matrix.MatrixContext in project angel by Tencent.
the class ParameterServerServiceTest method setup.
@Before
public void setup() throws Exception {
try {
// set basic configuration keys
conf = new Configuration();
conf.setBoolean("mapred.mapper.new-api", true);
conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
// use local deploy mode and dummy dataspliter
conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
// get a angel client
angelClient = AngelClientFactory.get(conf);
// add matrix
MatrixContext mMatrix = new MatrixContext();
mMatrix.setName("w1");
mMatrix.setRowNum(1);
mMatrix.setColNum(100000);
mMatrix.setMaxRowNumInBlock(1);
mMatrix.setMaxColNumInBlock(50000);
mMatrix.setRowType(RowType.T_INT_DENSE);
mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
angelClient.addMatrix(mMatrix);
mMatrix = new MatrixContext();
mMatrix.setName("w2");
mMatrix.setRowNum(1);
mMatrix.setColNum(100000);
mMatrix.setMaxRowNumInBlock(1);
mMatrix.setMaxColNumInBlock(50000);
mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
mMatrix.set(MatrixConf.MATRIX_HOGWILD, "false");
mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
angelClient.addMatrix(mMatrix);
angelClient.startPSServer();
angelClient.run();
Thread.sleep(5000);
// group0Id = new WorkerGroupId(0);
// worker0Id = new WorkerId(group0Id, 0);
// worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
// task0Id = new TaskId(0);
// task1Id = new TaskId(1);
psId = new ParameterServerId(0);
psAttempt0Id = new PSAttemptId(psId, 0);
} catch (Exception x) {
LOG.error("setup failed ", x);
throw x;
}
}
use of com.tencent.angel.ml.matrix.MatrixContext in project angel by Tencent.
the class TransportTest method setup.
@BeforeClass
public static void setup() throws Exception {
try {
// Set basic configuration keys
Configuration conf = new Configuration();
conf.setBoolean("mapred.mapper.new-api", true);
conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
// Use local deploy mode and dummy data spliter
conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
// Create an Angel client
angelClient = AngelClientFactory.get(conf);
// Add different types of matrix
MatrixContext matrix = new MatrixContext();
matrix.setName("dense_double_mat");
matrix.setRowNum(ddRow);
matrix.setColNum(ddCol);
matrix.setMaxRowNumInBlock(ddRow / 2);
matrix.setMaxColNumInBlock(ddCol / 2);
matrix.setRowType(RowType.T_DOUBLE_DENSE);
matrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
matrix.set(MatrixConf.MATRIX_HOGWILD, "false");
matrix.set(MatrixConf.MATRIX_AVERAGE, "false");
matrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
angelClient.addMatrix(matrix);
matrix = new MatrixContext();
matrix.setName("dense_double_mat_1");
matrix.setRowNum(ddRow);
matrix.setColNum(ddCol);
matrix.setMaxRowNumInBlock(ddRow / 2);
matrix.setMaxColNumInBlock(ddCol / 2);
matrix.setRowType(RowType.T_DOUBLE_DENSE);
matrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
matrix.set(MatrixConf.MATRIX_HOGWILD, "false");
matrix.set(MatrixConf.MATRIX_AVERAGE, "false");
matrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
angelClient.addMatrix(matrix);
matrix = new MatrixContext();
matrix.setName("dense_int_mat");
matrix.setRowNum(diRow);
matrix.setColNum(diCol);
matrix.setMaxRowNumInBlock(diRow / 2);
matrix.setMaxColNumInBlock(diCol / 2);
matrix.setRowType(RowType.T_INT_DENSE);
matrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
matrix.set(MatrixConf.MATRIX_HOGWILD, "false");
matrix.set(MatrixConf.MATRIX_AVERAGE, "false");
matrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
angelClient.addMatrix(matrix);
matrix = new MatrixContext();
matrix.setName("dense_int_mat_1");
matrix.setRowNum(diRow);
matrix.setColNum(diCol);
matrix.setMaxRowNumInBlock(diRow / 2);
matrix.setMaxColNumInBlock(diCol / 2);
matrix.setRowType(RowType.T_INT_DENSE);
matrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
matrix.set(MatrixConf.MATRIX_HOGWILD, "false");
matrix.set(MatrixConf.MATRIX_AVERAGE, "false");
matrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
angelClient.addMatrix(matrix);
matrix = new MatrixContext();
matrix.setName("dense_float_mat");
matrix.setRowNum(dfRow);
matrix.setColNum(dfCol);
matrix.setMaxRowNumInBlock(dfRow / 2);
matrix.setMaxColNumInBlock(dfCol / 2);
matrix.setRowType(RowType.T_FLOAT_DENSE);
matrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
matrix.set(MatrixConf.MATRIX_HOGWILD, "false");
matrix.set(MatrixConf.MATRIX_AVERAGE, "false");
matrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_FLOAT");
angelClient.addMatrix(matrix);
angelClient.startPSServer();
angelClient.run();
Thread.sleep(10000);
group0Id = new WorkerGroupId(0);
worker0Id = new WorkerId(group0Id, 0);
worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
task0Id = new TaskId(0);
psId = new ParameterServerId(0);
psAttempt0Id = new PSAttemptId(psId, 0);
} catch (Exception x) {
LOG.error("setup failed ", x);
throw x;
}
}
use of com.tencent.angel.ml.matrix.MatrixContext in project angel by Tencent.
the class WorkerTest method setup.
@BeforeClass
public static void setup() throws Exception {
try {
// set basic configuration keys
conf = new Configuration();
conf.setBoolean("mapred.mapper.new-api", true);
conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
// use local deploy mode and dummy dataspliter
conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
// get a angel client
angelClient = AngelClientFactory.get(conf);
// add matrix
MatrixContext mMatrix = new MatrixContext();
mMatrix.setName("w1");
mMatrix.setRowNum(1);
mMatrix.setColNum(100000);
mMatrix.setMaxRowNumInBlock(1);
mMatrix.setMaxColNumInBlock(50000);
mMatrix.setRowType(RowType.T_INT_DENSE);
mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
angelClient.addMatrix(mMatrix);
mMatrix = new MatrixContext();
mMatrix.setName("w2");
mMatrix.setRowNum(1);
mMatrix.setColNum(100000);
mMatrix.setMaxRowNumInBlock(1);
mMatrix.setMaxColNumInBlock(50000);
mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
mMatrix.set(MatrixConf.MATRIX_HOGWILD, "false");
mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
angelClient.addMatrix(mMatrix);
angelClient.startPSServer();
angelClient.run();
Thread.sleep(5000);
group0Id = new WorkerGroupId(0);
worker0Id = new WorkerId(group0Id, 0);
worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
task0Id = new TaskId(0);
task1Id = new TaskId(1);
} catch (Exception x) {
LOG.error("setup failed ", x);
throw x;
}
}
use of com.tencent.angel.ml.matrix.MatrixContext in project angel by Tencent.
the class PSFailedReportTest method setup.
@Before
public void setup() throws Exception {
try {
// set basic configuration keys
Configuration conf = new Configuration();
conf.setBoolean("mapred.mapper.new-api", true);
conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
// use local deploy mode and dummy dataspliter
conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_PS_NUMBER, 2);
conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_PS_HA_REPLICATION_NUMBER, 2);
conf.setInt(AngelConf.ANGEL_PS_HA_PUSH_INTERVAL_MS, 500);
// conf.setBoolean(AngelConf.ANGEL_PS_HA_USE_EVENT_PUSH, true);
// conf.setBoolean(AngelConf.ANGEL_PS_HA_PUSH_SYNC, true);
// get a angel client
angelClient = AngelClientFactory.get(conf);
// add matrix
MatrixContext mMatrix = new MatrixContext();
mMatrix.setName("w1");
mMatrix.setRowNum(1);
mMatrix.setColNum(dim);
mMatrix.setMaxRowNumInBlock(1);
mMatrix.setMaxColNumInBlock(dim / 2);
mMatrix.setRowType(RowType.T_INT_DENSE);
mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
angelClient.addMatrix(mMatrix);
angelClient.startPSServer();
angelClient.run();
Thread.sleep(5000);
task0Id = new TaskId(0);
} catch (Exception x) {
LOG.error("setup failed ", x);
throw x;
}
}
use of com.tencent.angel.ml.matrix.MatrixContext in project angel by Tencent.
the class IndexGetRowTest method submit.
@Override
public void submit(Configuration conf) throws Exception {
AngelClient angelClient = AngelClientFactory.get(conf);
rowNum = conf.getInt("row", 1);
colNum = conf.getInt("col", 1000000);
nnz = conf.getInt("nnz", 1000000);
blockRowNum = conf.getInt("blockRow", 1);
blockColNum = conf.getInt("blockCol", 1000000);
// add dense double matrix
MatrixContext dMat = new MatrixContext();
dMat.setName(DENSE_DOUBLE_MAT);
dMat.setRowNum(rowNum);
dMat.setColNum(colNum);
dMat.setMaxRowNumInBlock(blockRowNum);
dMat.setMaxColNumInBlock(blockColNum);
dMat.setRowType(RowType.T_DOUBLE_DENSE);
angelClient.addMatrix(dMat);
angelClient.startPSServer();
angelClient.run();
angelClient.waitForCompletion();
angelClient.stop(0);
}
Aggregations