use of com.tencent.angel.ml.matrix.MatrixContext in project angel by Tencent.
the class TaskManagerTest method setup.
@BeforeClass
public static void setup() throws Exception {
try {
// set basic configuration keys
Configuration conf = new Configuration();
conf.setBoolean("mapred.mapper.new-api", true);
conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
// use local deploy mode and dummy dataspliter
conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
// get a angel client
angelClient = AngelClientFactory.get(conf);
// add matrix
MatrixContext mMatrix = new MatrixContext();
mMatrix.setName("w1");
mMatrix.setRowNum(1);
mMatrix.setColNum(100000);
mMatrix.setMaxRowNumInBlock(1);
mMatrix.setMaxColNumInBlock(50000);
mMatrix.setRowType(RowType.T_INT_DENSE);
mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
angelClient.addMatrix(mMatrix);
MatrixContext mMatrix2 = new MatrixContext();
mMatrix2.setName("w2");
mMatrix2.setRowNum(1);
mMatrix2.setColNum(100000);
mMatrix2.setMaxRowNumInBlock(1);
mMatrix2.setMaxColNumInBlock(50000);
mMatrix2.setRowType(RowType.T_DOUBLE_DENSE);
mMatrix2.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
mMatrix2.set(MatrixConf.MATRIX_HOGWILD, "false");
mMatrix2.set(MatrixConf.MATRIX_AVERAGE, "false");
mMatrix2.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
angelClient.addMatrix(mMatrix2);
angelClient.startPSServer();
angelClient.run();
Thread.sleep(5000);
group0Id = new WorkerGroupId(0);
worker0Id = new WorkerId(group0Id, 0);
worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
task0Id = new TaskId(0);
task1Id = new TaskId(1);
} catch (Exception x) {
LOG.error("setup failed ", x);
throw x;
}
}
use of com.tencent.angel.ml.matrix.MatrixContext in project angel by Tencent.
the class AngelClient method saveModel.
@SuppressWarnings("rawtypes")
@Override
public void saveModel(MLModel model) throws AngelException {
if (master == null) {
throw new AngelException("parameter servers are not started, you must execute startPSServer first!!");
}
SaveRequest.Builder builder = SaveRequest.newBuilder();
Map<String, PSModel> psModels = model.getPSModels();
for (Map.Entry<String, PSModel> entry : psModels.entrySet()) {
MatrixContext context = entry.getValue().getContext();
String savePath = context.getAttributes().get(MatrixConf.MATRIX_SAVE_PATH);
if (savePath != null) {
builder.addMatrixNames(context.getName());
}
}
try {
master.save(null, builder.build());
} catch (ServiceException e) {
LOG.error("save model failed.", e);
throw new AngelException(e);
}
while (!isCompleted()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
throw new AngelException("Interrupted as waiting app complete");
}
}
if (appFailedMessage != null) {
throw new AngelException("app run failed, " + appFailedMessage);
}
}
use of com.tencent.angel.ml.matrix.MatrixContext in project angel by Tencent.
the class IndexGetFuncTest method setup.
@Before
public void setup() throws Exception {
// set basic configuration keys
Configuration conf = new Configuration();
conf.setBoolean("mapred.mapper.new-api", true);
conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
// use local deploy mode and dummy dataspliter
conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
// get a angel client
angelClient = AngelClientFactory.get(conf);
// add dense double matrix
MatrixContext dMat = new MatrixContext();
dMat.setName(DENSE_DOUBLE_MAT);
dMat.setRowNum(1);
dMat.setColNum(feaNum);
dMat.setMaxColNumInBlock(feaNum / 3);
dMat.setRowType(RowType.T_DOUBLE_DENSE);
angelClient.addMatrix(dMat);
// add sparse double matrix
MatrixContext sMat = new MatrixContext();
sMat.setName(SPARSE_DOUBLE_MAT);
sMat.setRowNum(1);
sMat.setColNum(feaNum);
sMat.setMaxColNumInBlock(feaNum / 3);
sMat.setRowType(RowType.T_DOUBLE_SPARSE);
angelClient.addMatrix(sMat);
// add component sparse double matrix
MatrixContext sCompMat = new MatrixContext();
sCompMat.setName(SPARSE_DOUBLE_MAT_COMP);
sCompMat.setRowNum(1);
sCompMat.setColNum(feaNum);
sCompMat.setMaxColNumInBlock(feaNum / 3);
sCompMat.setRowType(RowType.T_DOUBLE_SPARSE_COMPONENT);
angelClient.addMatrix(sCompMat);
// add sparse long-key double matrix
MatrixContext dLongKeysMatrix = new MatrixContext();
dLongKeysMatrix.setName(SPARSE_DOUBLE_LONG_MAT);
dLongKeysMatrix.setRowNum(1);
dLongKeysMatrix.setColNum(feaNum);
dLongKeysMatrix.setMaxColNumInBlock(feaNum / 3);
dLongKeysMatrix.setRowType(RowType.T_DOUBLE_SPARSE_LONGKEY);
angelClient.addMatrix(dLongKeysMatrix);
// add component long-key sparse double matrix
MatrixContext dLongKeysCompMatrix = new MatrixContext();
dLongKeysCompMatrix.setName(SPARSE_DOUBLE_LONG_MAT_COMP);
dLongKeysCompMatrix.setRowNum(1);
dLongKeysCompMatrix.setColNum(feaNum);
dLongKeysCompMatrix.setMaxColNumInBlock(feaNum / 3);
dLongKeysCompMatrix.setRowType(RowType.T_DOUBLE_SPARSE_LONGKEY_COMPONENT);
angelClient.addMatrix(dLongKeysCompMatrix);
// add dense float matrix
MatrixContext dfMat = new MatrixContext();
dfMat.setName(DENSE_FLOAT_MAT);
dfMat.setRowNum(1);
dfMat.setColNum(feaNum);
dfMat.setMaxColNumInBlock(feaNum / 3);
dfMat.setRowType(RowType.T_FLOAT_DENSE);
angelClient.addMatrix(dfMat);
// add sparse float matrix
MatrixContext sfMat = new MatrixContext();
sfMat.setName(SPARSE_FLOAT_MAT);
sfMat.setRowNum(1);
sfMat.setColNum(feaNum);
sfMat.setMaxColNumInBlock(feaNum / 3);
sfMat.setRowType(RowType.T_FLOAT_SPARSE);
angelClient.addMatrix(sfMat);
// add component sparse float matrix
MatrixContext sfCompMat = new MatrixContext();
sfCompMat.setName(SPARSE_FLOAT_MAT_COMP);
sfCompMat.setRowNum(1);
sfCompMat.setColNum(feaNum);
sfCompMat.setMaxColNumInBlock(feaNum / 3);
sfCompMat.setRowType(RowType.T_FLOAT_SPARSE_COMPONENT);
angelClient.addMatrix(sfCompMat);
// add dense float matrix
MatrixContext diMat = new MatrixContext();
diMat.setName(DENSE_INT_MAT);
diMat.setRowNum(1);
diMat.setColNum(feaNum);
diMat.setMaxColNumInBlock(feaNum / 3);
diMat.setRowType(RowType.T_INT_DENSE);
angelClient.addMatrix(diMat);
// add sparse float matrix
MatrixContext siMat = new MatrixContext();
siMat.setName(SPARSE_INT_MAT);
siMat.setRowNum(1);
siMat.setColNum(feaNum);
siMat.setMaxColNumInBlock(feaNum / 3);
siMat.setRowType(RowType.T_INT_SPARSE);
angelClient.addMatrix(siMat);
// add component sparse float matrix
MatrixContext siCompMat = new MatrixContext();
siCompMat.setName(SPARSE_INT_MAT_COMP);
siCompMat.setRowNum(1);
siCompMat.setColNum(feaNum);
siCompMat.setMaxColNumInBlock(feaNum / 3);
siCompMat.setRowType(RowType.T_INT_SPARSE_COMPONENT);
angelClient.addMatrix(siCompMat);
// Start PS
angelClient.startPSServer();
// Start to run application
angelClient.run();
Thread.sleep(5000);
psId = new ParameterServerId(0);
psAttempt0Id = new PSAttemptId(psId, 0);
WorkerGroupId workerGroupId = new WorkerGroupId(0);
workerId = new WorkerId(workerGroupId, 0);
workerAttempt0Id = new WorkerAttemptId(workerId, 0);
}
use of com.tencent.angel.ml.matrix.MatrixContext in project angel by Tencent.
the class ServerPartitionTest method testWriteTo.
@Test
public void testWriteTo() throws Exception {
// set basic configuration keys
conf = new Configuration();
conf.setBoolean("mapred.mapper.new-api", true);
conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
// use local deploy mode and dummy dataspliter
conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
// get a angel client
angelClient = AngelClientFactory.get(conf);
// add matrix
MatrixContext mMatrix = new MatrixContext();
mMatrix.setName("w1");
mMatrix.setRowNum(1);
mMatrix.setColNum(100000);
mMatrix.setMaxRowNumInBlock(1);
mMatrix.setMaxColNumInBlock(50000);
mMatrix.setRowType(RowType.T_INT_DENSE);
mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
angelClient.addMatrix(mMatrix);
angelClient.startPSServer();
angelClient.runTask(DummyTask.class);
Thread.sleep(5000);
group0Id = new WorkerGroupId(0);
worker0Id = new WorkerId(group0Id, 0);
worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
task0Id = new TaskId(0);
task1Id = new TaskId(1);
psId = new ParameterServerId(0);
psAttempt0Id = new PSAttemptId(psId, 0);
DataOutputStream out = new DataOutputStream(new FileOutputStream("data"));
ByteBuf buf = Unpooled.buffer(16);
buf.writeDouble(0.00);
buf.writeDouble(1.00);
buf.writeDouble(-1.00);
buf.writeDouble(-2.00);
buf.writeDouble(-5.00);
buf.writeDouble(-6.00);
buf.writeDouble(-7.00);
buf.writeDouble(-8.00);
serverPartition.getRow(6).update(RowType.T_DOUBLE_DENSE, buf, 8);
serverPartition.save(out);
out.close();
DataInputStream in = new DataInputStream(new FileInputStream("data"));
PartitionKey partitionKeyNew = new PartitionKey(2, 1, 1, 2, 8, 10);
ServerPartition serverPartitionNew = new ServerPartition(partitionKeyNew, RowType.T_DOUBLE_DENSE);
serverPartitionNew.init();
assertNotEquals(((ServerDenseDoubleRow) serverPartition.getRow(6)).getData(), ((ServerDenseDoubleRow) serverPartitionNew.getRow(6)).getData());
serverPartitionNew.load(in);
in.close();
assertEquals(((ServerDenseDoubleRow) serverPartition.getRow(6)).getData(), ((ServerDenseDoubleRow) serverPartitionNew.getRow(6)).getData());
angelClient.stop();
}
use of com.tencent.angel.ml.matrix.MatrixContext in project angel by Tencent.
the class MatrixOpLogTest method setup.
@Before
public void setup() throws Exception {
// set basic configuration keys
Configuration conf = new Configuration();
conf.setBoolean("mapred.mapper.new-api", true);
conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
// use local deploy mode and dummy dataspliter
conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
// get a angel client
angelClient = AngelClientFactory.get(conf);
// add matrix
MatrixContext mMatrix = new MatrixContext();
mMatrix.setName("w1");
mMatrix.setRowNum(100);
mMatrix.setColNum(100000);
mMatrix.setMaxRowNumInBlock(10);
mMatrix.setMaxColNumInBlock(50000);
mMatrix.setRowType(RowType.T_INT_DENSE);
mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
angelClient.addMatrix(mMatrix);
angelClient.startPSServer();
angelClient.run();
Thread.sleep(5000);
psId = new ParameterServerId(0);
psAttempt0Id = new PSAttemptId(psId, 0);
WorkerGroupId workerGroupId = new WorkerGroupId(0);
workerId = new WorkerId(workerGroupId, 0);
workerAttempt0Id = new WorkerAttemptId(workerId, 0);
}
Aggregations