use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.
the class ReadTagParam method split.
@Override
public List<PartitionGetParam> split() {
MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
PartitionKey[] parts = meta.getPartitionKeys();
KeyPart[] nodeIdsParts = RouterUtils.split(meta, rowId, nodeIds, false);
List<PartitionGetParam> partParams = new ArrayList<>(parts.length);
assert parts.length == nodeIdsParts.length;
for (int i = 0; i < parts.length; i++) {
if (nodeIdsParts[i] != null && nodeIdsParts[i].size() > 0) {
partParams.add(new GeneralPartGetParam(matrixId, parts[i], nodeIdsParts[i]));
}
}
return partParams;
}
use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.
the class SampleParam method split.
@Override
public List<PartitionGetParam> split() {
// Get matrix meta
MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
PartitionKey[] parts = meta.getPartitionKeys();
// Split
KeyPart[] nodeIdsParts = RouterUtils.split(meta, 0, nodeIds, false);
// Generate Part psf get param
List<PartitionGetParam> partParams = new ArrayList<>(parts.length);
assert parts.length == nodeIdsParts.length;
for (int i = 0; i < parts.length; i++) {
if (nodeIdsParts[i] != null && nodeIdsParts[i].size() > 0) {
partParams.add(new PartSampleParam(matrixId, parts[i], nodeIdsParts[i], sampleType));
}
}
return partParams;
}
use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.
the class ParameterServer method buildMatrixReports.
private List<MatrixReportProto> buildMatrixReports() {
MatrixReportProto.Builder matrixBuilder = MatrixReportProto.newBuilder();
PartReportProto.Builder partBuilder = PartReportProto.newBuilder();
List<MatrixReportProto> ret = new ArrayList<>();
for (MatrixMeta matrix : matrixMetaManager.getMatrixMetas().values()) {
matrixBuilder.setMatrixId(matrix.getId()).setMatrixName(matrix.getName());
if (context.getPartReplication() > 1) {
for (PartitionMeta part : matrix.getPartitionMetas().values()) {
partBuilder.setPartId(part.getPartId()).setStatus(context.getMatrixStorageManager().getPart(matrix.getId(), part.getPartId()).getState().getNumber());
matrixBuilder.addPartReports(partBuilder.build());
}
}
ret.add(matrixBuilder.build());
matrixBuilder.clear();
}
return ret;
}
use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.
the class PSAgentTest method testMatrixMetaManager.
@Test
public void testMatrixMetaManager() throws Exception {
try {
AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
assertTrue(angelAppMaster != null);
AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
assertTrue(taskManager != null);
WorkerManager workerManager = angelAppMaster.getAppContext().getWorkerManager();
assertTrue(workerManager != null);
Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
assertTrue(worker != null);
PSAgent psAgent = worker.getPSAgent();
assertTrue(psAgent != null);
PSAgentMatrixMetaManager matrixMetaManager = psAgent.getMatrixMetaManager();
assertTrue(matrixMetaManager != null);
// test all matrix ids
assertEquals(matrixMetaManager.getMatrixMetas().size(), 2);
// test all matrix names
assertNotNull(matrixMetaManager.getMatrixMeta("w1"));
assertNotNull(matrixMetaManager.getMatrixMeta("w2"));
// test matrix attribute
int matrixId1 = matrixMetaManager.getMatrixId("w1");
int matrixId2 = matrixMetaManager.getMatrixId("w2");
String hogwildAttr = matrixMetaManager.getMatrixMeta(matrixId1).getAttribute(MatrixConf.MATRIX_HOGWILD, "true");
assertEquals(hogwildAttr, "true");
hogwildAttr = matrixMetaManager.getMatrixMeta(matrixId2).getAttribute(MatrixConf.MATRIX_HOGWILD, "true");
assertEquals(hogwildAttr, "true");
int matrix1Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w1").getId();
int matrix2Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w2").getId();
// test matrix meta
MatrixMeta matrixMetaById = matrixMetaManager.getMatrixMeta(matrix1Id);
MatrixMeta matrixMetaByName = matrixMetaManager.getMatrixMeta("w1");
assertEquals(matrixMetaById, matrixMetaByName);
assertEquals(matrixMetaById.getName(), "w1");
assertEquals(matrixMetaById.getRowNum(), 1);
assertEquals(matrixMetaById.getColNum(), 100000);
assertEquals(matrixMetaById.getRowType(), RowType.T_DOUBLE_DENSE);
assertEquals(matrixMetaById.getAttribute(MatrixConf.MATRIX_HOGWILD, "true"), "true");
assertEquals(matrixMetaById.getStaleness(), 0);
} catch (Exception x) {
LOG.error("run testMatrixMetaManager failed ", x);
throw x;
}
}
use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.
the class MasterService method getMatrices.
/**
* Get matrices metadata
*/
@Override
public GetMatricesResponse getMatrices(RpcController controller, GetMatricesRequest request) throws ServiceException {
GetMatricesResponse.Builder builder = GetMatricesResponse.newBuilder();
AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
List<String> matrixNames = request.getMatrixNamesList();
int size = matrixNames.size();
for (int i = 0; i < size; i++) {
MatrixMeta matrixMeta = matrixMetaManager.getMatrix(matrixNames.get(i));
if (matrixMeta == null) {
throw new ServiceException("Can not find matrix " + matrixNames.get(i));
}
builder.addMatrixMetas(ProtobufUtil.convertToMatrixMetaProto(matrixMeta));
}
return builder.build();
}
Aggregations