Examples with MatrixMeta com.tencent.angel.ml.matrix.MatrixMeta used on opensource projects

Example 36 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ReadTagParam method split.

@Override
public List<PartitionGetParam> split() {
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = meta.getPartitionKeys();
    KeyPart[] nodeIdsParts = RouterUtils.split(meta, rowId, nodeIds, false);
    List<PartitionGetParam> partParams = new ArrayList<>(parts.length);
    assert parts.length == nodeIdsParts.length;
    for (int i = 0; i < parts.length; i++) {
        if (nodeIdsParts[i] != null && nodeIdsParts[i].size() > 0) {
            partParams.add(new GeneralPartGetParam(matrixId, parts[i], nodeIdsParts[i]));
        }
    }
    return partParams;
}

Example 37 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class SampleParam method split.

@Override
public List<PartitionGetParam> split() {
    // Get matrix meta
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = meta.getPartitionKeys();
    // Split
    KeyPart[] nodeIdsParts = RouterUtils.split(meta, 0, nodeIds, false);
    // Generate Part psf get param
    List<PartitionGetParam> partParams = new ArrayList<>(parts.length);
    assert parts.length == nodeIdsParts.length;
    for (int i = 0; i < parts.length; i++) {
        if (nodeIdsParts[i] != null && nodeIdsParts[i].size() > 0) {
            partParams.add(new PartSampleParam(matrixId, parts[i], nodeIdsParts[i], sampleType));
        }
    }
    return partParams;
}

Example 38 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ParameterServer method buildMatrixReports.

private List<MatrixReportProto> buildMatrixReports() {
    MatrixReportProto.Builder matrixBuilder = MatrixReportProto.newBuilder();
    PartReportProto.Builder partBuilder = PartReportProto.newBuilder();
    List<MatrixReportProto> ret = new ArrayList<>();
    for (MatrixMeta matrix : matrixMetaManager.getMatrixMetas().values()) {
        matrixBuilder.setMatrixId(matrix.getId()).setMatrixName(matrix.getName());
        if (context.getPartReplication() > 1) {
            for (PartitionMeta part : matrix.getPartitionMetas().values()) {
                partBuilder.setPartId(part.getPartId()).setStatus(context.getMatrixStorageManager().getPart(matrix.getId(), part.getPartId()).getState().getNumber());
                matrixBuilder.addPartReports(partBuilder.build());
            }
        }
        ret.add(matrixBuilder.build());
        matrixBuilder.clear();
    }
    return ret;
}

Example 39 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class PSAgentTest method testMatrixMetaManager.

@Test
public void testMatrixMetaManager() throws Exception {
    try {
        AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
        assertTrue(angelAppMaster != null);
        AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
        assertTrue(taskManager != null);
        WorkerManager workerManager = angelAppMaster.getAppContext().getWorkerManager();
        assertTrue(workerManager != null);
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        assertTrue(worker != null);
        PSAgent psAgent = worker.getPSAgent();
        assertTrue(psAgent != null);
        PSAgentMatrixMetaManager matrixMetaManager = psAgent.getMatrixMetaManager();
        assertTrue(matrixMetaManager != null);
        // test all matrix ids
        assertEquals(matrixMetaManager.getMatrixMetas().size(), 2);
        // test all matrix names
        assertNotNull(matrixMetaManager.getMatrixMeta("w1"));
        assertNotNull(matrixMetaManager.getMatrixMeta("w2"));
        // test matrix attribute
        int matrixId1 = matrixMetaManager.getMatrixId("w1");
        int matrixId2 = matrixMetaManager.getMatrixId("w2");
        String hogwildAttr = matrixMetaManager.getMatrixMeta(matrixId1).getAttribute(MatrixConf.MATRIX_HOGWILD, "true");
        assertEquals(hogwildAttr, "true");
        hogwildAttr = matrixMetaManager.getMatrixMeta(matrixId2).getAttribute(MatrixConf.MATRIX_HOGWILD, "true");
        assertEquals(hogwildAttr, "true");
        int matrix1Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w1").getId();
        int matrix2Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w2").getId();
        // test matrix meta
        MatrixMeta matrixMetaById = matrixMetaManager.getMatrixMeta(matrix1Id);
        MatrixMeta matrixMetaByName = matrixMetaManager.getMatrixMeta("w1");
        assertEquals(matrixMetaById, matrixMetaByName);
        assertEquals(matrixMetaById.getName(), "w1");
        assertEquals(matrixMetaById.getRowNum(), 1);
        assertEquals(matrixMetaById.getColNum(), 100000);
        assertEquals(matrixMetaById.getRowType(), RowType.T_DOUBLE_DENSE);
        assertEquals(matrixMetaById.getAttribute(MatrixConf.MATRIX_HOGWILD, "true"), "true");
        assertEquals(matrixMetaById.getStaleness(), 0);
    } catch (Exception x) {
        LOG.error("run testMatrixMetaManager failed ", x);
        throw x;
    }
}

Example 40 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class MasterService method getMatrices.

/**
 * Get matrices metadata
 */
@Override
public GetMatricesResponse getMatrices(RpcController controller, GetMatricesRequest request) throws ServiceException {
    GetMatricesResponse.Builder builder = GetMatricesResponse.newBuilder();
    AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
    List<String> matrixNames = request.getMatrixNamesList();
    int size = matrixNames.size();
    for (int i = 0; i < size; i++) {
        MatrixMeta matrixMeta = matrixMetaManager.getMatrix(matrixNames.get(i));
        if (matrixMeta == null) {
            throw new ServiceException("Can not find matrix " + matrixNames.get(i));
        }
        builder.addMatrixMetas(ProtobufUtil.convertToMatrixMetaProto(matrixMeta));
    }
    return builder.build();
}