Search in sources :

Example 36 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ReadTagParam method split.

@Override
public List<PartitionGetParam> split() {
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = meta.getPartitionKeys();
    KeyPart[] nodeIdsParts = RouterUtils.split(meta, rowId, nodeIds, false);
    List<PartitionGetParam> partParams = new ArrayList<>(parts.length);
    assert parts.length == nodeIdsParts.length;
    for (int i = 0; i < parts.length; i++) {
        if (nodeIdsParts[i] != null && nodeIdsParts[i].size() > 0) {
            partParams.add(new GeneralPartGetParam(matrixId, parts[i], nodeIdsParts[i]));
        }
    }
    return partParams;
}
Also used : GeneralPartGetParam(com.tencent.angel.ml.matrix.psf.get.base.GeneralPartGetParam) MatrixMeta(com.tencent.angel.ml.matrix.MatrixMeta) ArrayList(java.util.ArrayList) PartitionKey(com.tencent.angel.PartitionKey) KeyPart(com.tencent.angel.psagent.matrix.transport.router.KeyPart) PartitionGetParam(com.tencent.angel.ml.matrix.psf.get.base.PartitionGetParam)

Example 37 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class SampleParam method split.

@Override
public List<PartitionGetParam> split() {
    // Get matrix meta
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = meta.getPartitionKeys();
    // Split
    KeyPart[] nodeIdsParts = RouterUtils.split(meta, 0, nodeIds, false);
    // Generate Part psf get param
    List<PartitionGetParam> partParams = new ArrayList<>(parts.length);
    assert parts.length == nodeIdsParts.length;
    for (int i = 0; i < parts.length; i++) {
        if (nodeIdsParts[i] != null && nodeIdsParts[i].size() > 0) {
            partParams.add(new PartSampleParam(matrixId, parts[i], nodeIdsParts[i], sampleType));
        }
    }
    return partParams;
}
Also used : MatrixMeta(com.tencent.angel.ml.matrix.MatrixMeta) ArrayList(java.util.ArrayList) PartitionKey(com.tencent.angel.PartitionKey) KeyPart(com.tencent.angel.psagent.matrix.transport.router.KeyPart) PartitionGetParam(com.tencent.angel.ml.matrix.psf.get.base.PartitionGetParam)

Example 38 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ParameterServer method buildMatrixReports.

private List<MatrixReportProto> buildMatrixReports() {
    MatrixReportProto.Builder matrixBuilder = MatrixReportProto.newBuilder();
    PartReportProto.Builder partBuilder = PartReportProto.newBuilder();
    List<MatrixReportProto> ret = new ArrayList<>();
    for (MatrixMeta matrix : matrixMetaManager.getMatrixMetas().values()) {
        matrixBuilder.setMatrixId(matrix.getId()).setMatrixName(matrix.getName());
        if (context.getPartReplication() > 1) {
            for (PartitionMeta part : matrix.getPartitionMetas().values()) {
                partBuilder.setPartId(part.getPartId()).setStatus(context.getMatrixStorageManager().getPart(matrix.getId(), part.getPartId()).getState().getNumber());
                matrixBuilder.addPartReports(partBuilder.build());
            }
        }
        ret.add(matrixBuilder.build());
        matrixBuilder.clear();
    }
    return ret;
}
Also used : MatrixReportProto(com.tencent.angel.protobuf.generated.PSMasterServiceProtos.MatrixReportProto) PartReportProto(com.tencent.angel.protobuf.generated.PSMasterServiceProtos.PartReportProto) MatrixMeta(com.tencent.angel.ml.matrix.MatrixMeta) ArrayList(java.util.ArrayList) PartitionMeta(com.tencent.angel.ml.matrix.PartitionMeta)

Example 39 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class PSAgentTest method testMatrixMetaManager.

@Test
public void testMatrixMetaManager() throws Exception {
    try {
        AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
        assertTrue(angelAppMaster != null);
        AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
        assertTrue(taskManager != null);
        WorkerManager workerManager = angelAppMaster.getAppContext().getWorkerManager();
        assertTrue(workerManager != null);
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        assertTrue(worker != null);
        PSAgent psAgent = worker.getPSAgent();
        assertTrue(psAgent != null);
        PSAgentMatrixMetaManager matrixMetaManager = psAgent.getMatrixMetaManager();
        assertTrue(matrixMetaManager != null);
        // test all matrix ids
        assertEquals(matrixMetaManager.getMatrixMetas().size(), 2);
        // test all matrix names
        assertNotNull(matrixMetaManager.getMatrixMeta("w1"));
        assertNotNull(matrixMetaManager.getMatrixMeta("w2"));
        // test matrix attribute
        int matrixId1 = matrixMetaManager.getMatrixId("w1");
        int matrixId2 = matrixMetaManager.getMatrixId("w2");
        String hogwildAttr = matrixMetaManager.getMatrixMeta(matrixId1).getAttribute(MatrixConf.MATRIX_HOGWILD, "true");
        assertEquals(hogwildAttr, "true");
        hogwildAttr = matrixMetaManager.getMatrixMeta(matrixId2).getAttribute(MatrixConf.MATRIX_HOGWILD, "true");
        assertEquals(hogwildAttr, "true");
        int matrix1Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w1").getId();
        int matrix2Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w2").getId();
        // test matrix meta
        MatrixMeta matrixMetaById = matrixMetaManager.getMatrixMeta(matrix1Id);
        MatrixMeta matrixMetaByName = matrixMetaManager.getMatrixMeta("w1");
        assertEquals(matrixMetaById, matrixMetaByName);
        assertEquals(matrixMetaById.getName(), "w1");
        assertEquals(matrixMetaById.getRowNum(), 1);
        assertEquals(matrixMetaById.getColNum(), 100000);
        assertEquals(matrixMetaById.getRowType(), RowType.T_DOUBLE_DENSE);
        assertEquals(matrixMetaById.getAttribute(MatrixConf.MATRIX_HOGWILD, "true"), "true");
        assertEquals(matrixMetaById.getStaleness(), 0);
    } catch (Exception x) {
        LOG.error("run testMatrixMetaManager failed ", x);
        throw x;
    }
}
Also used : WorkerManager(com.tencent.angel.master.worker.WorkerManager) AMTaskManager(com.tencent.angel.master.task.AMTaskManager) AngelApplicationMaster(com.tencent.angel.master.AngelApplicationMaster) PSAgentMatrixMetaManager(com.tencent.angel.psagent.matrix.PSAgentMatrixMetaManager) MatrixMeta(com.tencent.angel.ml.matrix.MatrixMeta) Worker(com.tencent.angel.worker.Worker) Test(org.junit.Test)

Example 40 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class MasterService method getMatrices.

/**
 * Get matrices metadata
 */
@Override
public GetMatricesResponse getMatrices(RpcController controller, GetMatricesRequest request) throws ServiceException {
    GetMatricesResponse.Builder builder = GetMatricesResponse.newBuilder();
    AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
    List<String> matrixNames = request.getMatrixNamesList();
    int size = matrixNames.size();
    for (int i = 0; i < size; i++) {
        MatrixMeta matrixMeta = matrixMetaManager.getMatrix(matrixNames.get(i));
        if (matrixMeta == null) {
            throw new ServiceException("Can not find matrix " + matrixNames.get(i));
        }
        builder.addMatrixMetas(ProtobufUtil.convertToMatrixMetaProto(matrixMeta));
    }
    return builder.build();
}
Also used : ServiceException(com.google.protobuf.ServiceException) AMMatrixMetaManager(com.tencent.angel.master.matrixmeta.AMMatrixMetaManager) MatrixMeta(com.tencent.angel.ml.matrix.MatrixMeta) GetMatricesResponse(com.tencent.angel.protobuf.generated.PSAgentMasterServiceProtos.GetMatricesResponse)

Aggregations

MatrixMeta (com.tencent.angel.ml.matrix.MatrixMeta)78 PartitionKey (com.tencent.angel.PartitionKey)40 ArrayList (java.util.ArrayList)25 PartitionGetParam (com.tencent.angel.ml.matrix.psf.get.base.PartitionGetParam)13 KeyPart (com.tencent.angel.psagent.matrix.transport.router.KeyPart)13 PartitionGetResult (com.tencent.angel.ml.matrix.psf.get.base.PartitionGetResult)12 AngelException (com.tencent.angel.exception.AngelException)7 PartitionMeta (com.tencent.angel.ml.matrix.PartitionMeta)7 RowType (com.tencent.angel.ml.matrix.RowType)7 MatrixTransportClient (com.tencent.angel.psagent.matrix.transport.MatrixTransportClient)7 KeyValuePart (com.tencent.angel.psagent.matrix.transport.router.KeyValuePart)7 PartitionUpdateParam (com.tencent.angel.ml.matrix.psf.update.base.PartitionUpdateParam)6 Path (org.apache.hadoop.fs.Path)6 GeneralPartGetParam (com.tencent.angel.ml.matrix.psf.get.base.GeneralPartGetParam)5 ParameterServerId (com.tencent.angel.ps.ParameterServerId)5 FutureResult (com.tencent.angel.psagent.matrix.transport.FutureResult)5 MapResponseCache (com.tencent.angel.psagent.matrix.transport.response.MapResponseCache)5 ResponseCache (com.tencent.angel.psagent.matrix.transport.response.ResponseCache)5 AMMatrixMetaManager (com.tencent.angel.master.matrixmeta.AMMatrixMetaManager)4 Vector (com.tencent.angel.ml.math2.vector.Vector)4