Search in sources :

Example 6 with AMMatrixMetaManager

use of com.tencent.angel.master.matrixmeta.AMMatrixMetaManager in project angel by Tencent.

the class MasterService method getMatrices.

/**
 * Get matrices metadata
 */
@Override
public GetMatricesResponse getMatrices(RpcController controller, GetMatricesRequest request) throws ServiceException {
    GetMatricesResponse.Builder builder = GetMatricesResponse.newBuilder();
    AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
    List<String> matrixNames = request.getMatrixNamesList();
    int size = matrixNames.size();
    for (int i = 0; i < size; i++) {
        MatrixMeta matrixMeta = matrixMetaManager.getMatrix(matrixNames.get(i));
        if (matrixMeta == null) {
            throw new ServiceException("Can not find matrix " + matrixNames.get(i));
        }
        builder.addMatrixMetas(ProtobufUtil.convertToMatrixMetaProto(matrixMeta));
    }
    return builder.build();
}
Also used : ServiceException(com.google.protobuf.ServiceException) AMMatrixMetaManager(com.tencent.angel.master.matrixmeta.AMMatrixMetaManager) MatrixMeta(com.tencent.angel.ml.matrix.MatrixMeta) GetMatricesResponse(com.tencent.angel.protobuf.generated.PSAgentMasterServiceProtos.GetMatricesResponse)

Example 7 with AMMatrixMetaManager

use of com.tencent.angel.master.matrixmeta.AMMatrixMetaManager in project angel by Tencent.

the class AMModelSaver method getLoadPath.

private SaveResult getLoadPath(int matrixId) throws IOException {
    SaveResult result = matrixIdToLoadPathResult.get(matrixId);
    if (result == null) {
        String loadPath = context.getConf().get(AngelConf.ANGEL_LOAD_MODEL_PATH);
        if (loadPath != null) {
            AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
            MatrixMeta meta = matrixMetaManager.getMatrix(matrixId);
            if (meta != null) {
                Path matrixPath = new Path(loadPath, meta.getName());
                FileSystem fs = matrixPath.getFileSystem(context.getConf());
                if (fs.exists(matrixPath)) {
                    result = new SaveResult(loadPath, matrixPath.toString(), -1);
                    matrixIdToLoadPathResult.putIfAbsent(matrixId, result);
                }
            }
        }
    }
    return result;
}
Also used : Path(org.apache.hadoop.fs.Path) AMMatrixMetaManager(com.tencent.angel.master.matrixmeta.AMMatrixMetaManager) MatrixMeta(com.tencent.angel.ml.matrix.MatrixMeta) FileSystem(org.apache.hadoop.fs.FileSystem) ModelSaveResult(com.tencent.angel.model.ModelSaveResult) PSMatricesSaveResult(com.tencent.angel.model.PSMatricesSaveResult)

Example 8 with AMMatrixMetaManager

use of com.tencent.angel.master.matrixmeta.AMMatrixMetaManager in project angel by Tencent.

the class AMModelLoader method split.

private Map<ParameterServerId, PSMatrixLoadContext> split(MatrixLoadContext matrixLoadContext, ModelLoadContext modelLoadContext) throws IOException {
    Path matrixPath = new Path(modelLoadContext.getLoadPath(), matrixLoadContext.getMatrixName());
    Path metaFilePath = new Path(matrixPath, ModelFilesConstent.modelMetaFileName);
    MatrixFilesMeta matrixFilesMeta = new MatrixFilesMeta();
    FileSystem fs = metaFilePath.getFileSystem(context.getConf());
    if (fs.exists(metaFilePath)) {
        FSDataInputStream input = fs.open(metaFilePath);
        try {
            matrixFilesMeta.read(input);
        } catch (Throwable e) {
            throw new IOException("Read matrix meta failed ", e);
        } finally {
            input.close();
        }
    } else {
        throw new IOException("Can not find meta file " + metaFilePath);
    }
    AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
    MatrixMeta meta = matrixMetaManager.getMatrix(matrixLoadContext.getMatrixName());
    if (meta == null) {
        throw new IllegalStateException("Can not find matrix " + matrixLoadContext.getMatrixName());
    }
    Map<Integer, PartitionMeta> partitions = meta.getPartitionMetas();
    Map<ParameterServerId, Set<Integer>> psIdToPartIdsMap = new HashMap<>();
    for (Map.Entry<Integer, PartitionMeta> partEntry : partitions.entrySet()) {
        ParameterServerId psId = partEntry.getValue().getMasterPs();
        if (psId == null) {
            throw new IllegalStateException("Can not get ps for partition " + partEntry.getKey());
        }
        Set partIds = psIdToPartIdsMap.get(psId);
        if (partIds == null) {
            partIds = new HashSet();
            psIdToPartIdsMap.put(psId, partIds);
        }
        partIds.add(partEntry.getKey());
    }
    int matrixId = meta.getId();
    Map<ParameterServerId, PSMatrixLoadContext> ret = new HashMap<>(psIdToPartIdsMap.size());
    for (Map.Entry<ParameterServerId, Set<Integer>> entry : psIdToPartIdsMap.entrySet()) {
        List<Integer> partIds = new ArrayList<>(entry.getValue());
        partIds.sort(new Comparator<Integer>() {

            @Override
            public int compare(Integer id1, Integer id2) {
                return id1 - id2;
            }
        });
        PSMatrixLoadContext psMatrixLoadContext = new PSMatrixLoadContext(matrixId, matrixPath.toString(), partIds, matrixFilesMeta.getFormatClassName());
        ret.put(entry.getKey(), psMatrixLoadContext);
    }
    return ret;
}
Also used : ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) MatrixMeta(com.tencent.angel.ml.matrix.MatrixMeta) AMMatrixMetaManager(com.tencent.angel.master.matrixmeta.AMMatrixMetaManager) FileSystem(org.apache.hadoop.fs.FileSystem) Path(org.apache.hadoop.fs.Path) PartitionMeta(com.tencent.angel.ml.matrix.PartitionMeta) IOException(java.io.IOException) MatrixFilesMeta(com.tencent.angel.model.output.format.MatrixFilesMeta) FSDataInputStream(org.apache.hadoop.fs.FSDataInputStream) ParameterServerId(com.tencent.angel.ps.ParameterServerId) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Aggregations

AMMatrixMetaManager (com.tencent.angel.master.matrixmeta.AMMatrixMetaManager)8 MatrixMeta (com.tencent.angel.ml.matrix.MatrixMeta)4 AngelException (com.tencent.angel.exception.AngelException)2 PartitionMeta (com.tencent.angel.ml.matrix.PartitionMeta)2 ParameterServerId (com.tencent.angel.ps.ParameterServerId)2 IOException (java.io.IOException)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 FileSystem (org.apache.hadoop.fs.FileSystem)2 Path (org.apache.hadoop.fs.Path)2 Test (org.junit.Test)2 ServiceException (com.google.protobuf.ServiceException)1 InvalidParameterException (com.tencent.angel.exception.InvalidParameterException)1 AMTask (com.tencent.angel.master.task.AMTask)1 AMTaskManager (com.tencent.angel.master.task.AMTaskManager)1 IntDoubleDenseVectorStorage (com.tencent.angel.ml.math2.storage.IntDoubleDenseVectorStorage)1 IntDoubleVector (com.tencent.angel.ml.math2.vector.IntDoubleVector)1 ModelSaveResult (com.tencent.angel.model.ModelSaveResult)1 PSMatricesSaveResult (com.tencent.angel.model.PSMatricesSaveResult)1 PSMatrixSaveContext (com.tencent.angel.model.PSMatrixSaveContext)1 MatrixFilesMeta (com.tencent.angel.model.output.format.MatrixFilesMeta)1