Examples with PartitionMeta com.tencent.angel.ml.matrix.PartitionMeta used on opensource projects

Example 11 with PartitionMeta

use of com.tencent.angel.ml.matrix.PartitionMeta in project angel by Tencent.

the class AMModelLoader method split.

private Map<ParameterServerId, PSMatrixLoadContext> split(MatrixLoadContext matrixLoadContext, ModelLoadContext modelLoadContext) throws IOException {
    Path matrixPath = new Path(modelLoadContext.getLoadPath(), matrixLoadContext.getMatrixName());
    Path metaFilePath = new Path(matrixPath, ModelFilesConstent.modelMetaFileName);
    MatrixFilesMeta matrixFilesMeta = new MatrixFilesMeta();
    FileSystem fs = metaFilePath.getFileSystem(context.getConf());
    if (fs.exists(metaFilePath)) {
        FSDataInputStream input = fs.open(metaFilePath);
        try {
            matrixFilesMeta.read(input);
        } catch (Throwable e) {
            throw new IOException("Read matrix meta failed ", e);
        } finally {
            input.close();
        }
    } else {
        throw new IOException("Can not find meta file " + metaFilePath);
    }
    AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
    MatrixMeta meta = matrixMetaManager.getMatrix(matrixLoadContext.getMatrixName());
    if (meta == null) {
        throw new IllegalStateException("Can not find matrix " + matrixLoadContext.getMatrixName());
    }
    Map<Integer, PartitionMeta> partitions = meta.getPartitionMetas();
    Map<ParameterServerId, Set<Integer>> psIdToPartIdsMap = new HashMap<>();
    for (Map.Entry<Integer, PartitionMeta> partEntry : partitions.entrySet()) {
        ParameterServerId psId = partEntry.getValue().getMasterPs();
        if (psId == null) {
            throw new IllegalStateException("Can not get ps for partition " + partEntry.getKey());
        }
        Set partIds = psIdToPartIdsMap.get(psId);
        if (partIds == null) {
            partIds = new HashSet();
            psIdToPartIdsMap.put(psId, partIds);
        }
        partIds.add(partEntry.getKey());
    }
    int matrixId = meta.getId();
    Map<ParameterServerId, PSMatrixLoadContext> ret = new HashMap<>(psIdToPartIdsMap.size());
    for (Map.Entry<ParameterServerId, Set<Integer>> entry : psIdToPartIdsMap.entrySet()) {
        List<Integer> partIds = new ArrayList<>(entry.getValue());
        partIds.sort(new Comparator<Integer>() {

            @Override
            public int compare(Integer id1, Integer id2) {
                return id1 - id2;
            }
        });
        PSMatrixLoadContext psMatrixLoadContext = new PSMatrixLoadContext(matrixId, matrixPath.toString(), partIds, matrixFilesMeta.getFormatClassName());
        ret.put(entry.getKey(), psMatrixLoadContext);
    }
    return ret;
}

Example 12 with PartitionMeta

use of com.tencent.angel.ml.matrix.PartitionMeta in project angel by Tencent.

the class PSPartitioner method getPartitions.

@Override
public List<PartitionMeta> getPartitions() {
    List<PartitionMeta> partitions = new ArrayList<PartitionMeta>();
    int id = 0;
    int matrixId = mContext.getMatrixId();
    int row = mContext.getRowNum();
    long col = mContext.getColNum();
    long defaultPartSize = getDefaultPartSize();
    int blockRow = mContext.getMaxRowNumInBlock();
    long blockCol = mContext.getMaxColNumInBlock();
    if (blockRow == -1 || blockCol == -1) {
        int serverNum = conf.getInt(AngelConf.ANGEL_PS_NUMBER, AngelConf.DEFAULT_ANGEL_PS_NUMBER);
        if (row >= serverNum) {
            blockRow = (int) Math.min(row / serverNum, Math.max(1, defaultPartSize / col));
            blockCol = Math.min(defaultPartSize / blockRow, col);
        } else {
            blockRow = row;
            blockCol = Math.min(defaultPartSize / blockRow, Math.max(100, col / serverNum));
        }
    }
    LOG.info("blockRow = " + blockRow + ", blockCol=" + blockCol);
    for (int i = 0; i < row; ) {
        for (long j = 0; j < col; ) {
            int startRow = i;
            long startCol = j;
            int endRow = (i <= (row - blockRow)) ? (i + blockRow) : row;
            long endCol = (j <= (col - blockCol)) ? (j + blockCol) : col;
            partitions.add(new PartitionMeta(matrixId, id++, startRow, endRow, startCol, endCol));
            j = (j <= (col - blockCol)) ? (j + blockCol) : col;
        }
        i = (i <= (row - blockRow)) ? (i + blockRow) : row;
    }
    LOG.debug("partition count: " + partitions.size());
    return partitions;
}