Examples with PartitionMeta com.tencent.angel.ml.matrix.PartitionMeta used on opensource projects

Example 6 with PartitionMeta

use of com.tencent.angel.ml.matrix.PartitionMeta in project angel by Tencent.

the class RangePartitioner method getPartitions.

@Override
public List<PartitionMeta> getPartitions() {
    List<PartitionMeta> partitions = new ArrayList<PartitionMeta>();
    int id = 0;
    int matrixId = mContext.getMatrixId();
    int row = mContext.getRowNum();
    long col = mContext.getColNum();
    long start = mContext.getIndexStart();
    long end = mContext.getIndexEnd();
    long validIndexNum = mContext.getValidIndexNum();
    if (col > 0 && validIndexNum > col)
        validIndexNum = col;
    int blockRow = mContext.getMaxRowNumInBlock();
    long blockCol = mContext.getMaxColNumInBlock();
    int serverNum = conf.getInt(AngelConf.ANGEL_PS_NUMBER, AngelConf.DEFAULT_ANGEL_PS_NUMBER);
    LOG.info("start to split matrix " + mContext);
    double range = col;
    // range of long.
    if (col <= 0)
        range = ((double) end - (double) start);
    long partSize = DEFAULT_PARTITION_SIZE;
    if (validIndexNum > 0)
        partSize = (long) (DEFAULT_PARTITION_SIZE * (range / validIndexNum));
    if (blockRow < 0) {
        if (row > serverNum)
            blockRow = (int) Math.min(row / serverNum, Math.max(row / maxPartNum, Math.max(1, partSize / range)));
        else
            blockRow = row;
    }
    if (blockCol < 0)
        blockCol = Math.min(Math.max(100, (long) (range / serverNum)), Math.max(partSize / blockRow, (long) (row * (range / maxPartNum / blockRow))));
    LOG.info("blockRow = " + blockRow + ", blockCol=" + blockCol);
    mContext.setMaxRowNumInBlock(blockRow);
    mContext.setMaxColNumInBlock(blockCol);
    int startRow;
    int endRow;
    long startCol;
    long endCol;
    for (int i = 0; i < row; ) {
        for (long j = start; j < end; ) {
            startRow = i;
            startCol = j;
            endRow = (i <= (row - blockRow)) ? (i + blockRow) : row;
            endCol = (j <= (end - blockCol)) ? (j + blockCol) : end;
            partitions.add(new PartitionMeta(matrixId, id++, startRow, endRow, startCol, endCol));
            j = (j <= (end - blockCol)) ? (j + blockCol) : end;
        }
        i = (i <= (row - blockRow)) ? (i + blockRow) : row;
    }
    LOG.info("partition count: " + partitions.size());
    return partitions;
}

Example 7 with PartitionMeta

use of com.tencent.angel.ml.matrix.PartitionMeta in project angel by Tencent.

the class AMModelSaver method split.

private Map<ParameterServerId, PSMatrixSaveContext> split(MatrixSaveContext matrixSaveContext) {
    AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
    MatrixMeta meta = matrixMetaManager.getMatrix(matrixSaveContext.getMatrixName());
    if (meta == null) {
        throw new IllegalStateException("Can not find matrix " + matrixSaveContext.getMatrixName());
    }
    Map<Integer, PartitionMeta> partitions = meta.getPartitionMetas();
    List<Integer> rowIndexes = matrixSaveContext.getRowIndexes();
    Map<ParameterServerId, Set<Integer>> psIdToPartIdsMap = new HashMap<>();
    if (rowIndexes == null || rowIndexes.isEmpty()) {
        for (Map.Entry<Integer, PartitionMeta> partEntry : partitions.entrySet()) {
            ParameterServerId psId = partEntry.getValue().getMasterPs();
            if (psId == null) {
                throw new IllegalStateException("Can not get ps for partition " + partEntry.getKey());
            }
            Set partIds = psIdToPartIdsMap.get(psId);
            if (partIds == null) {
                partIds = new HashSet();
                psIdToPartIdsMap.put(psId, partIds);
            }
            partIds.add(partEntry.getKey());
        }
    } else {
        int size = rowIndexes.size();
        for (int i = 0; i < size; i++) {
            for (Map.Entry<Integer, PartitionMeta> partEntry : partitions.entrySet()) {
                if (!partEntry.getValue().contain(rowIndexes.get(i))) {
                    continue;
                }
                ParameterServerId psId = partEntry.getValue().getMasterPs();
                if (psId == null) {
                    throw new IllegalStateException("Can not get ps for partition " + partEntry.getKey());
                }
                Set partIds = psIdToPartIdsMap.get(psId);
                if (partIds == null) {
                    partIds = new HashSet();
                    psIdToPartIdsMap.put(psId, partIds);
                }
                partIds.add(partEntry.getKey());
            }
        }
    }
    int matrixId = meta.getId();
    Map<ParameterServerId, PSMatrixSaveContext> ret = new HashMap<>(psIdToPartIdsMap.size());
    for (Map.Entry<ParameterServerId, Set<Integer>> entry : psIdToPartIdsMap.entrySet()) {
        List<Integer> partIds = new ArrayList<>(entry.getValue());
        partIds.sort(new Comparator<Integer>() {

            @Override
            public int compare(Integer id1, Integer id2) {
                return id1 - id2;
            }
        });
        PSMatrixSaveContext psMatrixSaveContext = new PSMatrixSaveContext(matrixId, partIds, matrixSaveContext.getRowIndexes(), matrixSaveContext.getFormatClassName(), null, false, true);
        ret.put(entry.getKey(), psMatrixSaveContext);
    }
    return ret;
}

Example 8 with PartitionMeta

use of com.tencent.angel.ml.matrix.PartitionMeta in project angel by Tencent.

the class ProtobufUtil method convertToMatrixMeta.

public static MatrixMeta convertToMatrixMeta(MatrixMetaProto matrixMetaProto) throws ClassNotFoundException {
    MatrixContext matrixContext = convertToMatrixContext(matrixMetaProto.getMatrixContext());
    List<PartitionMetaProto> partMetaProtos = matrixMetaProto.getPartMetasList();
    int size = partMetaProtos.size();
    Map<Integer, PartitionMeta> partitionMetas = new HashMap<>(size);
    for (int i = 0; i < size; i++) {
        partitionMetas.put(partMetaProtos.get(i).getPartitionId(), convertToParitionMeta(matrixContext.getMatrixId(), partMetaProtos.get(i)));
    }
    return new MatrixMeta(matrixMetaProto.getTotalPartNum(), matrixContext, partitionMetas);
}

Example 9 with PartitionMeta

use of com.tencent.angel.ml.matrix.PartitionMeta in project angel by Tencent.

the class ParameterServer method buildMatrixReports.

private List<MatrixReportProto> buildMatrixReports() {
    MatrixReportProto.Builder matrixBuilder = MatrixReportProto.newBuilder();
    PartReportProto.Builder partBuilder = PartReportProto.newBuilder();
    List<MatrixReportProto> ret = new ArrayList<>();
    for (MatrixMeta matrix : matrixMetaManager.getMatrixMetas().values()) {
        matrixBuilder.setMatrixId(matrix.getId()).setMatrixName(matrix.getName());
        if (context.getPartReplication() > 1) {
            for (PartitionMeta part : matrix.getPartitionMetas().values()) {
                partBuilder.setPartId(part.getPartId()).setStatus(context.getMatrixStorageManager().getPart(matrix.getId(), part.getPartId()).getState().getNumber());
                matrixBuilder.addPartReports(partBuilder.build());
            }
        }
        ret.add(matrixBuilder.build());
        matrixBuilder.clear();
    }
    return ret;
}

Example 10 with PartitionMeta

use of com.tencent.angel.ml.matrix.PartitionMeta in project angel by Tencent.

the class HashPartitioner method getPartitions.

@Override
public List<PartitionMeta> getPartitions() {
    int psNum = conf.getInt(AngelConf.ANGEL_PS_NUMBER, AngelConf.DEFAULT_ANGEL_PS_NUMBER);
    List<PartitionMeta> partitions = new ArrayList<>(psNum * partNumPerServer);
    int matrixId = mContext.getMatrixId();
    for (int psIndex = 0; psIndex < psNum; psIndex++) {
        for (int partIndex = 0; partIndex < partNumPerServer; partIndex++) {
            int partId = psIndex * partNumPerServer + partIndex;
            PartitionMeta partMeta = new PartitionMeta(matrixId, partId, 0, mContext.getRowNum(), partId, partId + 1);
            partitions.add(partMeta);
            partId2serverIndex[partId] = partId % psNum;
        }
    }
    return partitions;
}