Examples with PartitionMeta com.tencent.angel.ml.matrix.PartitionMeta used on opensource projects

Example 1 with PartitionMeta

use of com.tencent.angel.ml.matrix.PartitionMeta in project angel by Tencent.

the class LongKeyPartitioner method getPartitions.

@Override
public List<PartitionMeta> getPartitions() {
    List<PartitionMeta> partitions = new ArrayList<PartitionMeta>();
    int id = 0;
    int matrixId = mContext.getMatrixId();
    int row = mContext.getRowNum();
    long col = mContext.getColNum();
    int blockRow = mContext.getMaxRowNumInBlock();
    long blockCol = mContext.getMaxColNumInBlock();
    int serverNum = conf.getInt(AngelConf.ANGEL_PS_NUMBER, AngelConf.DEFAULT_ANGEL_PS_NUMBER);
    if (col == -1) {
        blockRow = 1;
        blockCol = Long.MAX_VALUE / serverNum / partNumPerPS * 2;
    } else {
        if (blockRow == -1 || blockCol == -1) {
            if (row >= serverNum) {
                blockRow = (int) Math.min(row / serverNum, Math.max(1, DEFAULT_PARTITION_SIZE / col));
                blockCol = Math.min(DEFAULT_PARTITION_SIZE / blockRow, col);
            } else {
                blockRow = row;
                blockCol = Math.min(DEFAULT_PARTITION_SIZE / blockRow, Math.max(100, col / serverNum));
            }
        }
    }
    LOG.info("blockRow = " + blockRow + ", blockCol=" + blockCol);
    long minValue = 0;
    long maxValue = 0;
    if (col == -1) {
        minValue = Long.MIN_VALUE;
        maxValue = Long.MAX_VALUE;
    } else {
        minValue = 0;
        maxValue = col;
    }
    int startRow = 0;
    int endRow = 0;
    long startCol = 0;
    long endCol = 0;
    for (int i = 0; i < row; ) {
        for (long j = minValue; j < maxValue; ) {
            startRow = i;
            startCol = j;
            endRow = (i <= (row - blockRow)) ? (i + blockRow) : row;
            endCol = (j <= (col - blockCol)) ? (j + blockCol) : col;
            partitions.add(new PartitionMeta(matrixId, id++, startRow, endRow, startCol, endCol));
            j = (j <= (col - blockCol)) ? (j + blockCol) : col;
        }
        i = (i <= (row - blockRow)) ? (i + blockRow) : row;
    }
    LOG.debug("partition count: " + partitions.size());
    return partitions;
}

Example 2 with PartitionMeta

use of com.tencent.angel.ml.matrix.PartitionMeta in project angel by Tencent.

the class ParameterServer method buildMatrixReports.

private List<MatrixReportProto> buildMatrixReports() {
    MatrixReportProto.Builder matrixBuilder = MatrixReportProto.newBuilder();
    PartReportProto.Builder partBuilder = PartReportProto.newBuilder();
    List<MatrixReportProto> ret = new ArrayList<>();
    for (MatrixMeta matrix : matrixMetaManager.getMatrixMetas().values()) {
        matrixBuilder.setMatrixId(matrix.getId()).setMatrixName(matrix.getName());
        if (context.getPartReplication() > 1) {
            for (PartitionMeta part : matrix.getPartitionMetas().values()) {
                partBuilder.setPartId(part.getPartId()).setStatus(context.getMatrixStorageManager().getPart(matrix.getId(), part.getPartId()).getState().getNumber());
                matrixBuilder.addPartReports(partBuilder.build());
            }
        }
        ret.add(matrixBuilder.build());
        matrixBuilder.clear();
    }
    return ret;
}

Example 3 with PartitionMeta

use of com.tencent.angel.ml.matrix.PartitionMeta in project angel by Tencent.

the class ServerMatrix method init.

public void init() {
    MatrixMeta matrixMeta = context.getMatrixMetaManager().getMatrixMeta(matrixId);
    Map<Integer, PartitionMeta> partMetas = matrixMeta.getPartitionMetas();
    for (PartitionMeta partMeta : partMetas.values()) {
        ServerPartition part = new ServerPartition(partMeta.getPartitionKey(), matrixMeta.getRowType());
        partitionMaps.put(partMeta.getPartId(), part);
        part.init();
    }
}

Example 4 with PartitionMeta

use of com.tencent.angel.ml.matrix.PartitionMeta in project angel by Tencent.

the class PSAgentMatrixMetaManager method getPartitionsFromMeta.

private List<PartitionKey> getPartitionsFromMeta(int matrixId, int rowIndex) {
    List<PartitionKey> partitionKeys = new ArrayList<>();
    Iterator<PartitionMeta> iter = matrixMetaManager.getMatrixMeta(matrixId).getPartitionMetas().values().iterator();
    while (iter.hasNext()) {
        PartitionKey partitionKey = iter.next().getPartitionKey();
        if (partitionKey.getMatrixId() == matrixId && partitionKey.getStartRow() <= rowIndex && partitionKey.getEndRow() > rowIndex)
            partitionKeys.add(partitionKey);
    }
    partitionKeys.sort(new Comparator<PartitionKey>() {

        @Override
        public int compare(PartitionKey p1, PartitionKey p2) {
            return 0;
        }
    });
    return partitionKeys;
}

Example 5 with PartitionMeta

use of com.tencent.angel.ml.matrix.PartitionMeta in project angel by Tencent.

the class ServerMatrix method init.

public void init() {
    MatrixMeta matrixMeta = context.getMatrixMetaManager().getMatrixMeta(matrixId);
    Map<Integer, PartitionMeta> partMetas = matrixMeta.getPartitionMetas();
    String sourceClass = matrixMeta.getAttribute(AngelConf.ANGEL_PS_PARTITION_SOURCE_CLASS, AngelConf.DEFAULT_ANGEL_PS_PARTITION_SOURCE_CLASS);
    // Get server partition class
    Class<? extends IServerPartition> partClass;
    try {
        partClass = matrixMeta.getPartitionClass();
        // If partition class is not set, just use the default partition class
        if (partClass == null) {
            partClass = MatrixConf.DEFAULT_SERVER_PARTITION_CLASS;
        }
    } catch (Throwable e) {
        LOG.fatal("Server partition class failed ", e);
        throw new RuntimeException(e);
    }
    // Get server partition storage class type
    Class<? extends IServerPartitionStorage> storageClass;
    try {
        storageClass = matrixMeta.getPartitionStorageClass();
    } catch (Throwable e) {
        LOG.fatal("Server partition class failed ", e);
        throw new RuntimeException(e);
    }
    RowType rowType = matrixMeta.getRowType();
    // Get value class
    Class<? extends IElement> valueClass = null;
    if (rowType.isComplexValue()) {
        try {
            valueClass = matrixMeta.getValueClass();
        } catch (Throwable e) {
            LOG.fatal("Init value class failed ", e);
            throw new RuntimeException(e);
        }
        if (valueClass == null) {
            throw new RuntimeException("Complex type must set value type class!!");
        }
    }
    for (PartitionMeta partMeta : partMetas.values()) {
        ServerPartition part = ServerPartitionFactory.getPartition(partMeta.getPartitionKey(), partClass, storageClass, matrixMeta.getRowType(), valueClass, matrixMeta.getValidIndexNumInOnePart(), matrixMeta.isHash() ? RouterType.HASH : RouterType.RANGE);
        partitionMaps.put(partMeta.getPartId(), part);
        part.init();
        part.setState(PartitionState.READ_AND_WRITE);
    }
}