Examples with MatrixMeta com.tencent.angel.ml.matrix.MatrixMeta used on opensource projects

Example 56 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseDoubleCompVector.

// TODO : subDim set
public static CompIntDoubleVector mergeSparseDoubleCompVector(IndexGetParam param, List<PartitionGetResult> partResults) {
    Map<PartitionKey, PartitionGetResult> partKeyToResultMap = mapPartKeyToResult(partResults);
    List<PartitionKey> partKeys = getSortedPartKeys(param.matrixId, param.getRowId());
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.matrixId);
    int dim = (int) meta.getColNum();
    int subDim = (int) meta.getBlockColNum();
    int size = partKeys.size();
    IntDoubleVector[] splitVecs = new IntDoubleVector[size];
    for (int i = 0; i < size; i++) {
        if (param.getPartKeyToIndexesMap().containsKey(partKeys.get(i))) {
            double[] values = ((IndexPartGetDoubleResult) partKeyToResultMap.get(partKeys.get(i))).getValues();
            int[] indices = param.getPartKeyToIndexesMap().get(partKeys.get(i));
            transformIndices(indices, partKeys.get(i));
            splitVecs[i] = VFactory.sparseDoubleVector(subDim, indices, values);
        } else {
            splitVecs[i] = VFactory.sparseDoubleVector(subDim, 0);
        }
    }
    CompIntDoubleVector vector = VFactory.compIntDoubleVector(dim, splitVecs, subDim);
    vector.setMatrixId(param.getMatrixId());
    vector.setRowId(param.getRowId());
    return vector;
}

Example 57 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseFloatCompVector.

public static CompIntFloatVector mergeSparseFloatCompVector(IndexGetParam param, List<PartitionGetResult> partResults) {
    Map<PartitionKey, PartitionGetResult> partKeyToResultMap = mapPartKeyToResult(partResults);
    List<PartitionKey> partKeys = getSortedPartKeys(param.matrixId, param.getRowId());
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.matrixId);
    int dim = (int) meta.getColNum();
    int subDim = (int) meta.getBlockColNum();
    int size = partKeys.size();
    IntFloatVector[] splitVecs = new IntFloatVector[size];
    for (int i = 0; i < size; i++) {
        if (param.getPartKeyToIndexesMap().containsKey(partKeys.get(i))) {
            float[] values = ((IndexPartGetFloatResult) partKeyToResultMap.get(partKeys.get(i))).getValues();
            int[] indices = param.getPartKeyToIndexesMap().get(partKeys.get(i));
            transformIndices(indices, partKeys.get(i));
            splitVecs[i] = VFactory.sparseFloatVector(subDim, indices, values);
        } else {
            splitVecs[i] = VFactory.sparseFloatVector(subDim, 0);
        }
    }
    CompIntFloatVector vector = VFactory.compIntFloatVector(dim, splitVecs, subDim);
    vector.setMatrixId(param.getMatrixId());
    vector.setRowId(param.getRowId());
    return vector;
}

Example 58 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class MatrixClientImpl method generateEmptyVec.

private Vector generateEmptyVec(int rowId) {
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    RowType rowType = matrixMeta.getRowType();
    Vector vector;
    if (rowType.isInt()) {
        if (rowType.isLongKey())
            vector = VFactory.sparseLongKeyIntVector(0, 0);
        else
            vector = VFactory.sparseIntVector(0, 0);
    } else if (rowType.isLong()) {
        if (rowType.isLongKey())
            vector = VFactory.sparseLongKeyLongVector(0, 0);
        else
            vector = VFactory.sparseLongVector(0, 0);
    } else if (rowType.isFloat()) {
        if (rowType.isLongKey())
            vector = VFactory.sparseLongKeyFloatVector(0, 0);
        else
            vector = VFactory.sparseFloatVector(0, 0);
    } else if (rowType.isDouble()) {
        if (rowType.isLongKey())
            vector = VFactory.sparseLongKeyDoubleVector(0, 0);
        else
            vector = VFactory.sparseDoubleVector(0, 0);
    } else {
        throw new AngelException("Unsupport row type");
    }
    vector.setRowId(rowId);
    vector.setMatrixId(matrixId);
    return vector;
}

Example 59 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class PSAgent method releaseMatrix.

/**
 * Release a matrix
 *
 * @param matrixId matrix id
 * @throws AngelException exception come from master
 */
public void releaseMatrix(int matrixId) throws AngelException {
    MatrixMeta meta = matrixMetaManager.getMatrixMeta(matrixId);
    if (meta == null) {
        return;
    }
    releaseMatrix(meta.getName());
}

Example 60 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class IndexGetRowTask method run.

@Override
public void run(TaskContext taskContext) throws AngelException {
    try {
        MatrixClient matrixClient = taskContext.getMatrix(IndexGetRowTest.DENSE_DOUBLE_MAT);
        MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(IndexGetRowTest.DENSE_DOUBLE_MAT);
        int[] indices = genIndexs((int) matrixMeta.getColNum(), IndexGetRowTest.nnz);
        IntDoubleVector delta = VFactory.sparseDoubleVector((int) matrixMeta.getColNum(), IndexGetRowTest.nnz);
        for (int i = 0; i < IndexGetRowTest.nnz; i++) {
            delta.set(indices[i], indices[i]);
        }
        IntDoubleVector delta1 = VFactory.denseDoubleVector((int) matrixMeta.getColNum());
        for (int i = 0; i < IndexGetRowTest.colNum; i++) {
            delta1.set(i, i);
        }
        LOG.info("delta use " + delta.getType() + " type storage");
        int testNum = 500;
        long startTs = System.currentTimeMillis();
        LOG.info("for sparse delta type");
        conf.setBoolean("use.new.split", false);
        for (int i = 0; i < testNum; i++) {
            matrixClient.increment(0, delta, true);
            if (i > 0 && i % 10 == 0) {
                LOG.info("increment old use time = " + (System.currentTimeMillis() - startTs) / i);
            }
        }
        conf.setBoolean("use.new.split", true);
        startTs = System.currentTimeMillis();
        for (int i = 0; i < testNum; i++) {
            matrixClient.increment(0, delta, true);
            // IntDoubleVector vector = (IntDoubleVector) ((GetRowResult) matrixClient.get(func)).getRow();
            if (i > 0 && i % 10 == 0) {
                LOG.info("increment new use time = " + (System.currentTimeMillis() - startTs) / i);
            }
        }
        LOG.info("for dense delta type");
        conf.setBoolean("use.new.split", false);
        for (int i = 0; i < testNum; i++) {
            matrixClient.increment(0, delta1, true);
            if (i > 0 && i % 10 == 0) {
                LOG.info("increment old use time = " + (System.currentTimeMillis() - startTs) / i);
            }
        }
        conf.setBoolean("use.new.split", true);
        startTs = System.currentTimeMillis();
        for (int i = 0; i < testNum; i++) {
            matrixClient.increment(0, delta1, true);
            // IntDoubleVector vector = (IntDoubleVector) ((GetRowResult) matrixClient.get(func)).getRow();
            if (i > 0 && i % 10 == 0) {
                LOG.info("increment new use time = " + (System.currentTimeMillis() - startTs) / i);
            }
        }
    } catch (InvalidParameterException e) {
        LOG.error("get matrix failed ", e);
        throw new AngelException(e);
    }
}