Examples with MatrixMeta com.tencent.angel.ml.matrix.MatrixMeta used on opensource projects

Example 76 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseDoubleCompVector.

public static CompSparseDoubleVector mergeSparseDoubleCompVector(IndexGetParam param, List<PartitionGetResult> partResults) {
    Map<PartitionKey, PartitionGetResult> partKeyToResultMap = mapPartKeyToResult(partResults);
    List<PartitionKey> partKeys = getSortedPartKeys(param.matrixId, param.getRowId());
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.matrixId);
    int size = partKeys.size();
    SparseDoubleVector[] splitVecs = new SparseDoubleVector[size];
    for (int i = 0; i < size; i++) {
        if (param.getPartKeyToIndexesMap().containsKey(partKeys.get(i))) {
            splitVecs[i] = new SparseDoubleVector((int) meta.getColNum(), param.getPartKeyToIndexesMap().get(partKeys.get(i)), ((IndexPartGetDoubleResult) partKeyToResultMap.get(partKeys.get(i))).getValues());
        }
    }
    return new CompSparseDoubleVector(meta.getId(), param.getRowId(), (int) meta.getColNum(), partKeys.toArray(new PartitionKey[0]), splitVecs);
}

Example 77 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseIntCompVector.

public static CompSparseIntVector mergeSparseIntCompVector(IndexGetParam param, List<PartitionGetResult> partResults) {
    Map<PartitionKey, PartitionGetResult> partKeyToResultMap = mapPartKeyToResult(partResults);
    List<PartitionKey> partKeys = getSortedPartKeys(param.matrixId, param.getRowId());
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.matrixId);
    int size = partKeys.size();
    SparseIntVector[] splitVecs = new SparseIntVector[size];
    for (int i = 0; i < size; i++) {
        if (param.getPartKeyToIndexesMap().containsKey(partKeys.get(i))) {
            splitVecs[i] = new SparseIntVector((int) meta.getColNum(), param.getPartKeyToIndexesMap().get(partKeys.get(i)), ((IndexPartGetIntResult) partKeyToResultMap.get(partKeys.get(i))).getValues());
        }
    }
    return new CompSparseIntVector(meta.getId(), param.getRowId(), (int) meta.getColNum(), partKeys.toArray(new PartitionKey[0]), splitVecs);
}

Example 78 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class CompSparseIntMatrixOpLog method split.

/**
 * Split the update according to the matrix partitions
 *
 * @param psUpdateData partition -> row split list map
 */
public void split(Map<PartitionKey, List<RowUpdateSplit>> psUpdateData) {
    long startTime = System.currentTimeMillis();
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    List<PartitionKey> partitions = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId);
    int row = matrixMeta.getRowNum();
    for (int rowId = 0; rowId < row; rowId++) {
        TVector vector = getRow(rowId);
        if (vector == null)
            continue;
        // Filter it, removing zero values
        if (enableFilter && isNeedFilter(vector)) {
            vector = vector.filter(0.0);
        }
        // Doing average or not
        if (matrixMeta.isAverage()) {
            vector.timesBy(1.0 / PSAgentContext.get().getTotalTaskNum());
        }
        // Split this row according the matrix partitions
        Map<PartitionKey, RowUpdateSplit> splits = RowUpdateSplitUtils.split(vector, partitions);
        removeRow(rowId);
        // Add the splits to the result container
        for (Map.Entry<PartitionKey, RowUpdateSplit> entry : splits.entrySet()) {
            List<RowUpdateSplit> rowSplits = psUpdateData.get(entry.getKey());
            if (rowSplits == null) {
                rowSplits = new ArrayList<>();
                psUpdateData.put(entry.getKey(), rowSplits);
            }
            rowSplits.add(entry.getValue());
        }
    }
    LOG.debug("taking " + (System.currentTimeMillis() - startTime) + " ms to split logs for matrix=" + matrixId);
}