Examples with MatrixMeta com.tencent.angel.ml.matrix.MatrixMeta used on opensource projects

Example 71 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class GetHyperLogLogParam method split.

@Override
public List<PartitionGetParam> split() {
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = matrixMeta.getPartitionKeys();
    KeyPart[] keyParts = RouterUtils.split(matrixMeta, 0, nodes);
    List<PartitionGetParam> params = new ArrayList<>(parts.length);
    for (int i = 0; i < parts.length; i++) {
        if (keyParts[i] != null && keyParts[i].size() > 0) {
            params.add(new GetHyperLogLogPartParam(matrixId, parts[i], keyParts[i], n, isDirected));
        }
    }
    return params;
}

Example 72 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class GetNeighborWithCountParam method split.

@Override
public List<PartitionGetParam> split() {
    // Get matrix meta
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = meta.getPartitionKeys();
    // Split
    KeyValuePart[] nodeIdsParts = RouterUtils.split(meta, 0, nodeIds, count, false);
    // Generate Part psf get param
    List<PartitionGetParam> partParams = new ArrayList<>(parts.length);
    assert parts.length == nodeIdsParts.length;
    for (int i = 0; i < parts.length; i++) {
        if (nodeIdsParts[i] != null && nodeIdsParts[i].size() > 0) {
            partParams.add(new PartGetNeighborWithCountParam(matrixId, parts[i], nodeIdsParts[i]));
        }
    }
    return partParams;
}

Example 73 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class UpdateHyperLogLogParam method split.

@Override
public List<PartitionUpdateParam> split() {
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = meta.getPartitionKeys();
    KeyValuePart[] splits = splitHLLMap(meta, updates);
    assert parts.length == splits.length;
    List<PartitionUpdateParam> partParams = new ArrayList<>(parts.length);
    for (int i = 0; i < parts.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            partParams.add(new UpdateHyperLogLogPartParam(matrixId, parts[i], splits[i], p, sp, seed));
        }
    }
    return partParams;
}

Example 74 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class MatrixClientAdapter method getRow.

/**
 * Get a matrix row from parameter servers
 *
 * @param matrixId matrix id
 * @param rowIndex row index
 * @param clock    clock value
 * @return TVector matrix row
 * @throws ExecutionException   exception thrown when attempting to retrieve the result of a task
 *                              that aborted by throwing an exception
 * @throws InterruptedException interrupted while wait the result
 */
public TVector getRow(int matrixId, int rowIndex, int clock) throws InterruptedException, ExecutionException {
    LOG.debug("start to getRow request, matrix=" + matrixId + ", rowIndex=" + rowIndex + ", clock=" + clock);
    long startTs = System.currentTimeMillis();
    // Wait until the clock value of this row is greater than or equal to the value
    PSAgentContext.get().getConsistencyController().waitForClock(matrixId, rowIndex, clock);
    LOG.debug("getRow wait clock time=" + (System.currentTimeMillis() - startTs));
    startTs = System.currentTimeMillis();
    // Get partitions for this row
    List<PartitionKey> partList = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId, rowIndex);
    GetRowRequest request = new GetRowRequest(matrixId, rowIndex, clock);
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    GetRowPipelineCache responseCache = (GetRowPipelineCache) requestToResponseMap.get(request);
    if (responseCache == null) {
        responseCache = new GetRowPipelineCache(partList.size(), meta.getRowType());
        GetRowPipelineCache oldCache = (GetRowPipelineCache) requestToResponseMap.putIfAbsent(request, responseCache);
        if (oldCache != null) {
            responseCache = oldCache;
        }
    }
    // First get this row from matrix storage
    MatrixStorage matrixStorage = PSAgentContext.get().getMatrixStorageManager().getMatrixStoage(matrixId);
    try {
        responseCache.getDistinctLock().lock();
        // If the row exists in the matrix storage and the clock value meets the requirements, just
        // return
        TVector row = matrixStorage.getRow(rowIndex);
        if (row != null && row.getClock() >= clock) {
            return row;
        }
        // Get row splits of this row from the matrix cache first
        MatricesCache matricesCache = PSAgentContext.get().getMatricesCache();
        MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
        int size = partList.size();
        for (int i = 0; i < size; i++) {
            ServerRow rowSplit = matricesCache.getRowSplit(matrixId, partList.get(i), rowIndex);
            if (rowSplit != null && rowSplit.getClock() >= clock) {
                responseCache.addRowSplit(rowSplit);
            } else {
                // If the row split does not exist in cache, get it from parameter server
                responseCache.addRowSplit(matrixClient.getRowSplit(partList.get(i), rowIndex, clock));
            }
        }
        // Wait the final result
        row = responseCache.getMergedResult().get();
        LOG.debug("get row use time=" + (System.currentTimeMillis() - startTs));
        // Put it to the matrix cache
        matrixStorage.addRow(rowIndex, row);
        return row;
    } finally {
        responseCache.getDistinctLock().unlock();
        requestToResponseMap.remove(request);
    }
}

Example 75 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class RowSplitCombineUtils method combineRowSplitsPipeline.

/**
 * Combine the row splits use pipeline mode.
 *
 * @param cache the result cache for GET_ROW sub-requests
 * @param matrixId matrix id
 * @param rowIndex row index
 * @return TVector merged row
 * @throws InterruptedException interrupted while waiting for row splits
 */
public static TVector combineRowSplitsPipeline(GetRowPipelineCache cache, int matrixId, int rowIndex) throws InterruptedException {
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    RowType rowType = matrixMeta.getRowType();
    switch(rowType) {
        case T_DOUBLE_DENSE:
            return combineDenseDoubleRowSplits(cache, matrixMeta, rowIndex);
        case T_DOUBLE_SPARSE:
            return combineServerSparseDoubleRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_DOUBLE_SPARSE_LONGKEY:
            return combineServerSparseDoubleLongKeyRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_FLOAT_SPARSE:
            return combineServerSparseFloatRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_INT_DENSE:
            return combineDenseIntRowSplits(cache, matrixMeta, rowIndex);
        case T_FLOAT_DENSE:
            return combineDenseFloatRowSplits(cache, matrixMeta, rowIndex);
        case T_INT_SPARSE:
            return combineServerSparseIntRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_DOUBLE_SPARSE_COMPONENT:
            return combineComponentServerSparseDoubleRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_FLOAT_SPARSE_COMPONENT:
            return combineComponentServerSparseFloatRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_INT_SPARSE_COMPONENT:
            return combineComponentServerSparseIntRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_DOUBLE_SPARSE_LONGKEY_COMPONENT:
            return combineCompServerSparseDoubleLongKeyRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        default:
            return null;
    }
}