Examples with PartitionKey com.tencent.angel.PartitionKey used on opensource projects

Example 86 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class UserRequestAdapter method getRows.

public FutureResult<Vector[]> getRows(int matrixId, int[] rowIds) {
    checkParams(matrixId, rowIds);
    // Get partitions for this row
    List<PartitionKey> parts = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId, rowIds[0]);
    GetRowsRequest request = new GetRowsRequest(matrixId, rowIds);
    int requestId = request.getRequestId();
    FutureResult<Vector[]> result = new FutureResult<>();
    ResponseCache responseCache = new MapResponseCache(parts.size());
    requests.put(requestId, request);
    requestIdToResultMap.put(requestId, result);
    requestIdToResponseCache.put(requestId, responseCache);
    MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
    for (PartitionKey part : parts) {
        LOG.info("Get row " + part);
        sendGetRowsRequest(matrixClient, requestId, part.getMatrixId(), rowIds, part.getPartitionId());
    }
    return result;
}

Example 87 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class UserRequestAdapter method update.

public Future<VoidResult> update(int matrixId, Matrix delta, UpdateOp op) {
    checkParams(matrixId);
    delta.setMatrixId(matrixId);
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = matrixMeta.getPartitionKeys();
    CompStreamKeyValuePart[] splits = RouterUtils.splitStream(matrixMeta, delta);
    int needRequestPartNum = noEmptyPartNum(splits);
    FutureResult<VoidResult> result = new FutureResult<>();
    if (needRequestPartNum == 0) {
        result.set(null);
        return result;
    }
    UpdateMatrixRequest request = new UpdateMatrixRequest(matrixId, op);
    ResponseCache cache = new MapResponseCache(needRequestPartNum);
    int requestId = request.getRequestId();
    requestIdToResponseCache.put(requestId, cache);
    requestIdToResultMap.put(requestId, result);
    requests.put(requestId, request);
    MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
    for (int i = 0; i < splits.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            sendUpdateRequest(matrixClient, requestId, matrixId, parts[i].getPartitionId(), splits[i], op);
        }
    }
    return result;
}

Example 88 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class UserRequestAdapter method get.

private FutureResult<Vector> get(IndexGetRowRequest request) {
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(request.getMatrixId());
    PartitionKey[] parts = matrixMeta.getPartitionKeys();
    // Split the user request to partition requests
    FutureResult<Vector> result = new FutureResult<>();
    KeyPart[] splits;
    long startTs = System.currentTimeMillis();
    if (request instanceof IntIndexGetRowRequest) {
        splits = RouterUtils.split(matrixMeta, -1, ((IntIndexGetRowRequest) request).getKeys());
    } else if (request instanceof LongIndexGetRowRequest) {
        splits = RouterUtils.split(matrixMeta, -1, ((LongIndexGetRowRequest) request).getKeys());
    } else {
        throw new UnsupportedOperationException("Unsupport index request type " + request.getClass().toString());
    }
    LOG.info("Get by indices split use time: " + (System.currentTimeMillis() - startTs));
    assert parts.length == splits.length;
    // filter empty partition requests
    int needRequestPartNum = noEmptyPartNum(splits);
    if (needRequestPartNum == 0) {
        result.set(null);
        return result;
    }
    // Create partition results cache
    ResponseCache cache = new MapResponseCache(needRequestPartNum);
    int requestId = request.getRequestId();
    requestIdToResponseCache.put(requestId, cache);
    requestIdToResultMap.put(requestId, result);
    requests.put(requestId, request);
    // Send all the partition requests
    MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
    for (int i = 0; i < splits.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            sendIndexGetRowRequest(matrixClient, requestId, request.getMatrixId(), request.getRowId(), parts[i].getPartitionId(), splits[i], request.getFunc());
        }
    }
    return result;
}

Example 89 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class MatrixOpLog method split.

/**
 * Split the update according to the matrix partitions
 *
 * @param psUpdateData partition -> row split list map
 */
public void split(Map<PartitionKey, List<RowUpdateSplit>> psUpdateData) {
    long startTime = System.currentTimeMillis();
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    int row = matrixMeta.getRowNum();
    boolean enableFilter = matrixMeta.getAttribute(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, MatrixConf.DEFAULT_MATRIX_OPLOG_ENABLEFILTER).equalsIgnoreCase("true");
    double filterThreshold = Double.valueOf(matrixMeta.getAttribute(MatrixConf.MATRIX_OPLOG_FILTER_THRESHOLD, MatrixConf.DEFAULT_MATRIX_OPLOG_FILTER_THRESHOLD));
    for (int rowId = 0; rowId < row; rowId++) {
        Vector vector = getRow(rowId);
        if (vector == null) {
            continue;
        }
        List<PartitionKey> partitions = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId, rowId);
        // Doing average or not
        if (matrixMeta.isAverage()) {
            vector.div(PSAgentContext.get().getTotalTaskNum());
        }
        // Filter un-important update
        if (enableFilter) {
            vector = vector.filter(filterThreshold);
        }
        // Split this row according the matrix partitions
        Map<PartitionKey, RowUpdateSplit> splits = RowUpdateSplitUtils.split(vector, partitions);
        // Set split context
        for (Map.Entry<PartitionKey, RowUpdateSplit> entry : splits.entrySet()) {
            RowUpdateSplitContext context = new RowUpdateSplitContext();
            context.setEnableFilter(enableFilter);
            context.setFilterThreshold(filterThreshold);
            context.setPartKey(entry.getKey());
            entry.getValue().setSplitContext(context);
            List<RowUpdateSplit> rowSplits = psUpdateData.get(entry.getKey());
            if (rowSplits == null) {
                rowSplits = new ArrayList<>();
                psUpdateData.put(entry.getKey(), rowSplits);
            }
            rowSplits.add(entry.getValue());
        }
        // Remove the row from matrix
        removeRow(rowId);
    }
    LOG.debug("taking " + (System.currentTimeMillis() - startTime) + " ms to split logs for matrix=" + matrixId);
}

Example 90 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class CompIntLongVectorSplitter method split.

@Override
public Map<PartitionKey, RowUpdateSplit> split(Vector vector, List<PartitionKey> parts) {
    IntLongVector[] vecParts = ((CompIntLongVector) vector).getPartitions();
    assert vecParts.length == parts.size();
    Map<PartitionKey, RowUpdateSplit> updateSplitMap = new HashMap<>(parts.size());
    for (int i = 0; i < vecParts.length; i++) {
        updateSplitMap.put(parts.get(i), new CompIntLongRowUpdateSplit(vector.getRowId(), vecParts[i], (int) (parts.get(i).getEndCol() - parts.get(i).getStartCol())));
    }
    return updateSplitMap;
}