Examples with CompStreamKeyValuePart com.tencent.angel.psagent.matrix.transport.router.CompStreamKeyValuePart used on opensource projects

Example 1 with CompStreamKeyValuePart

use of com.tencent.angel.psagent.matrix.transport.router.CompStreamKeyValuePart in project angel by Tencent.

the class HashRouterUtils method splitStream.

/**
 * Split keys by matrix partition
 *
 * @param matrixMeta matrix meta data
 * @param vectors Matrix vectors
 * @return partition key to key partition map
 */
public static CompStreamKeyValuePart[] splitStream(MatrixMeta matrixMeta, Vector[] vectors) {
    PartitionKey[] matrixParts = matrixMeta.getPartitionKeys();
    // Use comp key value part
    CompStreamKeyValuePart[] dataParts = new CompStreamKeyValuePart[matrixParts.length];
    KeyValuePart[][] subDataParts = new KeyValuePart[vectors.length][];
    // Split each vector
    for (int i = 0; i < subDataParts.length; i++) {
        subDataParts[i] = split(matrixMeta, vectors[i]);
    }
    // Combine sub data part
    for (int i = 0; i < dataParts.length; i++) {
        dataParts[i] = new CompStreamKeyValuePart(vectors.length);
        for (int j = 0; j < vectors.length; j++) {
            dataParts[i].add(subDataParts[j][i]);
        }
    }
    return dataParts;
}

Example 2 with CompStreamKeyValuePart

use of com.tencent.angel.psagent.matrix.transport.router.CompStreamKeyValuePart in project angel by Tencent.

the class UserRequestAdapter method update.

public Future<VoidResult> update(int matrixId, int rowId, Vector delta, UpdateOp op) {
    checkParams(matrixId, rowId);
    delta.setMatrixId(matrixId);
    delta.setRowId(rowId);
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = matrixMeta.getPartitionKeys();
    CompStreamKeyValuePart[] splits = RouterUtils.splitStream(matrixMeta, delta);
    FutureResult<VoidResult> result = new FutureResult<>();
    int needRequestPartNum = noEmptyPartNum(splits);
    if (needRequestPartNum == 0) {
        result.set(null);
        return result;
    }
    UpdateRowRequest request = new UpdateRowRequest(matrixId, rowId, op);
    ResponseCache cache = new MapResponseCache(needRequestPartNum);
    int requestId = request.getRequestId();
    requestIdToResponseCache.put(requestId, cache);
    requestIdToResultMap.put(requestId, result);
    requests.put(requestId, request);
    MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
    for (int i = 0; i < splits.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            sendUpdateRequest(matrixClient, requestId, matrixId, parts[i].getPartitionId(), splits[i], op);
        }
    }
    return result;
}

Example 3 with CompStreamKeyValuePart

use of com.tencent.angel.psagent.matrix.transport.router.CompStreamKeyValuePart in project angel by Tencent.

the class ByteBufSerdeUtils method deserializeKeyValuePart.

public static KeyValuePart deserializeKeyValuePart(ByteBuf in) {
    boolean isComp = ByteBufSerdeUtils.deserializeBoolean(in);
    KeyValuePart keyValuePart;
    if (isComp) {
        keyValuePart = new CompStreamKeyValuePart();
    } else {
        RouterType routerType = RouterType.valueOf(deserializeInt(in));
        RowType keyValueType = RowType.valueOf(deserializeInt(in));
        keyValuePart = DataPartFactory.createKeyValuePart(keyValueType, routerType);
    }
    keyValuePart.deserialize(in);
    return keyValuePart;
}

Example 4 with CompStreamKeyValuePart

use of com.tencent.angel.psagent.matrix.transport.router.CompStreamKeyValuePart in project angel by Tencent.

the class RangeRouterUtils method splitStream.

/**
 * Split keys by matrix partition
 *
 * @param matrixMeta matrix meta data
 * @param vectors Matrix vectors
 * @return partition key to key partition map
 */
public static CompStreamKeyValuePart[] splitStream(MatrixMeta matrixMeta, Vector[] vectors) {
    CompStreamKeyValuePart[] dataParts = new CompStreamKeyValuePart[matrixMeta.getPartitionNum()];
    KeyValuePart[][] subDataParts = new KeyValuePart[vectors.length][];
    for (int i = 0; i < vectors.length; i++) {
        subDataParts[i] = split(matrixMeta, vectors[i]);
    }
    for (int i = 0; i < dataParts.length; i++) {
        dataParts[i] = new CompStreamKeyValuePart(vectors.length);
        for (int j = 0; j < vectors.length; j++) {
            dataParts[i].add(subDataParts[j][i]);
        }
    }
    return dataParts;
}

Example 5 with CompStreamKeyValuePart

use of com.tencent.angel.psagent.matrix.transport.router.CompStreamKeyValuePart in project angel by Tencent.

the class UserRequestAdapter method update.

public Future<VoidResult> update(int matrixId, int[] rowIds, Vector[] rows, UpdateOp op) {
    assert rowIds.length == rows.length;
    checkParams(matrixId, rowIds);
    for (int i = 0; i < rowIds.length; i++) {
        rows[i].setRowId(rowIds[i]);
    }
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = matrixMeta.getPartitionKeys();
    CompStreamKeyValuePart[] splits = RouterUtils.splitStream(matrixMeta, rows);
    int needRequestPartNum = noEmptyPartNum(splits);
    FutureResult<VoidResult> result = new FutureResult<>();
    if (needRequestPartNum == 0) {
        result.set(null);
        return result;
    }
    UpdateRowsRequest request = new UpdateRowsRequest(matrixId, op);
    ResponseCache cache = new MapResponseCache(needRequestPartNum);
    int requestId = request.getRequestId();
    requestIdToResponseCache.put(requestId, cache);
    requestIdToResultMap.put(requestId, result);
    requests.put(requestId, request);
    MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
    for (int i = 0; i < splits.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            sendUpdateRequest(matrixClient, requestId, matrixId, parts[i].getPartitionId(), splits[i], op);
        }
    }
    return result;
}