Examples with KeyValuePart com.tencent.angel.psagent.matrix.transport.router.KeyValuePart used on opensource projects

Example 1 with KeyValuePart

use of com.tencent.angel.psagent.matrix.transport.router.KeyValuePart in project angel by Tencent.

the class HashRouterUtils method splitStream.

/**
 * Split keys by matrix partition
 *
 * @param matrixMeta matrix meta data
 * @param vectors Matrix vectors
 * @return partition key to key partition map
 */
public static CompStreamKeyValuePart[] splitStream(MatrixMeta matrixMeta, Vector[] vectors) {
    PartitionKey[] matrixParts = matrixMeta.getPartitionKeys();
    // Use comp key value part
    CompStreamKeyValuePart[] dataParts = new CompStreamKeyValuePart[matrixParts.length];
    KeyValuePart[][] subDataParts = new KeyValuePart[vectors.length][];
    // Split each vector
    for (int i = 0; i < subDataParts.length; i++) {
        subDataParts[i] = split(matrixMeta, vectors[i]);
    }
    // Combine sub data part
    for (int i = 0; i < dataParts.length; i++) {
        dataParts[i] = new CompStreamKeyValuePart(vectors.length);
        for (int j = 0; j < vectors.length; j++) {
            dataParts[i].add(subDataParts[j][i]);
        }
    }
    return dataParts;
}

Example 2 with KeyValuePart

use of com.tencent.angel.psagent.matrix.transport.router.KeyValuePart in project angel by Tencent.

the class GeneralInitParam method split.

@Override
public List<PartitionUpdateParam> split() {
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = meta.getPartitionKeys();
    KeyValuePart[] splits = RouterUtils.split(meta, 0, nodeIds, features);
    assert parts.length == splits.length;
    List<PartitionUpdateParam> partParams = new ArrayList<>(parts.length);
    for (int i = 0; i < parts.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            partParams.add(new GeneralPartUpdateParam(matrixId, parts[i], splits[i]));
        }
    }
    return partParams;
}

Example 3 with KeyValuePart

use of com.tencent.angel.psagent.matrix.transport.router.KeyValuePart in project angel by Tencent.

the class SampleNeighborWithFilterParam method split.

@Override
public List<PartitionGetParam> split() {
    // Get matrix meta
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] partitions = meta.getPartitionKeys();
    // Split nodeIds
    KeyValuePart[] splits = RouterUtils.split(meta, 0, nodeIds, filterWithNeighKeys);
    assert partitions.length == splits.length;
    // Generate node ids
    List<PartitionGetParam> partParams = new ArrayList<>(partitions.length);
    for (int i = 0; i < partitions.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            partParams.add(new PartSampleNeighborWithFilterParam(matrixId, partitions[i], count, sampleType, splits[i], filterWithoutNeighKeys));
        }
    }
    return partParams;
}

Example 4 with KeyValuePart

use of com.tencent.angel.psagent.matrix.transport.router.KeyValuePart in project angel by Tencent.

the class ByteBufSerdeUtils method deserializeKeyValuePart.

public static KeyValuePart deserializeKeyValuePart(ByteBuf in) {
    boolean isComp = ByteBufSerdeUtils.deserializeBoolean(in);
    KeyValuePart keyValuePart;
    if (isComp) {
        keyValuePart = new CompStreamKeyValuePart();
    } else {
        RouterType routerType = RouterType.valueOf(deserializeInt(in));
        RowType keyValueType = RowType.valueOf(deserializeInt(in));
        keyValuePart = DataPartFactory.createKeyValuePart(keyValueType, routerType);
    }
    keyValuePart.deserialize(in);
    return keyValuePart;
}

Example 5 with KeyValuePart

use of com.tencent.angel.psagent.matrix.transport.router.KeyValuePart in project angel by Tencent.

the class HashRouterUtils method split.

/**
 * Split keys by matrix partition
 *
 * @param matrixMeta matrix meta data
 * @param vector Matrix vector
 * @return partition key to key partition map
 */
public static KeyValuePart[] split(MatrixMeta matrixMeta, Vector vector) {
    KeyHash hasher = HasherFactory.getHasher(matrixMeta.getRouterHash());
    PartitionKey[] matrixParts = matrixMeta.getPartitionKeys();
    KeyValuePart[] dataParts = new KeyValuePart[matrixParts.length];
    int estSize = (int) (vector.getSize() / matrixMeta.getPartitionNum());
    for (int i = 0; i < dataParts.length; i++) {
        dataParts[i] = generateDataPart(vector.getRowId(), vector.getType(), estSize);
    }
    switch(vector.getType()) {
        case T_DOUBLE_DENSE:
        case T_DOUBLE_SPARSE:
            {
                splitIntDoubleVector(hasher, matrixMeta, (IntDoubleVector) vector, dataParts);
                break;
            }
        case T_FLOAT_DENSE:
        case T_FLOAT_SPARSE:
            {
                splitIntFloatVector(hasher, matrixMeta, (IntFloatVector) vector, dataParts);
                break;
            }
        case T_INT_DENSE:
        case T_INT_SPARSE:
            {
                splitIntIntVector(hasher, matrixMeta, (IntIntVector) vector, dataParts);
                break;
            }
        case T_LONG_DENSE:
        case T_LONG_SPARSE:
            {
                splitIntLongVector(hasher, matrixMeta, (IntLongVector) vector, dataParts);
                break;
            }
        case T_DOUBLE_SPARSE_LONGKEY:
            {
                splitLongDoubleVector(hasher, matrixMeta, (LongDoubleVector) vector, dataParts);
                break;
            }
        case T_FLOAT_SPARSE_LONGKEY:
            {
                splitLongFloatVector(hasher, matrixMeta, (LongFloatVector) vector, dataParts);
                break;
            }
        case T_INT_SPARSE_LONGKEY:
            {
                splitLongIntVector(hasher, matrixMeta, (LongIntVector) vector, dataParts);
                break;
            }
        case T_LONG_SPARSE_LONGKEY:
            {
                splitLongLongVector(hasher, matrixMeta, (LongLongVector) vector, dataParts);
                break;
            }
        default:
            {
                throw new UnsupportedOperationException("Unsupport vector type " + vector.getType());
            }
    }
    for (int i = 0; i < dataParts.length; i++) {
        if (dataParts[i] != null) {
            dataParts[i].setRowId(vector.getRowId());
        }
    }
    return dataParts;
}