Examples with PartitionUpdateParam com.tencent.angel.ml.matrix.psf.update.base.PartitionUpdateParam used on opensource projects

Example 21 with PartitionUpdateParam

use of com.tencent.angel.ml.matrix.psf.update.base.PartitionUpdateParam in project angel by Tencent.

the class PushNeighborParam method split.

@Override
public List<PartitionUpdateParam> split() {
    List<PartitionKey> parts = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId);
    List<PartitionUpdateParam> partParams = new ArrayList<>(parts.size());
    int count = 0;
    int nodeIndex = 0;
    for (PartitionKey part : parts) {
        // include start
        int start = nodeIndex;
        while (nodeIndex < keyIds.length && keyIds[nodeIndex] < part.getEndCol()) {
            nodeIndex++;
        }
        // exclude end
        int end = nodeIndex;
        int sizePart = end - start;
        count += sizePart;
        if (sizePart > 0) {
            partParams.add(new PushNeighborPartitionParam(matrixId, part, nodeIdToNeighborIndices, keyIds, start, end));
        }
    }
    assert (count == nodeIdToNeighborIndices.size());
    return partParams;
}

Example 22 with PartitionUpdateParam

use of com.tencent.angel.ml.matrix.psf.update.base.PartitionUpdateParam in project angel by Tencent.

the class InitNeighborParam method split.

@Override
public List<PartitionUpdateParam> split() {
    LongIndexComparator comparator = new LongIndexComparator(keys);
    int size = end - start;
    int[] index = new int[size];
    for (int i = 0; i < size; i++) index[i] = i + start;
    IntArrays.quickSort(index, comparator);
    List<PartitionUpdateParam> params = new ArrayList<>();
    List<PartitionKey> parts = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId);
    if (!RowUpdateSplitUtils.isInRange(keys, index, parts)) {
        throw new AngelException("node id is not in range [" + parts.get(0).getStartCol() + ", " + parts.get(parts.size() - 1).getEndCol());
    }
    int nodeIndex = start;
    int partIndex = 0;
    while (nodeIndex < end || partIndex < parts.size()) {
        int length = 0;
        long endOffset = parts.get(partIndex).getEndCol();
        while (nodeIndex < end && keys[index[nodeIndex - start]] < endOffset) {
            nodeIndex++;
            length++;
        }
        if (length > 0)
            params.add(new InitNeighborPartParam(matrixId, parts.get(partIndex), keys, index, indptr, neighbors, edgeTypes, dstTypes, nodeIndex - length - start, nodeIndex - start));
        partIndex++;
    }
    return params;
}

Example 23 with PartitionUpdateParam

use of com.tencent.angel.ml.matrix.psf.update.base.PartitionUpdateParam in project angel by Tencent.

the class UpdateHyperLogLogParam method split.

@Override
public List<PartitionUpdateParam> split() {
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = meta.getPartitionKeys();
    KeyValuePart[] splits = splitHLLMap(meta, updates);
    assert parts.length == splits.length;
    List<PartitionUpdateParam> partParams = new ArrayList<>(parts.length);
    for (int i = 0; i < parts.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            partParams.add(new UpdateHyperLogLogPartParam(matrixId, parts[i], splits[i], p, sp, seed));
        }
    }
    return partParams;
}