Examples with PartitionKey com.tencent.angel.PartitionKey used on opensource projects

Example 16 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class MUpdateParam method split.

@Override
public List<PartitionUpdateParam> split() {
    List<PartitionKey> partList = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId);
    int size = partList.size();
    List<PartitionUpdateParam> partParams = new ArrayList<PartitionUpdateParam>(size);
    for (PartitionKey part : partList) {
        if (Utils.withinPart(part, rowIds)) {
            partParams.add(new MPartitionUpdateParam(matrixId, part, rowIds));
        }
    }
    if (partParams.isEmpty()) {
        System.out.println("Rows must in same partition.");
    }
    return partParams;
}

Example 17 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class CompSparseIntVectorTest method initVector.

private CompSparseIntVector initVector() {
    PartitionKey[] partKeys = new PartitionKey[4];
    TIntVector[] vectors = new TIntVector[4];
    long blockCol = dim / 4;
    partKeys[0] = new PartitionKey(0, 0, 0, 0, 0, blockCol);
    partKeys[1] = new PartitionKey(0, 0, 0, blockCol, 0, blockCol * 2);
    partKeys[2] = new PartitionKey(0, 0, 0, blockCol * 2, 0, blockCol * 3);
    partKeys[3] = new PartitionKey(0, 0, 0, blockCol * 3, 0, dim);
    vectors[0] = new SparseIntVector(dim);
    vectors[1] = new SparseIntVector(dim);
    vectors[2] = new SparseIntVector(dim);
    vectors[3] = new SparseIntVector(dim);
    return new CompSparseIntVector(0, 0, dim, partKeys, vectors);
}

Example 18 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class RowUpdateSplitUtils method split.

public static HashMap<PartitionKey, RowUpdateSplit> split(int rowId, int[] indices, double[] values, List<PartitionKey> partitionInfos, boolean sorted) {
    if (!sorted) {
        Sort.quickSort(indices, values, 0, indices.length - 1);
    }
    HashMap<PartitionKey, RowUpdateSplit> ret = new HashMap<>();
    ArrayList<PartitionKey> partitionOfVector = new ArrayList<>();
    LOG.debug("split sparse double vector, rowId=" + rowId);
    for (PartitionKey partitionKey : partitionInfos) {
        LOG.debug("split sparse double vector, rowId=" + rowId + ", partitionKey.getStartRow()=" + partitionKey.getStartRow() + ", partitionKey.getEndRow()=" + partitionKey.getEndRow());
        if (rowId >= partitionKey.getStartRow() && rowId < partitionKey.getEndRow()) {
            partitionOfVector.add(partitionKey);
        }
    }
    Collections.sort(partitionOfVector, new Comparator<PartitionKey>() {

        @Override
        public int compare(PartitionKey key1, PartitionKey key2) {
            return key1.getStartCol() < key2.getStartCol() ? -1 : 1;
        }
    });
    int ii = 0;
    int keyIndex = 0;
    while (ii < indices.length || keyIndex < partitionOfVector.size()) {
        int length = 0;
        int endOffset = (int) partitionOfVector.get(keyIndex).getEndCol();
        while (ii < indices.length && indices[ii] < endOffset) {
            ii++;
            length++;
        }
        RowUpdateSplit split = new SparseDoubleRowUpdateSplit(rowId, ii - length, ii, indices, values);
        ret.put(partitionOfVector.get(keyIndex), split);
        keyIndex++;
    }
    return ret;
}

Example 19 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class RowUpdateSplitUtils method split.

public static HashMap<PartitionKey, RowUpdateSplit> split(int rowId, long[] indices, double[] values, List<PartitionKey> partitionInfos, boolean sorted) {
    long startTs = System.currentTimeMillis();
    if (!sorted) {
        Sort.quickSort(indices, values, 0, indices.length - 1);
    }
    HashMap<PartitionKey, RowUpdateSplit> ret = new HashMap<>();
    ArrayList<PartitionKey> partitionOfVector = new ArrayList<>();
    for (PartitionKey partitionKey : partitionInfos) {
        if (rowId >= partitionKey.getStartRow() && rowId < partitionKey.getEndRow()) {
            partitionOfVector.add(partitionKey);
        }
    }
    Collections.sort(partitionOfVector, new Comparator<PartitionKey>() {

        @Override
        public int compare(PartitionKey key1, PartitionKey key2) {
            return key1.getStartCol() < key2.getStartCol() ? -1 : 1;
        }
    });
    int ii = 0;
    int keyIndex = 0;
    startTs = System.currentTimeMillis();
    // we still need to generate a update split to update the clock info on ps.
    while (ii < indices.length || keyIndex < partitionOfVector.size()) {
        int length = 0;
        long endOffset = partitionOfVector.get(keyIndex).getEndCol();
        while (ii < indices.length && indices[ii] < endOffset) {
            ii++;
            length++;
        }
        RowUpdateSplit split = new LongKeySparseDoubleRowUpdateSplit(rowId, RowType.T_DOUBLE_SPARSE_LONGKEY, ii - length, ii, indices, values);
        ret.put(partitionOfVector.get(keyIndex), split);
        keyIndex++;
    }
    return ret;
}

Example 20 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class MatrixClientAdapter method chooseRpcBatchSize.

private int chooseRpcBatchSize(RowIndex rowIndex) {
    PartitionKey part = PSAgentContext.get().getMatrixMetaManager().getPartitions(rowIndex.getMatrixId()).get(0);
    int rowNumInPart = part.getEndRow() - part.getStartRow();
    return Math.max(rowNumInPart / 4, 10);
}