Examples with PartitionKey com.tencent.angel.PartitionKey used on opensource projects

Example 51 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class FullUpdateParam method split.

@Override
public List<PartitionUpdateParam> split() {
    List<PartitionKey> partList = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId);
    int size = partList.size();
    List<PartitionUpdateParam> partParams = new ArrayList<PartitionUpdateParam>(size);
    for (PartitionKey part : partList) {
        partParams.add(new FullPartitionUpdateParam(matrixId, part, values));
    }
    return partParams;
}

Example 52 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class MMUpdateParam method split.

@Override
public List<PartitionUpdateParam> split() {
    List<PartitionKey> parts = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId);
    int size = parts.size();
    List<PartitionUpdateParam> partParams = new ArrayList<PartitionUpdateParam>(size);
    for (PartitionKey part : parts) {
        if (Utils.withinPart(part, rowIds)) {
            partParams.add(new MMPartitionUpdateParam(matrixId, part, rowIds, scalars));
        }
    }
    if (partParams.isEmpty()) {
        System.out.println("Rows must in same partition.");
    }
    return partParams;
}

Example 53 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class VAUpdateParam method split.

@Override
public List<PartitionUpdateParam> split() {
    List<PartitionKey> partList = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId);
    int size = partList.size();
    List<PartitionUpdateParam> partParams = new ArrayList<PartitionUpdateParam>(size);
    for (PartitionKey part : partList) {
        long colNum = part.getEndCol() - part.getStartCol();
        double[] sliceArray = new double[(int) colNum];
        for (int i = 0; i < colNum; i++) {
            sliceArray[i] = array[(int) (part.getStartCol() + i)];
        }
        partParams.add(new VAPartitionUpdateParam(matrixId, part, rowId, sliceArray));
    }
    return partParams;
}

Example 54 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class RowUpdateSplitUtils method split.

public static HashMap<PartitionKey, RowUpdateSplit> split(int rowId, int[] indices, float[] values, List<PartitionKey> partitionInfos, boolean sorted) {
    if (!sorted) {
        Sort.quickSort(indices, values, 0, indices.length - 1);
    }
    HashMap<PartitionKey, RowUpdateSplit> ret = new HashMap<>();
    ArrayList<PartitionKey> partitionOfVector = new ArrayList<>();
    for (PartitionKey partitionKey : partitionInfos) {
        if (rowId >= partitionKey.getStartRow() && rowId < partitionKey.getEndRow()) {
            partitionOfVector.add(partitionKey);
        }
    }
    Collections.sort(partitionOfVector, new Comparator<PartitionKey>() {

        @Override
        public int compare(PartitionKey key1, PartitionKey key2) {
            return key1.getStartCol() < key2.getStartCol() ? -1 : 1;
        }
    });
    int ii = 0;
    int keyIndex = 0;
    while (ii < indices.length || keyIndex < partitionOfVector.size()) {
        int length = 0;
        int endOffset = (int) partitionOfVector.get(keyIndex).getEndCol();
        while (ii < indices.length && indices[ii] < endOffset) {
            ii++;
            length++;
        }
        RowUpdateSplit split = new SparseFloatRowUpdateSplit(rowId, ii - length, ii, indices, values);
        ret.put(partitionOfVector.get(keyIndex), split);
        keyIndex++;
    }
    return ret;
}

Example 55 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class RowUpdateSplitUtils method split.

public static HashMap<PartitionKey, RowUpdateSplit> split(int rowId, int[] indices, int[] values, List<PartitionKey> partitionInfos, boolean sorted) {
    if (!sorted) {
        Sort.quickSort(indices, values, 0, indices.length - 1);
    }
    HashMap<PartitionKey, RowUpdateSplit> ret = new HashMap<>();
    ArrayList<PartitionKey> partitionOfVector = new ArrayList<>();
    for (PartitionKey partitionKey : partitionInfos) {
        if (rowId >= partitionKey.getStartRow() && rowId < partitionKey.getEndRow()) {
            partitionOfVector.add(partitionKey);
        }
    }
    Collections.sort(partitionOfVector, new Comparator<PartitionKey>() {

        @Override
        public int compare(PartitionKey key1, PartitionKey key2) {
            return key1.getStartCol() < key2.getStartCol() ? -1 : 1;
        }
    });
    int ii = 0;
    int keyIndex = 0;
    // we still need to generate a update split to update the clock info on ps.
    while (ii < indices.length || keyIndex < partitionOfVector.size()) {
        int length = 0;
        int endOffset = (int) partitionOfVector.get(keyIndex).getEndCol();
        while (ii < indices.length && indices[ii] < endOffset) {
            ii++;
            length++;
        }
        RowUpdateSplit split = new SparseIntRowUpdateSplit(rowId, ii - length, ii, indices, values);
        ret.put(partitionOfVector.get(keyIndex), split);
        keyIndex++;
    }
    return ret;
}