Examples with PartitionKey com.tencent.angel.PartitionKey used on opensource projects

Example 91 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class CompLongLongVectorSplitter method split.

@Override
public Map<PartitionKey, RowUpdateSplit> split(Vector vector, List<PartitionKey> parts) {
    LongLongVector[] vecParts = ((CompLongLongVector) vector).getPartitions();
    assert vecParts.length == parts.size();
    Map<PartitionKey, RowUpdateSplit> updateSplitMap = new HashMap<>(parts.size());
    for (int i = 0; i < vecParts.length; i++) {
        updateSplitMap.put(parts.get(i), new CompLongLongRowUpdateSplit(vector.getRowId(), vecParts[i]));
    }
    return updateSplitMap;
}

Example 92 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class RowUpdateSplitUtils method split.

/**
 * Split a array to vector splits by vector partitions in PS
 *
 * @param rowId need update row id
 * @param values column values, the dimension must equals to indices
 * @param parts partitions that contain this vector, this partition must be sorted by part start
 * column index
 * @return part to split map
 */
public static Map<PartitionKey, RowUpdateSplit> split(int rowId, int[] values, List<PartitionKey> parts) {
    Map<PartitionKey, RowUpdateSplit> ret = new HashMap<>();
    for (PartitionKey part : parts) {
        if (rowId >= part.getStartRow() && rowId < part.getEndRow()) {
            RowUpdateSplit split = new DenseIntRowUpdateSplit(rowId, (int) part.getStartCol(), (int) part.getEndCol(), values);
            ret.put(part, split);
        }
    }
    return ret;
}

Example 93 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class RowUpdateSplitUtils method split.

public static Map<PartitionKey, HashIndicesView> split(int[] indices, List<PartitionKey> parts) {
    Map<PartitionKey, HashIndicesView> ret = new HashMap<>();
    Map<Integer, IntArrayList> partIndex2Indices = new HashMap<>();
    SortedMap<Integer, Integer> key2partIndex = new TreeMap<>();
    for (int i = 0; i < parts.size(); i++) {
        key2partIndex.put((int) parts.get(i).getEndCol(), parts.get(i).getPartitionId());
    }
    for (int j = 0; j < indices.length; j++) {
        int hash = HashUtils.getFNV1_32_HashCode(indices[j]);
        SortedMap<Integer, Integer> subMap = key2partIndex.tailMap(hash);
        int key = subMap.firstKey();
        int partIndex = subMap.get(key);
        if (partIndex2Indices.containsKey(partIndex)) {
            partIndex2Indices.get(partIndex).add(indices[j]);
        } else {
            IntArrayList splitIndices = new IntArrayList();
            splitIndices.add(indices[j]);
            partIndex2Indices.put(partIndex, splitIndices);
        }
    }
    for (Map.Entry<Integer, IntArrayList> entry : partIndex2Indices.entrySet()) {
        ret.put(parts.get(entry.getKey()), new HashIndicesView(entry.getValue().toIntArray()));
    }
    return ret;
}

Example 94 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class RowUpdateSplitUtils method split.

/**
 * Split a array to vector splits by vector partitions in PS
 *
 * @param rowId need update row id
 * @param values column values, the dimension must equals to indices
 * @param parts partitions that contain this vector, this partition must be sorted by part start
 * column index
 * @return part to split map
 */
public static Map<PartitionKey, RowUpdateSplit> split(int rowId, float[] values, List<PartitionKey> parts) {
    Map<PartitionKey, RowUpdateSplit> ret = new HashMap<>();
    for (PartitionKey part : parts) {
        if (rowId >= part.getStartRow() && rowId < part.getEndRow()) {
            RowUpdateSplit split = new DenseFloatRowUpdateSplit(rowId, (int) part.getStartCol(), (int) part.getEndCol(), values);
            ret.put(part, split);
        }
    }
    return ret;
}

Example 95 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class RowUpdateSplitUtils method split.

/**
 * Split a array to vector splits by vector partitions in PS
 *
 * @param rowId need update row id
 * @param values column values, the dimension must equals to indices
 * @param parts partitions that contain this vector, this partition must be sorted by part start
 * column index
 * @return part to split map
 */
public static Map<PartitionKey, RowUpdateSplit> split(int rowId, double[] values, List<PartitionKey> parts) {
    Map<PartitionKey, RowUpdateSplit> ret = new HashMap<>();
    for (PartitionKey part : parts) {
        if (rowId >= part.getStartRow() && rowId < part.getEndRow()) {
            RowUpdateSplit split = new DenseDoubleRowUpdateSplit(rowId, (int) part.getStartCol(), (int) part.getEndCol(), values);
            ret.put(part, split);
        }
    }
    return ret;
}