Examples with MatrixMeta com.tencent.angel.ml.matrix.MatrixMeta used on opensource projects

Example 31 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class TaskManager method combineUpdateIndex.

/**
 * Combine update index.
 */
@SuppressWarnings("rawtypes")
public void combineUpdateIndex() {
    IntOpenHashSet indexSet = null;
    MatrixMeta meta = null;
    for (Entry<TaskId, Task> entry : runningTask.entrySet()) {
        LabeledUpdateIndexBaseTask task = (LabeledUpdateIndexBaseTask) entry.getValue().getUserTask();
        IntOpenHashSet taskIndexSet = task.getIndexSet();
        if (taskIndexSet != null) {
            if (indexSet == null) {
                indexSet = taskIndexSet;
                meta = task.getMatrixMeta();
            } else {
                indexSet.addAll(taskIndexSet);
                task.setIndexSet(null);
            }
        }
    }
    if (indexSet != null && meta != null) {
        int size = indexSet.size();
        int[] indexArray = new int[size];
        int index = 0;
        IntIterator iter = indexSet.iterator();
        while (iter.hasNext()) {
            indexArray[index++] = iter.nextInt();
        }
        Arrays.sort(indexArray);
        List<PartitionKey> partKeyList = WorkerContext.get().getPSAgent().getMatrixMetaManager().getPartitions(meta.getId());
        Collections.sort(partKeyList);
        int partNum = partKeyList.size();
        int lastPos = 0;
        for (int i = 0; i < partNum; i++) {
            PartitionKey partKey = partKeyList.get(i);
            long endCol = partKey.getEndCol();
            for (int j = lastPos; j < size; j++) {
                if (indexArray[j] >= endCol) {
                    lastPos = j;
                    break;
                }
            }
        }
    // Bitmap bitmap = new Bitmap();
    // int max = indexArray[size - 1];
    // byte [] bitIndexArray = new byte[max / 8 + 1];
    // for(int i = 0; i < size; i++){
    // int bitIndex = indexArray[i] >> 3;
    // int bitOffset = indexArray[i] - (bitIndex << 3);
    // switch(bitOffset){
    // case 0:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x01);break;
    // case 1:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x02);break;
    // case 2:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x04);break;
    // case 3:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x08);break;
    // case 4:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x10);break;
    // case 5:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x20);break;
    // case 6:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x40);break;
    // case 7:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x80);break;
    // }
    // }
    }
}

Example 32 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class SampleNeighborParam method split.

@Override
public List<PartitionGetParam> split() {
    // Get matrix meta
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] partitions = meta.getPartitionKeys();
    // Split nodeIds
    KeyPart[] splits = RouterUtils.split(meta, 0, nodeIds);
    assert partitions.length == splits.length;
    // Generate rpc params
    List<PartitionGetParam> partParams = new ArrayList<>(partitions.length);
    for (int i = 0; i < partitions.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            partParams.add(new PartSampleNeighborParam(matrixId, partitions[i], splits[i], count));
        }
    }
    return partParams;
}

Example 33 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class SampleNeighborWithFilterParam method split.

@Override
public List<PartitionGetParam> split() {
    // Get matrix meta
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] partitions = meta.getPartitionKeys();
    // Split nodeIds
    KeyValuePart[] splits = RouterUtils.split(meta, 0, nodeIds, filterWithNeighKeys);
    assert partitions.length == splits.length;
    // Generate node ids
    List<PartitionGetParam> partParams = new ArrayList<>(partitions.length);
    for (int i = 0; i < partitions.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            partParams.add(new PartSampleNeighborWithFilterParam(matrixId, partitions[i], count, sampleType, splits[i], filterWithoutNeighKeys));
        }
    }
    return partParams;
}

Example 34 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class SampleNeighborWithTypeParam method split.

@Override
public List<PartitionGetParam> split() {
    // Get matrix meta
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] partitions = meta.getPartitionKeys();
    // Split nodeIds
    KeyPart[] splits = RouterUtils.split(meta, 0, nodeIds);
    assert partitions.length == splits.length;
    // Generate node ids
    List<PartitionGetParam> partParams = new ArrayList<>(partitions.length);
    for (int i = 0; i < partitions.length; i++) {
        if (splits[i] != null && splits[i].size() > 0)
            partParams.add(new PartSampleNeighborWithTypeParam(matrixId, partitions[i], splits[i], count, sampleType));
    }
    return partParams;
}

Example 35 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class InitHyperLogLogParam method split.

@Override
public List<PartitionUpdateParam> split() {
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = matrixMeta.getPartitionKeys();
    KeyPart[] keyParts = RouterUtils.split(matrixMeta, 0, nodes);
    List<PartitionUpdateParam> params = new ArrayList<>(parts.length);
    for (int i = 0; i < parts.length; i++) {
        if (keyParts[i] != null && keyParts[i].size() > 0) {
            params.add(new InitHyperLogLogPartParam(matrixId, parts[i], keyParts[i], p, sp, seed));
        }
    }
    return params;
}