Examples with IntDummyVector com.tencent.angel.ml.math2.vector.IntDummyVector used on opensource projects

Example 21 with IntDummyVector

use of com.tencent.angel.ml.math2.vector.IntDummyVector in project angel by Tencent.

the class SimpleBinaryOutNonZAExecutor method apply.

public static Vector apply(IntFloatVector v1, IntDummyVector v2, Binary op) {
    IntFloatVectorStorage newStorage = (IntFloatVectorStorage) StorageSwitch.apply(v1, v2, op);
    if (v1.isDense()) {
        float[] resValues = newStorage.getValues();
        int[] v2Indices = v2.getIndices();
        for (int idx : v2Indices) {
            resValues[idx] = op.apply(resValues[idx], 1);
        }
    } else if (v1.isSparse()) {
        int[] v2Indices = v2.getIndices();
        if (((v1.size() + v2.size()) * Constant.intersectionCoeff > Constant.sparseDenseStorageThreshold * v1.getDim())) {
            float[] resValues = newStorage.getValues();
            ObjectIterator<Int2FloatMap.Entry> iter = v1.getStorage().entryIterator();
            while (iter.hasNext()) {
                Int2FloatMap.Entry entry = iter.next();
                newStorage.set(entry.getIntKey(), entry.getFloatValue());
            }
            for (int idx : v2Indices) {
                newStorage.set(idx, op.apply(v1.get(idx), 1));
            }
        } else {
            // to avoid multi-rehash
            int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
            if (v1.size() + v2.size() < 1.5 * capacity) {
                int size = v2.size();
                for (int i = 0; i < size; i++) {
                    int idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), 1));
                }
            } else {
                ObjectIterator<Int2FloatMap.Entry> iter1 = v1.getStorage().entryIterator();
                while (iter1.hasNext()) {
                    Int2FloatMap.Entry entry = iter1.next();
                    int idx = entry.getIntKey();
                    newStorage.set(idx, entry.getFloatValue());
                }
                int size = v2.size();
                for (int i = 0; i < size; i++) {
                    int idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), 1));
                }
            }
        }
    } else {
        // sorted
        int[] v1Indices = v1.getStorage().getIndices();
        int[] v2Indices = v2.getIndices();
        if (!op.isKeepStorage() && ((v1.size() + v2.size()) * Constant.intersectionCoeff > Constant.sortedDenseStorageThreshold * v1.getDim())) {
            float[] v1Values = v1.getStorage().getValues();
            int size = v1.size();
            for (int i = 0; i < size; i++) {
                newStorage.set(v1Indices[i], v1Values[i]);
            }
            size = v2.size();
            for (int i = 0; i < size; i++) {
                int idx = v2Indices[i];
                newStorage.set(idx, op.apply(newStorage.get(idx), 1));
            }
        } else {
            int v1Pointor = 0;
            int v2Pointor = 0;
            int size1 = v1.size();
            int size2 = v2.size();
            float[] v1Values = v1.getStorage().getValues();
            while (v1Pointor < size1 || v2Pointor < size2) {
                if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                    newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], 1));
                    v1Pointor++;
                    v2Pointor++;
                } else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
                    newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
                    v1Pointor++;
                } else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
                    newStorage.set(v2Indices[v2Pointor], op.apply(0, 1));
                    v2Pointor++;
                }
            }
        }
    }
    return new IntFloatVector(v1.getMatrixId(), v1.getRowId(), v1.getClock(), v1.getDim(), newStorage);
}

Example 22 with IntDummyVector

use of com.tencent.angel.ml.math2.vector.IntDummyVector in project angel by Tencent.

the class LabeledUpdateIndexBaseTask method preProcess.

@Override
public void preProcess(TaskContext taskContext) {
    try {
        Reader<KEYIN, VALUEIN> reader = taskContext.getReader();
        while (reader.nextKeyValue()) {
            LabeledData out = parse(reader.getCurrentKey(), reader.getCurrentValue());
            if (out != null) {
                taskDataBlock.put(out);
                if (updateIndexEnable) {
                    Vector vector = out.getX();
                    int[] indexes;
                    if (vector instanceof IntDummyVector) {
                        indexes = ((IntDummyVector) vector).getIndices();
                    } else if (vector.getStorage() instanceof IntKeyVectorStorage) {
                        indexes = ((IntKeyVectorStorage) vector).getIndices();
                    } else {
                        throw new AngelException("");
                    }
                    for (int i = 0; i < indexes.length; i++) {
                        indexSet.add(indexes[i]);
                    }
                }
            }
        }
        taskDataBlock.flush();
    } catch (Exception e) {
        throw new AngelException("Pre-Process Error.", e);
    }
}

Example 23 with IntDummyVector

use of com.tencent.angel.ml.math2.vector.IntDummyVector in project angel by Tencent.

the class MixedBinaryInAllExecutor method apply.

private static Vector apply(CompIntLongVector v1, IntDummyVector v2, Binary op) {
    IntLongVector[] parts = v1.getPartitions();
    Storage[] resParts = StorageSwitch.applyComp(v1, v2, op);
    if (!op.isKeepStorage()) {
        for (int i = 0; i < parts.length; i++) {
            if (parts[i].getStorage() instanceof IntLongSortedVectorStorage) {
                resParts[i] = new IntLongSparseVectorStorage(parts[i].getDim(), parts[i].getStorage().getIndices(), parts[i].getStorage().getValues());
            }
        }
    }
    int subDim = (v1.getDim() + v1.getNumPartitions() - 1) / v1.getNumPartitions();
    for (int i = 0; i < v1.getDim(); i++) {
        int pidx = (int) (i / subDim);
        int subidx = i % subDim;
        ((IntLongVectorStorage) resParts[pidx]).set(subidx, op.apply(parts[pidx].get(subidx), v2.get(i)));
    }
    IntLongVector[] res = new IntLongVector[parts.length];
    int i = 0;
    for (IntLongVector part : parts) {
        res[i] = new IntLongVector(part.getMatrixId(), part.getRowId(), part.getClock(), part.getDim(), (IntLongVectorStorage) resParts[i]);
        i++;
    }
    v1.setPartitions(res);
    return v1;
}

Example 24 with IntDummyVector

use of com.tencent.angel.ml.math2.vector.IntDummyVector in project angel by Tencent.

the class MixedBinaryInAllExecutor method apply.

private static Vector apply(CompIntIntVector v1, IntDummyVector v2, Binary op) {
    IntIntVector[] parts = v1.getPartitions();
    Storage[] resParts = StorageSwitch.applyComp(v1, v2, op);
    if (!op.isKeepStorage()) {
        for (int i = 0; i < parts.length; i++) {
            if (parts[i].getStorage() instanceof IntIntSortedVectorStorage) {
                resParts[i] = new IntIntSparseVectorStorage(parts[i].getDim(), parts[i].getStorage().getIndices(), parts[i].getStorage().getValues());
            }
        }
    }
    int subDim = (v1.getDim() + v1.getNumPartitions() - 1) / v1.getNumPartitions();
    for (int i = 0; i < v1.getDim(); i++) {
        int pidx = (int) (i / subDim);
        int subidx = i % subDim;
        ((IntIntVectorStorage) resParts[pidx]).set(subidx, op.apply(parts[pidx].get(subidx), v2.get(i)));
    }
    IntIntVector[] res = new IntIntVector[parts.length];
    int i = 0;
    for (IntIntVector part : parts) {
        res[i] = new IntIntVector(part.getMatrixId(), part.getRowId(), part.getClock(), part.getDim(), (IntIntVectorStorage) resParts[i]);
        i++;
    }
    v1.setPartitions(res);
    return v1;
}

Example 25 with IntDummyVector

use of com.tencent.angel.ml.math2.vector.IntDummyVector in project angel by Tencent.

the class MixedBinaryOutNonZAExecutor method apply.

private static Vector apply(CompIntFloatVector v1, IntDummyVector v2, Binary op) {
    IntFloatVector[] parts = v1.getPartitions();
    Storage[] resParts = StorageSwitch.applyComp(v1, v2, op);
    if (!op.isKeepStorage()) {
        for (int i = 0; i < parts.length; i++) {
            if (parts[i].getStorage() instanceof IntFloatSortedVectorStorage) {
                resParts[i] = new IntFloatSparseVectorStorage(parts[i].getDim(), parts[i].getStorage().getIndices(), parts[i].getStorage().getValues());
            }
        }
    }
    int subDim = (v1.getDim() + v1.getNumPartitions() - 1) / v1.getNumPartitions();
    int[] v2Indices = v2.getIndices();
    for (int i = 0; i < v2Indices.length; i++) {
        int gidx = v2Indices[i];
        int pidx = (int) (gidx / subDim);
        int subidx = gidx % subDim;
        ((IntFloatVectorStorage) resParts[pidx]).set(subidx, op.apply(parts[pidx].get(subidx), 1));
    }
    IntFloatVector[] res = new IntFloatVector[parts.length];
    int i = 0;
    for (IntFloatVector part : parts) {
        res[i] = new IntFloatVector(part.getMatrixId(), part.getRowId(), part.getClock(), part.getDim(), (IntFloatVectorStorage) resParts[i]);
        i++;
    }
    return new CompIntFloatVector(v1.getMatrixId(), v1.getRowId(), v1.getClock(), v1.getDim(), res, v1.getSubDim());
}