Examples with LongIntVector com.tencent.angel.ml.math2.vector.LongIntVector used on opensource projects

Example 16 with LongIntVector

use of com.tencent.angel.ml.math2.vector.LongIntVector in project angel by Tencent.

the class SimpleBinaryOutNonZAExecutor method apply.

public static Vector apply(LongDoubleVector v1, LongIntVector v2, Binary op) {
    LongDoubleVectorStorage newStorage = (LongDoubleVectorStorage) StorageSwitch.apply(v1, v2, op);
    if (v1.isSparse() && v2.isSparse()) {
        long v1Size = v1.size();
        long v2Size = v2.size();
        if (v1Size >= v2Size * Constant.sparseThreshold && (v1Size + v2Size) * Constant.intersectionCoeff <= Constant.sparseDenseStorageThreshold * v1.dim()) {
            // we gauss the indices of v2 maybe is a subset of v1, or overlap is very large
            ObjectIterator<Long2IntMap.Entry> iter = v2.getStorage().entryIterator();
            while (iter.hasNext()) {
                Long2IntMap.Entry entry = iter.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, op.apply(v1.get(idx), entry.getIntValue()));
            }
        } else if ((v1Size + v2Size) * Constant.intersectionCoeff >= Constant.sparseDenseStorageThreshold * v1.dim()) {
            // we gauss dense storage is more efficient
            ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
            while (iter1.hasNext()) {
                Long2DoubleMap.Entry entry = iter1.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, entry.getDoubleValue());
            }
            ObjectIterator<Long2IntMap.Entry> iter2 = v2.getStorage().entryIterator();
            while (iter2.hasNext()) {
                Long2IntMap.Entry entry = iter2.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, op.apply(v1.get(idx), entry.getIntValue()));
            }
        } else {
            // to avoid multi-rehash
            int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
            if (v1.size() + v2.size() <= 1.5 * capacity) {
                // no rehashor one onle rehash is required, nothing to optimization
                ObjectIterator<Long2IntMap.Entry> iter = v2.getStorage().entryIterator();
                while (iter.hasNext()) {
                    Long2IntMap.Entry entry = iter.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(v1.get(idx), entry.getIntValue()));
                }
            } else {
                // multi-rehash
                ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
                while (iter1.hasNext()) {
                    Long2DoubleMap.Entry entry = iter1.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, entry.getDoubleValue());
                }
                ObjectIterator<Long2IntMap.Entry> iter2 = v2.getStorage().entryIterator();
                while (iter2.hasNext()) {
                    Long2IntMap.Entry entry = iter2.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(v1.get(idx), entry.getIntValue()));
                }
            }
        }
    } else if (v1.isSparse() && v2.isSorted()) {
        long v1Size = v1.size();
        long v2Size = v2.size();
        if (v1Size >= v2Size * Constant.sparseThreshold && (v1Size + v2Size) * Constant.intersectionCoeff <= Constant.sparseDenseStorageThreshold * v1.dim()) {
            // we gauss the indices of v2 maybe is a subset of v1, or overlap is very large
            long[] v2Indices = v2.getStorage().getIndices();
            int[] v2Values = v2.getStorage().getValues();
            for (int i = 0; i < v2.size(); i++) {
                long idx = v2Indices[i];
                newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
            }
        } else if ((v1Size + v2Size) * Constant.intersectionCoeff >= Constant.sparseDenseStorageThreshold * v1.dim()) {
            ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
            while (iter1.hasNext()) {
                Long2DoubleMap.Entry entry = iter1.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, entry.getDoubleValue());
            }
            long[] v2Indices = v2.getStorage().getIndices();
            int[] v2Values = v2.getStorage().getValues();
            long size = v2.size();
            for (int i = 0; i < size; i++) {
                long idx = v2Indices[i];
                newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
            }
        } else {
            // to avoid multi-rehash
            int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
            if (v1.size() + v2.size() <= 1.5 * capacity) {
                long[] v2Indices = v2.getStorage().getIndices();
                int[] v2Values = v2.getStorage().getValues();
                for (int i = 0; i < v2.size(); i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
                }
            } else {
                ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
                while (iter1.hasNext()) {
                    Long2DoubleMap.Entry entry = iter1.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, entry.getDoubleValue());
                }
                long[] v2Indices = v2.getStorage().getIndices();
                int[] v2Values = v2.getStorage().getValues();
                long size = v2.size();
                for (int i = 0; i < size; i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
                }
            }
        }
    } else if (v1.isSorted() && v2.isSparse()) {
        long v1Size = v1.size();
        long v2Size = v2.size();
        if ((v1Size + v2Size) * Constant.intersectionCoeff >= Constant.sortedDenseStorageThreshold * v1.dim()) {
            if (op.isKeepStorage()) {
                long[] v1Indices = v1.getStorage().getIndices();
                long[] idxiter = v2.getStorage().indexIterator().toLongArray();
                long[] indices = new long[(int) (v1Size + v2Size)];
                System.arraycopy(v1Indices, 0, indices, 0, (int) v1.size());
                System.arraycopy(idxiter, 0, indices, (int) v1.size(), (int) v2.size());
                LongAVLTreeSet avl = new LongAVLTreeSet(indices);
                LongBidirectionalIterator iter = avl.iterator();
                double[] values = new double[indices.length];
                int i = 0;
                while (iter.hasNext()) {
                    long idx = iter.nextLong();
                    indices[i] = idx;
                    values[i] = op.apply(v1.get(idx), v2.get(idx));
                    i++;
                }
                while (i < indices.length) {
                    indices[i] = 0;
                    i++;
                }
                newStorage = new LongDoubleSortedVectorStorage(v1.getDim(), (int) avl.size(), indices, values);
            } else {
                long[] v1Indices = v1.getStorage().getIndices();
                double[] v1Values = v1.getStorage().getValues();
                long size = v1.size();
                for (int i = 0; i < size; i++) {
                    long idx = v1Indices[i];
                    newStorage.set(idx, v1Values[i]);
                }
                ObjectIterator<Long2IntMap.Entry> iter = v2.getStorage().entryIterator();
                while (iter.hasNext()) {
                    Long2IntMap.Entry entry = iter.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(newStorage.get(idx), entry.getIntValue()));
                }
            }
        } else {
            if (op.isKeepStorage()) {
                long[] v1Indices = v1.getStorage().getIndices();
                long[] idxiter = v2.getStorage().indexIterator().toLongArray();
                long[] indices = new long[(int) (v1Size + v2Size)];
                System.arraycopy(v1Indices, 0, indices, 0, (int) v1.size());
                System.arraycopy(idxiter, 0, indices, (int) v1.size(), (int) v2.size());
                LongAVLTreeSet avl = new LongAVLTreeSet(indices);
                LongBidirectionalIterator iter = avl.iterator();
                double[] values = new double[indices.length];
                int i = 0;
                while (iter.hasNext()) {
                    long idx = iter.nextLong();
                    indices[i] = idx;
                    values[i] = op.apply(v1.get(idx), v2.get(idx));
                    i++;
                }
                while (i < indices.length) {
                    indices[i] = 0;
                    i++;
                }
                newStorage = new LongDoubleSortedVectorStorage(v1.getDim(), (int) avl.size(), indices, values);
            } else {
                long[] v1Indices = v1.getStorage().getIndices();
                double[] v1Values = v1.getStorage().getValues();
                long size = v1.size();
                for (int i = 0; i < size; i++) {
                    long idx = v1Indices[i];
                    newStorage.set(idx, v1Values[i]);
                }
                ObjectIterator<Long2IntMap.Entry> iter = v2.getStorage().entryIterator();
                while (iter.hasNext()) {
                    Long2IntMap.Entry entry = iter.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(newStorage.get(idx), entry.getIntValue()));
                }
            }
        }
    } else if (v1.isSorted() && v2.isSorted()) {
        int v1Pointor = 0;
        int v2Pointor = 0;
        long size1 = v1.size();
        long size2 = v2.size();
        long[] v1Indices = v1.getStorage().getIndices();
        double[] v1Values = v1.getStorage().getValues();
        long[] v2Indices = v2.getStorage().getIndices();
        int[] v2Values = v2.getStorage().getValues();
        if ((size1 + size2) * Constant.intersectionCoeff >= Constant.sortedDenseStorageThreshold * v1.dim()) {
            if (op.isKeepStorage()) {
                // sorted
                long[] resIndices = newStorage.getIndices();
                double[] resValues = newStorage.getValues();
                int global = 0;
                while (v1Pointor < size1 && v2Pointor < size2) {
                    if (v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        resIndices[global] = v1Indices[v1Pointor];
                        resValues[global] = op.apply(v1Values[v1Pointor], v2Values[v2Pointor]);
                        global++;
                        v1Pointor++;
                        v2Pointor++;
                    } else if (v1Indices[v1Pointor] < v2Indices[v2Pointor]) {
                        resIndices[global] = v1Indices[v1Pointor];
                        resValues[global] = v1Values[v1Pointor];
                        global++;
                        v1Pointor++;
                    } else {
                        // v1Indices[v1Pointor] > v2Indices[v2Pointor]
                        resIndices[global] = v2Indices[v2Pointor];
                        resValues[global] = op.apply(0, v2Values[v2Pointor]);
                        global++;
                        v2Pointor++;
                    }
                }
            } else {
                // dense
                while (v1Pointor < size1 || v2Pointor < size2) {
                    if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], v2Values[v2Pointor]));
                        v1Pointor++;
                        v2Pointor++;
                    } else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
                        newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
                        v1Pointor++;
                    } else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
                        newStorage.set(v2Indices[v2Pointor], op.apply(0, v2Values[v2Pointor]));
                        v2Pointor++;
                    }
                }
            }
        } else {
            if (op.isKeepStorage()) {
                long[] resIndices = newStorage.getIndices();
                double[] resValues = newStorage.getValues();
                int globalPointor = 0;
                while (v1Pointor < size1 && v2Pointor < size2) {
                    if (v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        resIndices[globalPointor] = v1Indices[v1Pointor];
                        resValues[globalPointor] = op.apply(v1Values[v1Pointor], v2Values[v2Pointor]);
                        v1Pointor++;
                        v2Pointor++;
                        globalPointor++;
                    } else if (v1Indices[v1Pointor] < v2Indices[v2Pointor]) {
                        resIndices[globalPointor] = v1Indices[v1Pointor];
                        resValues[globalPointor] = v1Values[v1Pointor];
                        v1Pointor++;
                        globalPointor++;
                    } else {
                        // v1Indices[v1Pointor] > v2Indices[v2Pointor]
                        resIndices[globalPointor] = v2Indices[v2Pointor];
                        resValues[globalPointor] = op.apply(0, v2Values[v2Pointor]);
                        v2Pointor++;
                        globalPointor++;
                    }
                }
            } else {
                while (v1Pointor < size1 || v2Pointor < size2) {
                    if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], v2Values[v2Pointor]));
                        v1Pointor++;
                        v2Pointor++;
                    } else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
                        newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
                        v1Pointor++;
                    } else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
                        newStorage.set(v2Indices[v2Pointor], op.apply(0, v2Values[v2Pointor]));
                        v2Pointor++;
                    }
                }
            }
        }
    } else {
        throw new AngelException("The operation is not support!");
    }
    return new LongDoubleVector(v1.getMatrixId(), v1.getRowId(), v1.getClock(), v1.getDim(), newStorage);
}

Example 17 with LongIntVector

use of com.tencent.angel.ml.math2.vector.LongIntVector in project angel by Tencent.

the class SimpleBinaryOutNonZAExecutor method apply.

public static Vector apply(LongLongVector v1, LongIntVector v2, Binary op) {
    LongLongVectorStorage newStorage = (LongLongVectorStorage) StorageSwitch.apply(v1, v2, op);
    if (v1.isSparse() && v2.isSparse()) {
        long v1Size = v1.size();
        long v2Size = v2.size();
        if (v1Size >= v2Size * Constant.sparseThreshold && (v1Size + v2Size) * Constant.intersectionCoeff <= Constant.sparseDenseStorageThreshold * v1.dim()) {
            // we gauss the indices of v2 maybe is a subset of v1, or overlap is very large
            ObjectIterator<Long2IntMap.Entry> iter = v2.getStorage().entryIterator();
            while (iter.hasNext()) {
                Long2IntMap.Entry entry = iter.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, op.apply(v1.get(idx), entry.getIntValue()));
            }
        } else if ((v1Size + v2Size) * Constant.intersectionCoeff >= Constant.sparseDenseStorageThreshold * v1.dim()) {
            // we gauss dense storage is more efficient
            ObjectIterator<Long2LongMap.Entry> iter1 = v1.getStorage().entryIterator();
            while (iter1.hasNext()) {
                Long2LongMap.Entry entry = iter1.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, entry.getLongValue());
            }
            ObjectIterator<Long2IntMap.Entry> iter2 = v2.getStorage().entryIterator();
            while (iter2.hasNext()) {
                Long2IntMap.Entry entry = iter2.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, op.apply(v1.get(idx), entry.getIntValue()));
            }
        } else {
            // to avoid multi-rehash
            int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
            if (v1.size() + v2.size() <= 1.5 * capacity) {
                // no rehashor one onle rehash is required, nothing to optimization
                ObjectIterator<Long2IntMap.Entry> iter = v2.getStorage().entryIterator();
                while (iter.hasNext()) {
                    Long2IntMap.Entry entry = iter.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(v1.get(idx), entry.getIntValue()));
                }
            } else {
                // multi-rehash
                ObjectIterator<Long2LongMap.Entry> iter1 = v1.getStorage().entryIterator();
                while (iter1.hasNext()) {
                    Long2LongMap.Entry entry = iter1.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, entry.getLongValue());
                }
                ObjectIterator<Long2IntMap.Entry> iter2 = v2.getStorage().entryIterator();
                while (iter2.hasNext()) {
                    Long2IntMap.Entry entry = iter2.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(v1.get(idx), entry.getIntValue()));
                }
            }
        }
    } else if (v1.isSparse() && v2.isSorted()) {
        long v1Size = v1.size();
        long v2Size = v2.size();
        if (v1Size >= v2Size * Constant.sparseThreshold && (v1Size + v2Size) * Constant.intersectionCoeff <= Constant.sparseDenseStorageThreshold * v1.dim()) {
            // we gauss the indices of v2 maybe is a subset of v1, or overlap is very large
            long[] v2Indices = v2.getStorage().getIndices();
            int[] v2Values = v2.getStorage().getValues();
            for (int i = 0; i < v2.size(); i++) {
                long idx = v2Indices[i];
                newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
            }
        } else if ((v1Size + v2Size) * Constant.intersectionCoeff >= Constant.sparseDenseStorageThreshold * v1.dim()) {
            ObjectIterator<Long2LongMap.Entry> iter1 = v1.getStorage().entryIterator();
            while (iter1.hasNext()) {
                Long2LongMap.Entry entry = iter1.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, entry.getLongValue());
            }
            long[] v2Indices = v2.getStorage().getIndices();
            int[] v2Values = v2.getStorage().getValues();
            long size = v2.size();
            for (int i = 0; i < size; i++) {
                long idx = v2Indices[i];
                newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
            }
        } else {
            // to avoid multi-rehash
            int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
            if (v1.size() + v2.size() <= 1.5 * capacity) {
                long[] v2Indices = v2.getStorage().getIndices();
                int[] v2Values = v2.getStorage().getValues();
                for (int i = 0; i < v2.size(); i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
                }
            } else {
                ObjectIterator<Long2LongMap.Entry> iter1 = v1.getStorage().entryIterator();
                while (iter1.hasNext()) {
                    Long2LongMap.Entry entry = iter1.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, entry.getLongValue());
                }
                long[] v2Indices = v2.getStorage().getIndices();
                int[] v2Values = v2.getStorage().getValues();
                long size = v2.size();
                for (int i = 0; i < size; i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
                }
            }
        }
    } else if (v1.isSorted() && v2.isSparse()) {
        long v1Size = v1.size();
        long v2Size = v2.size();
        if ((v1Size + v2Size) * Constant.intersectionCoeff >= Constant.sortedDenseStorageThreshold * v1.dim()) {
            if (op.isKeepStorage()) {
                long[] v1Indices = v1.getStorage().getIndices();
                long[] idxiter = v2.getStorage().indexIterator().toLongArray();
                long[] indices = new long[(int) (v1Size + v2Size)];
                System.arraycopy(v1Indices, 0, indices, 0, (int) v1.size());
                System.arraycopy(idxiter, 0, indices, (int) v1.size(), (int) v2.size());
                LongAVLTreeSet avl = new LongAVLTreeSet(indices);
                LongBidirectionalIterator iter = avl.iterator();
                long[] values = new long[indices.length];
                int i = 0;
                while (iter.hasNext()) {
                    long idx = iter.nextLong();
                    indices[i] = idx;
                    values[i] = op.apply(v1.get(idx), v2.get(idx));
                    i++;
                }
                while (i < indices.length) {
                    indices[i] = 0;
                    i++;
                }
                newStorage = new LongLongSortedVectorStorage(v1.getDim(), (int) avl.size(), indices, values);
            } else {
                long[] v1Indices = v1.getStorage().getIndices();
                long[] v1Values = v1.getStorage().getValues();
                long size = v1.size();
                for (int i = 0; i < size; i++) {
                    long idx = v1Indices[i];
                    newStorage.set(idx, v1Values[i]);
                }
                ObjectIterator<Long2IntMap.Entry> iter = v2.getStorage().entryIterator();
                while (iter.hasNext()) {
                    Long2IntMap.Entry entry = iter.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(newStorage.get(idx), entry.getIntValue()));
                }
            }
        } else {
            if (op.isKeepStorage()) {
                long[] v1Indices = v1.getStorage().getIndices();
                long[] idxiter = v2.getStorage().indexIterator().toLongArray();
                long[] indices = new long[(int) (v1Size + v2Size)];
                System.arraycopy(v1Indices, 0, indices, 0, (int) v1.size());
                System.arraycopy(idxiter, 0, indices, (int) v1.size(), (int) v2.size());
                LongAVLTreeSet avl = new LongAVLTreeSet(indices);
                LongBidirectionalIterator iter = avl.iterator();
                long[] values = new long[indices.length];
                int i = 0;
                while (iter.hasNext()) {
                    long idx = iter.nextLong();
                    indices[i] = idx;
                    values[i] = op.apply(v1.get(idx), v2.get(idx));
                    i++;
                }
                while (i < indices.length) {
                    indices[i] = 0;
                    i++;
                }
                newStorage = new LongLongSortedVectorStorage(v1.getDim(), (int) avl.size(), indices, values);
            } else {
                long[] v1Indices = v1.getStorage().getIndices();
                long[] v1Values = v1.getStorage().getValues();
                long size = v1.size();
                for (int i = 0; i < size; i++) {
                    long idx = v1Indices[i];
                    newStorage.set(idx, v1Values[i]);
                }
                ObjectIterator<Long2IntMap.Entry> iter = v2.getStorage().entryIterator();
                while (iter.hasNext()) {
                    Long2IntMap.Entry entry = iter.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(newStorage.get(idx), entry.getIntValue()));
                }
            }
        }
    } else if (v1.isSorted() && v2.isSorted()) {
        int v1Pointor = 0;
        int v2Pointor = 0;
        long size1 = v1.size();
        long size2 = v2.size();
        long[] v1Indices = v1.getStorage().getIndices();
        long[] v1Values = v1.getStorage().getValues();
        long[] v2Indices = v2.getStorage().getIndices();
        int[] v2Values = v2.getStorage().getValues();
        if ((size1 + size2) * Constant.intersectionCoeff >= Constant.sortedDenseStorageThreshold * v1.dim()) {
            if (op.isKeepStorage()) {
                // sorted
                long[] resIndices = newStorage.getIndices();
                long[] resValues = newStorage.getValues();
                int global = 0;
                while (v1Pointor < size1 && v2Pointor < size2) {
                    if (v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        resIndices[global] = v1Indices[v1Pointor];
                        resValues[global] = op.apply(v1Values[v1Pointor], v2Values[v2Pointor]);
                        global++;
                        v1Pointor++;
                        v2Pointor++;
                    } else if (v1Indices[v1Pointor] < v2Indices[v2Pointor]) {
                        resIndices[global] = v1Indices[v1Pointor];
                        resValues[global] = v1Values[v1Pointor];
                        global++;
                        v1Pointor++;
                    } else {
                        // v1Indices[v1Pointor] > v2Indices[v2Pointor]
                        resIndices[global] = v2Indices[v2Pointor];
                        resValues[global] = op.apply(0, v2Values[v2Pointor]);
                        global++;
                        v2Pointor++;
                    }
                }
            } else {
                // dense
                while (v1Pointor < size1 || v2Pointor < size2) {
                    if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], v2Values[v2Pointor]));
                        v1Pointor++;
                        v2Pointor++;
                    } else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
                        newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
                        v1Pointor++;
                    } else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
                        newStorage.set(v2Indices[v2Pointor], op.apply(0, v2Values[v2Pointor]));
                        v2Pointor++;
                    }
                }
            }
        } else {
            if (op.isKeepStorage()) {
                long[] resIndices = newStorage.getIndices();
                long[] resValues = newStorage.getValues();
                int globalPointor = 0;
                while (v1Pointor < size1 && v2Pointor < size2) {
                    if (v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        resIndices[globalPointor] = v1Indices[v1Pointor];
                        resValues[globalPointor] = op.apply(v1Values[v1Pointor], v2Values[v2Pointor]);
                        v1Pointor++;
                        v2Pointor++;
                        globalPointor++;
                    } else if (v1Indices[v1Pointor] < v2Indices[v2Pointor]) {
                        resIndices[globalPointor] = v1Indices[v1Pointor];
                        resValues[globalPointor] = v1Values[v1Pointor];
                        v1Pointor++;
                        globalPointor++;
                    } else {
                        // v1Indices[v1Pointor] > v2Indices[v2Pointor]
                        resIndices[globalPointor] = v2Indices[v2Pointor];
                        resValues[globalPointor] = op.apply(0, v2Values[v2Pointor]);
                        v2Pointor++;
                        globalPointor++;
                    }
                }
            } else {
                while (v1Pointor < size1 || v2Pointor < size2) {
                    if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], v2Values[v2Pointor]));
                        v1Pointor++;
                        v2Pointor++;
                    } else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
                        newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
                        v1Pointor++;
                    } else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
                        newStorage.set(v2Indices[v2Pointor], op.apply(0, v2Values[v2Pointor]));
                        v2Pointor++;
                    }
                }
            }
        }
    } else {
        throw new AngelException("The operation is not support!");
    }
    return new LongLongVector(v1.getMatrixId(), v1.getRowId(), v1.getClock(), v1.getDim(), newStorage);
}

Example 18 with LongIntVector

use of com.tencent.angel.ml.math2.vector.LongIntVector in project angel by Tencent.

the class SimpleBinaryOutNonZAExecutor method apply.

public static Vector apply(LongIntVector v1, LongDummyVector v2, Binary op) {
    LongIntVectorStorage newStorage = (LongIntVectorStorage) StorageSwitch.apply(v1, v2, op);
    if (v1.isSparse()) {
        long[] v2Indices = v2.getIndices();
        if (((v1.size() + v2.size()) * Constant.intersectionCoeff > Constant.sparseDenseStorageThreshold * v1.getDim())) {
            int[] resValues = newStorage.getValues();
            ObjectIterator<Long2IntMap.Entry> iter = v1.getStorage().entryIterator();
            while (iter.hasNext()) {
                Long2IntMap.Entry entry = iter.next();
                newStorage.set(entry.getLongKey(), entry.getIntValue());
            }
            for (long idx : v2Indices) {
                newStorage.set(idx, op.apply(v1.get(idx), 1));
            }
        } else {
            // to avoid multi-rehash
            int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
            if (v1.size() + v2.size() < 1.5 * capacity) {
                long size = v2.size();
                for (int i = 0; i < size; i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), 1));
                }
            } else {
                ObjectIterator<Long2IntMap.Entry> iter1 = v1.getStorage().entryIterator();
                while (iter1.hasNext()) {
                    Long2IntMap.Entry entry = iter1.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, entry.getIntValue());
                }
                long size = v2.size();
                for (int i = 0; i < size; i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), 1));
                }
            }
        }
    } else {
        // sorted
        long[] v1Indices = v1.getStorage().getIndices();
        long[] v2Indices = v2.getIndices();
        if (!op.isKeepStorage() && ((v1.size() + v2.size()) * Constant.intersectionCoeff > Constant.sortedDenseStorageThreshold * v1.getDim())) {
            int[] v1Values = v1.getStorage().getValues();
            long size = v1.size();
            for (int i = 0; i < size; i++) {
                newStorage.set(v1Indices[i], v1Values[i]);
            }
            size = v2.size();
            for (int i = 0; i < size; i++) {
                long idx = v2Indices[i];
                newStorage.set(idx, op.apply(newStorage.get(idx), 1));
            }
        } else {
            int v1Pointor = 0;
            int v2Pointor = 0;
            long size1 = v1.size();
            long size2 = v2.size();
            int[] v1Values = v1.getStorage().getValues();
            while (v1Pointor < size1 || v2Pointor < size2) {
                if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                    newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], 1));
                    v1Pointor++;
                    v2Pointor++;
                } else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
                    newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
                    v1Pointor++;
                } else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
                    newStorage.set(v2Indices[v2Pointor], op.apply(0, 1));
                    v2Pointor++;
                }
            }
        }
    }
    return new LongIntVector(v1.getMatrixId(), v1.getRowId(), v1.getClock(), v1.getDim(), newStorage);
}

Example 19 with LongIntVector

use of com.tencent.angel.ml.math2.vector.LongIntVector in project angel by Tencent.

the class HashRouterUtils method splitLongIntVector.

public static void splitLongIntVector(KeyHash hasher, MatrixMeta matrixMeta, LongIntVector vector, KeyValuePart[] dataParts) {
    int dataPartNum = dataParts.length;
    int dataPartNumMinus1 = dataPartNum - 1;
    if (isPow2(dataPartNum)) {
        LongIntVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            LongIntSparseVectorStorage sparseStorage = (LongIntSparseVectorStorage) storage;
            ObjectIterator<Long2IntMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Long2IntMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getLongKey()) & dataPartNumMinus1;
                ((HashLongKeysIntValuesPart) dataParts[partId]).add(keyValue.getLongKey(), keyValue.getIntValue());
            }
        } else {
            // Key and value array pair
            LongIntSortedVectorStorage sortStorage = (LongIntSortedVectorStorage) storage;
            long[] keys = sortStorage.getIndices();
            int[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) & dataPartNumMinus1;
                ((HashLongKeysIntValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    } else {
        LongIntVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            LongIntSparseVectorStorage sparseStorage = (LongIntSparseVectorStorage) storage;
            ObjectIterator<Long2IntMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Long2IntMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getLongKey()) % dataPartNum;
                ((HashLongKeysIntValuesPart) dataParts[partId]).add(keyValue.getLongKey(), keyValue.getIntValue());
            }
        } else {
            // Key and value array pair
            LongIntSortedVectorStorage sortStorage = (LongIntSortedVectorStorage) storage;
            long[] keys = sortStorage.getIndices();
            int[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) % dataPartNum;
                ((HashLongKeysIntValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    }
}

Example 20 with LongIntVector

use of com.tencent.angel.ml.math2.vector.LongIntVector in project angel by Tencent.

the class UpdatePSFTest method testSparseIntLongKeyUDF.

public void testSparseIntLongKeyUDF() throws Exception {
    Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
    MatrixClient client1 = worker.getPSAgent().getMatrixClient(SPARSE_INT_LONG_MAT, 0);
    int matrixW1Id = client1.getMatrixId();
    long[] index = genLongIndexs(feaNum, nnz);
    LongIntVector deltaVec = new LongIntVector(feaNum, new LongIntSparseVectorStorage(feaNum, nnz));
    for (int i = 0; i < nnz; i++) {
        deltaVec.set(index[i], (int) index[i]);
    }
    deltaVec.setRowId(0);
    Vector[] updates = new Vector[1];
    updates[0] = deltaVec;
    client1.asyncUpdate(new IncrementRows(new IncrementRowsParam(matrixW1Id, updates))).get();
    LongIntVector row = (LongIntVector) client1.getRow(0);
    for (long id : index) {
        // System.out.println("id=" + id + ", value=" + row.get(id));
        Assert.assertTrue(row.get(id) == deltaVec.get(id));
    }
// Assert.assertTrue(index.length == row.size());
}