Examples with LongDoubleVector com.tencent.angel.ml.math2.vector.LongDoubleVector used on opensource projects

Example 26 with LongDoubleVector

use of com.tencent.angel.ml.math2.vector.LongDoubleVector in project angel by Tencent.

the class SimpleBinaryOutNonZAExecutor method apply.

public static Vector apply(LongDoubleVector v1, LongDummyVector v2, Binary op) {
    LongDoubleVectorStorage newStorage = (LongDoubleVectorStorage) StorageSwitch.apply(v1, v2, op);
    if (v1.isSparse()) {
        long[] v2Indices = v2.getIndices();
        if (((v1.size() + v2.size()) * Constant.intersectionCoeff > Constant.sparseDenseStorageThreshold * v1.getDim())) {
            double[] resValues = newStorage.getValues();
            ObjectIterator<Long2DoubleMap.Entry> iter = v1.getStorage().entryIterator();
            while (iter.hasNext()) {
                Long2DoubleMap.Entry entry = iter.next();
                newStorage.set(entry.getLongKey(), entry.getDoubleValue());
            }
            for (long idx : v2Indices) {
                newStorage.set(idx, op.apply(v1.get(idx), 1));
            }
        } else {
            // to avoid multi-rehash
            int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
            if (v1.size() + v2.size() < 1.5 * capacity) {
                long size = v2.size();
                for (int i = 0; i < size; i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), 1));
                }
            } else {
                ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
                while (iter1.hasNext()) {
                    Long2DoubleMap.Entry entry = iter1.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, entry.getDoubleValue());
                }
                long size = v2.size();
                for (int i = 0; i < size; i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), 1));
                }
            }
        }
    } else {
        // sorted
        long[] v1Indices = v1.getStorage().getIndices();
        long[] v2Indices = v2.getIndices();
        if (!op.isKeepStorage() && ((v1.size() + v2.size()) * Constant.intersectionCoeff > Constant.sortedDenseStorageThreshold * v1.getDim())) {
            double[] v1Values = v1.getStorage().getValues();
            long size = v1.size();
            for (int i = 0; i < size; i++) {
                newStorage.set(v1Indices[i], v1Values[i]);
            }
            size = v2.size();
            for (int i = 0; i < size; i++) {
                long idx = v2Indices[i];
                newStorage.set(idx, op.apply(newStorage.get(idx), 1));
            }
        } else {
            int v1Pointor = 0;
            int v2Pointor = 0;
            long size1 = v1.size();
            long size2 = v2.size();
            double[] v1Values = v1.getStorage().getValues();
            while (v1Pointor < size1 || v2Pointor < size2) {
                if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                    newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], 1));
                    v1Pointor++;
                    v2Pointor++;
                } else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
                    newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
                    v1Pointor++;
                } else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
                    newStorage.set(v2Indices[v2Pointor], op.apply(0, 1));
                    v2Pointor++;
                }
            }
        }
    }
    return new LongDoubleVector(v1.getMatrixId(), v1.getRowId(), v1.getClock(), v1.getDim(), newStorage);
}

Example 27 with LongDoubleVector

use of com.tencent.angel.ml.math2.vector.LongDoubleVector in project angel by Tencent.

the class SimpleBinaryOutNonZAExecutor method apply.

public static Vector apply(LongDoubleVector v1, LongIntVector v2, Binary op) {
    LongDoubleVectorStorage newStorage = (LongDoubleVectorStorage) StorageSwitch.apply(v1, v2, op);
    if (v1.isSparse() && v2.isSparse()) {
        long v1Size = v1.size();
        long v2Size = v2.size();
        if (v1Size >= v2Size * Constant.sparseThreshold && (v1Size + v2Size) * Constant.intersectionCoeff <= Constant.sparseDenseStorageThreshold * v1.dim()) {
            // we gauss the indices of v2 maybe is a subset of v1, or overlap is very large
            ObjectIterator<Long2IntMap.Entry> iter = v2.getStorage().entryIterator();
            while (iter.hasNext()) {
                Long2IntMap.Entry entry = iter.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, op.apply(v1.get(idx), entry.getIntValue()));
            }
        } else if ((v1Size + v2Size) * Constant.intersectionCoeff >= Constant.sparseDenseStorageThreshold * v1.dim()) {
            // we gauss dense storage is more efficient
            ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
            while (iter1.hasNext()) {
                Long2DoubleMap.Entry entry = iter1.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, entry.getDoubleValue());
            }
            ObjectIterator<Long2IntMap.Entry> iter2 = v2.getStorage().entryIterator();
            while (iter2.hasNext()) {
                Long2IntMap.Entry entry = iter2.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, op.apply(v1.get(idx), entry.getIntValue()));
            }
        } else {
            // to avoid multi-rehash
            int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
            if (v1.size() + v2.size() <= 1.5 * capacity) {
                // no rehashor one onle rehash is required, nothing to optimization
                ObjectIterator<Long2IntMap.Entry> iter = v2.getStorage().entryIterator();
                while (iter.hasNext()) {
                    Long2IntMap.Entry entry = iter.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(v1.get(idx), entry.getIntValue()));
                }
            } else {
                // multi-rehash
                ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
                while (iter1.hasNext()) {
                    Long2DoubleMap.Entry entry = iter1.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, entry.getDoubleValue());
                }
                ObjectIterator<Long2IntMap.Entry> iter2 = v2.getStorage().entryIterator();
                while (iter2.hasNext()) {
                    Long2IntMap.Entry entry = iter2.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(v1.get(idx), entry.getIntValue()));
                }
            }
        }
    } else if (v1.isSparse() && v2.isSorted()) {
        long v1Size = v1.size();
        long v2Size = v2.size();
        if (v1Size >= v2Size * Constant.sparseThreshold && (v1Size + v2Size) * Constant.intersectionCoeff <= Constant.sparseDenseStorageThreshold * v1.dim()) {
            // we gauss the indices of v2 maybe is a subset of v1, or overlap is very large
            long[] v2Indices = v2.getStorage().getIndices();
            int[] v2Values = v2.getStorage().getValues();
            for (int i = 0; i < v2.size(); i++) {
                long idx = v2Indices[i];
                newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
            }
        } else if ((v1Size + v2Size) * Constant.intersectionCoeff >= Constant.sparseDenseStorageThreshold * v1.dim()) {
            ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
            while (iter1.hasNext()) {
                Long2DoubleMap.Entry entry = iter1.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, entry.getDoubleValue());
            }
            long[] v2Indices = v2.getStorage().getIndices();
            int[] v2Values = v2.getStorage().getValues();
            long size = v2.size();
            for (int i = 0; i < size; i++) {
                long idx = v2Indices[i];
                newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
            }
        } else {
            // to avoid multi-rehash
            int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
            if (v1.size() + v2.size() <= 1.5 * capacity) {
                long[] v2Indices = v2.getStorage().getIndices();
                int[] v2Values = v2.getStorage().getValues();
                for (int i = 0; i < v2.size(); i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
                }
            } else {
                ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
                while (iter1.hasNext()) {
                    Long2DoubleMap.Entry entry = iter1.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, entry.getDoubleValue());
                }
                long[] v2Indices = v2.getStorage().getIndices();
                int[] v2Values = v2.getStorage().getValues();
                long size = v2.size();
                for (int i = 0; i < size; i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
                }
            }
        }
    } else if (v1.isSorted() && v2.isSparse()) {
        long v1Size = v1.size();
        long v2Size = v2.size();
        if ((v1Size + v2Size) * Constant.intersectionCoeff >= Constant.sortedDenseStorageThreshold * v1.dim()) {
            if (op.isKeepStorage()) {
                long[] v1Indices = v1.getStorage().getIndices();
                long[] idxiter = v2.getStorage().indexIterator().toLongArray();
                long[] indices = new long[(int) (v1Size + v2Size)];
                System.arraycopy(v1Indices, 0, indices, 0, (int) v1.size());
                System.arraycopy(idxiter, 0, indices, (int) v1.size(), (int) v2.size());
                LongAVLTreeSet avl = new LongAVLTreeSet(indices);
                LongBidirectionalIterator iter = avl.iterator();
                double[] values = new double[indices.length];
                int i = 0;
                while (iter.hasNext()) {
                    long idx = iter.nextLong();
                    indices[i] = idx;
                    values[i] = op.apply(v1.get(idx), v2.get(idx));
                    i++;
                }
                while (i < indices.length) {
                    indices[i] = 0;
                    i++;
                }
                newStorage = new LongDoubleSortedVectorStorage(v1.getDim(), (int) avl.size(), indices, values);
            } else {
                long[] v1Indices = v1.getStorage().getIndices();
                double[] v1Values = v1.getStorage().getValues();
                long size = v1.size();
                for (int i = 0; i < size; i++) {
                    long idx = v1Indices[i];
                    newStorage.set(idx, v1Values[i]);
                }
                ObjectIterator<Long2IntMap.Entry> iter = v2.getStorage().entryIterator();
                while (iter.hasNext()) {
                    Long2IntMap.Entry entry = iter.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(newStorage.get(idx), entry.getIntValue()));
                }
            }
        } else {
            if (op.isKeepStorage()) {
                long[] v1Indices = v1.getStorage().getIndices();
                long[] idxiter = v2.getStorage().indexIterator().toLongArray();
                long[] indices = new long[(int) (v1Size + v2Size)];
                System.arraycopy(v1Indices, 0, indices, 0, (int) v1.size());
                System.arraycopy(idxiter, 0, indices, (int) v1.size(), (int) v2.size());
                LongAVLTreeSet avl = new LongAVLTreeSet(indices);
                LongBidirectionalIterator iter = avl.iterator();
                double[] values = new double[indices.length];
                int i = 0;
                while (iter.hasNext()) {
                    long idx = iter.nextLong();
                    indices[i] = idx;
                    values[i] = op.apply(v1.get(idx), v2.get(idx));
                    i++;
                }
                while (i < indices.length) {
                    indices[i] = 0;
                    i++;
                }
                newStorage = new LongDoubleSortedVectorStorage(v1.getDim(), (int) avl.size(), indices, values);
            } else {
                long[] v1Indices = v1.getStorage().getIndices();
                double[] v1Values = v1.getStorage().getValues();
                long size = v1.size();
                for (int i = 0; i < size; i++) {
                    long idx = v1Indices[i];
                    newStorage.set(idx, v1Values[i]);
                }
                ObjectIterator<Long2IntMap.Entry> iter = v2.getStorage().entryIterator();
                while (iter.hasNext()) {
                    Long2IntMap.Entry entry = iter.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(newStorage.get(idx), entry.getIntValue()));
                }
            }
        }
    } else if (v1.isSorted() && v2.isSorted()) {
        int v1Pointor = 0;
        int v2Pointor = 0;
        long size1 = v1.size();
        long size2 = v2.size();
        long[] v1Indices = v1.getStorage().getIndices();
        double[] v1Values = v1.getStorage().getValues();
        long[] v2Indices = v2.getStorage().getIndices();
        int[] v2Values = v2.getStorage().getValues();
        if ((size1 + size2) * Constant.intersectionCoeff >= Constant.sortedDenseStorageThreshold * v1.dim()) {
            if (op.isKeepStorage()) {
                // sorted
                long[] resIndices = newStorage.getIndices();
                double[] resValues = newStorage.getValues();
                int global = 0;
                while (v1Pointor < size1 && v2Pointor < size2) {
                    if (v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        resIndices[global] = v1Indices[v1Pointor];
                        resValues[global] = op.apply(v1Values[v1Pointor], v2Values[v2Pointor]);
                        global++;
                        v1Pointor++;
                        v2Pointor++;
                    } else if (v1Indices[v1Pointor] < v2Indices[v2Pointor]) {
                        resIndices[global] = v1Indices[v1Pointor];
                        resValues[global] = v1Values[v1Pointor];
                        global++;
                        v1Pointor++;
                    } else {
                        // v1Indices[v1Pointor] > v2Indices[v2Pointor]
                        resIndices[global] = v2Indices[v2Pointor];
                        resValues[global] = op.apply(0, v2Values[v2Pointor]);
                        global++;
                        v2Pointor++;
                    }
                }
            } else {
                // dense
                while (v1Pointor < size1 || v2Pointor < size2) {
                    if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], v2Values[v2Pointor]));
                        v1Pointor++;
                        v2Pointor++;
                    } else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
                        newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
                        v1Pointor++;
                    } else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
                        newStorage.set(v2Indices[v2Pointor], op.apply(0, v2Values[v2Pointor]));
                        v2Pointor++;
                    }
                }
            }
        } else {
            if (op.isKeepStorage()) {
                long[] resIndices = newStorage.getIndices();
                double[] resValues = newStorage.getValues();
                int globalPointor = 0;
                while (v1Pointor < size1 && v2Pointor < size2) {
                    if (v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        resIndices[globalPointor] = v1Indices[v1Pointor];
                        resValues[globalPointor] = op.apply(v1Values[v1Pointor], v2Values[v2Pointor]);
                        v1Pointor++;
                        v2Pointor++;
                        globalPointor++;
                    } else if (v1Indices[v1Pointor] < v2Indices[v2Pointor]) {
                        resIndices[globalPointor] = v1Indices[v1Pointor];
                        resValues[globalPointor] = v1Values[v1Pointor];
                        v1Pointor++;
                        globalPointor++;
                    } else {
                        // v1Indices[v1Pointor] > v2Indices[v2Pointor]
                        resIndices[globalPointor] = v2Indices[v2Pointor];
                        resValues[globalPointor] = op.apply(0, v2Values[v2Pointor]);
                        v2Pointor++;
                        globalPointor++;
                    }
                }
            } else {
                while (v1Pointor < size1 || v2Pointor < size2) {
                    if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], v2Values[v2Pointor]));
                        v1Pointor++;
                        v2Pointor++;
                    } else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
                        newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
                        v1Pointor++;
                    } else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
                        newStorage.set(v2Indices[v2Pointor], op.apply(0, v2Values[v2Pointor]));
                        v2Pointor++;
                    }
                }
            }
        }
    } else {
        throw new AngelException("The operation is not support!");
    }
    return new LongDoubleVector(v1.getMatrixId(), v1.getRowId(), v1.getClock(), v1.getDim(), newStorage);
}

Example 28 with LongDoubleVector

use of com.tencent.angel.ml.math2.vector.LongDoubleVector in project angel by Tencent.

the class RangeRouterUtils method splitLongDoubleVector.

public static KeyValuePart[] splitLongDoubleVector(MatrixMeta matrixMeta, LongDoubleVector vector) {
    LongDoubleVectorStorage storage = vector.getStorage();
    if (storage.isSparse()) {
        // Get keys and values
        LongDoubleSparseVectorStorage sparseStorage = (LongDoubleSparseVectorStorage) storage;
        long[] keys = sparseStorage.getIndices();
        double[] values = sparseStorage.getValues();
        return split(matrixMeta, vector.getRowId(), keys, values, false);
    } else {
        // Key and value array pair
        LongDoubleSortedVectorStorage sortStorage = (LongDoubleSortedVectorStorage) storage;
        long[] keys = sortStorage.getIndices();
        double[] values = sortStorage.getValues();
        return split(matrixMeta, vector.getRowId(), keys, values, true);
    }
}

Example 29 with LongDoubleVector

use of com.tencent.angel.ml.math2.vector.LongDoubleVector in project angel by Tencent.

the class HashRouterUtils method splitLongDoubleVector.

public static void splitLongDoubleVector(KeyHash hasher, MatrixMeta matrixMeta, LongDoubleVector vector, KeyValuePart[] dataParts) {
    int dataPartNum = dataParts.length;
    int dataPartNumMinus1 = dataPartNum - 1;
    if (isPow2(dataPartNum)) {
        LongDoubleVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            LongDoubleSparseVectorStorage sparseStorage = (LongDoubleSparseVectorStorage) storage;
            ObjectIterator<Long2DoubleMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Long2DoubleMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getLongKey()) & dataPartNumMinus1;
                ((HashLongKeysDoubleValuesPart) dataParts[partId]).add(keyValue.getLongKey(), keyValue.getDoubleValue());
            }
        } else {
            // Key and value array pair
            LongDoubleSortedVectorStorage sortStorage = (LongDoubleSortedVectorStorage) storage;
            long[] keys = sortStorage.getIndices();
            double[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) & dataPartNumMinus1;
                ((HashLongKeysDoubleValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    } else {
        LongDoubleVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            LongDoubleSparseVectorStorage sparseStorage = (LongDoubleSparseVectorStorage) storage;
            ObjectIterator<Long2DoubleMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Long2DoubleMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getLongKey()) % dataPartNum;
                ((HashLongKeysDoubleValuesPart) dataParts[partId]).add(keyValue.getLongKey(), keyValue.getDoubleValue());
            }
        } else {
            // Key and value array pair
            LongDoubleSortedVectorStorage sortStorage = (LongDoubleSortedVectorStorage) storage;
            long[] keys = sortStorage.getIndices();
            double[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) % dataPartNum;
                ((HashLongKeysDoubleValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    }
}

Example 30 with LongDoubleVector

use of com.tencent.angel.ml.math2.vector.LongDoubleVector in project angel by Tencent.

the class RowSplitCombineUtils method combineServerLongDoubleRowSplits.

private static Vector combineServerLongDoubleRowSplits(List<ServerRow> rowSplits, MatrixMeta matrixMeta, int rowIndex) {
    long colNum = matrixMeta.getColNum();
    int elemNum = 0;
    int size = rowSplits.size();
    for (int i = 0; i < size; i++) {
        elemNum += rowSplits.get(i).size();
    }
    LongDoubleVector row = VFactory.sparseLongKeyDoubleVector(colNum, elemNum);
    row.setMatrixId(matrixMeta.getId());
    row.setRowId(rowIndex);
    Collections.sort(rowSplits, serverRowComp);
    int clock = Integer.MAX_VALUE;
    for (int i = 0; i < size; i++) {
        if (rowSplits.get(i) == null) {
            continue;
        }
        if (rowSplits.get(i).getClock() < clock) {
            clock = rowSplits.get(i).getClock();
        }
        ((ServerLongDoubleRow) rowSplits.get(i)).mergeTo(row);
    }
    row.setClock(clock);
    return row;
}