Examples with LongFloatVector com.tencent.angel.ml.math2.vector.LongFloatVector used on opensource projects

Example 26 with LongFloatVector

use of com.tencent.angel.ml.math2.vector.LongFloatVector in project angel by Tencent.

the class SimpleBinaryOutNonZAExecutor method apply.

public static Vector apply(LongDoubleVector v1, LongFloatVector v2, Binary op) {
    LongDoubleVectorStorage newStorage = (LongDoubleVectorStorage) StorageSwitch.apply(v1, v2, op);
    if (v1.isSparse() && v2.isSparse()) {
        long v1Size = v1.size();
        long v2Size = v2.size();
        if (v1Size >= v2Size * Constant.sparseThreshold && (v1Size + v2Size) * Constant.intersectionCoeff <= Constant.sparseDenseStorageThreshold * v1.dim()) {
            // we gauss the indices of v2 maybe is a subset of v1, or overlap is very large
            ObjectIterator<Long2FloatMap.Entry> iter = v2.getStorage().entryIterator();
            while (iter.hasNext()) {
                Long2FloatMap.Entry entry = iter.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, op.apply(v1.get(idx), entry.getFloatValue()));
            }
        } else if ((v1Size + v2Size) * Constant.intersectionCoeff >= Constant.sparseDenseStorageThreshold * v1.dim()) {
            // we gauss dense storage is more efficient
            ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
            while (iter1.hasNext()) {
                Long2DoubleMap.Entry entry = iter1.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, entry.getDoubleValue());
            }
            ObjectIterator<Long2FloatMap.Entry> iter2 = v2.getStorage().entryIterator();
            while (iter2.hasNext()) {
                Long2FloatMap.Entry entry = iter2.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, op.apply(v1.get(idx), entry.getFloatValue()));
            }
        } else {
            // to avoid multi-rehash
            int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
            if (v1.size() + v2.size() <= 1.5 * capacity) {
                // no rehashor one onle rehash is required, nothing to optimization
                ObjectIterator<Long2FloatMap.Entry> iter = v2.getStorage().entryIterator();
                while (iter.hasNext()) {
                    Long2FloatMap.Entry entry = iter.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(v1.get(idx), entry.getFloatValue()));
                }
            } else {
                // multi-rehash
                ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
                while (iter1.hasNext()) {
                    Long2DoubleMap.Entry entry = iter1.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, entry.getDoubleValue());
                }
                ObjectIterator<Long2FloatMap.Entry> iter2 = v2.getStorage().entryIterator();
                while (iter2.hasNext()) {
                    Long2FloatMap.Entry entry = iter2.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(v1.get(idx), entry.getFloatValue()));
                }
            }
        }
    } else if (v1.isSparse() && v2.isSorted()) {
        long v1Size = v1.size();
        long v2Size = v2.size();
        if (v1Size >= v2Size * Constant.sparseThreshold && (v1Size + v2Size) * Constant.intersectionCoeff <= Constant.sparseDenseStorageThreshold * v1.dim()) {
            // we gauss the indices of v2 maybe is a subset of v1, or overlap is very large
            long[] v2Indices = v2.getStorage().getIndices();
            float[] v2Values = v2.getStorage().getValues();
            for (int i = 0; i < v2.size(); i++) {
                long idx = v2Indices[i];
                newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
            }
        } else if ((v1Size + v2Size) * Constant.intersectionCoeff >= Constant.sparseDenseStorageThreshold * v1.dim()) {
            ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
            while (iter1.hasNext()) {
                Long2DoubleMap.Entry entry = iter1.next();
                long idx = entry.getLongKey();
                newStorage.set(idx, entry.getDoubleValue());
            }
            long[] v2Indices = v2.getStorage().getIndices();
            float[] v2Values = v2.getStorage().getValues();
            long size = v2.size();
            for (int i = 0; i < size; i++) {
                long idx = v2Indices[i];
                newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
            }
        } else {
            // to avoid multi-rehash
            int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
            if (v1.size() + v2.size() <= 1.5 * capacity) {
                long[] v2Indices = v2.getStorage().getIndices();
                float[] v2Values = v2.getStorage().getValues();
                for (int i = 0; i < v2.size(); i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
                }
            } else {
                ObjectIterator<Long2DoubleMap.Entry> iter1 = v1.getStorage().entryIterator();
                while (iter1.hasNext()) {
                    Long2DoubleMap.Entry entry = iter1.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, entry.getDoubleValue());
                }
                long[] v2Indices = v2.getStorage().getIndices();
                float[] v2Values = v2.getStorage().getValues();
                long size = v2.size();
                for (int i = 0; i < size; i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), v2Values[i]));
                }
            }
        }
    } else if (v1.isSorted() && v2.isSparse()) {
        long v1Size = v1.size();
        long v2Size = v2.size();
        if ((v1Size + v2Size) * Constant.intersectionCoeff >= Constant.sortedDenseStorageThreshold * v1.dim()) {
            if (op.isKeepStorage()) {
                long[] v1Indices = v1.getStorage().getIndices();
                long[] idxiter = v2.getStorage().indexIterator().toLongArray();
                long[] indices = new long[(int) (v1Size + v2Size)];
                System.arraycopy(v1Indices, 0, indices, 0, (int) v1.size());
                System.arraycopy(idxiter, 0, indices, (int) v1.size(), (int) v2.size());
                LongAVLTreeSet avl = new LongAVLTreeSet(indices);
                LongBidirectionalIterator iter = avl.iterator();
                double[] values = new double[indices.length];
                int i = 0;
                while (iter.hasNext()) {
                    long idx = iter.nextLong();
                    indices[i] = idx;
                    values[i] = op.apply(v1.get(idx), v2.get(idx));
                    i++;
                }
                while (i < indices.length) {
                    indices[i] = 0;
                    i++;
                }
                newStorage = new LongDoubleSortedVectorStorage(v1.getDim(), (int) avl.size(), indices, values);
            } else {
                long[] v1Indices = v1.getStorage().getIndices();
                double[] v1Values = v1.getStorage().getValues();
                long size = v1.size();
                for (int i = 0; i < size; i++) {
                    long idx = v1Indices[i];
                    newStorage.set(idx, v1Values[i]);
                }
                ObjectIterator<Long2FloatMap.Entry> iter = v2.getStorage().entryIterator();
                while (iter.hasNext()) {
                    Long2FloatMap.Entry entry = iter.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(newStorage.get(idx), entry.getFloatValue()));
                }
            }
        } else {
            if (op.isKeepStorage()) {
                long[] v1Indices = v1.getStorage().getIndices();
                long[] idxiter = v2.getStorage().indexIterator().toLongArray();
                long[] indices = new long[(int) (v1Size + v2Size)];
                System.arraycopy(v1Indices, 0, indices, 0, (int) v1.size());
                System.arraycopy(idxiter, 0, indices, (int) v1.size(), (int) v2.size());
                LongAVLTreeSet avl = new LongAVLTreeSet(indices);
                LongBidirectionalIterator iter = avl.iterator();
                double[] values = new double[indices.length];
                int i = 0;
                while (iter.hasNext()) {
                    long idx = iter.nextLong();
                    indices[i] = idx;
                    values[i] = op.apply(v1.get(idx), v2.get(idx));
                    i++;
                }
                while (i < indices.length) {
                    indices[i] = 0;
                    i++;
                }
                newStorage = new LongDoubleSortedVectorStorage(v1.getDim(), (int) avl.size(), indices, values);
            } else {
                long[] v1Indices = v1.getStorage().getIndices();
                double[] v1Values = v1.getStorage().getValues();
                long size = v1.size();
                for (int i = 0; i < size; i++) {
                    long idx = v1Indices[i];
                    newStorage.set(idx, v1Values[i]);
                }
                ObjectIterator<Long2FloatMap.Entry> iter = v2.getStorage().entryIterator();
                while (iter.hasNext()) {
                    Long2FloatMap.Entry entry = iter.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, op.apply(newStorage.get(idx), entry.getFloatValue()));
                }
            }
        }
    } else if (v1.isSorted() && v2.isSorted()) {
        int v1Pointor = 0;
        int v2Pointor = 0;
        long size1 = v1.size();
        long size2 = v2.size();
        long[] v1Indices = v1.getStorage().getIndices();
        double[] v1Values = v1.getStorage().getValues();
        long[] v2Indices = v2.getStorage().getIndices();
        float[] v2Values = v2.getStorage().getValues();
        if ((size1 + size2) * Constant.intersectionCoeff >= Constant.sortedDenseStorageThreshold * v1.dim()) {
            if (op.isKeepStorage()) {
                // sorted
                long[] resIndices = newStorage.getIndices();
                double[] resValues = newStorage.getValues();
                int global = 0;
                while (v1Pointor < size1 && v2Pointor < size2) {
                    if (v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        resIndices[global] = v1Indices[v1Pointor];
                        resValues[global] = op.apply(v1Values[v1Pointor], v2Values[v2Pointor]);
                        global++;
                        v1Pointor++;
                        v2Pointor++;
                    } else if (v1Indices[v1Pointor] < v2Indices[v2Pointor]) {
                        resIndices[global] = v1Indices[v1Pointor];
                        resValues[global] = v1Values[v1Pointor];
                        global++;
                        v1Pointor++;
                    } else {
                        // v1Indices[v1Pointor] > v2Indices[v2Pointor]
                        resIndices[global] = v2Indices[v2Pointor];
                        resValues[global] = op.apply(0, v2Values[v2Pointor]);
                        global++;
                        v2Pointor++;
                    }
                }
            } else {
                // dense
                while (v1Pointor < size1 || v2Pointor < size2) {
                    if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], v2Values[v2Pointor]));
                        v1Pointor++;
                        v2Pointor++;
                    } else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
                        newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
                        v1Pointor++;
                    } else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
                        newStorage.set(v2Indices[v2Pointor], op.apply(0, v2Values[v2Pointor]));
                        v2Pointor++;
                    }
                }
            }
        } else {
            if (op.isKeepStorage()) {
                long[] resIndices = newStorage.getIndices();
                double[] resValues = newStorage.getValues();
                int globalPointor = 0;
                while (v1Pointor < size1 && v2Pointor < size2) {
                    if (v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        resIndices[globalPointor] = v1Indices[v1Pointor];
                        resValues[globalPointor] = op.apply(v1Values[v1Pointor], v2Values[v2Pointor]);
                        v1Pointor++;
                        v2Pointor++;
                        globalPointor++;
                    } else if (v1Indices[v1Pointor] < v2Indices[v2Pointor]) {
                        resIndices[globalPointor] = v1Indices[v1Pointor];
                        resValues[globalPointor] = v1Values[v1Pointor];
                        v1Pointor++;
                        globalPointor++;
                    } else {
                        // v1Indices[v1Pointor] > v2Indices[v2Pointor]
                        resIndices[globalPointor] = v2Indices[v2Pointor];
                        resValues[globalPointor] = op.apply(0, v2Values[v2Pointor]);
                        v2Pointor++;
                        globalPointor++;
                    }
                }
            } else {
                while (v1Pointor < size1 || v2Pointor < size2) {
                    if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                        newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], v2Values[v2Pointor]));
                        v1Pointor++;
                        v2Pointor++;
                    } else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
                        newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
                        v1Pointor++;
                    } else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
                        newStorage.set(v2Indices[v2Pointor], op.apply(0, v2Values[v2Pointor]));
                        v2Pointor++;
                    }
                }
            }
        }
    } else {
        throw new AngelException("The operation is not support!");
    }
    return new LongDoubleVector(v1.getMatrixId(), v1.getRowId(), v1.getClock(), v1.getDim(), newStorage);
}

Example 27 with LongFloatVector

use of com.tencent.angel.ml.math2.vector.LongFloatVector in project angel by Tencent.

the class RowSplitCombineUtils method combineServerLongFloatRowSplits.

private static Vector combineServerLongFloatRowSplits(List<ServerRow> rowSplits, MatrixMeta matrixMeta, int rowIndex) {
    long colNum = matrixMeta.getColNum();
    int elemNum = 0;
    int size = rowSplits.size();
    for (int i = 0; i < size; i++) {
        elemNum += rowSplits.get(i).size();
    }
    LongFloatVector row = VFactory.sparseLongKeyFloatVector(colNum, elemNum);
    row.setMatrixId(matrixMeta.getId());
    row.setRowId(rowIndex);
    Collections.sort(rowSplits, serverRowComp);
    int clock = Integer.MAX_VALUE;
    for (int i = 0; i < size; i++) {
        if (rowSplits.get(i) == null) {
            continue;
        }
        if (rowSplits.get(i).getClock() < clock) {
            clock = rowSplits.get(i).getClock();
        }
        ((ServerLongFloatRow) rowSplits.get(i)).mergeTo(row);
    }
    row.setClock(clock);
    return row;
}

Example 28 with LongFloatVector

use of com.tencent.angel.ml.math2.vector.LongFloatVector in project angel by Tencent.

the class MergeUtils method combineServerLongFloatRowSplits.

private static Vector combineServerLongFloatRowSplits(List<ServerRow> rowSplits, MatrixMeta matrixMeta, int rowIndex) {
    long colNum = matrixMeta.getColNum();
    int elemNum = 0;
    int size = rowSplits.size();
    for (int i = 0; i < size; i++) {
        elemNum += rowSplits.get(i).size();
    }
    LongFloatVector row = VFactory.sparseLongKeyFloatVector(colNum, elemNum);
    row.setMatrixId(matrixMeta.getId());
    row.setRowId(rowIndex);
    Collections.sort(rowSplits, serverRowComp);
    for (int i = 0; i < size; i++) {
        if (rowSplits.get(i) == null) {
            continue;
        }
        ((ServerLongFloatRow) rowSplits.get(i)).mergeTo(row);
    }
    return row;
}

Example 29 with LongFloatVector

use of com.tencent.angel.ml.math2.vector.LongFloatVector in project angel by Tencent.

the class MergeUtils method combineLongFloatIndexRowSplits.

// //////////////////////////////////////////////////////////////////////////////
// Combine Long key Float value vector
// //////////////////////////////////////////////////////////////////////////////
public static Vector combineLongFloatIndexRowSplits(int matrixId, int rowId, int resultSize, KeyPart[] keyParts, ValuePart[] valueParts, MatrixMeta matrixMeta) {
    LongFloatVector vector = VFactory.sparseLongKeyFloatVector(matrixMeta.getColNum(), resultSize);
    for (int i = 0; i < keyParts.length; i++) {
        mergeTo(vector, keyParts[i], (FloatValuesPart) valueParts[i]);
    }
    vector.setRowId(rowId);
    vector.setMatrixId(matrixId);
    return vector;
}

Example 30 with LongFloatVector

use of com.tencent.angel.ml.math2.vector.LongFloatVector in project angel by Tencent.

the class CompLongFloatVectorSplitter method split.

@Override
public Map<PartitionKey, RowUpdateSplit> split(Vector vector, List<PartitionKey> parts) {
    LongFloatVector[] vecParts = ((CompLongFloatVector) vector).getPartitions();
    assert vecParts.length == parts.size();
    Map<PartitionKey, RowUpdateSplit> updateSplitMap = new HashMap<>(parts.size());
    for (int i = 0; i < vecParts.length; i++) {
        updateSplitMap.put(parts.get(i), new CompLongFloatRowUpdateSplit(vector.getRowId(), vecParts[i]));
    }
    return updateSplitMap;
}