Examples with LongIntVectorStorage com.tencent.angel.ml.math2.storage.LongIntVectorStorage used on opensource projects

Example 16 with LongIntVectorStorage

use of com.tencent.angel.ml.math2.storage.LongIntVectorStorage in project angel by Tencent.

the class SimpleBinaryOutNonZAExecutor method apply.

public static Vector apply(LongIntVector v1, LongDummyVector v2, Binary op) {
    LongIntVectorStorage newStorage = (LongIntVectorStorage) StorageSwitch.apply(v1, v2, op);
    if (v1.isSparse()) {
        long[] v2Indices = v2.getIndices();
        if (((v1.size() + v2.size()) * Constant.intersectionCoeff > Constant.sparseDenseStorageThreshold * v1.getDim())) {
            int[] resValues = newStorage.getValues();
            ObjectIterator<Long2IntMap.Entry> iter = v1.getStorage().entryIterator();
            while (iter.hasNext()) {
                Long2IntMap.Entry entry = iter.next();
                newStorage.set(entry.getLongKey(), entry.getIntValue());
            }
            for (long idx : v2Indices) {
                newStorage.set(idx, op.apply(v1.get(idx), 1));
            }
        } else {
            // to avoid multi-rehash
            int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
            if (v1.size() + v2.size() < 1.5 * capacity) {
                long size = v2.size();
                for (int i = 0; i < size; i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), 1));
                }
            } else {
                ObjectIterator<Long2IntMap.Entry> iter1 = v1.getStorage().entryIterator();
                while (iter1.hasNext()) {
                    Long2IntMap.Entry entry = iter1.next();
                    long idx = entry.getLongKey();
                    newStorage.set(idx, entry.getIntValue());
                }
                long size = v2.size();
                for (int i = 0; i < size; i++) {
                    long idx = v2Indices[i];
                    newStorage.set(idx, op.apply(v1.get(idx), 1));
                }
            }
        }
    } else {
        // sorted
        long[] v1Indices = v1.getStorage().getIndices();
        long[] v2Indices = v2.getIndices();
        if (!op.isKeepStorage() && ((v1.size() + v2.size()) * Constant.intersectionCoeff > Constant.sortedDenseStorageThreshold * v1.getDim())) {
            int[] v1Values = v1.getStorage().getValues();
            long size = v1.size();
            for (int i = 0; i < size; i++) {
                newStorage.set(v1Indices[i], v1Values[i]);
            }
            size = v2.size();
            for (int i = 0; i < size; i++) {
                long idx = v2Indices[i];
                newStorage.set(idx, op.apply(newStorage.get(idx), 1));
            }
        } else {
            int v1Pointor = 0;
            int v2Pointor = 0;
            long size1 = v1.size();
            long size2 = v2.size();
            int[] v1Values = v1.getStorage().getValues();
            while (v1Pointor < size1 || v2Pointor < size2) {
                if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
                    newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], 1));
                    v1Pointor++;
                    v2Pointor++;
                } else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
                    newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
                    v1Pointor++;
                } else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
                    newStorage.set(v2Indices[v2Pointor], op.apply(0, 1));
                    v2Pointor++;
                }
            }
        }
    }
    return new LongIntVector(v1.getMatrixId(), v1.getRowId(), v1.getClock(), v1.getDim(), newStorage);
}

Example 17 with LongIntVectorStorage

use of com.tencent.angel.ml.math2.storage.LongIntVectorStorage in project angel by Tencent.

the class CompLongIntRowUpdateSplit method serialize.

@Override
public void serialize(ByteBuf buf) {
    super.serialize(buf);
    LongIntVectorStorage storage = split.getStorage();
    buf.writeInt(storage.size());
    if (storage instanceof LongIntSparseVectorStorage) {
        ObjectIterator<Long2IntMap.Entry> iter = storage.entryIterator();
        Long2IntMap.Entry entry;
        while (iter.hasNext()) {
            entry = iter.next();
            buf.writeLong(entry.getLongKey());
            buf.writeInt(entry.getIntValue());
        }
    } else if (storage instanceof LongIntSortedVectorStorage) {
        long[] indices = storage.getIndices();
        int[] values = storage.getValues();
        for (int i = 0; i < indices.length; i++) {
            buf.writeLong(indices[i]);
            buf.writeInt(values[i]);
        }
    } else {
        throw new UnsupportedOperationException("unsupport split for storage " + storage.getClass().getName());
    }
}

Example 18 with LongIntVectorStorage

use of com.tencent.angel.ml.math2.storage.LongIntVectorStorage in project angel by Tencent.

the class RangeRouterUtils method splitLongIntVector.

public static KeyValuePart[] splitLongIntVector(MatrixMeta matrixMeta, LongIntVector vector) {
    LongIntVectorStorage storage = vector.getStorage();
    if (storage.isSparse()) {
        // Get keys and values
        LongIntSparseVectorStorage sparseStorage = (LongIntSparseVectorStorage) storage;
        long[] keys = sparseStorage.getIndices();
        int[] values = sparseStorage.getValues();
        return split(matrixMeta, vector.getRowId(), keys, values, false);
    } else {
        // Key and value array pair
        LongIntSortedVectorStorage sortStorage = (LongIntSortedVectorStorage) storage;
        long[] keys = sortStorage.getIndices();
        int[] values = sortStorage.getValues();
        return split(matrixMeta, vector.getRowId(), keys, values, true);
    }
}

Example 19 with LongIntVectorStorage

use of com.tencent.angel.ml.math2.storage.LongIntVectorStorage in project angel by Tencent.

the class HashRouterUtils method splitLongIntVector.

public static void splitLongIntVector(KeyHash hasher, MatrixMeta matrixMeta, LongIntVector vector, KeyValuePart[] dataParts) {
    int dataPartNum = dataParts.length;
    int dataPartNumMinus1 = dataPartNum - 1;
    if (isPow2(dataPartNum)) {
        LongIntVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            LongIntSparseVectorStorage sparseStorage = (LongIntSparseVectorStorage) storage;
            ObjectIterator<Long2IntMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Long2IntMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getLongKey()) & dataPartNumMinus1;
                ((HashLongKeysIntValuesPart) dataParts[partId]).add(keyValue.getLongKey(), keyValue.getIntValue());
            }
        } else {
            // Key and value array pair
            LongIntSortedVectorStorage sortStorage = (LongIntSortedVectorStorage) storage;
            long[] keys = sortStorage.getIndices();
            int[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) & dataPartNumMinus1;
                ((HashLongKeysIntValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    } else {
        LongIntVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            LongIntSparseVectorStorage sparseStorage = (LongIntSparseVectorStorage) storage;
            ObjectIterator<Long2IntMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Long2IntMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getLongKey()) % dataPartNum;
                ((HashLongKeysIntValuesPart) dataParts[partId]).add(keyValue.getLongKey(), keyValue.getIntValue());
            }
        } else {
            // Key and value array pair
            LongIntSortedVectorStorage sortStorage = (LongIntSortedVectorStorage) storage;
            long[] keys = sortStorage.getIndices();
            int[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) % dataPartNum;
                ((HashLongKeysIntValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    }
}

Example 20 with LongIntVectorStorage

use of com.tencent.angel.ml.math2.storage.LongIntVectorStorage in project angel by Tencent.

the class ByteBufSerdeUtils method serializeLongIntVector.

// LongFloatVector
private static void serializeLongIntVector(ByteBuf out, LongIntVector vector) {
    LongIntVectorStorage storage = vector.getStorage();
    if (storage.isSparse()) {
        serializeInt(out, SPARSE_STORAGE_TYPE);
        serializeInt(out, storage.size());
        ObjectIterator<Long2IntMap.Entry> iter = storage.entryIterator();
        while (iter.hasNext()) {
            Long2IntMap.Entry e = iter.next();
            serializeLong(out, e.getLongKey());
            serializeFloat(out, e.getIntValue());
        }
    } else if (storage.isSorted()) {
        serializeInt(out, SORTED_STORAGE_TYPE);
        long[] indices = vector.getStorage().getIndices();
        int[] values = vector.getStorage().getValues();
        serializeLongs(out, indices);
        serializeInts(out, values);
    } else {
        throw new UnsupportedOperationException("Unsupport storage type " + vector.getStorage().getClass());
    }
}