Examples with LongFloatVectorStorage com.tencent.angel.ml.math2.storage.LongFloatVectorStorage used on opensource projects

Example 21 with LongFloatVectorStorage

use of com.tencent.angel.ml.math2.storage.LongFloatVectorStorage in project angel by Tencent.

the class CompLongFloatRowUpdateSplit method serialize.

@Override
public void serialize(ByteBuf buf) {
    // TODO:
    super.serialize(buf);
    LongFloatVectorStorage storage = split.getStorage();
    buf.writeInt(storage.size());
    if (storage instanceof LongFloatSparseVectorStorage) {
        ObjectIterator<Long2FloatMap.Entry> iter = storage.entryIterator();
        Long2FloatMap.Entry entry;
        while (iter.hasNext()) {
            entry = iter.next();
            buf.writeLong(entry.getLongKey());
            buf.writeFloat(entry.getFloatValue());
        }
    } else if (storage instanceof LongFloatSortedVectorStorage) {
        long[] indices = storage.getIndices();
        float[] values = storage.getValues();
        for (int i = 0; i < indices.length; i++) {
            buf.writeLong(indices[i]);
            buf.writeFloat(values[i]);
        }
    } else {
        throw new UnsupportedOperationException("unsupport split for storage " + storage.getClass().getName());
    }
}

Example 22 with LongFloatVectorStorage

use of com.tencent.angel.ml.math2.storage.LongFloatVectorStorage in project angel by Tencent.

the class ByteBufSerdeUtils method serializeLongFloatVector.

// LongFloatVector
private static void serializeLongFloatVector(ByteBuf out, LongFloatVector vector) {
    LongFloatVectorStorage storage = vector.getStorage();
    if (storage.isSparse()) {
        serializeInt(out, SPARSE_STORAGE_TYPE);
        serializeInt(out, storage.size());
        ObjectIterator<Long2FloatMap.Entry> iter = storage.entryIterator();
        while (iter.hasNext()) {
            Long2FloatMap.Entry e = iter.next();
            serializeLong(out, e.getLongKey());
            serializeFloat(out, e.getFloatValue());
        }
    } else if (storage.isSorted()) {
        serializeInt(out, SORTED_STORAGE_TYPE);
        long[] indices = vector.getStorage().getIndices();
        float[] values = vector.getStorage().getValues();
        serializeLongs(out, indices);
        serializeFloats(out, values);
    } else {
        throw new UnsupportedOperationException("Unsupport storage type " + vector.getStorage().getClass());
    }
}

Example 23 with LongFloatVectorStorage

use of com.tencent.angel.ml.math2.storage.LongFloatVectorStorage in project angel by Tencent.

the class ByteBufSerdeUtils method serializedLongFloatVectorLen.

public static int serializedLongFloatVectorLen(LongFloatVector vector) {
    int len = 0;
    LongFloatVectorStorage storage = vector.getStorage();
    if (storage.isSparse()) {
        len += serializedIntLen(SPARSE_STORAGE_TYPE);
        len += serializedIntLen(storage.size());
        len += storage.size() * (INT_LENGTH + FLOAT_LENGTH);
    } else if (storage.isSorted()) {
        len += serializedIntLen(SORTED_STORAGE_TYPE);
        len += serializedLongsLen(vector.getStorage().getIndices());
        len += serializedFloatsLen(vector.getStorage().getValues());
    } else {
        throw new UnsupportedOperationException("Unsupport storage type " + vector.getStorage().getClass());
    }
    return len;
}

Example 24 with LongFloatVectorStorage

use of com.tencent.angel.ml.math2.storage.LongFloatVectorStorage in project angel by Tencent.

the class StreamSerdeUtils method serializeLongFloatVector.

// LongFloatVector
private static void serializeLongFloatVector(DataOutputStream out, LongFloatVector vector) throws IOException {
    LongFloatVectorStorage storage = vector.getStorage();
    if (storage.isSparse()) {
        serializeInt(out, SPARSE_STORAGE_TYPE);
        serializeInt(out, storage.size());
        ObjectIterator<Long2FloatMap.Entry> iter = storage.entryIterator();
        while (iter.hasNext()) {
            Long2FloatMap.Entry e = iter.next();
            serializeLong(out, e.getLongKey());
            serializeFloat(out, e.getFloatValue());
        }
    } else if (storage.isSorted()) {
        serializeInt(out, SORTED_STORAGE_TYPE);
        long[] indices = vector.getStorage().getIndices();
        float[] values = vector.getStorage().getValues();
        serializeLongs(out, indices);
        serializeFloats(out, values);
    } else {
        throw new UnsupportedOperationException("Unsupport storage type " + vector.getStorage().getClass());
    }
}

Example 25 with LongFloatVectorStorage

use of com.tencent.angel.ml.math2.storage.LongFloatVectorStorage in project angel by Tencent.

the class HashRouterUtils method splitLongFloatVector.

public static void splitLongFloatVector(KeyHash hasher, MatrixMeta matrixMeta, LongFloatVector vector, KeyValuePart[] dataParts) {
    int dataPartNum = dataParts.length;
    int dataPartNumMinus1 = dataPartNum - 1;
    if (isPow2(dataPartNum)) {
        LongFloatVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            LongFloatSparseVectorStorage sparseStorage = (LongFloatSparseVectorStorage) storage;
            ObjectIterator<Long2FloatMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Long2FloatMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getLongKey()) & dataPartNumMinus1;
                ((HashLongKeysFloatValuesPart) dataParts[partId]).add(keyValue.getLongKey(), keyValue.getFloatValue());
            }
        } else {
            // Key and value array pair
            LongFloatSortedVectorStorage sortStorage = (LongFloatSortedVectorStorage) storage;
            long[] keys = sortStorage.getIndices();
            float[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) & dataPartNumMinus1;
                ((HashLongKeysFloatValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    } else {
        LongFloatVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            LongFloatSparseVectorStorage sparseStorage = (LongFloatSparseVectorStorage) storage;
            ObjectIterator<Long2FloatMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Long2FloatMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getLongKey()) % dataPartNum;
                ((HashLongKeysFloatValuesPart) dataParts[partId]).add(keyValue.getLongKey(), keyValue.getFloatValue());
            }
        } else {
            // Key and value array pair
            LongFloatSortedVectorStorage sortStorage = (LongFloatSortedVectorStorage) storage;
            long[] keys = sortStorage.getIndices();
            float[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) % dataPartNum;
                ((HashLongKeysFloatValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    }
}