Examples with LongFloatVector com.tencent.angel.ml.math2.vector.LongFloatVector used on opensource projects

Example 36 with LongFloatVector

use of com.tencent.angel.ml.math2.vector.LongFloatVector in project angel by Tencent.

the class StreamSerdeUtils method deserializeLongFloatVector.

private static LongFloatVector deserializeLongFloatVector(DataInputStream in, long dim) throws IOException {
    int storageType = deserializeInt(in);
    switch(storageType) {
        case SPARSE_STORAGE_TYPE:
            int len = deserializeInt(in);
            Long2FloatOpenHashMap idToValueMap = new Long2FloatOpenHashMap(len);
            for (int i = 0; i < len; i++) {
                idToValueMap.put(deserializeInt(in), deserializeFloat(in));
            }
            return new LongFloatVector((int) dim, new LongFloatSparseVectorStorage((int) dim, idToValueMap));
        case SORTED_STORAGE_TYPE:
            return VFactory.sortedLongKeyFloatVector((int) dim, deserializeLongs(in), deserializeFloats(in));
        default:
            throw new UnsupportedOperationException("Unsupport storage type " + storageType);
    }
}

Example 37 with LongFloatVector

use of com.tencent.angel.ml.math2.vector.LongFloatVector in project angel by Tencent.

the class ByteBufSerdeUtils method deserializeLongFloatVector.

private static LongFloatVector deserializeLongFloatVector(ByteBuf in, long dim) {
    int storageType = deserializeInt(in);
    switch(storageType) {
        case SPARSE_STORAGE_TYPE:
            int len = deserializeInt(in);
            Long2FloatOpenHashMap idToValueMap = new Long2FloatOpenHashMap(len);
            for (int i = 0; i < len; i++) {
                idToValueMap.put(deserializeInt(in), deserializeFloat(in));
            }
            return new LongFloatVector((int) dim, new LongFloatSparseVectorStorage((int) dim, idToValueMap));
        case SORTED_STORAGE_TYPE:
            return VFactory.sortedLongKeyFloatVector((int) dim, deserializeLongs(in), deserializeFloats(in));
        default:
            throw new UnsupportedOperationException("Unsupport storage type " + storageType);
    }
}

Example 38 with LongFloatVector

use of com.tencent.angel.ml.math2.vector.LongFloatVector in project angel by Tencent.

the class SnapshotFormat method save.

private void save(ServerLongFloatRow row, PSMatrixSaveContext saveContext, MatrixPartitionMeta meta, DataOutputStream out) throws IOException {
    long startCol = meta.getStartCol();
    if (ServerRowUtils.getVector(row) instanceof IntFloatVector) {
        IntFloatVector vector = (IntFloatVector) ServerRowUtils.getVector(row);
        if (vector.isDense()) {
            float[] data = vector.getStorage().getValues();
            for (int i = 0; i < data.length; i++) {
                out.writeFloat(data[i]);
            }
        } else if (vector.isSorted()) {
            int[] indices = vector.getStorage().getIndices();
            float[] values = vector.getStorage().getValues();
            for (int i = 0; i < indices.length; i++) {
                out.writeLong(indices[i] + startCol);
                out.writeFloat(values[i]);
            }
        } else {
            ObjectIterator<Int2FloatMap.Entry> iter = vector.getStorage().entryIterator();
            Int2FloatMap.Entry entry;
            while (iter.hasNext()) {
                entry = iter.next();
                out.writeLong(entry.getIntKey() + startCol);
                out.writeFloat(entry.getFloatValue());
            }
        }
    } else {
        LongFloatVector vector = (LongFloatVector) ServerRowUtils.getVector(row);
        if (vector.isSorted()) {
            long[] indices = vector.getStorage().getIndices();
            float[] values = vector.getStorage().getValues();
            for (int i = 0; i < indices.length; i++) {
                out.writeLong(indices[i] + startCol);
                out.writeFloat(values[i]);
            }
        } else {
            ObjectIterator<Long2FloatMap.Entry> iter = vector.getStorage().entryIterator();
            Long2FloatMap.Entry entry;
            while (iter.hasNext()) {
                entry = iter.next();
                out.writeLong(entry.getLongKey() + startCol);
                out.writeFloat(entry.getFloatValue());
            }
        }
    }
}

Example 39 with LongFloatVector

use of com.tencent.angel.ml.math2.vector.LongFloatVector in project angel by Tencent.

the class HashRouterUtils method splitLongFloatVector.

public static void splitLongFloatVector(KeyHash hasher, MatrixMeta matrixMeta, LongFloatVector vector, KeyValuePart[] dataParts) {
    int dataPartNum = dataParts.length;
    int dataPartNumMinus1 = dataPartNum - 1;
    if (isPow2(dataPartNum)) {
        LongFloatVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            LongFloatSparseVectorStorage sparseStorage = (LongFloatSparseVectorStorage) storage;
            ObjectIterator<Long2FloatMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Long2FloatMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getLongKey()) & dataPartNumMinus1;
                ((HashLongKeysFloatValuesPart) dataParts[partId]).add(keyValue.getLongKey(), keyValue.getFloatValue());
            }
        } else {
            // Key and value array pair
            LongFloatSortedVectorStorage sortStorage = (LongFloatSortedVectorStorage) storage;
            long[] keys = sortStorage.getIndices();
            float[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) & dataPartNumMinus1;
                ((HashLongKeysFloatValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    } else {
        LongFloatVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            LongFloatSparseVectorStorage sparseStorage = (LongFloatSparseVectorStorage) storage;
            ObjectIterator<Long2FloatMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Long2FloatMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getLongKey()) % dataPartNum;
                ((HashLongKeysFloatValuesPart) dataParts[partId]).add(keyValue.getLongKey(), keyValue.getFloatValue());
            }
        } else {
            // Key and value array pair
            LongFloatSortedVectorStorage sortStorage = (LongFloatSortedVectorStorage) storage;
            long[] keys = sortStorage.getIndices();
            float[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) % dataPartNum;
                ((HashLongKeysFloatValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    }
}

Example 40 with LongFloatVector

use of com.tencent.angel.ml.math2.vector.LongFloatVector in project angel by Tencent.

the class HashRouterUtils method split.

/**
 * Split keys by matrix partition
 *
 * @param matrixMeta matrix meta data
 * @param vector Matrix vector
 * @return partition key to key partition map
 */
public static KeyValuePart[] split(MatrixMeta matrixMeta, Vector vector) {
    KeyHash hasher = HasherFactory.getHasher(matrixMeta.getRouterHash());
    PartitionKey[] matrixParts = matrixMeta.getPartitionKeys();
    KeyValuePart[] dataParts = new KeyValuePart[matrixParts.length];
    int estSize = (int) (vector.getSize() / matrixMeta.getPartitionNum());
    for (int i = 0; i < dataParts.length; i++) {
        dataParts[i] = generateDataPart(vector.getRowId(), vector.getType(), estSize);
    }
    switch(vector.getType()) {
        case T_DOUBLE_DENSE:
        case T_DOUBLE_SPARSE:
            {
                splitIntDoubleVector(hasher, matrixMeta, (IntDoubleVector) vector, dataParts);
                break;
            }
        case T_FLOAT_DENSE:
        case T_FLOAT_SPARSE:
            {
                splitIntFloatVector(hasher, matrixMeta, (IntFloatVector) vector, dataParts);
                break;
            }
        case T_INT_DENSE:
        case T_INT_SPARSE:
            {
                splitIntIntVector(hasher, matrixMeta, (IntIntVector) vector, dataParts);
                break;
            }
        case T_LONG_DENSE:
        case T_LONG_SPARSE:
            {
                splitIntLongVector(hasher, matrixMeta, (IntLongVector) vector, dataParts);
                break;
            }
        case T_DOUBLE_SPARSE_LONGKEY:
            {
                splitLongDoubleVector(hasher, matrixMeta, (LongDoubleVector) vector, dataParts);
                break;
            }
        case T_FLOAT_SPARSE_LONGKEY:
            {
                splitLongFloatVector(hasher, matrixMeta, (LongFloatVector) vector, dataParts);
                break;
            }
        case T_INT_SPARSE_LONGKEY:
            {
                splitLongIntVector(hasher, matrixMeta, (LongIntVector) vector, dataParts);
                break;
            }
        case T_LONG_SPARSE_LONGKEY:
            {
                splitLongLongVector(hasher, matrixMeta, (LongLongVector) vector, dataParts);
                break;
            }
        default:
            {
                throw new UnsupportedOperationException("Unsupport vector type " + vector.getType());
            }
    }
    for (int i = 0; i < dataParts.length; i++) {
        if (dataParts[i] != null) {
            dataParts[i].setRowId(vector.getRowId());
        }
    }
    return dataParts;
}