Examples with IntFloatVector com.tencent.angel.ml.math2.vector.IntFloatVector used on opensource projects

Example 66 with IntFloatVector

use of com.tencent.angel.ml.math2.vector.IntFloatVector in project angel by Tencent.

the class ByteBufSerdeUtils method serializeIntFloatVector.

// IntFloatVector
private static void serializeIntFloatVector(ByteBuf out, IntFloatVector vector) {
    IntFloatVectorStorage storage = vector.getStorage();
    if (storage.isDense()) {
        serializeInt(out, DENSE_STORAGE_TYPE);
        serializeFloats(out, storage.getValues());
    } else if (storage.isSparse()) {
        serializeInt(out, SPARSE_STORAGE_TYPE);
        serializeInt(out, storage.size());
        ObjectIterator<Int2FloatMap.Entry> iter = storage.entryIterator();
        while (iter.hasNext()) {
            Int2FloatMap.Entry e = iter.next();
            serializeInt(out, e.getIntKey());
            serializeFloat(out, e.getFloatValue());
        }
    } else if (storage.isSorted()) {
        serializeInt(out, SORTED_STORAGE_TYPE);
        int[] indices = vector.getStorage().getIndices();
        float[] values = vector.getStorage().getValues();
        serializeInts(out, indices);
        serializeFloats(out, values);
    } else {
        throw new UnsupportedOperationException("Unsupport storage type " + vector.getStorage().getClass());
    }
}

Example 67 with IntFloatVector

use of com.tencent.angel.ml.math2.vector.IntFloatVector in project angel by Tencent.

the class ByteBufSerdeUtils method serializedIntFloatVectorLen.

public static int serializedIntFloatVectorLen(IntFloatVector vector) {
    int len = 0;
    IntFloatVectorStorage storage = vector.getStorage();
    if (storage.isDense()) {
        len += serializedIntLen(DENSE_STORAGE_TYPE);
        len += serializedFloatsLen(storage.getValues());
    } else if (storage.isSparse()) {
        len += serializedIntLen(SPARSE_STORAGE_TYPE);
        len += serializedIntLen(storage.size());
        len += storage.size() * (INT_LENGTH + FLOAT_LENGTH);
    } else if (storage.isSorted()) {
        len += serializedIntLen(SORTED_STORAGE_TYPE);
        len += serializedIntsLen(vector.getStorage().getIndices());
        len += serializedFloatsLen(vector.getStorage().getValues());
    } else {
        throw new UnsupportedOperationException("Unsupport storage type " + vector.getStorage().getClass());
    }
    return len;
}

Example 68 with IntFloatVector

use of com.tencent.angel.ml.math2.vector.IntFloatVector in project angel by Tencent.

the class SnapshotFormat method save.

private void save(ServerLongFloatRow row, PSMatrixSaveContext saveContext, MatrixPartitionMeta meta, DataOutputStream out) throws IOException {
    long startCol = meta.getStartCol();
    if (ServerRowUtils.getVector(row) instanceof IntFloatVector) {
        IntFloatVector vector = (IntFloatVector) ServerRowUtils.getVector(row);
        if (vector.isDense()) {
            float[] data = vector.getStorage().getValues();
            for (int i = 0; i < data.length; i++) {
                out.writeFloat(data[i]);
            }
        } else if (vector.isSorted()) {
            int[] indices = vector.getStorage().getIndices();
            float[] values = vector.getStorage().getValues();
            for (int i = 0; i < indices.length; i++) {
                out.writeLong(indices[i] + startCol);
                out.writeFloat(values[i]);
            }
        } else {
            ObjectIterator<Int2FloatMap.Entry> iter = vector.getStorage().entryIterator();
            Int2FloatMap.Entry entry;
            while (iter.hasNext()) {
                entry = iter.next();
                out.writeLong(entry.getIntKey() + startCol);
                out.writeFloat(entry.getFloatValue());
            }
        }
    } else {
        LongFloatVector vector = (LongFloatVector) ServerRowUtils.getVector(row);
        if (vector.isSorted()) {
            long[] indices = vector.getStorage().getIndices();
            float[] values = vector.getStorage().getValues();
            for (int i = 0; i < indices.length; i++) {
                out.writeLong(indices[i] + startCol);
                out.writeFloat(values[i]);
            }
        } else {
            ObjectIterator<Long2FloatMap.Entry> iter = vector.getStorage().entryIterator();
            Long2FloatMap.Entry entry;
            while (iter.hasNext()) {
                entry = iter.next();
                out.writeLong(entry.getLongKey() + startCol);
                out.writeFloat(entry.getFloatValue());
            }
        }
    }
}

Example 69 with IntFloatVector

use of com.tencent.angel.ml.math2.vector.IntFloatVector in project angel by Tencent.

the class ColumnFormat method saveIntFloatRows.

private void saveIntFloatRows(ServerPartition part, ServerRow[] rows, MatrixPartitionMeta partMeta, PSMatrixSaveContext saveContext, DataOutputStream output) throws IOException {
    Vector vec = ServerRowUtils.getVector((ServerIntFloatRow) rows[0]);
    // int size = rows.length;
    int indexOffset = (int) part.getPartitionKey().getStartCol();
    IntFloatVectorStorage storage = ((IntFloatVector) vec).getStorage();
    IntFloatsCol col = new IntFloatsCol(0, new float[rows.length]);
    int startCol = (int) rows[0].getStartCol();
    int endCol = (int) rows[0].getEndCol();
    if (storage.isDense()) {
        for (int i = startCol; i < endCol; i++) {
            col.colId = i;
            for (int j = 0; j < rows.length; j++) {
                col.colElems[j] = ((ServerIntFloatRow) (rows[j])).get(col.colId);
            }
            save(col, output);
        }
    } else {
        if (saveContext.sortFirst()) {
            int[] indices = storage.getIndices();
            Sort.quickSort(indices, 0, indices.length - 1);
            for (int i = 0; i < indices.length; i++) {
                col.colId = indices[i] + indexOffset;
                for (int j = 0; j < rows.length; j++) {
                    col.colElems[j] = ((ServerIntFloatRow) (rows[j])).get(col.colId);
                }
                save(col, output);
            }
        } else {
            ObjectIterator<Int2FloatMap.Entry> iter = storage.entryIterator();
            while (iter.hasNext()) {
                col.colId = iter.next().getIntKey() + indexOffset;
                for (int j = 0; j < rows.length; j++) {
                    col.colElems[j] = ((ServerIntFloatRow) (rows[j])).get(col.colId);
                }
                save(col, output);
            }
        }
    }
}

Example 70 with IntFloatVector

use of com.tencent.angel.ml.math2.vector.IntFloatVector in project angel by Tencent.

the class HashRouterUtils method splitIntFloatVector.

public static void splitIntFloatVector(KeyHash hasher, MatrixMeta matrixMeta, IntFloatVector vector, KeyValuePart[] dataParts) {
    int dataPartNum = dataParts.length;
    int dataPartNumMinus1 = dataPartNum - 1;
    if (isPow2(dataPartNum)) {
        IntFloatVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            IntFloatSparseVectorStorage sparseStorage = (IntFloatSparseVectorStorage) storage;
            ObjectIterator<Int2FloatMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Int2FloatMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getIntKey()) & dataPartNumMinus1;
                ((HashIntKeysFloatValuesPart) dataParts[partId]).add(keyValue.getIntKey(), keyValue.getFloatValue());
            }
        } else if (storage.isDense()) {
            // Get values
            IntFloatDenseVectorStorage denseStorage = (IntFloatDenseVectorStorage) storage;
            float[] values = denseStorage.getValues();
            for (int i = 0; i < values.length; i++) {
                int partId = computeHashCode(hasher, i) & dataPartNumMinus1;
                ((HashIntKeysFloatValuesPart) dataParts[partId]).add(i, values[i]);
            }
        } else {
            // Key and value array pair
            IntFloatSortedVectorStorage sortStorage = (IntFloatSortedVectorStorage) storage;
            int[] keys = sortStorage.getIndices();
            float[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) & dataPartNumMinus1;
                ((HashIntKeysFloatValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    } else {
        IntFloatVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            IntFloatSparseVectorStorage sparseStorage = (IntFloatSparseVectorStorage) storage;
            ObjectIterator<Int2FloatMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Int2FloatMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getIntKey()) % dataPartNum;
                ((HashIntKeysFloatValuesPart) dataParts[partId]).add(keyValue.getIntKey(), keyValue.getFloatValue());
            }
        } else if (storage.isDense()) {
            // Get values
            IntFloatDenseVectorStorage denseStorage = (IntFloatDenseVectorStorage) storage;
            float[] values = denseStorage.getValues();
            for (int i = 0; i < values.length; i++) {
                int partId = computeHashCode(hasher, i) % dataPartNum;
                ((HashIntKeysFloatValuesPart) dataParts[partId]).add(i, values[i]);
            }
        } else {
            // Key and value array pair
            IntFloatSortedVectorStorage sortStorage = (IntFloatSortedVectorStorage) storage;
            int[] keys = sortStorage.getIndices();
            float[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) % dataPartNum;
                ((HashIntKeysFloatValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    }
}