Examples with IntLongVector com.tencent.angel.ml.math2.vector.IntLongVector used on opensource projects

Example 36 with IntLongVector

use of com.tencent.angel.ml.math2.vector.IntLongVector in project angel by Tencent.

the class ColumnFormat method saveLongLongRows.

private void saveLongLongRows(ServerPartition part, ServerRow[] rows, MatrixPartitionMeta partMeta, PSMatrixSaveContext saveContext, DataOutputStream output) throws IOException {
    Vector vec = ServerRowUtils.getVector((ServerLongLongRow) rows[0]);
    // int size = rows.size();
    long indexOffset = part.getPartitionKey().getStartCol();
    LongLongsCol col = new LongLongsCol(0, new long[rows.length]);
    if (vec instanceof IntLongVector) {
        IntLongVectorStorage storage = ((IntLongVector) vec).getStorage();
        long startCol = rows[0].getStartCol();
        long endCol = rows[0].getEndCol();
        if (storage.isDense()) {
            for (long i = startCol; i < endCol; i++) {
                col.colId = i;
                for (int j = 0; j < rows.length; j++) {
                    col.colElems[j] = ((ServerLongLongRow) (rows[j])).get(col.colId);
                }
                save(col, output);
            }
        } else {
            if (saveContext.sortFirst()) {
                int[] indices = storage.getIndices();
                Sort.quickSort(indices, 0, indices.length - 1);
                for (int i = 0; i < indices.length; i++) {
                    col.colId = indices[i] + indexOffset;
                    for (int j = 0; j < rows.length; j++) {
                        col.colElems[j] = ((ServerLongLongRow) (rows[j])).get(col.colId);
                    }
                    save(col, output);
                }
            } else {
                ObjectIterator<Int2LongMap.Entry> iter = storage.entryIterator();
                while (iter.hasNext()) {
                    col.colId = iter.next().getIntKey() + indexOffset;
                    for (int j = 0; j < rows.length; j++) {
                        col.colElems[j] = ((ServerLongLongRow) (rows[j])).get(col.colId);
                    }
                    save(col, output);
                }
            }
        }
    } else {
        LongLongVectorStorage storage = ((LongLongVector) vec).getStorage();
        if (saveContext.sortFirst()) {
            long[] indices = storage.getIndices();
            Sort.quickSort(indices, 0, indices.length - 1);
            for (int i = 0; i < indices.length; i++) {
                col.colId = indices[i] + indexOffset;
                for (int j = 0; j < rows.length; j++) {
                    col.colElems[j] = ((ServerLongLongRow) (rows[j])).get(col.colId);
                }
                save(col, output);
            }
        } else {
            ObjectIterator<Long2LongMap.Entry> iter = storage.entryIterator();
            while (iter.hasNext()) {
                col.colId = iter.next().getLongKey() + indexOffset;
                for (int j = 0; j < rows.length; j++) {
                    col.colElems[j] = ((ServerLongLongRow) (rows[j])).get(col.colId);
                }
                save(col, output);
            }
        }
    }
}

Example 37 with IntLongVector

use of com.tencent.angel.ml.math2.vector.IntLongVector in project angel by Tencent.

the class ColumnFormat method saveIntLongRows.

private void saveIntLongRows(ServerPartition part, ServerRow[] rows, MatrixPartitionMeta partMeta, PSMatrixSaveContext saveContext, DataOutputStream output) throws IOException {
    Vector vec = ServerRowUtils.getVector((ServerIntLongRow) rows[0]);
    // int size = rows.size();
    int indexOffset = (int) part.getPartitionKey().getStartCol();
    IntLongVectorStorage storage = ((IntLongVector) vec).getStorage();
    IntLongsCol col = new IntLongsCol(0, new long[rows.length]);
    int startCol = (int) rows[0].getStartCol();
    int endCol = (int) rows[0].getEndCol();
    if (storage.isDense()) {
        for (int i = startCol; i < endCol; i++) {
            col.colId = i;
            for (int j = 0; j < rows.length; j++) {
                col.colElems[j] = ((ServerIntLongRow) (rows[j])).get(col.colId);
            }
            save(col, output);
        }
    } else {
        if (saveContext.sortFirst()) {
            int[] indices = storage.getIndices();
            Sort.quickSort(indices, 0, indices.length - 1);
            for (int i = 0; i < indices.length; i++) {
                col.colId = indices[i] + indexOffset;
                for (int j = 0; j < rows.length; j++) {
                    col.colElems[j] = ((ServerIntLongRow) (rows[j])).get(col.colId);
                }
                save(col, output);
            }
        } else {
            ObjectIterator<Int2LongMap.Entry> iter = storage.entryIterator();
            while (iter.hasNext()) {
                col.colId = iter.next().getIntKey() + indexOffset;
                for (int j = 0; j < rows.length; j++) {
                    col.colElems[j] = ((ServerIntLongRow) (rows[j])).get(col.colId);
                }
                save(col, output);
            }
        }
    }
}

Example 38 with IntLongVector

use of com.tencent.angel.ml.math2.vector.IntLongVector in project angel by Tencent.

the class HashRouterUtils method splitIntLongVector.

public static void splitIntLongVector(KeyHash hasher, MatrixMeta matrixMeta, IntLongVector vector, KeyValuePart[] dataParts) {
    int dataPartNum = dataParts.length;
    int dataPartNumMinus1 = dataPartNum - 1;
    if (isPow2(dataPartNum)) {
        IntLongVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            IntLongSparseVectorStorage sparseStorage = (IntLongSparseVectorStorage) storage;
            ObjectIterator<Int2LongMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Int2LongMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getIntKey()) & dataPartNumMinus1;
                ((HashIntKeysLongValuesPart) dataParts[partId]).add(keyValue.getIntKey(), keyValue.getLongValue());
            }
        } else if (storage.isDense()) {
            // Get values
            IntLongDenseVectorStorage denseStorage = (IntLongDenseVectorStorage) storage;
            long[] values = denseStorage.getValues();
            for (int i = 0; i < values.length; i++) {
                int partId = computeHashCode(hasher, i) & dataPartNumMinus1;
                ((HashIntKeysLongValuesPart) dataParts[partId]).add(i, values[i]);
            }
        } else {
            // Key and value array pair
            IntLongSortedVectorStorage sortStorage = (IntLongSortedVectorStorage) storage;
            int[] keys = sortStorage.getIndices();
            long[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) & dataPartNumMinus1;
                ((HashIntKeysLongValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    } else {
        IntLongVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            IntLongSparseVectorStorage sparseStorage = (IntLongSparseVectorStorage) storage;
            ObjectIterator<Int2LongMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Int2LongMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getIntKey()) % dataPartNum;
                ((HashIntKeysLongValuesPart) dataParts[partId]).add(keyValue.getIntKey(), keyValue.getLongValue());
            }
        } else if (storage.isDense()) {
            // Get values
            IntLongDenseVectorStorage denseStorage = (IntLongDenseVectorStorage) storage;
            long[] values = denseStorage.getValues();
            for (int i = 0; i < values.length; i++) {
                int partId = computeHashCode(hasher, i) % dataPartNum;
                ((HashIntKeysLongValuesPart) dataParts[partId]).add(i, values[i]);
            }
        } else {
            // Key and value array pair
            IntLongSortedVectorStorage sortStorage = (IntLongSortedVectorStorage) storage;
            int[] keys = sortStorage.getIndices();
            long[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) % dataPartNum;
                ((HashIntKeysLongValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    }
}

Example 39 with IntLongVector

use of com.tencent.angel.ml.math2.vector.IntLongVector in project angel by Tencent.

the class HashRouterUtils method split.

/**
 * Split keys by matrix partition
 *
 * @param matrixMeta matrix meta data
 * @param vector Matrix vector
 * @return partition key to key partition map
 */
public static KeyValuePart[] split(MatrixMeta matrixMeta, Vector vector) {
    KeyHash hasher = HasherFactory.getHasher(matrixMeta.getRouterHash());
    PartitionKey[] matrixParts = matrixMeta.getPartitionKeys();
    KeyValuePart[] dataParts = new KeyValuePart[matrixParts.length];
    int estSize = (int) (vector.getSize() / matrixMeta.getPartitionNum());
    for (int i = 0; i < dataParts.length; i++) {
        dataParts[i] = generateDataPart(vector.getRowId(), vector.getType(), estSize);
    }
    switch(vector.getType()) {
        case T_DOUBLE_DENSE:
        case T_DOUBLE_SPARSE:
            {
                splitIntDoubleVector(hasher, matrixMeta, (IntDoubleVector) vector, dataParts);
                break;
            }
        case T_FLOAT_DENSE:
        case T_FLOAT_SPARSE:
            {
                splitIntFloatVector(hasher, matrixMeta, (IntFloatVector) vector, dataParts);
                break;
            }
        case T_INT_DENSE:
        case T_INT_SPARSE:
            {
                splitIntIntVector(hasher, matrixMeta, (IntIntVector) vector, dataParts);
                break;
            }
        case T_LONG_DENSE:
        case T_LONG_SPARSE:
            {
                splitIntLongVector(hasher, matrixMeta, (IntLongVector) vector, dataParts);
                break;
            }
        case T_DOUBLE_SPARSE_LONGKEY:
            {
                splitLongDoubleVector(hasher, matrixMeta, (LongDoubleVector) vector, dataParts);
                break;
            }
        case T_FLOAT_SPARSE_LONGKEY:
            {
                splitLongFloatVector(hasher, matrixMeta, (LongFloatVector) vector, dataParts);
                break;
            }
        case T_INT_SPARSE_LONGKEY:
            {
                splitLongIntVector(hasher, matrixMeta, (LongIntVector) vector, dataParts);
                break;
            }
        case T_LONG_SPARSE_LONGKEY:
            {
                splitLongLongVector(hasher, matrixMeta, (LongLongVector) vector, dataParts);
                break;
            }
        default:
            {
                throw new UnsupportedOperationException("Unsupport vector type " + vector.getType());
            }
    }
    for (int i = 0; i < dataParts.length; i++) {
        if (dataParts[i] != null) {
            dataParts[i].setRowId(vector.getRowId());
        }
    }
    return dataParts;
}

Example 40 with IntLongVector

use of com.tencent.angel.ml.math2.vector.IntLongVector in project angel by Tencent.

the class RangeRouterUtils method splitIntLongVector.

public static KeyValuePart[] splitIntLongVector(MatrixMeta matrixMeta, IntLongVector vector) {
    IntLongVectorStorage storage = vector.getStorage();
    if (storage.isSparse()) {
        // Get keys and values
        IntLongSparseVectorStorage sparseStorage = (IntLongSparseVectorStorage) storage;
        int[] keys = sparseStorage.getIndices();
        long[] values = sparseStorage.getValues();
        return split(matrixMeta, vector.getRowId(), keys, values, false);
    } else if (storage.isDense()) {
        // Get values
        IntLongDenseVectorStorage denseStorage = (IntLongDenseVectorStorage) storage;
        long[] values = denseStorage.getValues();
        return split(matrixMeta, vector.getRowId(), values);
    } else {
        // Key and value array pair
        IntLongSortedVectorStorage sortStorage = (IntLongSortedVectorStorage) storage;
        int[] keys = sortStorage.getIndices();
        long[] values = sortStorage.getValues();
        return split(matrixMeta, vector.getRowId(), keys, values, true);
    }
}