Examples with LongLongVector com.tencent.angel.ml.math2.vector.LongLongVector used on opensource projects

Example 31 with LongLongVector

use of com.tencent.angel.ml.math2.vector.LongLongVector in project angel by Tencent.

the class MergeUtils method combineLongLongIndexRowSplits.

// //////////////////////////////////////////////////////////////////////////////
// Combine Long key Long value vector
// //////////////////////////////////////////////////////////////////////////////
public static Vector combineLongLongIndexRowSplits(int matrixId, int rowId, int resultSize, KeyPart[] keyParts, ValuePart[] valueParts, MatrixMeta matrixMeta) {
    LongLongVector vector = VFactory.sparseLongKeyLongVector(matrixMeta.getColNum(), resultSize);
    for (int i = 0; i < keyParts.length; i++) {
        mergeTo(vector, keyParts[i], (LongValuesPart) valueParts[i]);
    }
    vector.setRowId(rowId);
    vector.setMatrixId(matrixId);
    return vector;
}

Example 32 with LongLongVector

use of com.tencent.angel.ml.math2.vector.LongLongVector in project angel by Tencent.

the class RowSplitCombineUtils method combineLongLongIndexRowSplits.

// //////////////////////////////////////////////////////////////////////////////
// Combine Long key Long value vector
// //////////////////////////////////////////////////////////////////////////////
public static Vector combineLongLongIndexRowSplits(int matrixId, int rowId, int resultSize, KeyPart[] keyParts, ValuePart[] valueParts, MatrixMeta matrixMeta) {
    LongLongVector vector = VFactory.sparseLongKeyLongVector(matrixMeta.getColNum(), resultSize);
    for (int i = 0; i < keyParts.length; i++) {
        mergeTo(vector, keyParts[i], (LongValuesPart) valueParts[i]);
    }
    vector.setRowId(rowId);
    vector.setMatrixId(matrixId);
    return vector;
}

Example 33 with LongLongVector

use of com.tencent.angel.ml.math2.vector.LongLongVector in project angel by Tencent.

the class ColumnFormat method saveLongLongRows.

private void saveLongLongRows(ServerPartition part, ServerRow[] rows, MatrixPartitionMeta partMeta, PSMatrixSaveContext saveContext, DataOutputStream output) throws IOException {
    Vector vec = ServerRowUtils.getVector((ServerLongLongRow) rows[0]);
    // int size = rows.size();
    long indexOffset = part.getPartitionKey().getStartCol();
    LongLongsCol col = new LongLongsCol(0, new long[rows.length]);
    if (vec instanceof IntLongVector) {
        IntLongVectorStorage storage = ((IntLongVector) vec).getStorage();
        long startCol = rows[0].getStartCol();
        long endCol = rows[0].getEndCol();
        if (storage.isDense()) {
            for (long i = startCol; i < endCol; i++) {
                col.colId = i;
                for (int j = 0; j < rows.length; j++) {
                    col.colElems[j] = ((ServerLongLongRow) (rows[j])).get(col.colId);
                }
                save(col, output);
            }
        } else {
            if (saveContext.sortFirst()) {
                int[] indices = storage.getIndices();
                Sort.quickSort(indices, 0, indices.length - 1);
                for (int i = 0; i < indices.length; i++) {
                    col.colId = indices[i] + indexOffset;
                    for (int j = 0; j < rows.length; j++) {
                        col.colElems[j] = ((ServerLongLongRow) (rows[j])).get(col.colId);
                    }
                    save(col, output);
                }
            } else {
                ObjectIterator<Int2LongMap.Entry> iter = storage.entryIterator();
                while (iter.hasNext()) {
                    col.colId = iter.next().getIntKey() + indexOffset;
                    for (int j = 0; j < rows.length; j++) {
                        col.colElems[j] = ((ServerLongLongRow) (rows[j])).get(col.colId);
                    }
                    save(col, output);
                }
            }
        }
    } else {
        LongLongVectorStorage storage = ((LongLongVector) vec).getStorage();
        if (saveContext.sortFirst()) {
            long[] indices = storage.getIndices();
            Sort.quickSort(indices, 0, indices.length - 1);
            for (int i = 0; i < indices.length; i++) {
                col.colId = indices[i] + indexOffset;
                for (int j = 0; j < rows.length; j++) {
                    col.colElems[j] = ((ServerLongLongRow) (rows[j])).get(col.colId);
                }
                save(col, output);
            }
        } else {
            ObjectIterator<Long2LongMap.Entry> iter = storage.entryIterator();
            while (iter.hasNext()) {
                col.colId = iter.next().getLongKey() + indexOffset;
                for (int j = 0; j < rows.length; j++) {
                    col.colElems[j] = ((ServerLongLongRow) (rows[j])).get(col.colId);
                }
                save(col, output);
            }
        }
    }
}

Example 34 with LongLongVector

use of com.tencent.angel.ml.math2.vector.LongLongVector in project angel by Tencent.

the class HashRouterUtils method split.

/**
 * Split keys by matrix partition
 *
 * @param matrixMeta matrix meta data
 * @param vector Matrix vector
 * @return partition key to key partition map
 */
public static KeyValuePart[] split(MatrixMeta matrixMeta, Vector vector) {
    KeyHash hasher = HasherFactory.getHasher(matrixMeta.getRouterHash());
    PartitionKey[] matrixParts = matrixMeta.getPartitionKeys();
    KeyValuePart[] dataParts = new KeyValuePart[matrixParts.length];
    int estSize = (int) (vector.getSize() / matrixMeta.getPartitionNum());
    for (int i = 0; i < dataParts.length; i++) {
        dataParts[i] = generateDataPart(vector.getRowId(), vector.getType(), estSize);
    }
    switch(vector.getType()) {
        case T_DOUBLE_DENSE:
        case T_DOUBLE_SPARSE:
            {
                splitIntDoubleVector(hasher, matrixMeta, (IntDoubleVector) vector, dataParts);
                break;
            }
        case T_FLOAT_DENSE:
        case T_FLOAT_SPARSE:
            {
                splitIntFloatVector(hasher, matrixMeta, (IntFloatVector) vector, dataParts);
                break;
            }
        case T_INT_DENSE:
        case T_INT_SPARSE:
            {
                splitIntIntVector(hasher, matrixMeta, (IntIntVector) vector, dataParts);
                break;
            }
        case T_LONG_DENSE:
        case T_LONG_SPARSE:
            {
                splitIntLongVector(hasher, matrixMeta, (IntLongVector) vector, dataParts);
                break;
            }
        case T_DOUBLE_SPARSE_LONGKEY:
            {
                splitLongDoubleVector(hasher, matrixMeta, (LongDoubleVector) vector, dataParts);
                break;
            }
        case T_FLOAT_SPARSE_LONGKEY:
            {
                splitLongFloatVector(hasher, matrixMeta, (LongFloatVector) vector, dataParts);
                break;
            }
        case T_INT_SPARSE_LONGKEY:
            {
                splitLongIntVector(hasher, matrixMeta, (LongIntVector) vector, dataParts);
                break;
            }
        case T_LONG_SPARSE_LONGKEY:
            {
                splitLongLongVector(hasher, matrixMeta, (LongLongVector) vector, dataParts);
                break;
            }
        default:
            {
                throw new UnsupportedOperationException("Unsupport vector type " + vector.getType());
            }
    }
    for (int i = 0; i < dataParts.length; i++) {
        if (dataParts[i] != null) {
            dataParts[i].setRowId(vector.getRowId());
        }
    }
    return dataParts;
}

Example 35 with LongLongVector

use of com.tencent.angel.ml.math2.vector.LongLongVector in project angel by Tencent.

the class MixedBinaryInAllExecutor method apply.

private static Vector apply(CompLongLongVector v1, LongDummyVector v2, Binary op) {
    LongLongVector[] parts = v1.getPartitions();
    Storage[] resParts = StorageSwitch.applyComp(v1, v2, op);
    if (!op.isKeepStorage()) {
        for (int i = 0; i < parts.length; i++) {
            if (parts[i].getStorage() instanceof LongLongSortedVectorStorage) {
                resParts[i] = new LongLongSparseVectorStorage(parts[i].getDim(), parts[i].getStorage().getIndices(), parts[i].getStorage().getValues());
            }
        }
    }
    long subDim = (v1.getDim() + v1.getNumPartitions() - 1) / v1.getNumPartitions();
    for (int i = 0; i < v1.getDim(); i++) {
        int pidx = (int) (i / subDim);
        long subidx = i % subDim;
        ((LongLongVectorStorage) resParts[pidx]).set(subidx, op.apply(parts[pidx].get(subidx), v2.get(i)));
    }
    LongLongVector[] res = new LongLongVector[parts.length];
    int i = 0;
    for (LongLongVector part : parts) {
        res[i] = new LongLongVector(part.getMatrixId(), part.getRowId(), part.getClock(), part.getDim(), (LongLongVectorStorage) resParts[i]);
        i++;
    }
    v1.setPartitions(res);
    return v1;
}