Examples with IntCSRStorage com.tencent.angel.ps.storage.partition.storage.IntCSRStorage used on opensource projects

Example 1 with IntCSRStorage

use of com.tencent.angel.ps.storage.partition.storage.IntCSRStorage in project angel by Tencent.

the class InitNeighbor method partitionUpdate.

@Override
public void partitionUpdate(PartitionUpdateParam partParam) {
    PartInitNeighborParam param = (PartInitNeighborParam) partParam;
    int[] nodeIds = param.getNodeIds();
    int[] neighborNums = param.getNeighborNums();
    int[] neighbors = param.getNeighbors();
    ServerMatrix matrix = psContext.getMatrixStorageManager().getMatrix(param.getMatrixId());
    CSRPartition part = (CSRPartition) matrix.getPartition(param.getPartKey().getPartitionId());
    IntCSRStorage storage = (IntCSRStorage) part.getStorage();
    synchronized (storage) {
        int startOffset = (int) param.getPartKey().getStartCol();
        // Store the total neighbor number of all nodes in rowOffsets
        int[] rowOffsets = storage.getRowOffsets();
        for (int i = 0; i < nodeIds.length; i++) {
            rowOffsets[nodeIds[i] - startOffset] += neighborNums[i];
        }
        // Put the node ids, node neighbor number, node neighbors to the cache
        List<int[]> tempRowIds = storage.getTempRowIds();
        List<int[]> tempRowLens = storage.getTempRowLens();
        List<int[]> tempColumnOffsets = storage.getTempColumnIndices();
        if (tempRowIds == null) {
            tempRowIds = new ArrayList<>();
            tempRowLens = new ArrayList<>();
            tempColumnOffsets = new ArrayList<>();
            storage.setTempRowIds(tempRowIds);
            storage.setTempRowLens(tempRowLens);
            storage.setTempColumnIndices(tempColumnOffsets);
        }
        tempRowIds.add(param.getNodeIds());
        tempRowLens.add(neighborNums);
        tempColumnOffsets.add(neighbors);
    }
}

Example 2 with IntCSRStorage

use of com.tencent.angel.ps.storage.partition.storage.IntCSRStorage in project angel by Tencent.

the class BinaryCSRFormat method load.

@Override
public void load(ServerPartition part, MatrixPartitionMeta partMeta, PSMatrixLoadContext loadContext, DataInputStream input) throws IOException {
    CSRPartition csrPart = (CSRPartition) part;
    CSRStorage storage = csrPart.getStorage();
    if (storage instanceof IntCSRStorage) {
        load((IntCSRStorage) storage, input);
    }
}

Example 3 with IntCSRStorage

use of com.tencent.angel.ps.storage.partition.storage.IntCSRStorage in project angel by Tencent.

the class BinaryCSRFormat method save.

@Override
public void save(ServerPartition part, MatrixPartitionMeta partMeta, PSMatrixSaveContext saveContext, DataOutputStream output) throws IOException {
    CSRPartition csrPart = (CSRPartition) part;
    CSRStorage storage = csrPart.getStorage();
    if (storage instanceof IntCSRStorage) {
        save((IntCSRStorage) storage, output);
    }
}

Example 4 with IntCSRStorage

use of com.tencent.angel.ps.storage.partition.storage.IntCSRStorage in project angel by Tencent.

the class InitNeighborOver method partitionUpdate.

@Override
public void partitionUpdate(PartitionUpdateParam param) {
    ServerMatrix matrix = psContext.getMatrixStorageManager().getMatrix(param.getMatrixId());
    CSRPartition part = (CSRPartition) matrix.getPartition(param.getPartKey().getPartitionId());
    IntCSRStorage storage = (IntCSRStorage) part.getStorage();
    int startCol = (int) param.getPartKey().getStartCol();
    synchronized (storage) {
        // No data in this partition
        if (storage.getTempRowIds() == null) {
            return;
        }
        // Get total neighbor number
        int[] rowOffsets = storage.getRowOffsets();
        int accumOffset = 0;
        for (int i = 0; i < rowOffsets.length - 1; i++) {
            int offset = rowOffsets[i];
            rowOffsets[i] = accumOffset;
            accumOffset += offset;
        }
        rowOffsets[rowOffsets.length - 1] = accumOffset;
        // Final matrix column indices: neighbors node ids
        int[] cloumnIndices = new int[accumOffset];
        // Write positions in cloumnIndices for nodes
        int[] copyOffsets = new int[rowOffsets.length - 1];
        System.arraycopy(rowOffsets, 0, copyOffsets, 0, rowOffsets.length - 1);
        List<int[]> tempRowIds = storage.getTempRowIds();
        List<int[]> tempRowLens = storage.getTempRowLens();
        List<int[]> tempColumnIndices = storage.getTempColumnIndices();
        // Copy all cached sub column indices to final column indices
        int size = tempRowIds.size();
        for (int i = 0; i < size; i++) {
            // Read position for a sub column indices
            int copyLen = 0;
            int[] subTempRowIds = tempRowIds.get(i);
            int[] subTempLens = tempRowLens.get(i);
            int[] subTempColumnIndices = tempColumnIndices.get(i);
            for (int j = 0; j < subTempRowIds.length; j++) {
                // Copy column indices for a node to final column indices
                System.arraycopy(subTempColumnIndices, copyLen, cloumnIndices, copyOffsets[subTempRowIds[j] - startCol], subTempLens[j]);
                // Update write position for this node in final column indices
                copyOffsets[subTempRowIds[j] - startCol] += subTempLens[j];
                // Update the read position in sub column indices
                copyLen += subTempLens[j];
            }
        }
        storage.setColumnIndices(cloumnIndices);
        // Clear all temp data
        storage.setTempRowIds(null);
        storage.setTempRowLens(null);
        storage.setTempColumnIndices(null);
    }
}

Example 5 with IntCSRStorage

use of com.tencent.angel.ps.storage.partition.storage.IntCSRStorage in project angel by Tencent.

the class SampleNeighbor method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    PartSampleNeighborParam param = (PartSampleNeighborParam) partParam;
    ServerMatrix matrix = psContext.getMatrixStorageManager().getMatrix(partParam.getMatrixId());
    CSRPartition part = (CSRPartition) matrix.getPartition(partParam.getPartKey().getPartitionId());
    IntCSRStorage storage = (IntCSRStorage) (part.getStorage());
    Int2ObjectOpenHashMap<int[]> results = new Int2ObjectOpenHashMap<>();
    int[] neighborOffsets = storage.getRowOffsets();
    int[] neighbors = storage.getColumnIndices();
    int startCol = (int) partParam.getPartKey().getStartCol();
    int[] nodeIds = param.getNodeIds();
    int count = param.getCount();
    Random r = new Random();
    for (int i = 0; i < nodeIds.length; i++) {
        int nodeId = nodeIds[i];
        // Get node neighbor number
        int num = neighborOffsets[nodeId - startCol + 1] - neighborOffsets[nodeId - startCol];
        int[] result;
        if (num == 0) {
            // If the neighbor number is 0, just return a int[0]
            result = new int[0];
        } else if (count <= 0 || num <= count) {
            // If count <= 0 or the neighbor number is less or equal then count, just copy all neighbors to the result array
            result = new int[num];
            System.arraycopy(neighbors, neighborOffsets[nodeId - startCol], result, 0, num);
        } else {
            // If the neighbor number > count, just copy a range of neighbors to the result array, the copy position is random
            int startPos = Math.abs(r.nextInt()) % num;
            result = new int[count];
            if (startPos + count <= num) {
                System.arraycopy(neighbors, neighborOffsets[nodeId - startCol] + startPos, result, 0, count);
            } else {
                System.arraycopy(neighbors, neighborOffsets[nodeId - startCol] + startPos, result, 0, num - startPos);
                System.arraycopy(neighbors, neighborOffsets[nodeId - startCol], result, num - startPos, count - (num - startPos));
            }
        }
        results.put(nodeIds[i], result);
    }
    return new PartSampleNeighborResult(results);
}