Examples with ServerRow com.tencent.angel.ps.storage.vector.ServerRow used on opensource projects

Example 6 with ServerRow

use of com.tencent.angel.ps.storage.vector.ServerRow in project angel by Tencent.

the class ComplexRowFormat method save.

@Override
public void save(RowBasedPartition part, MatrixPartitionMeta partMeta, PSMatrixSaveContext saveContext, DataOutputStream output) throws IOException {
    List<Integer> rowIds = saveContext.getRowIndexes();
    if (rowIds == null || rowIds.isEmpty()) {
        Iterator<Entry<Integer, ServerRow>> iter = part.getRowsStorage().iterator();
        rowIds = new ArrayList<>();
        while (iter.hasNext()) {
            rowIds.add(iter.next().getKey());
        }
    } else {
        rowIds = filter(part, rowIds);
    }
    FSDataOutputStream dataOutputStream = new FSDataOutputStream(output, null, partMeta != null ? partMeta.getOffset() : 0);
    partMeta.setSaveRowNum(rowIds.size());
    for (int rowId : rowIds) {
        ServerRow row = part.getRow(rowId);
        RowPartitionMeta rowMeta = new RowPartitionMeta(rowId, 0, 0);
        if (row != null) {
            rowMeta.setElementNum(row.size());
            rowMeta.setOffset(dataOutputStream.getPos());
            save(part.getRow(rowId), saveContext, partMeta, output);
        } else {
            rowMeta.setElementNum(0);
            rowMeta.setOffset(dataOutputStream.getPos());
        }
        partMeta.setRowMeta(rowMeta);
    }
}

Example 7 with ServerRow

use of com.tencent.angel.ps.storage.vector.ServerRow in project angel by Tencent.

the class SnapshotFormat method save.

/**
 * Matrix partition data
 *
 * @param part matrix partition
 * @param partMeta matrix partition data meta
 * @param saveContext save context
 * @param output output stream
 */
public void save(RowBasedPartition part, MatrixPartitionMeta partMeta, PSMatrixSaveContext saveContext, DataOutputStream output) throws IOException {
    List<Integer> rowIds = saveContext.getRowIndexes();
    ServerRowsStorage rows = part.getRowsStorage();
    if (rowIds == null || rowIds.isEmpty()) {
        int rowStart = part.getPartitionKey().getStartRow();
        int rowEnd = part.getPartitionKey().getEndRow();
        rowIds = new ArrayList<>(rowEnd - rowStart);
        for (int i = rowStart; i < rowEnd; i++) {
            rowIds.add(i);
        }
    } else {
        rowIds = filter(part, rowIds);
    }
    FSDataOutputStream dataOutputStream = new FSDataOutputStream(output, null, partMeta != null ? partMeta.getOffset() : 0);
    partMeta.setSaveRowNum(rowIds.size());
    for (int rowId : rowIds) {
        ServerRow row = rows.getRow(rowId);
        RowPartitionMeta rowMeta = new RowPartitionMeta(rowId, 0, 0);
        if (row != null) {
            rowMeta.setElementNum(row.size());
            rowMeta.setOffset(dataOutputStream.getPos());
            if (row.isDense()) {
                rowMeta.setSaveType(SaveType.DENSE.getTypeId());
            } else {
                rowMeta.setSaveType(SaveType.SPARSE.getTypeId());
            }
            save(rows.getRow(rowId), saveContext, partMeta, output);
        } else {
            rowMeta.setElementNum(0);
            rowMeta.setOffset(dataOutputStream.getPos());
        }
        partMeta.setRowMeta(rowMeta);
    }
}

Example 8 with ServerRow

use of com.tencent.angel.ps.storage.vector.ServerRow in project angel by Tencent.

the class GetRowsHandler method merge.

private Vector[] merge(int matrixId, int[] rowIds, Collection<Response> responses) {
    Map<Integer, List<ServerRow>> rowIdToserverRows = new HashMap<>(rowIds.length);
    int valueNum = responses.size();
    for (Response response : responses) {
        ServerRow[] serverRows = ((GetRowsSplitResponse) (response.getData())).getRowsSplit();
        for (int j = 0; j < serverRows.length; j++) {
            int rowId = serverRows[j].getRowId();
            List<ServerRow> rowSplits = rowIdToserverRows.get(rowId);
            if (rowSplits == null) {
                rowSplits = new ArrayList<>(valueNum);
                rowIdToserverRows.put(rowId, rowSplits);
            }
            rowSplits.add(serverRows[j]);
        }
    }
    Vector[] vectors = new Vector[rowIds.length];
    for (int i = 0; i < rowIds.length; i++) {
        vectors[i] = MergeUtils.combineServerRowSplits(rowIdToserverRows.get(rowIds[i]), matrixId, rowIds[i]);
        vectors[i].setMatrixId(matrixId);
        vectors[i].setRowId(rowIds[i]);
    }
    return vectors;
}

Example 9 with ServerRow

use of com.tencent.angel.ps.storage.vector.ServerRow in project angel by Tencent.

the class GetRowHandler method handle.

@Override
public void handle(FutureResult finalResult, UserRequest userRequest, ResponseCache responseCache) {
    GetRowRequest getRowRequest = (GetRowRequest) userRequest;
    MapResponseCache cache = (MapResponseCache) responseCache;
    // Merge the sub-response
    List<ServerRow> serverRows = new ArrayList<>(cache.expectedResponseNum);
    for (Response response : cache.getResponses().values()) {
        serverRows.add(((GetRowSplitResponse) (response.getData())).getRowSplit());
    }
    Vector vector = MergeUtils.combineServerRowSplits(serverRows, getRowRequest.getMatrixId(), getRowRequest.getRowId());
    // Set matrix/row information
    vector.setMatrixId(getRowRequest.getMatrixId());
    vector.setRowId(getRowRequest.getRowId());
    // Set result
    finalResult.set(vector);
}

Example 10 with ServerRow

use of com.tencent.angel.ps.storage.vector.ServerRow in project angel by Tencent.

the class UpdatePartFunc method partitionUpdate.

@Override
public void partitionUpdate(PartitionUpdateParam partParam) {
    CSRPartUpdateParam param = (CSRPartUpdateParam) partParam;
    PartitionKey pkey = partParam.getPartKey();
    pkey = psContext.getMatrixMetaManager().getMatrixMeta(pkey.getMatrixId()).getPartitionMeta(pkey.getPartitionId()).getPartitionKey();
    try {
        while (param.buf.isReadable()) {
            int row = param.buf.readInt();
            int len = param.buf.readInt();
            ServerRow serverRow = psContext.getMatrixStorageManager().getRow(pkey, row);
            try {
                serverRow.startWrite();
                if (serverRow.isDense())
                    updateDenseIntRow((ServerIntIntRow) serverRow, param.buf, len);
                if (serverRow.isSparse())
                    updateSparseIntRow((ServerIntIntRow) serverRow, param.buf, len);
            } finally {
                serverRow.endWrite();
            }
        }
    } finally {
        param.buf.release();
    }
}