Examples with ServerRowsStorage com.tencent.angel.ps.storage.partition.storage.ServerRowsStorage used on opensource projects

Example 1 with ServerRowsStorage

use of com.tencent.angel.ps.storage.partition.storage.ServerRowsStorage in project angel by Tencent.

the class SnapshotFormat method load.

/**
 * Load a matrix partition
 *
 * @param part matrix partition
 * @param partMeta matrix partition data meta
 * @param loadContext load context
 * @param input input stream
 */
public void load(RowBasedPartition part, MatrixPartitionMeta partMeta, PSMatrixLoadContext loadContext, DataInputStream input) throws IOException {
    ServerRowsStorage rows = part.getRowsStorage();
    try {
        Map<Integer, RowPartitionMeta> rowMetas = partMeta.getRowMetas();
        for (RowPartitionMeta rowMeta : rowMetas.values()) {
            ServerRow row = rows.getRow(rowMeta.getRowId());
            load(row, partMeta, loadContext, input);
        }
    } finally {
        part.setState(PartitionState.READ_AND_WRITE);
    }
}

Example 2 with ServerRowsStorage

use of com.tencent.angel.ps.storage.partition.storage.ServerRowsStorage in project angel by Tencent.

the class SnapshotFormat method save.

/**
 * Matrix partition data
 *
 * @param part matrix partition
 * @param partMeta matrix partition data meta
 * @param saveContext save context
 * @param output output stream
 */
public void save(RowBasedPartition part, MatrixPartitionMeta partMeta, PSMatrixSaveContext saveContext, DataOutputStream output) throws IOException {
    List<Integer> rowIds = saveContext.getRowIndexes();
    ServerRowsStorage rows = part.getRowsStorage();
    if (rowIds == null || rowIds.isEmpty()) {
        int rowStart = part.getPartitionKey().getStartRow();
        int rowEnd = part.getPartitionKey().getEndRow();
        rowIds = new ArrayList<>(rowEnd - rowStart);
        for (int i = rowStart; i < rowEnd; i++) {
            rowIds.add(i);
        }
    } else {
        rowIds = filter(part, rowIds);
    }
    FSDataOutputStream dataOutputStream = new FSDataOutputStream(output, null, partMeta != null ? partMeta.getOffset() : 0);
    partMeta.setSaveRowNum(rowIds.size());
    for (int rowId : rowIds) {
        ServerRow row = rows.getRow(rowId);
        RowPartitionMeta rowMeta = new RowPartitionMeta(rowId, 0, 0);
        if (row != null) {
            rowMeta.setElementNum(row.size());
            rowMeta.setOffset(dataOutputStream.getPos());
            if (row.isDense()) {
                rowMeta.setSaveType(SaveType.DENSE.getTypeId());
            } else {
                rowMeta.setSaveType(SaveType.SPARSE.getTypeId());
            }
            save(rows.getRow(rowId), saveContext, partMeta, output);
        } else {
            rowMeta.setElementNum(0);
            rowMeta.setOffset(dataOutputStream.getPos());
        }
        partMeta.setRowMeta(rowMeta);
    }
}