Examples with ServerPartition com.tencent.angel.ps.storage.partition.ServerPartition used on opensource projects

Example 21 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class MatrixFormatImpl method save.

private void save(ServerMatrix matrix, Path matrixPath, FileSystem fs, List<Integer> partIds, PSMatrixSaveContext saveContext, int startPos, int endPos, PSMatrixFilesMeta dataFilesMeta) throws IOException {
    Path destFile = new Path(matrixPath, ModelFilesUtils.fileName(partIds.get(startPos)));
    Path tmpDestFile = HdfsUtil.toTmpPath(destFile);
    if (fs.exists(tmpDestFile)) {
        fs.delete(tmpDestFile, true);
    }
    FSDataOutputStream out = fs.create(tmpDestFile, true, conf.getInt(AngelConf.ANGEL_PS_IO_FILE_BUFFER_SIZE, AngelConf.DEFAULT_ANGEL_PS_IO_FILE_BUFFER_SIZE));
    long streamPos = 0;
    ServerPartition partition;
    for (int i = startPos; i < endPos; i++) {
        LOG.info("Write partition " + partIds.get(i) + " of matrix " + matrix.getName() + " to " + tmpDestFile);
        streamPos = out.getPos();
        partition = matrix.getPartition(partIds.get(i));
        PartitionKey partKey = partition.getPartitionKey();
        MatrixPartitionMeta partMeta;
        long startCol = 0;
        long endCol = 0;
        if (useRange) {
            startCol = partKey.getStartCol();
            endCol = partKey.getEndCol();
        }
        partMeta = new MatrixPartitionMeta(partKey.getPartitionId(), partKey.getStartRow(), partKey.getEndRow(), startCol, endCol, partition.getElemNum(), destFile.getName(), streamPos, 0);
        save(partition, partMeta, saveContext, out);
        partMeta.setLength(out.getPos() - streamPos);
        dataFilesMeta.addPartitionMeta(partIds.get(i), partMeta);
    }
    out.flush();
    out.close();
    HdfsUtil.rename(tmpDestFile, destFile, fs);
}

Example 22 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class GetPartHandler method handle.

@Override
public ResponseData handle(RequestHeader header, RequestData data) throws Exception {
    GetPartitionRequest request = (GetPartitionRequest) data;
    ServerPartition partition = MatrixUtils.getPart(context.getMatrixStorageManager(), header.matrixId, header.partId);
    return new GetPartitionResponse(partition);
}

Example 23 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class MatrixUtils method getPart.

public static ServerPartition getPart(MatrixStorageManager storageManager, int matrixId, int partId) {
    ServerMatrix matrix = storageManager.getMatrix(matrixId);
    if (matrix == null) {
        throw new ObjectNotFoundException("Can not find matrix " + matrixId);
    }
    ServerPartition part = matrix.getPartition(partId);
    if (part == null) {
        throw new ObjectNotFoundException("Can not find partition " + partId + " of matrix " + matrixId);
    }
    return part;
}

Example 24 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class MatrixUtils method getRow.

public static ServerRow getRow(MatrixStorageManager storageManager, int matrixId, int partId, int rowId) {
    ServerMatrix matrix = storageManager.getMatrix(matrixId);
    if (matrix == null) {
        throw new ObjectNotFoundException("Can not find matrix " + matrixId);
    }
    ServerPartition part = matrix.getPartition(partId);
    if (part == null) {
        throw new ObjectNotFoundException("Can not find partition " + partId + " of matrix " + matrixId);
    }
    if (!(part instanceof RowBasedPartition)) {
        throw new UnsupportedOperationException("Get row only support for RowBasedPartition");
    }
    ServerRow row = ((RowBasedPartition) part).getRow(rowId);
    if (row == null) {
        throw new ObjectNotFoundException("Can not find row " + rowId + " of matrix " + matrixId + " in partition " + partId);
    }
    return row;
}

Example 25 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class GraphMatrixUtils method getPSLongKeyRow.

public static ServerLongAnyRow getPSLongKeyRow(PSContext psContext, PartitionUpdateParam partParam) {
    ServerMatrix matrix = psContext.getMatrixStorageManager().getMatrix(partParam.getMatrixId());
    ServerPartition part = matrix.getPartition(partParam.getPartKey().getPartitionId());
    return (ServerLongAnyRow) (((RowBasedPartition) part).getRow(0));
}