Examples with ServerPartition com.tencent.angel.ps.storage.partition.ServerPartition used on opensource projects

Example 1 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class ServerMatrix method init.

public void init() {
    MatrixMeta matrixMeta = context.getMatrixMetaManager().getMatrixMeta(matrixId);
    Map<Integer, PartitionMeta> partMetas = matrixMeta.getPartitionMetas();
    String sourceClass = matrixMeta.getAttribute(AngelConf.ANGEL_PS_PARTITION_SOURCE_CLASS, AngelConf.DEFAULT_ANGEL_PS_PARTITION_SOURCE_CLASS);
    // Get server partition class
    Class<? extends IServerPartition> partClass;
    try {
        partClass = matrixMeta.getPartitionClass();
        // If partition class is not set, just use the default partition class
        if (partClass == null) {
            partClass = MatrixConf.DEFAULT_SERVER_PARTITION_CLASS;
        }
    } catch (Throwable e) {
        LOG.fatal("Server partition class failed ", e);
        throw new RuntimeException(e);
    }
    // Get server partition storage class type
    Class<? extends IServerPartitionStorage> storageClass;
    try {
        storageClass = matrixMeta.getPartitionStorageClass();
    } catch (Throwable e) {
        LOG.fatal("Server partition class failed ", e);
        throw new RuntimeException(e);
    }
    RowType rowType = matrixMeta.getRowType();
    // Get value class
    Class<? extends IElement> valueClass = null;
    if (rowType.isComplexValue()) {
        try {
            valueClass = matrixMeta.getValueClass();
        } catch (Throwable e) {
            LOG.fatal("Init value class failed ", e);
            throw new RuntimeException(e);
        }
        if (valueClass == null) {
            throw new RuntimeException("Complex type must set value type class!!");
        }
    }
    for (PartitionMeta partMeta : partMetas.values()) {
        ServerPartition part = ServerPartitionFactory.getPartition(partMeta.getPartitionKey(), partClass, storageClass, matrixMeta.getRowType(), valueClass, matrixMeta.getValidIndexNumInOnePart(), matrixMeta.isHash() ? RouterType.HASH : RouterType.RANGE);
        partitionMaps.put(partMeta.getPartId(), part);
        part.init();
        part.setState(PartitionState.READ_AND_WRITE);
    }
}

Example 2 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class BasicStreamUpdateHandler method handle.

@Override
public ResponseData handle(RequestHeader header, RequestData data) throws Exception {
    UpdateRequest request = (UpdateRequest) data;
    ServerPartition part = MatrixUtils.getPart(context.getMatrixStorageManager(), header.matrixId, header.partId);
    ByteBuf in = request.getInputBuffer();
    // Filter comp key value
    ByteBufSerdeUtils.deserializeBoolean(in);
    part.update(in, request.getOp());
    return new UpdateResponse();
}

Example 3 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class MatrixFormatImpl method load.

@Override
public void load(ServerMatrix matrix, PSMatrixLoadContext loadContext, Configuration conf) throws IOException {
    LOG.info("load matrix " + matrix.getName() + " from path " + loadContext.getLoadPath());
    Path matrixFilesPath = new Path(loadContext.getLoadPath());
    FileSystem fs = matrixFilesPath.getFileSystem(conf);
    if (!fs.exists(matrixFilesPath)) {
        LOG.error("Can not find matrix " + matrix.getName() + " in directory " + loadContext.getLoadPath());
        throw new IOException("Can not find matrix " + matrix.getName() + " in directory " + loadContext.getLoadPath());
    // matrix.startServering();
    // return;
    }
    // Read matrix meta from meta file
    PSMatrixFilesMeta psMatrixFilesMeta = new PSMatrixFilesMeta(matrix.getId());
    Map<Integer, ServerPartition> partitionMaps = matrix.getPartitions();
    List<MatrixPartitionMeta> partFileMetas = new ArrayList<>(partitionMaps.size());
    Path metaFilePath = new Path(matrixFilesPath, ModelFilesConstent.modelMetaFileName);
    if (fs.exists(metaFilePath)) {
        FSDataInputStream input = fs.open(metaFilePath);
        MatrixFilesMeta matrixFilesMeta = new MatrixFilesMeta();
        try {
            matrixFilesMeta.read(input);
        } catch (Throwable e) {
            throw new IOException("Read meta failed ", e);
        } finally {
            input.close();
        }
        Map<Integer, MatrixPartitionMeta> partIdToFileMetaMap = matrixFilesMeta.getPartMetas();
        for (int partId : partitionMaps.keySet()) {
            partFileMetas.add(partIdToFileMetaMap.get(partId));
            psMatrixFilesMeta.addPartitionMeta(partId, partIdToFileMetaMap.get(partId));
        }
    } else {
        Path psMetaFilePath = new Path(matrixFilesPath, ModelFilesConstent.psModelMetaFileName);
        if (fs.exists(psMetaFilePath)) {
            FSDataInputStream input = fs.open(psMetaFilePath);
            try {
                psMatrixFilesMeta.read(input);
            } catch (Throwable e) {
                throw new IOException("Read meta failed ", e);
            } finally {
                input.close();
            }
            Map<Integer, MatrixPartitionMeta> partIdToFileMetaMap = psMatrixFilesMeta.getPartMetas();
            for (int partId : partitionMaps.keySet()) {
                partFileMetas.add(partIdToFileMetaMap.get(partId));
                psMatrixFilesMeta.addPartitionMeta(partId, partIdToFileMetaMap.get(partId));
            }
        } else {
            LOG.warn("Can not find matrix meta file in directory " + loadContext.getLoadPath());
            matrix.startServering();
            return;
        }
    }
    Collections.sort(partFileMetas, new Comparator<MatrixPartitionMeta>() {

        @Override
        public int compare(MatrixPartitionMeta p1, MatrixPartitionMeta p2) {
            if (p1.getFileName().compareTo(p2.getFileName()) < 0) {
                return -1;
            } else if (p1.getFileName().compareTo(p2.getFileName()) > 0) {
                return 1;
            } else {
                return (int) (p1.getOffset() - p2.getOffset());
            }
        }
    });
    int size = partFileMetas.size();
    List<Integer> parts = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        parts.add(partFileMetas.get(i).getPartId());
    }
    // Load partitions from file use fork-join
    Vector<String> errorLogs = new Vector<>();
    int maxPartsInSingleFile = conf.getInt(AngelConf.ANGEL_PS_MAX_PARTITION_NUM_SINGLE_FILE, AngelConf.DEFAULT_ANGEL_PS_MAX_PARTITION_NUM_SINGLE_FILE);
    PartitionDiskOp loadOp = new PartitionDiskOp(matrix, fs, matrixFilesPath, ACTION.LOAD, parts, loadContext, psMatrixFilesMeta, errorLogs, 0, parts.size(), maxPartsInSingleFile);
    loadContext.getWorkers().execute(loadOp);
    loadOp.join();
    if (!errorLogs.isEmpty()) {
        String errorLog = "load partitions for matrix " + matrix.getName() + " failed, error log is " + StringUtils.join("\n", errorLogs);
        LOG.error(errorLog);
        throw new IOException(errorLog);
    }
}

Example 4 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class GetNodeAttrs method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    PartGetNodeAttrsParam param = (PartGetNodeAttrsParam) partParam;
    ServerMatrix matrix = psContext.getMatrixStorageManager().getMatrix(partParam.getMatrixId());
    ServerPartition part = matrix.getPartition(partParam.getPartKey().getPartitionId());
    ServerLongAnyRow row = (ServerLongAnyRow) (((RowBasedPartition) part).getRow(0));
    long[] nodeIds = param.getNodeIds();
    float[][] attrs = new float[nodeIds.length][];
    int count = param.getCount();
    Random r = new Random();
    for (int i = 0; i < nodeIds.length; i++) {
        long nodeId = nodeIds[i];
        // Get node neighbor number
        FloatArrayElement element = (FloatArrayElement) (row.get(nodeId));
        if (element == null) {
            attrs[i] = null;
        } else {
            float[] nodeAttrs = element.getData();
            if (nodeAttrs == null || nodeAttrs.length == 0) {
                attrs[i] = null;
            } else if (count <= 0 || nodeAttrs.length <= count) {
                attrs[i] = nodeAttrs;
            } else {
                attrs[i] = new float[count];
                // If the neighbor number > count, just copy a range of neighbors to the result array, the copy position is random
                int startPos = Math.abs(r.nextInt()) % nodeAttrs.length;
                if (startPos + count <= nodeAttrs.length) {
                    System.arraycopy(nodeAttrs, startPos, attrs[i], 0, count);
                } else {
                    System.arraycopy(nodeAttrs, startPos, attrs[i], 0, nodeAttrs.length - startPos);
                    System.arraycopy(nodeAttrs, 0, attrs[i], nodeAttrs.length - startPos, count - (nodeAttrs.length - startPos));
                }
            }
        }
    }
    return new PartGetNodeAttrsResult(part.getPartitionKey().getPartitionId(), attrs);
}

Example 5 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class GetNodeFeats method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    PartGetNodeFeatsParam param = (PartGetNodeFeatsParam) partParam;
    ServerMatrix matrix = psContext.getMatrixStorageManager().getMatrix(partParam.getMatrixId());
    ServerPartition part = matrix.getPartition(partParam.getPartKey().getPartitionId());
    ServerLongAnyRow row = (ServerLongAnyRow) (((RowBasedPartition) part).getRow(0));
    long[] nodeIds = param.getNodeIds();
    IntFloatVector[] feats = new IntFloatVector[nodeIds.length];
    for (int i = 0; i < nodeIds.length; i++) {
        if (row.get(nodeIds[i]) == null) {
            continue;
        }
        feats[i] = ((Node) (row.get(nodeIds[i]))).getFeats();
    }
    return new PartGetNodeFeatsResult(part.getPartitionKey().getPartitionId(), feats);
}