Examples with ServerPartition com.tencent.angel.ps.storage.partition.ServerPartition used on opensource projects

Example 16 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class GetNodes method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    ServerPartition part = psContext.getMatrixStorageManager().getPart(partParam.getPartKey());
    ServerLongAnyRow row = (ServerLongAnyRow) psContext.getMatrixStorageManager().getRow(partParam.getPartKey(), 0);
    LongArrayList ret = new LongArrayList();
    row.startRead();
    try {
        ObjectIterator<Long2ObjectMap.Entry<IElement>> it = row.iterator();
        while (it.hasNext()) {
            Long2ObjectMap.Entry<IElement> entry = it.next();
            ret.add(entry.getLongKey() + partParam.getPartKey().getStartCol());
        }
    } finally {
        row.endRead();
    }
    return new IndexPartGetLongResult(part.getPartitionKey(), ret.toLongArray());
}

Example 17 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class GetNodes method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    ServerPartition part = psContext.getMatrixStorageManager().getPart(partParam.getPartKey());
    ServerRow ranks = psContext.getMatrixStorageManager().getRow(partParam.getPartKey(), 2);
    FloatVector ranksVector = ServerRowUtils.getVector((ServerLongFloatRow) ranks);
    long[] ret;
    if (ranksVector instanceof IntFloatVector)
        ret = gatherNodes((IntFloatVector) ranksVector, part.getPartitionKey().getStartCol());
    else if (ranksVector instanceof LongFloatVector)
        ret = gatherNodes((LongFloatVector) ranksVector, part.getPartitionKey().getStartCol());
    else {
        throw new AngelException("vector should be intfloat or longfloat but is " + ranksVector.getClass().getName());
    }
    return new IndexPartGetLongResult(part.getPartitionKey(), ret);
}

Example 18 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class GetOutDegreeFunc method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    PartGetOutDegreeParam param = (PartGetOutDegreeParam) partParam;
    ServerMatrix matrix = psContext.getMatrixStorageManager().getMatrix(partParam.getMatrixId());
    ServerPartition part = matrix.getPartition(partParam.getPartKey().getPartitionId());
    ServerLongIntRow row = (ServerLongIntRow) (((RowBasedPartition) part).getRow(0));
    long[] nodeIds = param.getNodeIds();
    int[] outDegrees = new int[nodeIds.length];
    for (int i = 0; i < nodeIds.length; i++) {
        outDegrees[i] = row.get(nodeIds[i]);
    }
    return new PartGetOutDegreeResult(part.getPartitionKey().getPartitionId(), outDegrees);
}

Example 19 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class ParameterServer method heartbeat.

private void heartbeat() throws Exception {
    PSReportRequest.Builder builder = PSReportRequest.newBuilder();
    builder.setPsAttemptId(attemptIdProto);
    if (heartbeatCount.incrementAndGet() % dataCollectionInterval == 0) {
        // calculate data size of all partitions of ps
        Map<Integer, ServerMatrix> serverMatrixMap = matrixStorageManager.getMatrices();
        long dataSize = 0;
        for (ServerMatrix serverMatrix : serverMatrixMap.values()) {
            Map<Integer, ServerPartition> partitions = serverMatrix.getPartitions();
            for (ServerPartition partition : partitions.values()) {
                dataSize += partition.dataSize();
            }
        }
        Pair.Builder pairBuilder = Pair.newBuilder();
        pairBuilder.setKey("key");
        pairBuilder.setValue("value");
        builder.addMetrics(pairBuilder.build());
        // totalRPC
        pairBuilder.setKey("totalRPC");
        pairBuilder.setValue(WorkerPool.total.toString());
        builder.addMetrics(pairBuilder.build());
        // request size
        pairBuilder.setKey("requestSize");
        pairBuilder.setValue(String.format("%.2f", WorkerPool.requestSize.longValue() * 1.0 / 1024 / 1024));
        builder.addMetrics(pairBuilder.build());
        // data size
        pairBuilder.setKey("dataSize");
        pairBuilder.setValue(String.format("%.2f", dataSize * 1.0 / 1024 / 1024));
        builder.addMetrics(pairBuilder.build());
    }
    builder.addAllMatrixReports(buildMatrixReports());
    PSReportResponse ret;
    PSReportRequest request = builder.build();
    LOG.debug("ps hb = " + request);
    ret = master.psReport(request);
    switch(ret.getPsCommand()) {
        case PSCOMMAND_REGISTER:
            try {
                register();
            } catch (Exception x) {
                LOG.error("register failed: ", x);
                stop(-1);
            }
            break;
        case PSCOMMAND_SHUTDOWN:
            LOG.error("shutdown command come from appmaster, exit now!!");
            stop(-1);
            break;
        default:
            break;
    }
    LOG.debug("ps hb ret = " + ret);
    if (ret.hasNeedSaveMatrices()) {
        saver.save(ProtobufUtil.convert(ret.getNeedSaveMatrices()));
    }
    if (ret.hasNeedLoadMatrices()) {
        loader.load(ProtobufUtil.convert(ret.getNeedLoadMatrices()));
    }
    syncMatrices(ret.getNeedCreateMatricesList(), ret.getNeedReleaseMatrixIdsList(), ret.getNeedRecoverPartsList());
}

Example 20 with ServerPartition

use of com.tencent.angel.ps.storage.partition.ServerPartition in project angel by Tencent.

the class MatrixFormatImpl method load.

private void load(ServerMatrix matrix, Path matrixPath, FileSystem fs, List<Integer> partIds, PSMatrixLoadContext loadContext, int startPos, int endPos, PSMatrixFilesMeta dataFilesMeta) throws IOException {
    ServerPartition partition;
    FSDataInputStream input = null;
    long offset = 0;
    String currentFileName = "";
    for (int i = startPos; i < endPos; i++) {
        partition = matrix.getPartition(partIds.get(i));
        MatrixPartitionMeta partMeta = dataFilesMeta.getPartitionMeta(partIds.get(i));
        String fileName = partMeta.getFileName();
        offset = partMeta.getOffset();
        if (!fileName.equals(currentFileName)) {
            currentFileName = fileName;
            if (input != null) {
                input.close();
            }
            input = fs.open(new Path(matrixPath, currentFileName));
        }
        input.seek(offset);
        load(partition, partMeta, loadContext, input);
    }
    if (input != null) {
        input.close();
    }
}