Examples with ServerMatrix com.tencent.angel.ps.storage.matrix.ServerMatrix used on opensource projects

Example 21 with ServerMatrix

use of com.tencent.angel.ps.storage.matrix.ServerMatrix in project angel by Tencent.

the class GetSort method partitionUpdate.

@Override
public void partitionUpdate(PartitionUpdateParam partParam) {
    ServerMatrix matrix = psContext.getMatrixStorageManager().getMatrix(partParam.getMatrixId());
    RowBasedPartition part = (RowBasedPartition) matrix.getPartition(partParam.getPartKey().getPartitionId());
    ServerLongAnyRow row = (ServerLongAnyRow) part.getRow(0);
    ObjectIterator<Long2ObjectMap.Entry<IElement>> it = row.getStorage().iterator();
    row.startWrite();
    try {
        while (it.hasNext()) {
            Long2ObjectMap.Entry<IElement> next = it.next();
            DynamicNeighborElement ele = (DynamicNeighborElement) next.getValue();
            if (ele != null) {
                ele.trans();
            }
        }
    } finally {
        row.endWrite();
    }
}

Example 22 with ServerMatrix

use of com.tencent.angel.ps.storage.matrix.ServerMatrix in project angel by Tencent.

the class GetOutDegreeFunc method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    PartGetOutDegreeParam param = (PartGetOutDegreeParam) partParam;
    ServerMatrix matrix = psContext.getMatrixStorageManager().getMatrix(partParam.getMatrixId());
    ServerPartition part = matrix.getPartition(partParam.getPartKey().getPartitionId());
    ServerLongIntRow row = (ServerLongIntRow) (((RowBasedPartition) part).getRow(0));
    long[] nodeIds = param.getNodeIds();
    int[] outDegrees = new int[nodeIds.length];
    for (int i = 0; i < nodeIds.length; i++) {
        outDegrees[i] = row.get(nodeIds[i]);
    }
    return new PartGetOutDegreeResult(part.getPartitionKey().getPartitionId(), outDegrees);
}

Example 23 with ServerMatrix

use of com.tencent.angel.ps.storage.matrix.ServerMatrix in project angel by Tencent.

the class ParameterServer method heartbeat.

private void heartbeat() throws Exception {
    PSReportRequest.Builder builder = PSReportRequest.newBuilder();
    builder.setPsAttemptId(attemptIdProto);
    if (heartbeatCount.incrementAndGet() % dataCollectionInterval == 0) {
        // calculate data size of all partitions of ps
        Map<Integer, ServerMatrix> serverMatrixMap = matrixStorageManager.getMatrices();
        long dataSize = 0;
        for (ServerMatrix serverMatrix : serverMatrixMap.values()) {
            Map<Integer, ServerPartition> partitions = serverMatrix.getPartitions();
            for (ServerPartition partition : partitions.values()) {
                dataSize += partition.dataSize();
            }
        }
        Pair.Builder pairBuilder = Pair.newBuilder();
        pairBuilder.setKey("key");
        pairBuilder.setValue("value");
        builder.addMetrics(pairBuilder.build());
        // totalRPC
        pairBuilder.setKey("totalRPC");
        pairBuilder.setValue(WorkerPool.total.toString());
        builder.addMetrics(pairBuilder.build());
        // request size
        pairBuilder.setKey("requestSize");
        pairBuilder.setValue(String.format("%.2f", WorkerPool.requestSize.longValue() * 1.0 / 1024 / 1024));
        builder.addMetrics(pairBuilder.build());
        // data size
        pairBuilder.setKey("dataSize");
        pairBuilder.setValue(String.format("%.2f", dataSize * 1.0 / 1024 / 1024));
        builder.addMetrics(pairBuilder.build());
    }
    builder.addAllMatrixReports(buildMatrixReports());
    PSReportResponse ret;
    PSReportRequest request = builder.build();
    LOG.debug("ps hb = " + request);
    ret = master.psReport(request);
    switch(ret.getPsCommand()) {
        case PSCOMMAND_REGISTER:
            try {
                register();
            } catch (Exception x) {
                LOG.error("register failed: ", x);
                stop(-1);
            }
            break;
        case PSCOMMAND_SHUTDOWN:
            LOG.error("shutdown command come from appmaster, exit now!!");
            stop(-1);
            break;
        default:
            break;
    }
    LOG.debug("ps hb ret = " + ret);
    if (ret.hasNeedSaveMatrices()) {
        saver.save(ProtobufUtil.convert(ret.getNeedSaveMatrices()));
    }
    if (ret.hasNeedLoadMatrices()) {
        loader.load(ProtobufUtil.convert(ret.getNeedLoadMatrices()));
    }
    syncMatrices(ret.getNeedCreateMatricesList(), ret.getNeedReleaseMatrixIdsList(), ret.getNeedRecoverPartsList());
}

Example 24 with ServerMatrix

use of com.tencent.angel.ps.storage.matrix.ServerMatrix in project angel by Tencent.

the class InitNeighborOver method partitionUpdate.

@Override
public void partitionUpdate(PartitionUpdateParam param) {
    ServerMatrix matrix = psContext.getMatrixStorageManager().getMatrix(param.getMatrixId());
    CSRPartition part = (CSRPartition) matrix.getPartition(param.getPartKey().getPartitionId());
    IntCSRStorage storage = (IntCSRStorage) part.getStorage();
    int startCol = (int) param.getPartKey().getStartCol();
    synchronized (storage) {
        // No data in this partition
        if (storage.getTempRowIds() == null) {
            return;
        }
        // Get total neighbor number
        int[] rowOffsets = storage.getRowOffsets();
        int accumOffset = 0;
        for (int i = 0; i < rowOffsets.length - 1; i++) {
            int offset = rowOffsets[i];
            rowOffsets[i] = accumOffset;
            accumOffset += offset;
        }
        rowOffsets[rowOffsets.length - 1] = accumOffset;
        // Final matrix column indices: neighbors node ids
        int[] cloumnIndices = new int[accumOffset];
        // Write positions in cloumnIndices for nodes
        int[] copyOffsets = new int[rowOffsets.length - 1];
        System.arraycopy(rowOffsets, 0, copyOffsets, 0, rowOffsets.length - 1);
        List<int[]> tempRowIds = storage.getTempRowIds();
        List<int[]> tempRowLens = storage.getTempRowLens();
        List<int[]> tempColumnIndices = storage.getTempColumnIndices();
        // Copy all cached sub column indices to final column indices
        int size = tempRowIds.size();
        for (int i = 0; i < size; i++) {
            // Read position for a sub column indices
            int copyLen = 0;
            int[] subTempRowIds = tempRowIds.get(i);
            int[] subTempLens = tempRowLens.get(i);
            int[] subTempColumnIndices = tempColumnIndices.get(i);
            for (int j = 0; j < subTempRowIds.length; j++) {
                // Copy column indices for a node to final column indices
                System.arraycopy(subTempColumnIndices, copyLen, cloumnIndices, copyOffsets[subTempRowIds[j] - startCol], subTempLens[j]);
                // Update write position for this node in final column indices
                copyOffsets[subTempRowIds[j] - startCol] += subTempLens[j];
                // Update the read position in sub column indices
                copyLen += subTempLens[j];
            }
        }
        storage.setColumnIndices(cloumnIndices);
        // Clear all temp data
        storage.setTempRowIds(null);
        storage.setTempRowLens(null);
        storage.setTempColumnIndices(null);
    }
}

Example 25 with ServerMatrix

use of com.tencent.angel.ps.storage.matrix.ServerMatrix in project angel by Tencent.

the class InitAliasTable method partitionUpdate.

@Override
public void partitionUpdate(PartitionUpdateParam partParam) {
    InitAliasTablePartParam param = (InitAliasTablePartParam) partParam;
    ServerMatrix matrix = psContext.getMatrixStorageManager().getMatrix(partParam.getMatrixId());
    RowBasedPartition part = (RowBasedPartition) matrix.getPartition(partParam.getPartKey().getPartitionId());
    ServerLongAnyRow row = (ServerLongAnyRow) part.getRow(0);
    ObjectIterator<Long2ObjectMap.Entry<AliasElement>> iter = param.getNodeId2Neighbors().long2ObjectEntrySet().iterator();
    row.startWrite();
    try {
        while (iter.hasNext()) {
            Long2ObjectMap.Entry<AliasElement> entry = iter.next();
            AliasElement element = entry.getValue();
            if (element == null) {
                row.set(entry.getLongKey(), null);
            } else {
                row.set(entry.getLongKey(), new AliasElement(element.getNeighborIds(), element.getAccept(), element.getAlias()));
            }
        }
    } finally {
        row.endWrite();
    }
}