Examples with ServerIntDoubleRow com.tencent.angel.ps.storage.vector.ServerIntDoubleRow used on opensource projects

Example 1 with ServerIntDoubleRow

use of com.tencent.angel.ps.storage.vector.ServerIntDoubleRow in project angel by Tencent.

the class GBDTGradHistGetRowFunc method merge.

@Override
public GetResult merge(List<PartitionGetResult> partResults) {
    int size = partResults.size();
    List<ServerRow> rowSplits = new ArrayList<ServerRow>(size);
    for (int i = 0; i < size; i++) {
        rowSplits.add(((PartitionGetRowResult) partResults.get(i)).getRowSplit());
    }
    SplitEntry splitEntry = new SplitEntry();
    for (int i = 0; i < size; i++) {
        ServerIntDoubleRow row = (ServerIntDoubleRow) ((PartitionGetRowResult) partResults.get(i)).getRowSplit();
        int fid = (int) row.get(0 + (int) row.getStartCol());
        if (fid != -1) {
            int splitIndex = (int) row.get(1 + (int) row.getStartCol());
            float lossGain = (float) row.get(2 + (int) row.getStartCol());
            float leftSumGrad = (float) row.get(3 + (int) row.getStartCol());
            float leftSumHess = (float) row.get(4 + (int) row.getStartCol());
            float rightSumGrad = (float) row.get(5 + (int) row.getStartCol());
            float rightSumHess = (float) row.get(6 + (int) row.getStartCol());
            LOG.info(String.format("psFunc: the best split after looping a split: fid[%d], fvalue[%d], loss gain[%f]" + ", leftSumGrad[%f], leftSumHess[%f], rightSumGrad[%f], rightSumHess[%f]", fid, splitIndex, lossGain, leftSumGrad, leftSumHess, rightSumGrad, rightSumHess));
            GradStats curLeftGradStat = new GradStats(leftSumGrad, leftSumHess);
            GradStats curRightGradStat = new GradStats(rightSumGrad, rightSumHess);
            SplitEntry curSplitEntry = new SplitEntry(fid, splitIndex, lossGain);
            curSplitEntry.leftGradStat = curLeftGradStat;
            curSplitEntry.rightGradStat = curRightGradStat;
            splitEntry.update(curSplitEntry);
        }
    }
    return new GBDTGradHistGetRowResult(ResponseType.SUCCESS, splitEntry);
}

Example 2 with ServerIntDoubleRow

use of com.tencent.angel.ps.storage.vector.ServerIntDoubleRow in project angel by Tencent.

the class RowSplitCombineUtils method combineServerIntDoubleRowSplits.

private static Vector combineServerIntDoubleRowSplits(List<ServerRow> rowSplits, MatrixMeta matrixMeta, int rowIndex) {
    int colNum = (int) matrixMeta.getColNum();
    int elemNum = 0;
    int size = rowSplits.size();
    for (int i = 0; i < size; i++) {
        elemNum += rowSplits.get(i).size();
    }
    IntDoubleVector row;
    if (elemNum >= (int) (storageConvFactor * colNum)) {
        row = VFactory.denseDoubleVector(colNum);
    } else {
        row = VFactory.sparseDoubleVector(colNum, elemNum);
    }
    row.setMatrixId(matrixMeta.getId());
    row.setRowId(rowIndex);
    Collections.sort(rowSplits, serverRowComp);
    int clock = Integer.MAX_VALUE;
    for (int i = 0; i < size; i++) {
        if (rowSplits.get(i) == null) {
            continue;
        }
        if (rowSplits.get(i).getClock() < clock) {
            clock = rowSplits.get(i).getClock();
        }
        ((ServerIntDoubleRow) rowSplits.get(i)).mergeTo(row);
    }
    row.setClock(clock);
    return row;
}

Example 3 with ServerIntDoubleRow

use of com.tencent.angel.ps.storage.vector.ServerIntDoubleRow in project angel by Tencent.

the class MergeUtils method combineServerIntDoubleRowSplits.

private static Vector combineServerIntDoubleRowSplits(List<ServerRow> rowSplits, MatrixMeta matrixMeta, int rowIndex) {
    int colNum = (int) matrixMeta.getColNum();
    int elemNum = 0;
    int size = rowSplits.size();
    for (int i = 0; i < size; i++) {
        elemNum += rowSplits.get(i).size();
    }
    IntDoubleVector row;
    if (matrixMeta.isHash()) {
        row = VFactory.sparseDoubleVector(colNum, elemNum);
    } else {
        if (elemNum >= (int) (storageConvFactor * colNum)) {
            row = VFactory.denseDoubleVector(colNum);
        } else {
            row = VFactory.sparseDoubleVector(colNum, elemNum);
        }
    }
    row.setMatrixId(matrixMeta.getId());
    row.setRowId(rowIndex);
    Collections.sort(rowSplits, serverRowComp);
    for (int i = 0; i < size; i++) {
        if (rowSplits.get(i) == null) {
            continue;
        }
        ((ServerIntDoubleRow) rowSplits.get(i)).mergeTo(row);
    }
    return row;
}

Example 4 with ServerIntDoubleRow

use of com.tencent.angel.ps.storage.vector.ServerIntDoubleRow in project angel by Tencent.

the class GBDTGradHistGetRowFunc method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    HistAggrParam.HistPartitionAggrParam param = (HistAggrParam.HistPartitionAggrParam) partParam;
    LOG.info("For the gradient histogram of GBDT, we use PS to find the optimal split");
    GBDTParam gbtparam = new GBDTParam();
    gbtparam.numSplit = param.getSplitNum();
    gbtparam.minChildWeight = param.getMinChildWeight();
    gbtparam.regAlpha = param.getRegAlpha();
    gbtparam.regLambda = param.getRegLambda();
    ServerIntDoubleRow row = (ServerIntDoubleRow) psContext.getMatrixStorageManager().getRow(param.getMatrixId(), param.getRowId(), param.getPartKey().getPartitionId());
    SplitEntry splitEntry = GradHistHelper.findSplitOfServerRow(row, gbtparam);
    int fid = splitEntry.getFid();
    int splitIndex = (int) splitEntry.getFvalue();
    double lossGain = splitEntry.getLossChg();
    GradStats leftGradStat = splitEntry.leftGradStat;
    GradStats rightGradStat = splitEntry.rightGradStat;
    double leftSumGrad = leftGradStat.sumGrad;
    double leftSumHess = leftGradStat.sumHess;
    double rightSumGrad = rightGradStat.sumGrad;
    double rightSumHess = rightGradStat.sumHess;
    LOG.info(String.format("split of matrix[%d] part[%d] row[%d]: fid[%d], split index[%d], loss gain[%f], " + "left sumGrad[%f], left sum hess[%f], right sumGrad[%f], right sum hess[%f]", param.getMatrixId(), param.getPartKey().getPartitionId(), param.getRowId(), fid, splitIndex, lossGain, leftSumGrad, leftSumHess, rightSumGrad, rightSumHess));
    int startFid = (int) row.getStartCol() / (2 * gbtparam.numSplit);
    // each split contains 7 doubles
    int sendStartCol = startFid * 7;
    // int sendStartCol = (int) row.getStartCol();
    int sendEndCol = sendStartCol + 7;
    ServerIntDoubleRow sendRow = new ServerIntDoubleRow(param.getRowId(), RowType.T_DOUBLE_DENSE, sendStartCol, sendEndCol, sendEndCol - sendStartCol, RouterType.RANGE);
    LOG.info(String.format("Create server row of split result: row id[%d], start col[%d], end col[%d]", param.getRowId(), sendStartCol, sendEndCol));
    sendRow.set(0 + sendStartCol, fid);
    sendRow.set(1 + sendStartCol, splitIndex);
    sendRow.set(2 + sendStartCol, lossGain);
    sendRow.set(3 + sendStartCol, leftSumGrad);
    sendRow.set(4 + sendStartCol, leftSumHess);
    sendRow.set(5 + sendStartCol, rightSumGrad);
    sendRow.set(6 + sendStartCol, rightSumHess);
    return new PartitionGetRowResult(sendRow);
}