Examples with IntDoubleDenseVectorStorage com.tencent.angel.ml.math2.storage.IntDoubleDenseVectorStorage used on opensource projects

Example 21 with IntDoubleDenseVectorStorage

use of com.tencent.angel.ml.math2.storage.IntDoubleDenseVectorStorage in project angel by Tencent.

the class GBDTController method findSplit.

// find split
public void findSplit() throws Exception {
    LOG.info("------Find split------");
    long startTime = System.currentTimeMillis();
    // 1. find responsible tree node, using RR scheme
    List<Integer> responsibleTNode = new ArrayList<>();
    int activeTNodeNum = 0;
    for (int nid = 0; nid < this.activeNode.length; nid++) {
        int isActive = this.activeNode[nid];
        if (isActive == 1) {
            if (this.taskContext.getTaskIndex() == activeTNodeNum) {
                responsibleTNode.add(nid);
            }
            if (++activeTNodeNum >= taskContext.getTotalTaskNum()) {
                activeTNodeNum = 0;
            }
        }
    }
    int[] tNodeId = Maths.intList2Arr(responsibleTNode);
    LOG.info(String.format("Task[%d] responsible tree node: %s", this.taskContext.getTaskId().getIndex(), responsibleTNode.toString()));
    // 2. pull gradient histogram
    // the updated indices of the parameter on PS
    int[] updatedIndices = new int[tNodeId.length];
    // the updated split features
    int[] updatedSplitFid = new int[tNodeId.length];
    // the updated split value
    double[] updatedSplitFvalue = new double[tNodeId.length];
    // the updated split gain
    double[] updatedSplitGain = new double[tNodeId.length];
    boolean isServerSplit = taskContext.getConf().getBoolean(MLConf.ML_GBDT_SERVER_SPLIT(), MLConf.DEFAULT_ML_GBDT_SERVER_SPLIT());
    int splitNum = taskContext.getConf().getInt(MLConf.ML_GBDT_SPLIT_NUM(), MLConf.DEFAULT_ML_GBDT_SPLIT_NUM());
    for (int i = 0; i < tNodeId.length; i++) {
        int nid = tNodeId[i];
        LOG.debug(String.format("Task[%d] find best split of tree node: %d", this.taskContext.getTaskIndex(), nid));
        // 2.1. get the name of this node's gradient histogram on PS
        String gradHistName = this.param.gradHistNamePrefix + nid;
        // 2.2. pull the histogram
        long pullStartTime = System.currentTimeMillis();
        PSModel histMat = model.getPSModel(gradHistName);
        IntDoubleVector histogram = null;
        SplitEntry splitEntry = null;
        if (isServerSplit) {
            int matrixId = histMat.getMatrixId();
            GBDTGradHistGetRowFunc func = new GBDTGradHistGetRowFunc(new HistAggrParam(matrixId, 0, param.numSplit, param.minChildWeight, param.regAlpha, param.regLambda));
            splitEntry = ((GBDTGradHistGetRowResult) histMat.get(func)).getSplitEntry();
        } else {
            histogram = (IntDoubleVector) histMat.getRow(0);
            LOG.debug("Get grad histogram without server split mode, histogram size" + histogram.getDim());
        }
        LOG.info(String.format("Pull histogram from PS cost %d ms", System.currentTimeMillis() - pullStartTime));
        GradHistHelper histHelper = new GradHistHelper(this, nid);
        // 2.3. find best split result of this tree node
        if (this.param.isServerSplit) {
            // 2.3.1 using server split
            if (splitEntry.getFid() != -1) {
                int trueSplitFid = this.fSet[splitEntry.getFid()];
                int splitIdx = (int) splitEntry.getFvalue();
                float trueSplitValue = this.sketches[trueSplitFid * this.param.numSplit + splitIdx];
                LOG.info(String.format("Best split of node[%d]: feature[%d], value[%f], " + "true feature[%d], true value[%f], losschg[%f]", nid, splitEntry.getFid(), splitEntry.getFvalue(), trueSplitFid, trueSplitValue, splitEntry.getLossChg()));
                splitEntry.setFid(trueSplitFid);
                splitEntry.setFvalue(trueSplitValue);
            }
            // update the grad stats of the root node on PS, only called once by leader worker
            if (nid == 0) {
                GradStats rootStats = new GradStats(splitEntry.leftGradStat);
                rootStats.add(splitEntry.rightGradStat);
                this.updateNodeGradStats(nid, rootStats);
            }
            // update the grad stats of children node
            if (splitEntry.fid != -1) {
                // update the left child
                this.updateNodeGradStats(2 * nid + 1, splitEntry.leftGradStat);
                // update the right child
                this.updateNodeGradStats(2 * nid + 2, splitEntry.rightGradStat);
            }
            // 2.3.2 the updated split result (tree node/feature/value/gain) on PS,
            updatedIndices[i] = nid;
            updatedSplitFid[i] = splitEntry.fid;
            updatedSplitFvalue[i] = splitEntry.fvalue;
            updatedSplitGain[i] = splitEntry.lossChg;
        } else {
            // 2.3.3 otherwise, the returned histogram contains the gradient info
            splitEntry = histHelper.findBestSplit(histogram);
            LOG.info(String.format("Best split of node[%d]: feature[%d], value[%f], losschg[%f]", nid, splitEntry.getFid(), splitEntry.getFvalue(), splitEntry.getLossChg()));
            // 2.3.4 the updated split result (tree node/feature/value/gain) on PS,
            updatedIndices[i] = nid;
            updatedSplitFid[i] = splitEntry.fid;
            updatedSplitFvalue[i] = splitEntry.fvalue;
            updatedSplitGain[i] = splitEntry.lossChg;
        }
        // 2.3.5 reset this tree node's gradient histogram to 0
        histMat.zero();
    }
    // 3. push split feature to PS
    IntIntVector splitFeatureVector = new IntIntVector(this.activeNode.length, new IntIntDenseVectorStorage(this.activeNode.length));
    // 4. push split value to PS
    IntDoubleVector splitValueVector = new IntDoubleVector(this.activeNode.length, new IntDoubleDenseVectorStorage(this.activeNode.length));
    // 5. push split gain to PS
    IntDoubleVector splitGainVector = new IntDoubleVector(this.activeNode.length, new IntDoubleDenseVectorStorage(this.activeNode.length));
    for (int i = 0; i < updatedIndices.length; i++) {
        splitFeatureVector.set(updatedIndices[i], updatedSplitFid[i]);
        splitValueVector.set(updatedIndices[i], updatedSplitFvalue[i]);
        splitGainVector.set(updatedIndices[i], updatedSplitGain[i]);
    }
    PSModel splitFeat = model.getPSModel(this.param.splitFeaturesName);
    splitFeat.increment(this.currentTree, splitFeatureVector);
    PSModel splitValue = model.getPSModel(this.param.splitValuesName);
    splitValue.increment(this.currentTree, splitValueVector);
    PSModel splitGain = model.getPSModel(this.param.splitGainsName);
    splitGain.increment(this.currentTree, splitGainVector);
    // 6. set phase to AFTER_SPLIT
    // this.phase = GBDTPhase.AFTER_SPLIT;
    LOG.info(String.format("Find split cost: %d ms", System.currentTimeMillis() - startTime));
    // clock
    Set<String> needFlushMatrixSet = new HashSet<String>(3);
    needFlushMatrixSet.add(this.param.splitFeaturesName);
    needFlushMatrixSet.add(this.param.splitValuesName);
    needFlushMatrixSet.add(this.param.splitGainsName);
    needFlushMatrixSet.add(this.param.nodeGradStatsName);
    clockAllMatrix(needFlushMatrixSet, true);
}

Example 22 with IntDoubleDenseVectorStorage

use of com.tencent.angel.ml.math2.storage.IntDoubleDenseVectorStorage in project angel by Tencent.

the class GBDTController method runActiveNode.

public void runActiveNode() throws Exception {
    LOG.info("------Run active node------");
    long startTime = System.currentTimeMillis();
    Set<String> needFlushMatrixSet = new HashSet<String>();
    // 1. decide nodes that should be calculated
    Set<Integer> calNodes = new HashSet<>();
    Set<Integer> subNodes = new HashSet<>();
    // 2. decide calculated and subtracted tree nodes
    for (int nid = 0; nid < this.maxNodeNum; nid++) {
        if (this.activeNode[nid] == 1) {
            if (nid == 0) {
                calNodes.add(nid);
            } else {
                int parentNid = (nid - 1) / 2;
                int siblingNid = 4 * parentNid + 3 - nid;
                int sampleNum = this.nodePosEnd[nid] - this.nodePosStart[nid] + 1;
                int siblingSampleNum = this.nodePosEnd[siblingNid] - this.nodePosStart[siblingNid] + 1;
                boolean ltSibling = sampleNum < siblingSampleNum || (sampleNum == siblingSampleNum && nid < siblingNid);
                if (ltSibling) {
                    calNodes.add(nid);
                    subNodes.add(siblingNid);
                } else {
                    calNodes.add(siblingNid);
                    subNodes.add(nid);
                }
            }
        }
    }
    // 3. calculate threads
    Map<Integer, List<Future<Boolean>>> calFutures = new HashMap<>();
    for (int nid : calNodes) {
        histCache[nid] = new IntDoubleVector(this.fSet.length * 2 * this.param.numSplit, new IntDoubleDenseVectorStorage(new double[this.param.numFeature * 2 * this.param.numSplit]));
        calFutures.put(nid, new ArrayList<>());
        int nodeStart = this.nodePosStart[nid];
        int nodeEnd = this.nodePosEnd[nid];
        int batchNum = (nodeEnd - nodeStart + 1) / this.param.batchSize + ((nodeEnd - nodeStart + 1) % this.param.batchSize == 0 ? 0 : 1);
        LOG.info(String.format("Node[%d], start[%d], end[%d], batch[%d]", nid, nodeStart, nodeEnd, batchNum));
        for (int batch = 0; batch < batchNum; batch++) {
            int start = nodeStart + batch * this.param.batchSize;
            int end = nodeStart + (batch + 1) * this.param.batchSize;
            if (end > nodeEnd) {
                end = nodeEnd;
            }
            LOG.info(String.format("Calculate thread: nid[%d], start[%d], end[%d]", nid, start, end));
            Future<Boolean> future = this.threadPool.submit(new HistCalThread(this, nid, start, end));
            calFutures.get(nid).add(future);
        }
    }
    // wait until all threads finish
    for (int nid : calNodes) {
        for (Future<Boolean> future : calFutures.get(nid)) {
            future.get();
        }
    }
    // 4. subtract threads
    Map<Integer, Future<Boolean>> subFutures = new HashMap<>();
    for (int nid : subNodes) {
        int parentId = (nid - 1) / 2;
        histCache[nid] = histCache[parentId].clone();
        LOG.info(String.format("Subtract thread: nid[%d]", nid));
        Future<Boolean> future = this.threadPool.submit(new HistSubThread(this, nid));
        subFutures.put(nid, future);
    }
    // wait until all threads finish
    for (int nid : subNodes) {
        subFutures.get(nid).get();
    }
    // 5. send histograms to PS
    Set<Integer> pushNodes = new HashSet<>(calNodes);
    pushNodes.addAll(subNodes);
    int bytesPerItem = this.taskContext.getConf().getInt(MLConf.ANGEL_COMPRESS_BYTES(), MLConf.DEFAULT_ANGEL_COMPRESS_BYTES());
    if (bytesPerItem < 1 || bytesPerItem > 8) {
        LOG.info("Invalid compress configuration: " + bytesPerItem + ", it should be [1,8].");
        bytesPerItem = MLConf.DEFAULT_ANGEL_COMPRESS_BYTES();
    }
    for (int nid : pushNodes) {
        pushHistogram(nid, bytesPerItem);
        needFlushMatrixSet.add(this.param.gradHistNamePrefix + nid);
    }
    // 6. update histogram cache
    for (int nid : calNodes) {
        if (nid == 0)
            break;
        int parentId = (nid - 1) / 2;
        this.histCache[parentId] = null;
    }
    LOG.info(String.format("Run active node cost: %d ms", System.currentTimeMillis() - startTime));
    // clock
    clockAllMatrix(needFlushMatrixSet, true);
}

Example 23 with IntDoubleDenseVectorStorage

use of com.tencent.angel.ml.math2.storage.IntDoubleDenseVectorStorage in project angel by Tencent.

the class GradHistHelper method buildHistogram.

public IntDoubleVector buildHistogram(int insStart, int insEnd) {
    // 1. new feature's histogram (grad + hess)
    // size: sampled_featureNum * (2 * splitNum)
    // in other words, concatenate each feature's histogram
    int featureNum = this.controller.fSet.length;
    int splitNum = this.controller.param.numSplit;
    IntDoubleVector histogram = new IntDoubleVector(featureNum * 2 * splitNum, new IntDoubleDenseVectorStorage(new double[featureNum * 2 * splitNum]));
    // 2. get the span of this node
    int nodeStart = insStart;
    // inclusive
    int nodeEnd = insEnd;
    LOG.debug(String.format("Build histogram of node[%d]: size[%d] instance span [%d - %d]", this.nid, histogram.getDim(), nodeStart, nodeEnd));
    // ------ 3. using sparse-aware method to build histogram ---
    // first add grads of all instances to the zero bin of all features, then loop the non-zero entries of all the instances
    float gradSum = 0.0f;
    float hessSum = 0.0f;
    long parseInstanceTime = 0;
    long startTime = System.currentTimeMillis();
    for (int idx = nodeStart; idx <= nodeEnd; idx++) {
        // 3.1. get the instance index
        int insIdx = this.controller.instancePos[idx];
        // 3.2. get the grad and hess of the instance
        GradPair gradPair = this.controller.gradPairs[insIdx];
        // 3.3. add to the sum
        gradSum += gradPair.getGrad();
        hessSum += gradPair.getHess();
        IntFloatVector instance = this.controller.trainDataStore.instances[insIdx];
        int numNnz = instance.getStorage().getIndices().length;
        long tmpTime = System.currentTimeMillis();
        int[] indices = instance.getStorage().getIndices();
        float[] values = instance.getStorage().getValues();
        parseInstanceTime += System.currentTimeMillis() - tmpTime;
        // 3.4. loop the non-zero entries
        for (int i = 0; i < numNnz; i++) {
            int fid = indices[i];
            // 3.4.1. get feature value
            float fv = values[i];
            // 3.4.2. current feature's position in the sampled feature set
            // int fPos = findFidPlace(this.controller.fSet, fid);
            int fPos = this.controller.fPos[fid];
            if (fPos == -1) {
                continue;
            }
            // 3.4.3. find the position of feature value in a histogram
            // the search area in the sketch is [fid * #splitNum, (fid+1) * #splitNum - 1]
            int start = fid * splitNum;
            // inclusive
            int end;
            if (this.controller.cateFeatNum.containsKey(fid)) {
                end = start + this.controller.cateFeatNum.get(fid) - 1;
            } else {
                end = start + splitNum - 1;
            }
            int fValueIdx = findFvaluePlace(this.controller.sketches, fv, start, end);
            assert fValueIdx >= 0 && fValueIdx < splitNum;
            int gradIdx = 2 * splitNum * fPos + fValueIdx;
            int hessIdx = gradIdx + splitNum;
            // 3.4.4. add the grad and hess to the corresponding bin
            histogram.set(gradIdx, histogram.get(gradIdx) + gradPair.getGrad());
            histogram.set(hessIdx, histogram.get(hessIdx) + gradPair.getHess());
            // 3.4.5. add the reverse to the bin that contains 0.0f
            int fZeroValueIdx = findFvaluePlace(this.controller.sketches, 0.0f, start, end);
            assert fZeroValueIdx >= 0 && fZeroValueIdx < splitNum;
            int gradZeroIdx = 2 * splitNum * fPos + fZeroValueIdx;
            int hessZeroIdx = gradZeroIdx + splitNum;
            double curGrad = histogram.get(gradZeroIdx);
            double curHess = histogram.get(hessZeroIdx);
            histogram.set(gradZeroIdx, curGrad - gradPair.getGrad());
            histogram.set(hessZeroIdx, curHess - gradPair.getHess());
        }
    }
    // 4. add the grad and hess sum to the zero bin of all features
    for (int fid = 0; fid < featureNum; fid++) {
        int fPos = findFidPlace(this.controller.fSet, fid);
        if (fPos == -1) {
            continue;
        }
        int start = fPos * splitNum;
        int end;
        if (this.controller.cateFeatNum.containsKey(fid)) {
            end = start + this.controller.cateFeatNum.get(fid) - 1;
        } else {
            end = start + splitNum - 1;
        }
        int fZeroValueIdx = findFvaluePlace(this.controller.sketches, 0.0f, start, end);
        int gradZeroIdx = 2 * splitNum * fPos + fZeroValueIdx;
        int hessZeroIdx = 2 * splitNum * fPos + fZeroValueIdx + splitNum;
        histogram.set(gradZeroIdx, histogram.get(gradZeroIdx) + gradSum);
        histogram.set(hessZeroIdx, histogram.get(hessZeroIdx) + hessSum);
    }
    LOG.debug(String.format("Build histogram cost %d ms, parse instance cost %d ms", System.currentTimeMillis() - startTime, parseInstanceTime));
    return histogram;
}

Example 24 with IntDoubleDenseVectorStorage

use of com.tencent.angel.ml.math2.storage.IntDoubleDenseVectorStorage in project angel by Tencent.

the class HashRouterUtils method splitIntDoubleVector.

public static void splitIntDoubleVector(KeyHash hasher, MatrixMeta matrixMeta, IntDoubleVector vector, KeyValuePart[] dataParts) {
    int dataPartNum = dataParts.length;
    int dataPartNumMinus1 = dataPartNum - 1;
    if (isPow2(dataPartNum)) {
        IntDoubleVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            IntDoubleSparseVectorStorage sparseStorage = (IntDoubleSparseVectorStorage) storage;
            ObjectIterator<Int2DoubleMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Int2DoubleMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getIntKey()) & dataPartNumMinus1;
                ((HashIntKeysDoubleValuesPart) dataParts[partId]).add(keyValue.getIntKey(), keyValue.getDoubleValue());
            }
        } else if (storage.isDense()) {
            // Get values
            IntDoubleDenseVectorStorage denseStorage = (IntDoubleDenseVectorStorage) storage;
            double[] values = denseStorage.getValues();
            for (int i = 0; i < values.length; i++) {
                int partId = computeHashCode(hasher, i) & dataPartNumMinus1;
                ((HashIntKeysDoubleValuesPart) dataParts[partId]).add(i, values[i]);
            }
        } else {
            // Key and value array pair
            IntDoubleSortedVectorStorage sortStorage = (IntDoubleSortedVectorStorage) storage;
            int[] keys = sortStorage.getIndices();
            double[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) & dataPartNumMinus1;
                ((HashIntKeysDoubleValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    } else {
        IntDoubleVectorStorage storage = vector.getStorage();
        if (storage.isSparse()) {
            // Use iterator
            IntDoubleSparseVectorStorage sparseStorage = (IntDoubleSparseVectorStorage) storage;
            ObjectIterator<Int2DoubleMap.Entry> iter = sparseStorage.entryIterator();
            while (iter.hasNext()) {
                Int2DoubleMap.Entry keyValue = iter.next();
                int partId = computeHashCode(hasher, keyValue.getIntKey()) % dataPartNum;
                ((HashIntKeysDoubleValuesPart) dataParts[partId]).add(keyValue.getIntKey(), keyValue.getDoubleValue());
            }
        } else if (storage.isDense()) {
            // Get values
            IntDoubleDenseVectorStorage denseStorage = (IntDoubleDenseVectorStorage) storage;
            double[] values = denseStorage.getValues();
            for (int i = 0; i < values.length; i++) {
                int partId = computeHashCode(hasher, i) % dataPartNum;
                ((HashIntKeysDoubleValuesPart) dataParts[partId]).add(i, values[i]);
            }
        } else {
            // Key and value array pair
            IntDoubleSortedVectorStorage sortStorage = (IntDoubleSortedVectorStorage) storage;
            int[] keys = sortStorage.getIndices();
            double[] values = sortStorage.getValues();
            for (int i = 0; i < keys.length; i++) {
                int partId = computeHashCode(hasher, keys[i]) % dataPartNum;
                ((HashIntKeysDoubleValuesPart) dataParts[partId]).add(keys[i], values[i]);
            }
        }
    }
}