Examples with HierarchyNode com.linkedin.thirdeye.client.diffsummary.HierarchyNode used on opensource projects

Example 1 with HierarchyNode

use of com.linkedin.thirdeye.client.diffsummary.HierarchyNode in project pinot by linkedin.

the class Summary method recomputeCostAndRemoveSmallNodes.

/**
   * Recompute costs of the nodes in a DPArray using targetRatio for calculating the cost.
   */
private void recomputeCostAndRemoveSmallNodes(HierarchyNode parentNode, DPArray dp, double targetRatio) {
    Set<HierarchyNode> removedNodes = new HashSet<>(dp.getAnswer());
    List<HierarchyNode> ans = new ArrayList<>(dp.getAnswer());
    ans.sort(NODE_COMPARATOR);
    dp.reset();
    for (HierarchyNode node : ans) {
        insertRowWithAdaptiveRatioNoOneSideError(dp, node, targetRatio);
    }
    removedNodes.removeAll(dp.getAnswer());
    if (removedNodes.size() != 0) {
        // Temporarily add parentNode to the answer so the baseline and current values of the removed small node can
        // successfully add back to parentNode by re-using the method updateWowValuesDueToRemoval.
        dp.getAnswer().add(parentNode);
        updateWowValuesDueToRemoval(parentNode.getParent(), dp.getAnswer(), removedNodes);
        dp.getAnswer().remove(parentNode);
    }
}

Example 2 with HierarchyNode

use of com.linkedin.thirdeye.client.diffsummary.HierarchyNode in project pinot by linkedin.

the class Summary method computeChildDPArray.

/**
   * Build the summary recursively. The parentTargetRatio for the root node can be any arbitrary value.
   * The calculated answer for each invocation is put at dpArrays[node.level].
   * So, the final answer is located at dpArray[0].
   */
private void computeChildDPArray(HierarchyNode node) {
    HierarchyNode parent = node.getParent();
    DPArray dpArray = dpArrays.get(node.getLevel());
    dpArray.fullReset();
    dpArray.targetRatio = node.targetRatio();
    // Otherwise, merge DPArrays from its children.
    if (node.getLevel() == levelCount - 1) {
        // Shrink answer size for getting a higher level view, which gives larger picture of the dataset
        if (node.childrenSize() < dpArray.size()) {
            dpArray.setShrinkSize(Math.max(1, (node.childrenSize() + 1) / 2));
        }
        for (HierarchyNode child : node.getChildren()) {
            leafRowInserter.insertRowToDPArray(dpArray, child, node.targetRatio());
            updateWowValues(node, dpArray.getAnswer());
            // get updated ratio
            dpArray.targetRatio = node.targetRatio();
        }
    } else {
        List<HierarchyNode> removedNodes = new ArrayList<>();
        boolean doRollback = false;
        do {
            doRollback = false;
            for (HierarchyNode child : node.getChildren()) {
                computeChildDPArray(child);
                removedNodes.addAll(mergeDPArray(node, dpArray, dpArrays.get(node.getLevel() + 1)));
                updateWowValues(node, dpArray.getAnswer());
                // get updated ratio
                dpArray.targetRatio = node.targetRatio();
            }
            // If the current node is kept being thinned out, it eventually aggregates all its children.
            if (nodeIsThinnedOut(node) && dpArray.getAnswer().size() < dpArray.maxSize()) {
                doRollback = true;
                rollbackInsertions(node, dpArray.getAnswer(), removedNodes);
                removedNodes.clear();
                dpArray.setShrinkSize(Math.max(1, (dpArray.getAnswer().size() * 2) / 3));
                dpArray.reset();
                dpArray.targetRatio = node.targetRatio();
            }
        } while (doRollback);
    }
    // Moreover, if a node is thinned out by its children, it won't be inserted to the answer.
    if (node.getLevel() != 0) {
        updateWowValues(parent, dpArray.getAnswer());
        double targetRatio = parent.targetRatio();
        recomputeCostAndRemoveSmallNodes(node, dpArray, targetRatio);
        dpArray.targetRatio = targetRatio;
        if (!nodeIsThinnedOut(node)) {
            // in order to insert the aggregated node to the answer.
            if (dpArray.size() == 1)
                dpArray.setShrinkSize(2);
            Set<HierarchyNode> removedNode = new HashSet<>(dpArray.getAnswer());
            basicRowInserter.insertRowToDPArray(dpArray, node, targetRatio);
            removedNode.removeAll(dpArray.getAnswer());
            if (removedNode.size() != 0) {
                updateWowValuesDueToRemoval(node, dpArray.getAnswer(), removedNode);
                updateWowValues(node, dpArray.getAnswer());
            }
        }
    } else {
        dpArray.getAnswer().add(node);
    }
}

Example 3 with HierarchyNode

use of com.linkedin.thirdeye.client.diffsummary.HierarchyNode in project pinot by linkedin.

the class Summary method updateWowValuesDueToRemoval.

/**
   * Update an internal node's baseline and current values if any of the nodes in its subtree is removed.
   * @param node The internal node to be updated.
   * @param answer The new answer.
   * @param removedNodes The nodes removed from the subtree of node.
   */
private static void updateWowValuesDueToRemoval(HierarchyNode node, Set<HierarchyNode> answer, Set<HierarchyNode> removedNodes) {
    List<HierarchyNode> removedNodesList = new ArrayList<>(removedNodes);
    // Process lower level nodes first
    removedNodesList.sort(NODE_COMPARATOR);
    for (HierarchyNode removedNode : removedNodesList) {
        HierarchyNode parents = findAncestor(removedNode, node, answer);
        if (parents != null)
            parents.addNodeValues(removedNode);
    }
}

Example 4 with HierarchyNode

use of com.linkedin.thirdeye.client.diffsummary.HierarchyNode in project pinot by linkedin.

the class Summary method computeSummary.

public SummaryResponse computeSummary(int answerSize, boolean doOneSideError, int userLevelCount) {
    if (answerSize <= 0)
        answerSize = 1;
    if (userLevelCount <= 0 || userLevelCount > this.maxLevelCount) {
        userLevelCount = this.maxLevelCount;
    }
    this.levelCount = userLevelCount;
    dpArrays = new ArrayList<>(this.levelCount);
    for (int i = 0; i < this.levelCount; ++i) {
        dpArrays.add(new DPArray(answerSize));
    }
    HierarchyNode root = cube.getRoot();
    if (doOneSideError) {
        oneSideErrorRowInserter = new OneSideErrorRowInserter(basicRowInserter, Double.compare(1., root.targetRatio()) <= 0);
        // otherwise, a row at different side is removed through internal nodes.
        if (this.levelCount == 1)
            leafRowInserter = oneSideErrorRowInserter;
    }
    computeChildDPArray(root);
    List<HierarchyNode> answer = new ArrayList<>(dpArrays.get(0).getAnswer());
    SummaryResponse response = new SummaryResponse();
    response.build(answer, this.levelCount, this.costSet);
    return response;
}

Example 5 with HierarchyNode

use of com.linkedin.thirdeye.client.diffsummary.HierarchyNode in project pinot by linkedin.

the class Summary method testCorrectnessOfWowValues.

/**
   * Check correctness of the sum of wow values. The check changes the wow values, so it should only be invoked after
   * SummaryResponse is generated.
   */
public void testCorrectnessOfWowValues() {
    List<HierarchyNode> nodeList = new ArrayList<>(dpArrays.get(0).getAnswer());
    // Process lower level nodes first
    nodeList.sort(NODE_COMPARATOR);
    for (HierarchyNode node : nodeList) {
        HierarchyNode parent = findAncestor(node, null, dpArrays.get(0).getAnswer());
        if (parent != null)
            parent.addNodeValues(node);
    }
    for (HierarchyNode node : nodeList) {
        if (Double.compare(node.getBaselineValue(), node.getOriginalBaselineValue()) != 0 || Double.compare(node.getCurrentValue(), node.getOriginalCurrentValue()) != 0) {
            Log.warn("Wrong Wow values at node: " + node.getDimensionValues() + ". Expected: " + node.getOriginalBaselineValue() + "," + node.getOriginalCurrentValue() + ", actual: " + node.getBaselineValue() + "," + node.getCurrentValue());
        }
    }
}