Examples with FilterQueryTree com.linkedin.pinot.common.utils.request.FilterQueryTree used on opensource projects

Example 21 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class FilterOptimizerTest method testPositive.

// Testing positive cases of flattening the query tree.
@Test
public void testPositive() throws Exception {
    Pql2Compiler pql2Compiler = new Pql2Compiler();
    BrokerRequest req;
    String timeColumn = null;
    FilterQueryTree tree;
    int numLeaves;
    int numOps;
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE (A = 4 AND B = 5) AND (C=7)");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), 3);
    Assert.assertEquals(tree.getOperator(), FilterOperator.AND);
    for (FilterQueryTree node : tree.getChildren()) {
        Assert.assertNull(node.getChildren());
        Assert.assertEquals(node.getOperator(), FilterOperator.EQUALITY);
    }
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE ((A = 4 AND B = 5) AND (C=7)) AND D=8");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), 4);
    Assert.assertEquals(tree.getOperator(), FilterOperator.AND);
    for (FilterQueryTree node : tree.getChildren()) {
        Assert.assertNull(node.getChildren());
        Assert.assertEquals(node.getOperator(), FilterOperator.EQUALITY);
    }
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE (A = 4 OR B = 5) OR (C=7)");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), 3);
    Assert.assertEquals(tree.getOperator(), FilterOperator.OR);
    for (FilterQueryTree node : tree.getChildren()) {
        Assert.assertNull(node.getChildren());
        Assert.assertEquals(node.getOperator(), FilterOperator.EQUALITY);
    }
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE ((A = 4 OR B = 5) OR (C=7)) OR D=8");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), 4);
    Assert.assertEquals(tree.getOperator(), FilterOperator.OR);
    for (FilterQueryTree node : tree.getChildren()) {
        Assert.assertNull(node.getChildren());
        Assert.assertEquals(node.getOperator(), FilterOperator.EQUALITY);
    }
    // 3-level test case
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE ((A = 4 OR (B = 5 OR D = 9)) OR (C=7)) OR E=8");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), 5);
    Assert.assertEquals(tree.getOperator(), FilterOperator.OR);
    for (FilterQueryTree node : tree.getChildren()) {
        Assert.assertNull(node.getChildren());
        Assert.assertEquals(node.getOperator(), FilterOperator.EQUALITY);
    }
    // Mixed case.
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE ((A = 4 OR (B = 5 AND D = 9)) OR (C=7)) OR E=8");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), 4);
    Assert.assertEquals(tree.getOperator(), FilterOperator.OR);
    numLeaves = 0;
    numOps = 0;
    for (FilterQueryTree node : tree.getChildren()) {
        if (node.getOperator().equals(FilterOperator.EQUALITY)) {
            Assert.assertNull(node.getChildren());
            numLeaves++;
        } else {
            Assert.assertNotNull(node.getChildren());
            Assert.assertEquals(node.getOperator(), FilterOperator.AND);
            numOps++;
        }
    }
    Assert.assertEquals(1, numOps);
    Assert.assertEquals(3, numLeaves);
    final int maxNodesAtTopLevel = FlattenNestedPredicatesFilterQueryTreeOptimizer.MAX_OPTIMIZING_DEPTH;
    String whereClause = constructWhereClause(FilterOperator.OR, maxNodesAtTopLevel + 50);
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE " + whereClause);
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), maxNodesAtTopLevel + 1);
    Assert.assertEquals(tree.getOperator(), FilterOperator.OR);
    numLeaves = 0;
    numOps = 0;
    for (FilterQueryTree node : tree.getChildren()) {
        if (node.getOperator().equals(FilterOperator.EQUALITY)) {
            Assert.assertNull(node.getChildren());
            numLeaves++;
        } else {
            Assert.assertNotNull(node.getChildren());
            numOps++;
        }
    }
    Assert.assertEquals(maxNodesAtTopLevel, numLeaves);
    Assert.assertEquals(1, numOps);
}

Example 22 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class FilterOptimizerTest method testNegative.

// Tests cases where we should not do any flattening.
@Test
public void testNegative() throws Exception {
    Pql2Compiler pql2Compiler = new Pql2Compiler();
    BrokerRequest req;
    FilterQueryTree tree;
    int numLeaves;
    int numOps;
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE (A = 4 AND (B = 5 OR D = 9))");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, null));
    Assert.assertEquals(tree.getChildren().size(), 2);
    Assert.assertEquals(tree.getOperator(), FilterOperator.AND);
    numOps = 0;
    numLeaves = 0;
    for (FilterQueryTree node : tree.getChildren()) {
        if (node.getOperator().equals(FilterOperator.OR)) {
            Assert.assertEquals(2, node.getChildren().size());
            numOps++;
        } else {
            numLeaves++;
        }
    }
    Assert.assertEquals(1, numOps);
    Assert.assertEquals(1, numLeaves);
}

Example 23 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class RangeMergeOptimizerTest method testRangeOptimizer.

@Test
public void testRangeOptimizer() {
    // Query with single >
    FilterQueryTree actualTree = buildFilterQueryTree("select * from table where time > 10", true);
    FilterQueryTree expectedTree = buildFilterQueryTree("select * from table where time > 10", false);
    compareTrees(actualTree, expectedTree);
    // Query with single >=
    actualTree = buildFilterQueryTree("select * from table where time >= 10", true);
    expectedTree = buildFilterQueryTree("select * from table where time >= 10", false);
    compareTrees(actualTree, expectedTree);
    // Query with single <
    actualTree = buildFilterQueryTree("select * from table where time < 10", true);
    expectedTree = buildFilterQueryTree("select * from table where time < 10", false);
    compareTrees(actualTree, expectedTree);
    // Query with single <=
    actualTree = buildFilterQueryTree("select * from table where time <= 10", true);
    expectedTree = buildFilterQueryTree("select * from table where time <= 10", false);
    compareTrees(actualTree, expectedTree);
    // Query with >= and <=
    actualTree = buildFilterQueryTree("select * from table where time >= 10 and time <= 20 and foo = 'bar'", true);
    expectedTree = buildFilterQueryTree("select * from table where time between 10 and 20 and foo = 'bar'", false);
    compareTrees(actualTree, expectedTree);
    // Query with no intersection
    actualTree = buildFilterQueryTree("select * from table where time >= 10 and time <= 5", true);
    expectedTree = buildFilterQueryTree("select * from table where time between 10 and 5", false);
    compareTrees(actualTree, expectedTree);
    // Query with multiple ranges on time column
    actualTree = buildFilterQueryTree("select * from table where time >= 10 and time <= 20 and time <= 15", true);
    expectedTree = buildFilterQueryTree("select * from table where time between 10 and 15", false);
    compareTrees(actualTree, expectedTree);
    // Query with multiple predicates
    actualTree = buildFilterQueryTree("select * from table where time >= 10 and time <= 20 and time <= 15 and foo = 'bar'", true);
    expectedTree = buildFilterQueryTree("select * from table where time between 10 and 15 and foo = 'bar'", false);
    compareTrees(actualTree, expectedTree);
    // Query with nested predicates
    actualTree = buildFilterQueryTree("select * from table where (time >= 10 and time <= 20) and ((time >= 5 and time <= 15))", true);
    expectedTree = buildFilterQueryTree("select * from table where time between 10 and 15", false);
    compareTrees(actualTree, expectedTree);
    // Query with nested predicates where not all ranges can be merged
    actualTree = buildFilterQueryTree("select * from table where (time >= 10 and time <= 20) and (foo1 = 'bar1' and (foo2 = 'bar2' and (time >= 5 and time <= 15)))", true);
    expectedTree = buildFilterQueryTree("select * from table where time between 10 and 20 and (foo1 = 'bar1' and (foo2 = 'bar2' and (time between 5 and 15)))", false);
    compareTrees(actualTree, expectedTree);
    // Query with time range in different levels of tree and all ranges can be merged
    actualTree = buildFilterQueryTree("select * from table where (((time >= 10 and time <= 20) and time >= 5) and time <= 15)", true);
    expectedTree = buildFilterQueryTree("select * from table where time between 10 and 15", false);
    compareTrees(actualTree, expectedTree);
    // Query with time range of one value.
    actualTree = buildFilterQueryTree("select * from table where (time > 10 and time <= 20) and (time >= 20 and time <= 30)", true);
    expectedTree = buildFilterQueryTree("select * from table where time between 20 and 20", false);
    compareTrees(actualTree, expectedTree);
    // Query with OR predicates
    actualTree = buildFilterQueryTree("select * from table where (foo1 = 'bar1' or (foo2 = 'bar2' and (time >= 10 and time <= 20)))", true);
    expectedTree = buildFilterQueryTree("select * from table where (foo1 = 'bar1' or (foo2 = 'bar2' and time between 10 and 20))", false);
    compareTrees(actualTree, expectedTree);
    // Query with same lower and upper range
    actualTree = buildFilterQueryTree("select * from table where (time >= 10 and time <= 20) and (time between 10 and 20)", true);
    expectedTree = buildFilterQueryTree("select * from table where time between 10 and 20", false);
    compareTrees(actualTree, expectedTree);
    // Query without time column
    actualTree = buildFilterQueryTree("select * from table where foo = 'bar'", true);
    expectedTree = buildFilterQueryTree("select * from table where foo = 'bar'", false);
    compareTrees(actualTree, expectedTree);
}

Example 24 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class BaseSumStarTreeIndexTest method testHardCodedQueries.

protected void testHardCodedQueries(IndexSegment segment, Schema schema) {
    // Test against all metric columns, instead of just the aggregation column in the query.
    List<String> metricNames = schema.getMetricNames();
    SegmentMetadata segmentMetadata = segment.getSegmentMetadata();
    for (int i = 0; i < _hardCodedQueries.length; i++) {
        Pql2Compiler compiler = new Pql2Compiler();
        BrokerRequest brokerRequest = compiler.compileToBrokerRequest(_hardCodedQueries[i]);
        FilterQueryTree filterQueryTree = RequestUtils.generateFilterQueryTree(brokerRequest);
        Assert.assertTrue(RequestUtils.isFitForStarTreeIndex(segmentMetadata, filterQueryTree, brokerRequest));
        Map<String, double[]> expectedResult = computeSumUsingRawDocs(segment, metricNames, brokerRequest);
        Map<String, double[]> actualResult = computeSumUsingAggregatedDocs(segment, metricNames, brokerRequest);
        Assert.assertEquals(expectedResult.size(), actualResult.size(), "Mis-match in number of groups");
        for (Map.Entry<String, double[]> entry : expectedResult.entrySet()) {
            String expectedKey = entry.getKey();
            Assert.assertTrue(actualResult.containsKey(expectedKey));
            double[] expectedSums = entry.getValue();
            double[] actualSums = actualResult.get(expectedKey);
            for (int j = 0; j < expectedSums.length; j++) {
                Assert.assertEquals(actualSums[j], expectedSums[j], "Mis-match sum for key '" + expectedKey + "', Metric: " + metricNames.get(j) + ", Random Seed: " + _randomSeed);
            }
        }
    }
}