Examples with Pql2Compiler com.linkedin.pinot.pql.parsers.Pql2Compiler used on opensource projects

Example 6 with Pql2Compiler

use of com.linkedin.pinot.pql.parsers.Pql2Compiler in project pinot by linkedin.

the class FilterOperatorBenchmark method main.

public static void main(String[] args) throws Exception {
    String rootDir = args[0];
    File[] segmentDirs = new File(rootDir).listFiles();
    String query = args[1];
    AtomicInteger totalDocsMatched = new AtomicInteger(0);
    Pql2Compiler pql2Compiler = new Pql2Compiler();
    BrokerRequest brokerRequest = pql2Compiler.compileToBrokerRequest(query);
    List<Callable<Void>> segmentProcessors = new ArrayList<>();
    long[] timesSpent = new long[segmentDirs.length];
    for (int i = 0; i < segmentDirs.length; i++) {
        File indexSegmentDir = segmentDirs[i];
        System.out.println("Loading " + indexSegmentDir.getName());
        Configuration tableDataManagerConfig = new PropertiesConfiguration();
        List<String> invertedColumns = new ArrayList<>();
        FilenameFilter filter = new FilenameFilter() {

            @Override
            public boolean accept(File dir, String name) {
                return name.endsWith(".bitmap.inv");
            }
        };
        String[] indexFiles = indexSegmentDir.list(filter);
        for (String indexFileName : indexFiles) {
            invertedColumns.add(indexFileName.replace(".bitmap.inv", ""));
        }
        tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_LOADING_INVERTED_INDEX, invertedColumns);
        IndexLoadingConfigMetadata indexLoadingConfigMetadata = new IndexLoadingConfigMetadata(tableDataManagerConfig);
        IndexSegmentImpl indexSegmentImpl = (IndexSegmentImpl) Loaders.IndexSegment.load(indexSegmentDir, ReadMode.heap, indexLoadingConfigMetadata);
        segmentProcessors.add(new SegmentProcessor(i, indexSegmentImpl, brokerRequest, totalDocsMatched, timesSpent));
    }
    ExecutorService executorService = Executors.newCachedThreadPool();
    for (int run = 0; run < 5; run++) {
        System.out.println("START RUN:" + run);
        totalDocsMatched.set(0);
        long start = System.currentTimeMillis();
        List<Future<Void>> futures = executorService.invokeAll(segmentProcessors);
        for (int i = 0; i < futures.size(); i++) {
            futures.get(i).get();
        }
        long end = System.currentTimeMillis();
        System.out.println("Total docs matched:" + totalDocsMatched + " took:" + (end - start));
        System.out.println("Times spent:" + Arrays.toString(timesSpent));
        System.out.println("END RUN:" + run);
    }
    System.exit(0);
}

Example 7 with Pql2Compiler

use of com.linkedin.pinot.pql.parsers.Pql2Compiler in project pinot by linkedin.

the class FilterTreeOptimizationTest method setup.

@BeforeClass
public void setup() throws Exception {
    Schema schema = buildSchema(DIMENSIONS);
    buildSegment(SEGMENT_DIR_NAME, SEGMENT_NAME, schema);
    _indexSegment = Loaders.IndexSegment.load(new File(SEGMENT_DIR_NAME, SEGMENT_NAME), ReadMode.heap);
    _compiler = new Pql2Compiler();
}

Example 8 with Pql2Compiler

use of com.linkedin.pinot.pql.parsers.Pql2Compiler in project pinot by linkedin.

the class BrokerRequestValidationTest method runQuery.

/**
   * Helper method to run the query.
   *
   * @param query
   * @return Returns exception string if any, null otherwise.
   * @throws Exception
   */
private String runQuery(String query) throws Exception {
    BrokerRequestHandler brokerRequestHandler = brokerBuilder.getBrokerRequestHandler();
    Pql2Compiler compiler = new Pql2Compiler();
    BrokerRequest brokerRequest = compiler.compileToBrokerRequest(query);
    String exceptionMessage = null;
    try {
        brokerRequestHandler.validateRequest(brokerRequest);
    } catch (Exception e) {
        exceptionMessage = e.getMessage();
    }
    return exceptionMessage;
}

Example 9 with Pql2Compiler

use of com.linkedin.pinot.pql.parsers.Pql2Compiler in project pinot by linkedin.

the class FilterOptimizerTest method testPositive.

// Testing positive cases of flattening the query tree.
@Test
public void testPositive() throws Exception {
    Pql2Compiler pql2Compiler = new Pql2Compiler();
    BrokerRequest req;
    String timeColumn = null;
    FilterQueryTree tree;
    int numLeaves;
    int numOps;
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE (A = 4 AND B = 5) AND (C=7)");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), 3);
    Assert.assertEquals(tree.getOperator(), FilterOperator.AND);
    for (FilterQueryTree node : tree.getChildren()) {
        Assert.assertNull(node.getChildren());
        Assert.assertEquals(node.getOperator(), FilterOperator.EQUALITY);
    }
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE ((A = 4 AND B = 5) AND (C=7)) AND D=8");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), 4);
    Assert.assertEquals(tree.getOperator(), FilterOperator.AND);
    for (FilterQueryTree node : tree.getChildren()) {
        Assert.assertNull(node.getChildren());
        Assert.assertEquals(node.getOperator(), FilterOperator.EQUALITY);
    }
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE (A = 4 OR B = 5) OR (C=7)");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), 3);
    Assert.assertEquals(tree.getOperator(), FilterOperator.OR);
    for (FilterQueryTree node : tree.getChildren()) {
        Assert.assertNull(node.getChildren());
        Assert.assertEquals(node.getOperator(), FilterOperator.EQUALITY);
    }
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE ((A = 4 OR B = 5) OR (C=7)) OR D=8");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), 4);
    Assert.assertEquals(tree.getOperator(), FilterOperator.OR);
    for (FilterQueryTree node : tree.getChildren()) {
        Assert.assertNull(node.getChildren());
        Assert.assertEquals(node.getOperator(), FilterOperator.EQUALITY);
    }
    // 3-level test case
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE ((A = 4 OR (B = 5 OR D = 9)) OR (C=7)) OR E=8");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), 5);
    Assert.assertEquals(tree.getOperator(), FilterOperator.OR);
    for (FilterQueryTree node : tree.getChildren()) {
        Assert.assertNull(node.getChildren());
        Assert.assertEquals(node.getOperator(), FilterOperator.EQUALITY);
    }
    // Mixed case.
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE ((A = 4 OR (B = 5 AND D = 9)) OR (C=7)) OR E=8");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), 4);
    Assert.assertEquals(tree.getOperator(), FilterOperator.OR);
    numLeaves = 0;
    numOps = 0;
    for (FilterQueryTree node : tree.getChildren()) {
        if (node.getOperator().equals(FilterOperator.EQUALITY)) {
            Assert.assertNull(node.getChildren());
            numLeaves++;
        } else {
            Assert.assertNotNull(node.getChildren());
            Assert.assertEquals(node.getOperator(), FilterOperator.AND);
            numOps++;
        }
    }
    Assert.assertEquals(1, numOps);
    Assert.assertEquals(3, numLeaves);
    final int maxNodesAtTopLevel = FlattenNestedPredicatesFilterQueryTreeOptimizer.MAX_OPTIMIZING_DEPTH;
    String whereClause = constructWhereClause(FilterOperator.OR, maxNodesAtTopLevel + 50);
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE " + whereClause);
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, timeColumn));
    Assert.assertEquals(tree.getChildren().size(), maxNodesAtTopLevel + 1);
    Assert.assertEquals(tree.getOperator(), FilterOperator.OR);
    numLeaves = 0;
    numOps = 0;
    for (FilterQueryTree node : tree.getChildren()) {
        if (node.getOperator().equals(FilterOperator.EQUALITY)) {
            Assert.assertNull(node.getChildren());
            numLeaves++;
        } else {
            Assert.assertNotNull(node.getChildren());
            numOps++;
        }
    }
    Assert.assertEquals(maxNodesAtTopLevel, numLeaves);
    Assert.assertEquals(1, numOps);
}

Example 10 with Pql2Compiler

use of com.linkedin.pinot.pql.parsers.Pql2Compiler in project pinot by linkedin.

the class FilterOptimizerTest method testNegative.

// Tests cases where we should not do any flattening.
@Test
public void testNegative() throws Exception {
    Pql2Compiler pql2Compiler = new Pql2Compiler();
    BrokerRequest req;
    FilterQueryTree tree;
    int numLeaves;
    int numOps;
    req = pql2Compiler.compileToBrokerRequest("SELECT * FROM T WHERE (A = 4 AND (B = 5 OR D = 9))");
    tree = RequestUtils.generateFilterQueryTree(_optimizer.optimize(req, null));
    Assert.assertEquals(tree.getChildren().size(), 2);
    Assert.assertEquals(tree.getOperator(), FilterOperator.AND);
    numOps = 0;
    numLeaves = 0;
    for (FilterQueryTree node : tree.getChildren()) {
        if (node.getOperator().equals(FilterOperator.OR)) {
            Assert.assertEquals(2, node.getChildren().size());
            numOps++;
        } else {
            numLeaves++;
        }
    }
    Assert.assertEquals(1, numOps);
    Assert.assertEquals(1, numLeaves);
}