Examples with DrillParseContext org.apache.drill.exec.planner.logical.DrillParseContext used on opensource projects

Example 26 with DrillParseContext

use of org.apache.drill.exec.planner.logical.DrillParseContext in project drill by apache.

the class IndexPlanUtils method queryCoveredByIndex.

/**
 * This method is called only when the index has at least one functional indexed field. If there is no function field,
 * we don't need to worry whether there could be paths not found in Scan.
 * In functional case, we have to check all available (if needed) operators to find out if the query is covered or not.
 * E.g. cast(a.b as INT) in project, a.b in Scan's rowType or columns, and cast(a.b as INT)
 * is an indexed field named '$0'. In this case, by looking at Scan, we see only 'a.b' which is not in index. We have to
 * look into Project, and if we see 'a.b' is only used in functional index expression cast(a.b as INT), then we know
 * this Project+Scan is covered.
 * @param indexContext the index call context
 * @param functionInfo functional index information that may impact rewrite
 * @return false if the query could not be covered by the index (should not create covering index plan)
 */
private static boolean queryCoveredByIndex(IndexCallContext indexContext, FunctionalIndexInfo functionInfo) {
    if (indexContext.getFilter() != null && indexContext.getUpperProject() == null) {
        if (!isFullQuery(indexContext)) {
            return false;
        }
    }
    DrillParseContext parserContext = new DrillParseContext(PrelUtil.getPlannerSettings(indexContext.getCall().rel(0).getCluster()));
    Set<LogicalExpression> exprs = Sets.newHashSet();
    if (indexContext.getUpperProject() != null) {
        if (indexContext.getLowerProject() == null) {
            for (RexNode rex : indexContext.getUpperProject().getProjects()) {
                LogicalExpression expr = RexToExpression.toDrill(parserContext, null, indexContext.getScan(), rex);
                exprs.add(expr);
            }
            // now collect paths in filter since upperProject may drop some paths in filter
            IndexableExprMarker filterMarker = new IndexableExprMarker(indexContext.getScan());
            indexContext.getFilterCondition().accept(filterMarker);
            for (RexNode rex : filterMarker.getIndexableExpression().keySet()) {
                LogicalExpression expr = RexToExpression.toDrill(parserContext, null, indexContext.getScan(), rex);
                exprs.add(expr);
            }
        } else {
            // we have underneath project, so we have to do more to convert expressions
            for (RexNode rex : indexContext.getUpperProject().getProjects()) {
                LogicalExpression expr = RexToExpression.toDrill(parserContext, indexContext.getLowerProject(), indexContext.getScan(), rex);
                exprs.add(expr);
            }
            // Now collect paths in filter since upperProject may drop some paths in filter.
            // Since this is (upper)Proj+Filter+(lower)Proj+Scan case, and IndexableExprMarker works
            // only with expressions that referencing directly to Scan, it has to use indexContext.origPushedCondition
            IndexableExprMarker filterMarker = new IndexableExprMarker(indexContext.getScan());
            indexContext.getOrigCondition().accept(filterMarker);
            for (RexNode rex : filterMarker.getIndexableExpression().keySet()) {
                // Since rex represents the filter expression directly referencing the scan row type,
                // (the condition has been pushed down of lowerProject), set the lowerProject as null.
                LogicalExpression expr = RexToExpression.toDrill(parserContext, null, indexContext.getScan(), rex);
                exprs.add(expr);
            }
        }
    } else if (indexContext.getLowerProject() != null) {
        for (RexNode rex : indexContext.getLowerProject().getProjects()) {
            LogicalExpression expr = DrillOptiq.toDrill(parserContext, indexContext.getScan(), rex);
            exprs.add(expr);
        }
    } else {
        // upperProject and lowerProject both are null, the only place to find columns being used in query is scan
        exprs.addAll(indexContext.getScanColumns());
    }
    Map<LogicalExpression, Set<SchemaPath>> exprPathMap = functionInfo.getPathsInFunctionExpr();
    PathInExpr exprSearch = new PathInExpr(exprPathMap);
    for (LogicalExpression expr : exprs) {
        if (expr.accept(exprSearch, null) == false) {
            return false;
        }
    }
    // if we come to here, paths in indexed function expressions are covered in capProject.
    // now we check other paths.
    // check the leftout paths (appear in capProject other than functional index expression) are covered by other index fields or not
    List<LogicalExpression> leftPaths = Lists.newArrayList(exprSearch.getRemainderPaths());
    indexContext.setLeftOutPathsInFunctions(exprSearch.getRemainderPathsInFunctions());
    return functionInfo.getIndexDesc().isCoveringIndex(leftPaths);
}

Example 27 with DrillParseContext

use of org.apache.drill.exec.planner.logical.DrillParseContext in project drill by apache.

the class IndexPlanUtils method updateSortExpression.

/**
 * generate logical expressions for sort rexNodes in SortRel, the result is store to IndexPlanCallContext
 * @param indexContext the index call context
 * @param coll list of field collations
 */
public static void updateSortExpression(IndexCallContext indexContext, List<RelFieldCollation> coll) {
    if (coll == null) {
        return;
    }
    DrillParseContext parserContext = new DrillParseContext(PrelUtil.getPlannerSettings(indexContext.getCall().rel(0).getCluster()));
    indexContext.createSortExprs();
    for (RelFieldCollation collation : coll) {
        int idx = collation.getFieldIndex();
        DrillProjectRelBase oneProject;
        if (indexContext.getUpperProject() != null && indexContext.getLowerProject() != null) {
            LogicalExpression expr = RexToExpression.toDrill(parserContext, indexContext.getLowerProject(), indexContext.getScan(), indexContext.getUpperProject().getProjects().get(idx));
            indexContext.getSortExprs().add(expr);
        } else {
            // one project is null now
            oneProject = (indexContext.getUpperProject() != null) ? indexContext.getUpperProject() : indexContext.getLowerProject();
            if (oneProject != null) {
                LogicalExpression expr = RexToExpression.toDrill(parserContext, null, indexContext.getScan(), getProjects(oneProject).get(idx));
                indexContext.getSortExprs().add(expr);
            } else {
                // two projects are null
                SchemaPath path;
                RelDataTypeField f = indexContext.getScan().getRowType().getFieldList().get(idx);
                String pathSeg = f.getName().replaceAll("`", "");
                final String[] segs = pathSeg.split("\\.");
                path = SchemaPath.getCompoundPath(segs);
                indexContext.getSortExprs().add(path);
            }
        }
    }
}

Example 28 with DrillParseContext

use of org.apache.drill.exec.planner.logical.DrillParseContext in project drill by apache.

the class IndexPlanUtils method getProjectExprs.

public static Map<LogicalExpression, Integer> getProjectExprs(List<RexNode> projectRexs, DrillProjectRelBase project, RelNode input) {
    Map<LogicalExpression, Integer> projectExprs = Maps.newLinkedHashMap();
    DrillParseContext parserContext = new DrillParseContext(PrelUtil.getPlannerSettings(input.getCluster()));
    int idx = 0;
    for (RexNode rex : projectRexs) {
        LogicalExpression expr;
        expr = RexToExpression.toDrill(parserContext, project, input, rex);
        projectExprs.put(expr, idx);
        idx++;
    }
    return projectExprs;
}

Example 29 with DrillParseContext

use of org.apache.drill.exec.planner.logical.DrillParseContext in project drill by apache.

the class IndexPlanUtils method updateSortExpression.

/**
 * generate logical expressions for sort rexNodes in SortRel, the result is store to IndexPlanCallContext
 * @param indexContext the index call context
 * @param coll list of field collations
 */
public static void updateSortExpression(IndexPhysicalPlanCallContext indexContext, List<RelFieldCollation> coll) {
    if (coll == null) {
        return;
    }
    DrillParseContext parserContext = new DrillParseContext(PrelUtil.getPlannerSettings(indexContext.call.rel(0).getCluster()));
    indexContext.sortExprs = Lists.newArrayList();
    for (RelFieldCollation collation : coll) {
        int idx = collation.getFieldIndex();
        ProjectPrel oneProject;
        if (indexContext.upperProject != null && indexContext.lowerProject != null) {
            LogicalExpression expr = RexToExpression.toDrill(parserContext, indexContext.lowerProject, indexContext.scan, indexContext.upperProject.getProjects().get(idx));
            indexContext.sortExprs.add(expr);
        } else {
            // one project is null now
            oneProject = (indexContext.upperProject != null) ? indexContext.upperProject : indexContext.lowerProject;
            if (oneProject != null) {
                LogicalExpression expr = RexToExpression.toDrill(parserContext, null, indexContext.scan, oneProject.getProjects().get(idx));
                indexContext.sortExprs.add(expr);
            } else {
                // two projects are null
                SchemaPath path;
                RelDataTypeField f = indexContext.scan.getRowType().getFieldList().get(idx);
                String pathSeg = f.getName().replaceAll("`", "");
                final String[] segs = pathSeg.split("\\.");
                path = SchemaPath.getCompoundPath(segs);
                indexContext.sortExprs.add(path);
            }
        }
    }
}

Example 30 with DrillParseContext

use of org.apache.drill.exec.planner.logical.DrillParseContext in project drill by apache.

the class CoveringIndexPlanGenerator method convertChild.

@Override
public RelNode convertChild(final RelNode filter, final RelNode input) throws InvalidRelException {
    if (indexGroupScan == null) {
        logger.error("Null indexgroupScan in CoveringIndexPlanGenerator.convertChild");
        return null;
    }
    RexNode coveringCondition;
    ScanPrel indexScanPrel = IndexPlanUtils.buildCoveringIndexScan(origScan, indexGroupScan, indexContext, indexDesc);
    // If remainder condition, then combine the index and remainder conditions. This is a covering plan so we can
    // pushed the entire condition into the index.
    coveringCondition = IndexPlanUtils.getTotalFilter(indexCondition, remainderCondition, indexScanPrel.getCluster().getRexBuilder());
    RexNode newIndexCondition = rewriteFunctionalCondition(coveringCondition, indexScanPrel.getRowType(), functionInfo);
    // build collation for filter
    RelTraitSet indexFilterTraitSet = indexScanPrel.getTraitSet();
    FilterPrel indexFilterPrel = new FilterPrel(indexScanPrel.getCluster(), indexFilterTraitSet, indexScanPrel, newIndexCondition);
    ProjectPrel indexProjectPrel = null;
    if (origProject != null) {
        RelCollation collation = IndexPlanUtils.buildCollationProject(IndexPlanUtils.getProjects(origProject), null, origScan, functionInfo, indexContext);
        indexProjectPrel = new ProjectPrel(origScan.getCluster(), indexFilterTraitSet.plus(collation), indexFilterPrel, IndexPlanUtils.getProjects(origProject), origProject.getRowType());
    }
    RelNode finalRel;
    if (indexProjectPrel != null) {
        finalRel = indexProjectPrel;
    } else {
        finalRel = indexFilterPrel;
    }
    if (upperProject != null) {
        RelCollation newCollation = IndexPlanUtils.buildCollationProject(IndexPlanUtils.getProjects(upperProject), origProject, origScan, functionInfo, indexContext);
        ProjectPrel cap = new ProjectPrel(upperProject.getCluster(), newCollation == null ? finalRel.getTraitSet() : finalRel.getTraitSet().plus(newCollation), finalRel, IndexPlanUtils.getProjects(upperProject), upperProject.getRowType());
        if (functionInfo.hasFunctional()) {
            // if there is functional index field, then a rewrite may be needed in upperProject/indexProject
            // merge upperProject with indexProjectPrel(from origProject) if both exist,
            ProjectPrel newProject = cap;
            if (indexProjectPrel != null) {
                newProject = (ProjectPrel) DrillMergeProjectRule.replace(newProject, indexProjectPrel);
            }
            // then rewrite functional expressions in new project.
            List<RexNode> newProjects = Lists.newArrayList();
            DrillParseContext parseContxt = new DrillParseContext(PrelUtil.getPlannerSettings(newProject.getCluster()));
            for (RexNode projectRex : newProject.getProjects()) {
                RexNode newRex = IndexPlanUtils.rewriteFunctionalRex(indexContext, parseContxt, null, origScan, projectRex, indexScanPrel.getRowType(), functionInfo);
                newProjects.add(newRex);
            }
            ProjectPrel rewrittenProject = new ProjectPrel(newProject.getCluster(), newCollation == null ? newProject.getTraitSet() : newProject.getTraitSet().plus(newCollation), indexFilterPrel, newProjects, newProject.getRowType());
            cap = rewrittenProject;
        }
        finalRel = cap;
    }
    if (indexContext.getSort() != null) {
        finalRel = getSortNode(indexContext, finalRel, false, true, true);
        Preconditions.checkArgument(finalRel != null);
    }
    finalRel = Prule.convert(finalRel, finalRel.getTraitSet().plus(Prel.DRILL_PHYSICAL));
    logger.debug("CoveringIndexPlanGenerator got finalRel {} from origScan {}, original digest {}, new digest {}.", finalRel.toString(), origScan.toString(), upperProject == null ? indexContext.getFilter().getDigest() : upperProject.getDigest(), finalRel.getDigest());
    return finalRel;
}