Examples with FilterOperator org.apache.hadoop.hive.ql.exec.FilterOperator used on opensource projects

Example 6 with FilterOperator

use of org.apache.hadoop.hive.ql.exec.FilterOperator in project hive by apache.

the class ConstantPropagateProcCtx method getPropagatedConstants.

/**
   * Get propagated constant map from parents.
   *
   * Traverse all parents of current operator, if there is propagated constant (determined by
   * assignment expression like column=constant value), resolve the column using RowResolver and add
   * it to current constant map.
   *
   * @param op
   *        operator getting the propagated constants.
   * @return map of ColumnInfo to ExprNodeDesc. The values of that map must be either
   *         ExprNodeConstantDesc or ExprNodeNullDesc.
   */
public Map<ColumnInfo, ExprNodeDesc> getPropagatedConstants(Operator<? extends Serializable> op) {
    // this map should map columnInfo to ExprConstantNodeDesc
    Map<ColumnInfo, ExprNodeDesc> constants = new HashMap<ColumnInfo, ExprNodeDesc>();
    if (op.getSchema() == null) {
        return constants;
    }
    RowSchema rs = op.getSchema();
    LOG.debug("Getting constants of op:" + op + " with rs:" + rs);
    if (op.getParentOperators() == null) {
        return constants;
    }
    // A previous solution is based on tableAlias and colAlias, which is
    // unsafe, esp. when CBO generates derived table names. see HIVE-13602.
    // For correctness purpose, we only trust colExpMap.
    // We assume that CBO can do the constantPropagation before this function is
    // called to help improve the performance.
    // UnionOperator, LimitOperator and FilterOperator are special, they should already be
    // column-position aligned.
    List<Map<Integer, ExprNodeDesc>> parentsToConstant = new ArrayList<>();
    boolean areAllParentsContainConstant = true;
    boolean noParentsContainConstant = true;
    for (Operator<?> parent : op.getParentOperators()) {
        Map<ColumnInfo, ExprNodeDesc> constMap = opToConstantExprs.get(parent);
        if (constMap == null) {
            LOG.debug("Constant of Op " + parent.getOperatorId() + " is not found");
            areAllParentsContainConstant = false;
        } else {
            noParentsContainConstant = false;
            Map<Integer, ExprNodeDesc> map = new HashMap<>();
            for (Entry<ColumnInfo, ExprNodeDesc> entry : constMap.entrySet()) {
                map.put(parent.getSchema().getPosition(entry.getKey().getInternalName()), entry.getValue());
            }
            parentsToConstant.add(map);
            LOG.debug("Constant of Op " + parent.getOperatorId() + " " + constMap);
        }
    }
    if (noParentsContainConstant) {
        return constants;
    }
    ArrayList<ColumnInfo> signature = op.getSchema().getSignature();
    if (op instanceof LimitOperator || op instanceof FilterOperator) {
        // there should be only one parent.
        if (op.getParentOperators().size() == 1) {
            Map<Integer, ExprNodeDesc> parentToConstant = parentsToConstant.get(0);
            for (int index = 0; index < signature.size(); index++) {
                if (parentToConstant.containsKey(index)) {
                    constants.put(signature.get(index), parentToConstant.get(index));
                }
            }
        }
    } else if (op instanceof UnionOperator && areAllParentsContainConstant) {
        for (int index = 0; index < signature.size(); index++) {
            ExprNodeDesc constant = null;
            for (Map<Integer, ExprNodeDesc> parentToConstant : parentsToConstant) {
                if (!parentToConstant.containsKey(index)) {
                    // if this parent does not contain a constant at this position, we
                    // continue to look at other positions.
                    constant = null;
                    break;
                } else {
                    if (constant == null) {
                        constant = parentToConstant.get(index);
                    } else {
                        // compare if they are the same constant.
                        ExprNodeDesc nextConstant = parentToConstant.get(index);
                        if (!nextConstant.isSame(constant)) {
                            // they are not the same constant. for example, union all of 1
                            // and 2.
                            constant = null;
                            break;
                        }
                    }
                }
            }
            // we have checked all the parents for the "index" position.
            if (constant != null) {
                constants.put(signature.get(index), constant);
            }
        }
    } else if (op instanceof JoinOperator) {
        JoinOperator joinOp = (JoinOperator) op;
        Iterator<Entry<Byte, List<ExprNodeDesc>>> itr = joinOp.getConf().getExprs().entrySet().iterator();
        while (itr.hasNext()) {
            Entry<Byte, List<ExprNodeDesc>> e = itr.next();
            int tag = e.getKey();
            Operator<?> parent = op.getParentOperators().get(tag);
            List<ExprNodeDesc> exprs = e.getValue();
            if (exprs == null) {
                continue;
            }
            for (ExprNodeDesc expr : exprs) {
                // we are only interested in ExprNodeColumnDesc
                if (expr instanceof ExprNodeColumnDesc) {
                    String parentColName = ((ExprNodeColumnDesc) expr).getColumn();
                    // find this parentColName in its parent's rs
                    int parentPos = parent.getSchema().getPosition(parentColName);
                    if (parentsToConstant.get(tag).containsKey(parentPos)) {
                        // reverse look up colExprMap to find the childColName
                        if (op.getColumnExprMap() != null && op.getColumnExprMap().entrySet() != null) {
                            for (Entry<String, ExprNodeDesc> entry : op.getColumnExprMap().entrySet()) {
                                if (entry.getValue().isSame(expr)) {
                                    // now propagate the constant from the parent to the child
                                    constants.put(signature.get(op.getSchema().getPosition(entry.getKey())), parentsToConstant.get(tag).get(parentPos));
                                }
                            }
                        }
                    }
                }
            }
        }
    } else {
        // there should be only one parent.
        if (op.getParentOperators().size() == 1) {
            Operator<?> parent = op.getParentOperators().get(0);
            if (op.getColumnExprMap() != null && op.getColumnExprMap().entrySet() != null) {
                for (Entry<String, ExprNodeDesc> entry : op.getColumnExprMap().entrySet()) {
                    if (op.getSchema().getPosition(entry.getKey()) == -1) {
                        // Not present
                        continue;
                    }
                    ExprNodeDesc expr = entry.getValue();
                    if (expr instanceof ExprNodeColumnDesc) {
                        String parentColName = ((ExprNodeColumnDesc) expr).getColumn();
                        // find this parentColName in its parent's rs
                        int parentPos = parent.getSchema().getPosition(parentColName);
                        if (parentsToConstant.get(0).containsKey(parentPos)) {
                            // this position in parent is a constant
                            // now propagate the constant from the parent to the child
                            constants.put(signature.get(op.getSchema().getPosition(entry.getKey())), parentsToConstant.get(0).get(parentPos));
                        }
                    }
                }
            }
        }
    }
    LOG.debug("Offering constants " + constants.keySet() + " to operator " + op.toString());
    return constants;
}

Example 7 with FilterOperator

use of org.apache.hadoop.hive.ql.exec.FilterOperator in project hive by apache.

the class GlobalLimitOptimizer method checkQbpForGlobalLimit.

/**
   * Check the limit number in all sub queries
   *
   * @return if there is one and only one limit for all subqueries, return the limit
   *         if there is no limit, return 0
   *         otherwise, return null
   */
private static LimitOperator checkQbpForGlobalLimit(TableScanOperator ts) {
    Set<Class<? extends Operator<?>>> searchedClasses = new ImmutableSet.Builder<Class<? extends Operator<?>>>().add(ReduceSinkOperator.class).add(GroupByOperator.class).add(FilterOperator.class).add(LimitOperator.class).build();
    Multimap<Class<? extends Operator<?>>, Operator<?>> ops = OperatorUtils.classifyOperators(ts, searchedClasses);
    // existsOrdering AND existsPartitioning should be false.
    for (Operator<?> op : ops.get(ReduceSinkOperator.class)) {
        ReduceSinkDesc reduceSinkConf = ((ReduceSinkOperator) op).getConf();
        if (reduceSinkConf.isOrdering() || reduceSinkConf.isPartitioning()) {
            return null;
        }
    }
    // - There cannot exist any (distinct) aggregate.
    for (Operator<?> op : ops.get(GroupByOperator.class)) {
        GroupByDesc groupByConf = ((GroupByOperator) op).getConf();
        if (groupByConf.isAggregate() || groupByConf.isDistinct()) {
            return null;
        }
    }
    // - There cannot exist any sampling predicate.
    for (Operator<?> op : ops.get(FilterOperator.class)) {
        FilterDesc filterConf = ((FilterOperator) op).getConf();
        if (filterConf.getIsSamplingPred()) {
            return null;
        }
    }
    // If there is one and only one limit starting at op, return the limit
    // If there is no limit, return 0
    // Otherwise, return null
    Collection<Operator<?>> limitOps = ops.get(LimitOperator.class);
    if (limitOps.size() == 1) {
        return (LimitOperator) limitOps.iterator().next();
    } else if (limitOps.size() == 0) {
        return null;
    }
    return null;
}

Example 8 with FilterOperator

use of org.apache.hadoop.hive.ql.exec.FilterOperator in project hive by apache.

the class HiveOpConverter method visit.

/**
   * TODO: 1) isSamplingPred 2) sampleDesc 3) isSortedFilter
   */
OpAttr visit(HiveFilter filterRel) throws SemanticException {
    OpAttr inputOpAf = dispatch(filterRel.getInput());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Translating operator rel#" + filterRel.getId() + ":" + filterRel.getRelTypeName() + " with row type: [" + filterRel.getRowType() + "]");
    }
    ExprNodeDesc filCondExpr = filterRel.getCondition().accept(new ExprNodeConverter(inputOpAf.tabAlias, filterRel.getInput().getRowType(), inputOpAf.vcolsInCalcite, filterRel.getCluster().getTypeFactory(), true));
    FilterDesc filDesc = new FilterDesc(filCondExpr, false);
    ArrayList<ColumnInfo> cinfoLst = createColInfos(inputOpAf.inputs.get(0));
    FilterOperator filOp = (FilterOperator) OperatorFactory.getAndMakeChild(filDesc, new RowSchema(cinfoLst), inputOpAf.inputs.get(0));
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated " + filOp + " with row schema: [" + filOp.getSchema() + "]");
    }
    return inputOpAf.clone(filOp);
}