Examples with ExprNodeGenericFuncDesc org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc used on opensource projects

Example 56 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class ConstantPropagateProcFactory method foldNegative.

/**
 * Combines the logical not() operator with the child operator if possible.
 * @param desc the expression to be evaluated
 * @return  the new expression to be replaced
 * @throws UDFArgumentException
 */
private static ExprNodeDesc foldNegative(ExprNodeDesc desc) throws UDFArgumentException {
    if (desc instanceof ExprNodeGenericFuncDesc) {
        ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc;
        GenericUDF udf = funcDesc.getGenericUDF();
        if (udf instanceof GenericUDFOPNot) {
            ExprNodeDesc child = funcDesc.getChildren().get(0);
            if (child instanceof ExprNodeGenericFuncDesc) {
                ExprNodeGenericFuncDesc childDesc = (ExprNodeGenericFuncDesc) child;
                GenericUDF childUDF = childDesc.getGenericUDF();
                List<ExprNodeDesc> grandChildren = child.getChildren();
                if (childUDF instanceof GenericUDFBaseCompare || childUDF instanceof GenericUDFOPNull || childUDF instanceof GenericUDFOPNotNull) {
                    List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
                    for (ExprNodeDesc grandChild : grandChildren) {
                        newGrandChildren.add(foldNegative(grandChild));
                    }
                    return ExprNodeGenericFuncDesc.newInstance(childUDF.negative(), newGrandChildren);
                } else if (childUDF instanceof GenericUDFOPAnd || childUDF instanceof GenericUDFOPOr) {
                    List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
                    for (ExprNodeDesc grandChild : grandChildren) {
                        newGrandChildren.add(foldNegative(ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), Arrays.asList(grandChild))));
                    }
                    return ExprNodeGenericFuncDesc.newInstance(childUDF.negative(), newGrandChildren);
                } else if (childUDF instanceof GenericUDFOPNot) {
                    return foldNegative(child.getChildren().get(0));
                } else {
                    // For operator like if() that cannot be handled, leave not() as it
                    // is and continue processing the children
                    List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
                    for (ExprNodeDesc grandChild : grandChildren) {
                        newGrandChildren.add(foldNegative(grandChild));
                    }
                    childDesc.setChildren(newGrandChildren);
                    return funcDesc;
                }
            }
        }
    }
    return desc;
}

Example 57 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class PartitionPruner method compactExpr.

/**
 * Taking a partition pruning expression, remove the null operands and non-partition columns.
 * The reason why there are null operands is ExprProcFactory classes, for example
 * PPRColumnExprProcessor.
 * @param expr original partition pruning expression.
 * @return partition pruning expression that only contains partition columns.
 */
@VisibleForTesting
static ExprNodeDesc compactExpr(ExprNodeDesc expr) {
    // If this is a constant boolean expression, return the value.
    if (expr == null) {
        return null;
    }
    if (expr instanceof ExprNodeConstantDesc) {
        if (((ExprNodeConstantDesc) expr).getValue() == null) {
            return null;
        }
        if (!isBooleanExpr(expr)) {
            throw new IllegalStateException("Unexpected non-boolean ExprNodeConstantDesc: " + expr.getExprString(true));
        }
        return expr;
    } else if (expr instanceof ExprNodeColumnDesc) {
        return expr;
    } else if (expr instanceof ExprNodeGenericFuncDesc) {
        GenericUDF udf = ((ExprNodeGenericFuncDesc) expr).getGenericUDF();
        boolean isAnd = udf instanceof GenericUDFOPAnd;
        boolean isOr = udf instanceof GenericUDFOPOr;
        List<ExprNodeDesc> children = expr.getChildren();
        if (isAnd) {
            // Non-partition expressions are converted to nulls.
            List<ExprNodeDesc> newChildren = new ArrayList<ExprNodeDesc>();
            boolean allTrue = true;
            for (ExprNodeDesc child : children) {
                ExprNodeDesc compactChild = compactExpr(child);
                if (compactChild != null) {
                    if (!isTrueExpr(compactChild)) {
                        newChildren.add(compactChild);
                        allTrue = false;
                    }
                    if (isFalseExpr(compactChild)) {
                        return new ExprNodeConstantDesc(Boolean.FALSE);
                    }
                } else {
                    allTrue = false;
                }
            }
            if (allTrue) {
                return new ExprNodeConstantDesc(Boolean.TRUE);
            }
            if (newChildren.size() == 0) {
                return null;
            }
            if (newChildren.size() == 1) {
                return newChildren.get(0);
            }
            // Nothing to compact, update expr with compacted children.
            ((ExprNodeGenericFuncDesc) expr).setChildren(newChildren);
        } else if (isOr) {
            // Non-partition expressions are converted to nulls.
            List<ExprNodeDesc> newChildren = new ArrayList<ExprNodeDesc>();
            boolean allFalse = true;
            boolean isNull = false;
            for (ExprNodeDesc child : children) {
                ExprNodeDesc compactChild = compactExpr(child);
                if (compactChild != null) {
                    if (isTrueExpr(compactChild)) {
                        return new ExprNodeConstantDesc(Boolean.TRUE);
                    }
                    if (!isNull && !isFalseExpr(compactChild)) {
                        newChildren.add(compactChild);
                        allFalse = false;
                    }
                } else {
                    isNull = true;
                }
            }
            if (isNull) {
                return null;
            }
            if (allFalse) {
                return new ExprNodeConstantDesc(Boolean.FALSE);
            }
            if (newChildren.size() == 1) {
                return newChildren.get(0);
            }
            // Nothing to compact, update expr with compacted children.
            ((ExprNodeGenericFuncDesc) expr).setChildren(newChildren);
        }
        return expr;
    } else {
        throw new IllegalStateException("Unexpected type of ExprNodeDesc: " + expr.getExprString(true));
    }
}

Example 58 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class PartitionPruner method prune.

/**
 * Get the partition list for the table that satisfies the partition pruner
 * condition.
 *
 * @param tab
 *          the table object for the alias
 * @param prunerExpr
 *          the pruner expression for the alias
 * @param conf
 *          for checking whether "strict" mode is on.
 * @param alias
 *          for generating error message only.
 * @param prunedPartitionsMap
 *          cached result for the table
 * @return the partition list for the table that satisfies the partition
 *         pruner condition.
 * @throws SemanticException
 */
public static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr, HiveConf conf, String alias, Map<String, PrunedPartitionList> prunedPartitionsMap) throws SemanticException {
    if (LOG.isTraceEnabled()) {
        LOG.trace("Started pruning partition");
        LOG.trace("dbname = " + tab.getDbName());
        LOG.trace("tabname = " + tab.getTableName());
        LOG.trace("prune Expression = " + (prunerExpr == null ? "" : prunerExpr));
    }
    String key = tab.getFullyQualifiedName() + ";";
    if (tab.getMetaTable() != null) {
        key = tab.getFullyQualifiedName() + "." + tab.getMetaTable() + ";";
    }
    if (!tab.isPartitioned()) {
        // If the table is not partitioned, return empty list.
        return getAllPartsFromCacheOrServer(tab, key, false, prunedPartitionsMap);
    }
    if (!hasColumnExpr(prunerExpr)) {
        // If the "strict" mode is on, we have to provide partition pruner for each table.
        String error = StrictChecks.checkNoPartitionFilter(conf);
        if (error != null) {
            throw new SemanticException(error + " No partition predicate for Alias \"" + alias + "\" Table \"" + tab.getTableName() + "\"");
        }
    }
    if (prunerExpr == null) {
        // In non-strict mode and there is no predicates at all - get everything.
        return getAllPartsFromCacheOrServer(tab, key, false, prunedPartitionsMap);
    }
    Set<String> partColsUsedInFilter = new LinkedHashSet<String>();
    // Replace virtual columns with nulls. See javadoc for details.
    prunerExpr = removeNonPartCols(prunerExpr, extractPartColNames(tab), partColsUsedInFilter);
    // Remove all parts that are not partition columns. See javadoc for details.
    ExprNodeDesc compactExpr = compactExpr(prunerExpr.clone());
    String oldFilter = prunerExpr.getExprString(true);
    if (compactExpr == null || isBooleanExpr(compactExpr)) {
        if (isFalseExpr(compactExpr)) {
            return new PrunedPartitionList(tab, key + compactExpr.getExprString(true), Collections.emptySet(), Collections.emptyList(), false);
        }
        // For null and true values, return every partition
        return getAllPartsFromCacheOrServer(tab, key, true, prunedPartitionsMap);
    }
    String compactExprString = compactExpr.getExprString(true);
    LOG.debug("Filter w/ compacting: {}; filter w/o compacting: {}", compactExprString, oldFilter);
    key = key + compactExprString;
    PrunedPartitionList ppList = prunedPartitionsMap.get(key);
    if (ppList != null) {
        return ppList;
    }
    ppList = getPartitionsFromServer(tab, key, (ExprNodeGenericFuncDesc) compactExpr, conf, alias, partColsUsedInFilter, oldFilter.equals(compactExprString));
    prunedPartitionsMap.put(key, ppList);
    return ppList;
}

Example 59 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class PartitionPruner method removeNonPartCols.

/**
 * See compactExpr. Some things in the expr are replaced with nulls for pruner, however
 * the virtual columns are not removed (ExprNodeColumnDesc cannot tell them apart from
 * partition columns), so we do it here.
 * The expression is only used to prune by partition name, so we have no business with VCs.
 * @param expr original partition pruning expression.
 * @param partCols list of partition columns for the table.
 * @param referred partition columns referred by expr
 * @return partition pruning expression that only contains partition columns from the list.
 */
private static ExprNodeDesc removeNonPartCols(ExprNodeDesc expr, List<String> partCols, Set<String> referred) {
    if (expr instanceof ExprNodeFieldDesc) {
        // list or struct fields.
        return new ExprNodeConstantDesc(expr.getTypeInfo(), null);
    } else if (expr instanceof ExprNodeColumnDesc) {
        String column = ((ExprNodeColumnDesc) expr).getColumn();
        if (!partCols.contains(column)) {
            // Column doesn't appear to be a partition column for the table.
            return new ExprNodeConstantDesc(expr.getTypeInfo(), null);
        }
        referred.add(column);
    } else if (expr instanceof ExprNodeGenericFuncDesc) {
        List<ExprNodeDesc> children = expr.getChildren();
        for (int i = 0; i < children.size(); ++i) {
            ExprNodeDesc other = removeNonPartCols(children.get(i), partCols, referred);
            if (ExprNodeDescUtils.isNullConstant(other)) {
                if (FunctionRegistry.isOpAnd(expr)) {
                    // partcol=... AND nonpartcol=...   is replaced with partcol=... AND TRUE
                    // which will be folded to partcol=...
                    // This cannot be done also for OR
                    Preconditions.checkArgument(expr.getTypeInfo().accept(TypeInfoFactory.booleanTypeInfo));
                    other = new ExprNodeConstantDesc(expr.getTypeInfo(), true);
                } else {
                    // and cause overaggressive prunning, missing data (incorrect result)
                    return new ExprNodeConstantDesc(expr.getTypeInfo(), null);
                }
            }
            children.set(i, other);
        }
    }
    return expr;
}

Example 60 with ExprNodeGenericFuncDesc

use of org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc in project hive by apache.

the class TestVectorizationContext method testBooleanColumnCompareBooleanScalar.

@Test
public void testBooleanColumnCompareBooleanScalar() throws HiveException {
    ExprNodeGenericFuncDesc colEqualScalar = new ExprNodeGenericFuncDesc();
    GenericUDFOPEqual gudf = new GenericUDFOPEqual();
    colEqualScalar.setGenericUDF(gudf);
    List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(2);
    ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, 20);
    ExprNodeColumnDesc colDesc = new ExprNodeColumnDesc(Boolean.class, "a", "table", false);
    children.add(colDesc);
    children.add(constDesc);
    colEqualScalar.setChildren(children);
    colEqualScalar.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
    List<String> columns = new ArrayList<String>();
    columns.add("a");
    VectorizationContext vc = new VectorizationContext("name", columns);
    VectorExpression ve = vc.getVectorExpression(colEqualScalar, VectorExpressionDescriptor.Mode.PROJECTION);
    assertEquals(LongColEqualLongScalar.class, ve.getClass());
}