Examples with FilterOperator org.apache.hadoop.hive.ql.exec.FilterOperator used on opensource projects

Example 26 with FilterOperator

use of org.apache.hadoop.hive.ql.exec.FilterOperator in project hive by apache.

the class SharedWorkOptimizer method pushFilterToTopOfTableScan.

private static void pushFilterToTopOfTableScan(SharedWorkOptimizerCache optimizerCache, DecomposedTs tsModel) throws UDFArgumentException {
    TableScanOperator tsOp = tsModel.ts;
    ExprNodeGenericFuncDesc tableScanExprNode = (ExprNodeGenericFuncDesc) tsModel.getFullFilterExpr();
    if (tableScanExprNode == null) {
        return;
    }
    List<Operator<? extends OperatorDesc>> allChildren = Lists.newArrayList(tsOp.getChildOperators());
    childOperators: for (Operator<? extends OperatorDesc> op : allChildren) {
        if (optimizerCache.isKnownFilteringOperator(op)) {
            continue;
        }
        if (op instanceof FilterOperator) {
            FilterOperator filterOp = (FilterOperator) op;
            ExprNodeDesc filterExprNode = filterOp.getConf().getPredicate();
            if (tableScanExprNode.isSame(filterExprNode)) {
                // We do not need to do anything
                optimizerCache.setKnownFilteringOperator(filterOp);
                continue;
            }
            if (tableScanExprNode.getGenericUDF() instanceof GenericUDFOPOr) {
                for (ExprNodeDesc childExprNode : tableScanExprNode.getChildren()) {
                    if (childExprNode.isSame(filterExprNode)) {
                        // We do not need to do anything, it is in the OR expression
                        // so probably we pushed previously
                        optimizerCache.setKnownFilteringOperator(filterOp);
                        continue childOperators;
                    }
                }
            }
            ExprNodeDesc newFilterExpr = conjunction(filterExprNode, tableScanExprNode);
            if (!isSame(filterOp.getConf().getPredicate(), newFilterExpr)) {
                filterOp.getConf().setPredicate(newFilterExpr);
            }
            optimizerCache.setKnownFilteringOperator(filterOp);
        } else {
            Operator<FilterDesc> newOp = OperatorFactory.get(tsOp.getCompilationOpContext(), new FilterDesc(tableScanExprNode.clone(), false), new RowSchema(tsOp.getSchema().getSignature()));
            tsOp.replaceChild(op, newOp);
            newOp.getParentOperators().add(tsOp);
            op.replaceParent(tsOp, newOp);
            newOp.getChildOperators().add(op);
            // Add to cache (same group as tsOp)
            optimizerCache.putIfWorkExists(newOp, tsOp);
            optimizerCache.setKnownFilteringOperator(newOp);
        }
    }
}

Example 27 with FilterOperator

use of org.apache.hadoop.hive.ql.exec.FilterOperator in project hive by apache.

the class SimpleFetchOptimizer method checkTree.

// all we can handle is LimitOperator, FilterOperator SelectOperator and final FS
// 
// for non-aggressive mode (minimal)
// 1. sampling is not allowed
// 2. for partitioned table, all filters should be targeted to partition column
// 3. SelectOperator should use only simple cast/column access
private FetchData checkTree(boolean aggressive, ParseContext pctx, String alias, TableScanOperator ts) throws HiveException {
    SplitSample splitSample = pctx.getNameToSplitSample().get(alias);
    if (!aggressive && splitSample != null) {
        return null;
    }
    if (!aggressive && ts.getConf().getTableSample() != null) {
        return null;
    }
    Table table = ts.getConf().getTableMetadata();
    if (table == null) {
        return null;
    }
    ReadEntity parent = PlanUtils.getParentViewInfo(alias, pctx.getViewAliasToInput());
    if (!table.isPartitioned()) {
        FetchData fetch = new FetchData(ts, parent, table, splitSample);
        return checkOperators(fetch, aggressive, false);
    }
    boolean bypassFilter = false;
    if (HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVEOPTPPD)) {
        ExprNodeDesc pruner = pctx.getOpToPartPruner().get(ts);
        if (PartitionPruner.onlyContainsPartnCols(table, pruner)) {
            bypassFilter = !pctx.getPrunedPartitions(alias, ts).hasUnknownPartitions();
        }
    }
    boolean onlyPruningFilter = bypassFilter;
    Operator<?> op = ts;
    while (onlyPruningFilter) {
        if (op instanceof FileSinkOperator || op.getChildOperators() == null) {
            break;
        } else if (op.getChildOperators().size() != 1) {
            onlyPruningFilter = false;
            break;
        } else {
            op = op.getChildOperators().get(0);
        }
        if (op instanceof FilterOperator) {
            ExprNodeDesc predicate = ((FilterOperator) op).getConf().getPredicate();
            if (predicate instanceof ExprNodeConstantDesc && "boolean".equals(predicate.getTypeInfo().getTypeName())) {
                continue;
            } else if (PartitionPruner.onlyContainsPartnCols(table, predicate)) {
                continue;
            } else {
                onlyPruningFilter = false;
            }
        }
    }
    if (!aggressive && !onlyPruningFilter) {
        return null;
    }
    PrunedPartitionList partitions = pctx.getPrunedPartitions(alias, ts);
    FetchData fetch = new FetchData(ts, parent, table, partitions, splitSample, onlyPruningFilter);
    return checkOperators(fetch, aggressive, bypassFilter);
}

Example 28 with FilterOperator

use of org.apache.hadoop.hive.ql.exec.FilterOperator in project hive by apache.

the class ExecuteStatementAnalyzer method bindOperatorsWithDynamicParams.

private void bindOperatorsWithDynamicParams(Map<Integer, ASTNode> parameterMap) throws SemanticException {
    // Push the node in the stack
    Collection<Operator> allOps = this.getParseContext().getAllOps();
    for (Operator op : allOps) {
        switch(op.getType()) {
            case FILTER:
                FilterOperator filterOp = (FilterOperator) op;
                ExprNodeDesc predicate = filterOp.getConf().getPredicate();
                filterOp.getConf().setPredicate(replaceDynamicParamsWithConstant(predicate, TypeInfoFactory.booleanTypeInfo, parameterMap));
                break;
        }
    }
}

Example 29 with FilterOperator

use of org.apache.hadoop.hive.ql.exec.FilterOperator in project hive by apache.

the class TestCounterMapping method testInConversion.

@Test
public void testInConversion() throws ParseException, CommandProcessorException {
    String query = "explain select sum(id_uv) from tu where u in (1,2) group by u";
    HiveConf conf = env_setup.getTestCtx().hiveConf;
    conf.setIntVar(ConfVars.HIVEPOINTLOOKUPOPTIMIZERMIN, 10);
    IDriver driver = createDriver();
    PlanMapper pm = getMapperForQuery(driver, query);
    List<FilterOperator> fos = pm.getAll(FilterOperator.class);
    OpTreeSignature filterSig = pm.lookup(OpTreeSignature.class, fos.get(0));
    Object pred = filterSig.getSig().getSigMap().get("getPredicateString");
    assertEquals("((u = 1) or (u = 2)) (type: boolean)", pred);
}

Example 30 with FilterOperator

use of org.apache.hadoop.hive.ql.exec.FilterOperator in project hive by apache.

the class TestCounterMapping method testUsageOfRuntimeInfo.

@Test
public void testUsageOfRuntimeInfo() throws ParseException, CommandProcessorException {
    IDriver driver = createDriver();
    String query = "select sum(u) from tu where u>1";
    PlanMapper pm1 = getMapperForQuery(driver, query);
    List<FilterOperator> filters1 = pm1.getAll(FilterOperator.class);
    filters1.sort(OPERATOR_ID_COMPARATOR.reversed());
    FilterOperator filter1 = filters1.get(0);
    driver = createDriver();
    ((ReExecDriver) driver).setStatsSource(StatsSources.getStatsSourceContaining(EmptyStatsSource.INSTANCE, pm1));
    PlanMapper pm2 = getMapperForQuery(driver, query);
    List<FilterOperator> filters2 = pm2.getAll(FilterOperator.class);
    filters2.sort(OPERATOR_ID_COMPARATOR.reversed());
    FilterOperator filter2 = filters2.get(0);
    assertEquals("original check", 7, filter1.getStatistics().getNumRows());
    assertEquals("optimized check", 6, filter2.getStatistics().getNumRows());
}