Examples with Dispatcher org.apache.hadoop.hive.ql.lib.Dispatcher used on opensource projects

Example 81 with Dispatcher

use of org.apache.hadoop.hive.ql.lib.Dispatcher in project hive by apache.

the class SemanticAnalyzer method replaceViewReferenceWithDefinition.

private void replaceViewReferenceWithDefinition(QB qb, Table tab, String tab_name, String alias) throws SemanticException {
    ASTNode viewTree;
    final ASTNodeOrigin viewOrigin = new ASTNodeOrigin("VIEW", tab.getTableName(), tab.getViewExpandedText(), alias, qb.getParseInfo().getSrcForAlias(alias));
    try {
        // Reparse text, passing null for context to avoid clobbering
        // the top-level token stream.
        String viewFullyQualifiedName = tab.getCompleteName();
        String viewText = tab.getViewExpandedText();
        TableMask viewMask = new TableMask(this, conf, false);
        viewTree = ParseUtils.parse(viewText, ctx, tab.getCompleteName());
        if (!unparseTranslator.isEnabled() && (viewMask.isEnabled() && analyzeRewrite == null)) {
            viewTree = rewriteASTWithMaskAndFilter(viewMask, viewTree, ctx.getViewTokenRewriteStream(viewFullyQualifiedName), ctx, db, tabNameToTabObject, ignoredTokens);
        }
        Dispatcher nodeOriginDispatcher = new Dispatcher() {

            @Override
            public Object dispatch(Node nd, java.util.Stack<Node> stack, Object... nodeOutputs) {
                ((ASTNode) nd).setOrigin(viewOrigin);
                return null;
            }
        };
        GraphWalker nodeOriginTagger = new DefaultGraphWalker(nodeOriginDispatcher);
        nodeOriginTagger.startWalking(java.util.Collections.<Node>singleton(viewTree), null);
    } catch (ParseException e) {
        // A user could encounter this if a stored view definition contains
        // an old SQL construct which has been eliminated in a later Hive
        // version, so we need to provide full debugging info to help
        // with fixing the view definition.
        LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
        StringBuilder sb = new StringBuilder();
        sb.append(e.getMessage());
        ErrorMsg.renderOrigin(sb, viewOrigin);
        throw new SemanticException(sb.toString(), e);
    }
    QBExpr qbexpr = new QBExpr(alias);
    doPhase1QBExpr(viewTree, qbexpr, qb.getId(), alias, true);
    // if HIVE_STATS_COLLECT_SCANCOLS is enabled, check.
    if ((!this.skipAuthorization() && !qb.isInsideView() && HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED)) || HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS)) {
        qb.rewriteViewToSubq(alias, tab_name, qbexpr, tab);
    } else {
        qb.rewriteViewToSubq(alias, tab_name, qbexpr, null);
    }
}

Example 82 with Dispatcher

use of org.apache.hadoop.hive.ql.lib.Dispatcher in project hive by apache.

the class Vectorizer method resolve.

@Override
public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticException {
    hiveConf = physicalContext.getConf();
    String vectorizationEnabledOverrideString = HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_TEST_VECTORIZATION_ENABLED_OVERRIDE);
    vectorizationEnabledOverride = VectorizationEnabledOverride.nameMap.get(vectorizationEnabledOverrideString);
    isVectorizationEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED);
    final boolean weCanAttemptVectorization;
    switch(vectorizationEnabledOverride) {
        case NONE:
            weCanAttemptVectorization = isVectorizationEnabled;
            break;
        case DISABLE:
            weCanAttemptVectorization = false;
            break;
        case ENABLE:
            weCanAttemptVectorization = true;
            break;
        default:
            throw new RuntimeException("Unexpected vectorization enabled override " + vectorizationEnabledOverride);
    }
    if (!weCanAttemptVectorization) {
        LOG.info("Vectorization is disabled");
        return physicalContext;
    }
    useVectorizedInputFileFormat = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTORIZED_INPUT_FILE_FORMAT);
    if (useVectorizedInputFileFormat) {
        initVectorizedInputFormatExcludeClasses();
    }
    useVectorDeserialize = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTOR_DESERIALIZE);
    useRowDeserialize = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_USE_ROW_DESERIALIZE);
    if (useRowDeserialize) {
        initRowDeserializeExcludeClasses();
    }
    // TODO: we could also vectorize some formats based on hive.llap.io.encode.formats if LLAP IO
    // is enabled and we are going to run in LLAP. However, we don't know if we end up in
    // LLAP or not at this stage, so don't do this now. We may need to add a 'force' option.
    isReduceVectorizationEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_ENABLED);
    isPtfVectorizationEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_PTF_ENABLED);
    isVectorizationComplexTypesEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_COMPLEX_TYPES_ENABLED);
    isVectorizationGroupByComplexTypesEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_GROUPBY_COMPLEX_TYPES_ENABLED);
    isVectorizedRowIdentifierEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ROW_IDENTIFIER_ENABLED);
    vectorizedPTFMaxMemoryBufferingBatchCount = HiveConf.getIntVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_PTF_MAX_MEMORY_BUFFERING_BATCH_COUNT);
    vectorizedTestingReducerBatchSize = HiveConf.getIntVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_TESTING_REDUCER_BATCH_SIZE);
    vectorizedInputFormatSupportEnabled = HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED);
    String[] supportEnabledStrings = vectorizedInputFormatSupportEnabled.toLowerCase().split(",");
    vectorizedInputFormatSupportEnabledSet = new TreeSet<Support>();
    for (String supportEnabledString : supportEnabledStrings) {
        Support support = Support.nameToSupportMap.get(supportEnabledString);
        // Known?
        if (support != null) {
            vectorizedInputFormatSupportEnabledSet.add(support);
        }
    }
    isLlapIoEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.LLAP_IO_ENABLED, LlapProxy.isDaemon());
    isSchemaEvolution = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_SCHEMA_EVOLUTION);
    hiveVectorAdaptorUsageMode = HiveVectorAdaptorUsageMode.getHiveConfValue(hiveConf);
    // create dispatcher and graph walker
    Dispatcher disp = new VectorizationDispatcher();
    TaskGraphWalker ogw = new TaskGraphWalker(disp);
    // get all the tasks nodes from root task
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(physicalContext.getRootTasks());
    // begin to walk through the task tree.
    ogw.startWalking(topNodes, null);
    return physicalContext;
}

Example 83 with Dispatcher

use of org.apache.hadoop.hive.ql.lib.Dispatcher in project hive by apache.

the class UnionProcessor method transform.

/**
 * Transform the query tree. For each union, store the fact whether both the
 * sub-queries are map-only
 *
 * @param pCtx
 *          the current parse context
 */
public ParseContext transform(ParseContext pCtx) throws SemanticException {
    // create a walker which walks the tree in a BFS manner while maintaining
    // the operator stack.
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", ReduceSinkOperator.getOperatorName() + "%.*" + UnionOperator.getOperatorName() + "%"), UnionProcFactory.getMapRedUnion());
    opRules.put(new RuleRegExp("R2", UnionOperator.getOperatorName() + "%.*" + UnionOperator.getOperatorName() + "%"), UnionProcFactory.getUnknownUnion());
    opRules.put(new RuleRegExp("R3", TableScanOperator.getOperatorName() + "%.*" + UnionOperator.getOperatorName() + "%"), UnionProcFactory.getMapUnion());
    // The dispatcher fires the processor for the matching rule and passes the
    // context along
    UnionProcContext uCtx = new UnionProcContext();
    uCtx.setParseContext(pCtx);
    Dispatcher disp = new DefaultRuleDispatcher(UnionProcFactory.getNoUnion(), opRules, uCtx);
    LevelOrderWalker ogw = new LevelOrderWalker(disp);
    ogw.setNodeTypes(UnionOperator.class);
    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pCtx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    pCtx.setUCtx(uCtx);
    // Walk the tree again to see if the union can be removed completely
    HiveConf conf = pCtx.getConf();
    opRules.clear();
    if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_UNION_REMOVE) && !conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
        opRules.put(new RuleRegExp("R5", UnionOperator.getOperatorName() + "%" + ".*" + FileSinkOperator.getOperatorName() + "%"), UnionProcFactory.getUnionNoProcessFile());
        disp = new DefaultRuleDispatcher(UnionProcFactory.getNoUnion(), opRules, uCtx);
        ogw = new LevelOrderWalker(disp);
        ogw.setNodeTypes(FileSinkOperator.class);
        // Create a list of topop nodes
        topNodes.clear();
        topNodes.addAll(pCtx.getTopOps().values());
        ogw.startWalking(topNodes, null);
    }
    return pCtx;
}

Example 84 with Dispatcher

use of org.apache.hadoop.hive.ql.lib.Dispatcher in project hive by apache.

the class CorrelationOptimizer method transform.

/**
 * Detect correlations and transform the query tree.
 *
 * @param pactx
 *          current parse context
 * @throws SemanticException
 */
public ParseContext transform(ParseContext pctx) throws SemanticException {
    pCtx = pctx;
    if (HiveConf.getBoolVar(pCtx.getConf(), HiveConf.ConfVars.HIVECONVERTJOIN)) {
        findPossibleAutoConvertedJoinOperators();
    }
    // detect correlations
    CorrelationNodeProcCtx corrCtx = new CorrelationNodeProcCtx(pCtx);
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", ReduceSinkOperator.getOperatorName() + "%"), new CorrelationNodeProc());
    Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, corrCtx);
    GraphWalker ogw = new DefaultGraphWalker(disp);
    // Create a list of topOp nodes
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pCtx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    // We have finished tree walking (correlation detection).
    // We will first see if we need to abort (the operator tree has not been changed).
    // If not, we will start to transform the operator tree.
    abort = corrCtx.isAbort();
    if (abort) {
        LOG.info("Abort. Reasons are ...");
        for (String reason : corrCtx.getAbortReasons()) {
            LOG.info("-- " + reason);
        }
    } else {
        // transform the operator tree
        LOG.info("Begain query plan transformation based on intra-query correlations. " + corrCtx.getCorrelations().size() + " correlation(s) to be applied");
        for (IntraQueryCorrelation correlation : corrCtx.getCorrelations()) {
            QueryPlanTreeTransformation.applyCorrelation(pCtx, corrCtx, correlation);
        }
    }
    return pCtx;
}

Example 85 with Dispatcher

use of org.apache.hadoop.hive.ql.lib.Dispatcher in project hive by apache.

the class ReduceSinkJoinDeDuplication method transform.

@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
    pGraphContext = pctx;
    ReduceSinkJoinDeDuplicateProcCtx cppCtx = new ReduceSinkJoinDeDuplicateProcCtx(pGraphContext);
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", ReduceSinkOperator.getOperatorName() + "%"), ReduceSinkJoinDeDuplicateProcFactory.getReducerMapJoinProc());
    Dispatcher disp = new DefaultRuleDispatcher(ReduceSinkJoinDeDuplicateProcFactory.getDefaultProc(), opRules, cppCtx);
    GraphWalker ogw = new ForwardWalker(disp);
    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pGraphContext.getTopOps().values());
    ogw.startWalking(topNodes, null);
    return pGraphContext;
}