Examples with SemanticDispatcher org.apache.hadoop.hive.ql.lib.SemanticDispatcher used on opensource projects

Example 86 with SemanticDispatcher

use of org.apache.hadoop.hive.ql.lib.SemanticDispatcher in project hive by apache.

the class Vectorizer method resolve.

@Override
public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticException {
    hiveConf = physicalContext.getConf();
    planMapper = physicalContext.getContext().getPlanMapper();
    String vectorizationEnabledOverrideString = HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_TEST_VECTORIZATION_ENABLED_OVERRIDE);
    vectorizationEnabledOverride = EnabledOverride.nameMap.get(vectorizationEnabledOverrideString);
    isVectorizationEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED);
    final boolean weCanAttemptVectorization;
    isTestForcedVectorizationEnable = false;
    switch(vectorizationEnabledOverride) {
        case NONE:
            weCanAttemptVectorization = isVectorizationEnabled;
            break;
        case DISABLE:
            weCanAttemptVectorization = false;
            break;
        case ENABLE:
            weCanAttemptVectorization = true;
            isTestForcedVectorizationEnable = !isVectorizationEnabled;
            // Different parts of the code rely on this being set...
            HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true);
            isVectorizationEnabled = true;
            break;
        default:
            throw new RuntimeException("Unexpected vectorization enabled override " + vectorizationEnabledOverride);
    }
    if (!weCanAttemptVectorization) {
        LOG.info("Vectorization is disabled");
        return physicalContext;
    }
    useVectorizedInputFileFormat = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTORIZED_INPUT_FILE_FORMAT);
    if (useVectorizedInputFileFormat) {
        initVectorizedInputFormatExcludeClasses();
    }
    useVectorDeserialize = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTOR_DESERIALIZE);
    useRowDeserialize = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_USE_ROW_DESERIALIZE);
    if (useRowDeserialize) {
        initRowDeserializeExcludeClasses();
    }
    // TODO: we could also vectorize some formats based on hive.llap.io.encode.formats if LLAP IO
    // is enabled and we are going to run in LLAP. However, we don't know if we end up in
    // LLAP or not at this stage, so don't do this now. We may need to add a 'force' option.
    isReduceVectorizationEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_ENABLED);
    isPtfVectorizationEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_PTF_ENABLED);
    isVectorizationComplexTypesEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_COMPLEX_TYPES_ENABLED);
    isVectorizationGroupByComplexTypesEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_GROUPBY_COMPLEX_TYPES_ENABLED);
    isVectorizedRowIdentifierEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ROW_IDENTIFIER_ENABLED);
    vectorizedPTFMaxMemoryBufferingBatchCount = HiveConf.getIntVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_PTF_MAX_MEMORY_BUFFERING_BATCH_COUNT);
    vectorizedTestingReducerBatchSize = HiveConf.getIntVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_TESTING_REDUCER_BATCH_SIZE);
    isTestVectorizerSuppressFatalExceptions = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_TEST_VECTORIZER_SUPPRESS_FATAL_EXCEPTIONS);
    vectorizedInputFormatSupportEnabled = HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED);
    String[] supportEnabledStrings = vectorizedInputFormatSupportEnabled.toLowerCase().split(",");
    vectorizedInputFormatSupportEnabledSet = new TreeSet<Support>();
    for (String supportEnabledString : supportEnabledStrings) {
        Support support = Support.nameToSupportMap.get(supportEnabledString);
        // Known?
        if (support != null) {
            vectorizedInputFormatSupportEnabledSet.add(support);
        }
    }
    /*
     * Notice the default value for LLAP_IO_ENABLED is overridden to be whether we are
     * executing under LLAP.
     */
    isLlapIoEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.LLAP_IO_ENABLED, LlapProxy.isDaemon());
    isSchemaEvolution = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_SCHEMA_EVOLUTION);
    hiveVectorAdaptorUsageMode = HiveVectorAdaptorUsageMode.getHiveConfValue(hiveConf);
    isTestVectorizationSuppressExplainExecutionMode = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_TEST_VECTORIZATION_SUPPRESS_EXPLAIN_EXECUTION_MODE);
    // create dispatcher and graph walker
    SemanticDispatcher disp = new VectorizationDispatcher();
    TaskGraphWalker ogw = new TaskGraphWalker(disp);
    // get all the tasks nodes from root task
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(physicalContext.getRootTasks());
    // begin to walk through the task tree.
    ogw.startWalking(topNodes, null);
    return physicalContext;
}

Example 87 with SemanticDispatcher

use of org.apache.hadoop.hive.ql.lib.SemanticDispatcher in project hive by apache.

the class MemoryDecider method resolve.

@Override
public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
    pctx.getConf();
    // create dispatcher and graph walker
    SemanticDispatcher disp = new MemoryCalculator(pctx);
    TaskGraphWalker ogw = new TaskGraphWalker(disp);
    // get all the tasks nodes from root task
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getRootTasks());
    // begin to walk through the task tree.
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 88 with SemanticDispatcher

use of org.apache.hadoop.hive.ql.lib.SemanticDispatcher in project hive by apache.

the class NullScanOptimizer method resolve.

@Override
public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
    Map<SemanticRule, SemanticNodeProcessor> opRules = new LinkedHashMap<>();
    opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%.*" + FilterOperator.getOperatorName() + "%"), new WhereFalseProcessor());
    SemanticDispatcher disp = new NullScanTaskDispatcher(pctx, opRules);
    SemanticGraphWalker ogw = new DefaultGraphWalker(disp);
    List<Node> topNodes = new ArrayList<>(pctx.getRootTasks());
    ogw.startWalking(topNodes, null);
    opRules.clear();
    opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%"), new TSMarker());
    opRules.put(new RuleRegExp("R2", LimitOperator.getOperatorName() + "%"), new Limit0Processor());
    disp = new NullScanTaskDispatcher(pctx, opRules);
    ogw = new DefaultGraphWalker(disp);
    topNodes = new ArrayList<>(pctx.getRootTasks());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 89 with SemanticDispatcher

use of org.apache.hadoop.hive.ql.lib.SemanticDispatcher in project hive by apache.

the class MapJoinResolver method resolve.

@Override
public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
    // create dispatcher and graph walker
    SemanticDispatcher disp = new LocalMapJoinTaskDispatcher(pctx);
    TaskGraphWalker ogw = new TaskGraphWalker(disp);
    // get all the tasks nodes from root task
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getRootTasks());
    // begin to walk through the task tree.
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 90 with SemanticDispatcher

use of org.apache.hadoop.hive.ql.lib.SemanticDispatcher in project hive by apache.

the class SortMergeJoinResolver method resolve.

@Override
public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
    // create dispatcher and graph walker
    SemanticDispatcher disp = new SortMergeJoinTaskDispatcher(pctx);
    TaskGraphWalker ogw = new TaskGraphWalker(disp);
    // get all the tasks nodes from root task
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.rootTasks);
    // begin to walk through the task tree.
    ogw.startWalking(topNodes, null);
    return pctx;
}