Examples with TaskGraphWalker org.apache.hadoop.hive.ql.lib.TaskGraphWalker used on opensource projects

Example 1 with TaskGraphWalker

use of org.apache.hadoop.hive.ql.lib.TaskGraphWalker in project hive by apache.

the class CrossProductCheck method resolve.

@Override
public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
    TaskGraphWalker ogw = new TaskGraphWalker(this);
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getRootTasks());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 2 with TaskGraphWalker

use of org.apache.hadoop.hive.ql.lib.TaskGraphWalker in project hive by apache.

the class CrossProductHandler method resolve.

@Override
public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
    HiveConf conf = pctx.getConf();
    cartesianProductEdgeEnabled = HiveConf.getBoolVar(conf, HiveConf.ConfVars.TEZ_CARTESIAN_PRODUCT_EDGE_ENABLED);
    // if max parallelism isn't set by user in llap mode, set it to number of executors
    if (cartesianProductEdgeEnabled && HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_MODE).equals("llap") && conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM) == null) {
        LlapClusterStateForCompile llapInfo = LlapClusterStateForCompile.getClusterInfo(conf);
        llapInfo.initClusterInfo();
        if (llapInfo.hasClusterInfo()) {
            conf.setInt(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM, llapInfo.getKnownExecutorCount());
        }
    }
    TaskGraphWalker ogw = new TaskGraphWalker(this);
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getRootTasks());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 3 with TaskGraphWalker

use of org.apache.hadoop.hive.ql.lib.TaskGraphWalker in project hive by apache.

the class CommonJoinResolver method resolve.

@Override
public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
    // create dispatcher and graph walker
    SemanticDispatcher disp = new CommonJoinTaskDispatcher(pctx);
    TaskGraphWalker ogw = new TaskGraphWalker(disp);
    // get all the tasks nodes from root task
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.rootTasks);
    // begin to walk through the task tree.
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 4 with TaskGraphWalker

use of org.apache.hadoop.hive.ql.lib.TaskGraphWalker in project hive by apache.

the class LlapDecider method resolve.

@Override
public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
    this.conf = pctx.getConf();
    this.mode = LlapMode.valueOf(HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_EXECUTION_MODE));
    Preconditions.checkState(this.mode != null, "Unrecognized LLAP mode configuration: " + HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_EXECUTION_MODE));
    LOG.info("llap mode: " + this.mode);
    if (mode == none) {
        LOG.info("LLAP disabled.");
        return pctx;
    }
    // create dispatcher and graph walker
    SemanticDispatcher disp = new LlapDecisionDispatcher(pctx, mode);
    TaskGraphWalker ogw = new TaskGraphWalker(disp);
    // get all the tasks nodes from root task
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getRootTasks());
    // begin to walk through the task tree.
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 5 with TaskGraphWalker

use of org.apache.hadoop.hive.ql.lib.TaskGraphWalker in project hive by apache.

the class Vectorizer method resolve.

@Override
public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticException {
    hiveConf = physicalContext.getConf();
    planMapper = physicalContext.getContext().getPlanMapper();
    String vectorizationEnabledOverrideString = HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_TEST_VECTORIZATION_ENABLED_OVERRIDE);
    vectorizationEnabledOverride = EnabledOverride.nameMap.get(vectorizationEnabledOverrideString);
    isVectorizationEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED);
    final boolean weCanAttemptVectorization;
    isTestForcedVectorizationEnable = false;
    switch(vectorizationEnabledOverride) {
        case NONE:
            weCanAttemptVectorization = isVectorizationEnabled;
            break;
        case DISABLE:
            weCanAttemptVectorization = false;
            break;
        case ENABLE:
            weCanAttemptVectorization = true;
            isTestForcedVectorizationEnable = !isVectorizationEnabled;
            // Different parts of the code rely on this being set...
            HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true);
            isVectorizationEnabled = true;
            break;
        default:
            throw new RuntimeException("Unexpected vectorization enabled override " + vectorizationEnabledOverride);
    }
    if (!weCanAttemptVectorization) {
        LOG.info("Vectorization is disabled");
        return physicalContext;
    }
    useVectorizedInputFileFormat = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTORIZED_INPUT_FILE_FORMAT);
    if (useVectorizedInputFileFormat) {
        initVectorizedInputFormatExcludeClasses();
    }
    useVectorDeserialize = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTOR_DESERIALIZE);
    useRowDeserialize = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_USE_ROW_DESERIALIZE);
    if (useRowDeserialize) {
        initRowDeserializeExcludeClasses();
    }
    // TODO: we could also vectorize some formats based on hive.llap.io.encode.formats if LLAP IO
    // is enabled and we are going to run in LLAP. However, we don't know if we end up in
    // LLAP or not at this stage, so don't do this now. We may need to add a 'force' option.
    isReduceVectorizationEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_ENABLED);
    isPtfVectorizationEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_PTF_ENABLED);
    isVectorizationComplexTypesEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_COMPLEX_TYPES_ENABLED);
    isVectorizationGroupByComplexTypesEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_GROUPBY_COMPLEX_TYPES_ENABLED);
    isVectorizedRowIdentifierEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ROW_IDENTIFIER_ENABLED);
    vectorizedPTFMaxMemoryBufferingBatchCount = HiveConf.getIntVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_PTF_MAX_MEMORY_BUFFERING_BATCH_COUNT);
    vectorizedTestingReducerBatchSize = HiveConf.getIntVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_TESTING_REDUCER_BATCH_SIZE);
    isTestVectorizerSuppressFatalExceptions = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_TEST_VECTORIZER_SUPPRESS_FATAL_EXCEPTIONS);
    vectorizedInputFormatSupportEnabled = HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED);
    String[] supportEnabledStrings = vectorizedInputFormatSupportEnabled.toLowerCase().split(",");
    vectorizedInputFormatSupportEnabledSet = new TreeSet<Support>();
    for (String supportEnabledString : supportEnabledStrings) {
        Support support = Support.nameToSupportMap.get(supportEnabledString);
        // Known?
        if (support != null) {
            vectorizedInputFormatSupportEnabledSet.add(support);
        }
    }
    /*
     * Notice the default value for LLAP_IO_ENABLED is overridden to be whether we are
     * executing under LLAP.
     */
    isLlapIoEnabled = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.LLAP_IO_ENABLED, LlapProxy.isDaemon());
    isSchemaEvolution = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_SCHEMA_EVOLUTION);
    hiveVectorAdaptorUsageMode = HiveVectorAdaptorUsageMode.getHiveConfValue(hiveConf);
    isTestVectorizationSuppressExplainExecutionMode = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_TEST_VECTORIZATION_SUPPRESS_EXPLAIN_EXECUTION_MODE);
    // create dispatcher and graph walker
    SemanticDispatcher disp = new VectorizationDispatcher();
    TaskGraphWalker ogw = new TaskGraphWalker(disp);
    // get all the tasks nodes from root task
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(physicalContext.getRootTasks());
    // begin to walk through the task tree.
    ogw.startWalking(topNodes, null);
    return physicalContext;
}