Examples with Explain org.apache.hadoop.hive.ql.plan.Explain used on opensource projects

Example 1 with Explain

use of org.apache.hadoop.hive.ql.plan.Explain in project hive by apache.

the class ExplainTask method outputPlan.

private JSONObject outputPlan(Object work, PrintStream out, boolean extended, boolean jsonOutput, int indent, String appendToHeader) throws Exception {
    // Check if work has an explain annotation
    Annotation note = AnnotationUtils.getAnnotation(work.getClass(), Explain.class);
    String keyJSONObject = null;
    if (note instanceof Explain) {
        Explain xpl_note = (Explain) note;
        boolean invokeFlag = false;
        if (this.work != null && this.work.isUserLevelExplain()) {
            invokeFlag = Level.USER.in(xpl_note.explainLevels());
        } else {
            if (extended) {
                invokeFlag = Level.EXTENDED.in(xpl_note.explainLevels());
            } else {
                invokeFlag = Level.DEFAULT.in(xpl_note.explainLevels());
            }
        }
        if (invokeFlag) {
            Vectorization vectorization = xpl_note.vectorization();
            if (this.work != null && this.work.isVectorization()) {
                // The EXPLAIN VECTORIZATION option was specified.
                final boolean desireOnly = this.work.isVectorizationOnly();
                final VectorizationDetailLevel desiredVecDetailLevel = this.work.isVectorizationDetailLevel();
                switch(vectorization) {
                    case NON_VECTORIZED:
                        // Display all non-vectorized leaf objects unless ONLY.
                        if (desireOnly) {
                            invokeFlag = false;
                        }
                        break;
                    case SUMMARY:
                    case OPERATOR:
                    case EXPRESSION:
                    case DETAIL:
                        if (vectorization.rank < desiredVecDetailLevel.rank) {
                            // This detail not desired.
                            invokeFlag = false;
                        }
                        break;
                    case SUMMARY_PATH:
                    case OPERATOR_PATH:
                        if (desireOnly) {
                            if (vectorization.rank < desiredVecDetailLevel.rank) {
                                // Suppress headers and all objects below.
                                invokeFlag = false;
                            }
                        }
                        break;
                    default:
                        throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization);
                }
            } else {
                // Do not display vectorization objects.
                switch(vectorization) {
                    case SUMMARY:
                    case OPERATOR:
                    case EXPRESSION:
                    case DETAIL:
                        invokeFlag = false;
                        break;
                    case NON_VECTORIZED:
                        // No action.
                        break;
                    case SUMMARY_PATH:
                    case OPERATOR_PATH:
                        // Always include headers since they contain non-vectorized objects, too.
                        break;
                    default:
                        throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization);
                }
            }
        }
        if (invokeFlag) {
            keyJSONObject = xpl_note.displayName();
            if (out != null) {
                out.print(indentString(indent));
                if (appendToHeader != null && !appendToHeader.isEmpty()) {
                    out.println(xpl_note.displayName() + appendToHeader);
                } else {
                    out.println(xpl_note.displayName());
                }
            }
        }
    }
    JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null;
    // conf and then the children
    if (work instanceof Operator) {
        Operator<? extends OperatorDesc> operator = (Operator<? extends OperatorDesc>) work;
        if (operator.getConf() != null) {
            String appender = isLogical ? " (" + operator.getOperatorId() + ")" : "";
            JSONObject jsonOut = outputPlan(operator.getConf(), out, extended, jsonOutput, jsonOutput ? 0 : indent, appender);
            if (this.work != null && (this.work.isUserLevelExplain() || this.work.isFormatted())) {
                if (jsonOut != null && jsonOut.length() > 0) {
                    ((JSONObject) jsonOut.get(JSONObject.getNames(jsonOut)[0])).put("OperatorId:", operator.getOperatorId());
                    if (!this.work.isUserLevelExplain() && this.work.isFormatted() && operator instanceof ReduceSinkOperator) {
                        List<String> outputOperators = ((ReduceSinkOperator) operator).getConf().getOutputOperators();
                        if (outputOperators != null) {
                            ((JSONObject) jsonOut.get(JSONObject.getNames(jsonOut)[0])).put(OUTPUT_OPERATORS, Arrays.toString(outputOperators.toArray()));
                        }
                    }
                }
            }
            if (jsonOutput) {
                json = jsonOut;
            }
        }
        if (!visitedOps.contains(operator) || !isLogical) {
            visitedOps.add(operator);
            if (operator.getChildOperators() != null) {
                int cindent = jsonOutput ? 0 : indent + 2;
                for (Operator<? extends OperatorDesc> op : operator.getChildOperators()) {
                    JSONObject jsonOut = outputPlan(op, out, extended, jsonOutput, cindent);
                    if (jsonOutput) {
                        ((JSONObject) json.get(JSONObject.getNames(json)[0])).accumulate("children", jsonOut);
                    }
                }
            }
        }
        if (jsonOutput) {
            return json;
        }
        return null;
    }
    // We look at all methods that generate values for explain
    Method[] methods = work.getClass().getMethods();
    Arrays.sort(methods, new MethodComparator());
    for (Method m : methods) {
        int prop_indents = jsonOutput ? 0 : indent + 2;
        note = AnnotationUtils.getAnnotation(m, Explain.class);
        if (note instanceof Explain) {
            Explain xpl_note = (Explain) note;
            boolean invokeFlag = false;
            if (this.work != null && this.work.isUserLevelExplain()) {
                invokeFlag = Level.USER.in(xpl_note.explainLevels());
            } else {
                if (extended) {
                    invokeFlag = Level.EXTENDED.in(xpl_note.explainLevels());
                } else {
                    invokeFlag = Level.DEFAULT.in(xpl_note.explainLevels());
                }
            }
            if (invokeFlag) {
                Vectorization vectorization = xpl_note.vectorization();
                if (this.work != null && this.work.isVectorization()) {
                    // The EXPLAIN VECTORIZATION option was specified.
                    final boolean desireOnly = this.work.isVectorizationOnly();
                    final VectorizationDetailLevel desiredVecDetailLevel = this.work.isVectorizationDetailLevel();
                    switch(vectorization) {
                        case NON_VECTORIZED:
                            // Display all non-vectorized leaf objects unless ONLY.
                            if (desireOnly) {
                                invokeFlag = false;
                            }
                            break;
                        case SUMMARY:
                        case OPERATOR:
                        case EXPRESSION:
                        case DETAIL:
                            if (vectorization.rank < desiredVecDetailLevel.rank) {
                                // This detail not desired.
                                invokeFlag = false;
                            }
                            break;
                        case SUMMARY_PATH:
                        case OPERATOR_PATH:
                            if (desireOnly) {
                                if (vectorization.rank < desiredVecDetailLevel.rank) {
                                    // Suppress headers and all objects below.
                                    invokeFlag = false;
                                }
                            }
                            break;
                        default:
                            throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization);
                    }
                } else {
                    // Do not display vectorization objects.
                    switch(vectorization) {
                        case SUMMARY:
                        case OPERATOR:
                        case EXPRESSION:
                        case DETAIL:
                            invokeFlag = false;
                            break;
                        case NON_VECTORIZED:
                            // No action.
                            break;
                        case SUMMARY_PATH:
                        case OPERATOR_PATH:
                            // Always include headers since they contain non-vectorized objects, too.
                            break;
                        default:
                            throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization);
                    }
                }
            }
            if (invokeFlag) {
                Object val = null;
                try {
                    val = m.invoke(work);
                } catch (InvocationTargetException ex) {
                    // Ignore the exception, this may be caused by external jars
                    val = null;
                }
                if (val == null) {
                    continue;
                }
                String header = null;
                boolean skipHeader = xpl_note.skipHeader();
                boolean emptyHeader = false;
                if (!xpl_note.displayName().equals("")) {
                    header = indentString(prop_indents) + xpl_note.displayName() + ":";
                } else {
                    emptyHeader = true;
                    prop_indents = indent;
                    header = indentString(prop_indents);
                }
                // Try the output as a primitive object
                if (isPrintable(val)) {
                    if (out != null && shouldPrint(xpl_note, val)) {
                        if (!skipHeader) {
                            out.print(header);
                            out.print(" ");
                        }
                        out.println(val);
                    }
                    if (jsonOutput && shouldPrint(xpl_note, val)) {
                        json.put(header, val.toString());
                    }
                    continue;
                }
                int ind = 0;
                if (!jsonOutput) {
                    if (!skipHeader) {
                        ind = prop_indents + 2;
                    } else {
                        ind = indent;
                    }
                }
                // Try this as a map
                if (val instanceof Map) {
                    // Go through the map and print out the stuff
                    Map<?, ?> mp = (Map<?, ?>) val;
                    if (out != null && !skipHeader && mp != null && !mp.isEmpty()) {
                        out.print(header);
                    }
                    JSONObject jsonOut = outputMap(mp, !skipHeader && !emptyHeader, out, extended, jsonOutput, ind);
                    if (jsonOutput && !mp.isEmpty()) {
                        json.put(header, jsonOut);
                    }
                    continue;
                }
                // Try this as a list
                if (val instanceof List || val instanceof Set) {
                    List l = val instanceof List ? (List) val : new ArrayList((Set) val);
                    if (out != null && !skipHeader && l != null && !l.isEmpty()) {
                        out.print(header);
                    }
                    JSONArray jsonOut = outputList(l, out, !skipHeader && !emptyHeader, extended, jsonOutput, ind);
                    if (jsonOutput && !l.isEmpty()) {
                        json.put(header, jsonOut);
                    }
                    continue;
                }
                // Finally check if it is serializable
                try {
                    if (!skipHeader && out != null) {
                        out.println(header);
                    }
                    JSONObject jsonOut = outputPlan(val, out, extended, jsonOutput, ind);
                    if (jsonOutput && jsonOut != null && jsonOut.length() != 0) {
                        if (!skipHeader) {
                            json.put(header, jsonOut);
                        } else {
                            for (String k : JSONObject.getNames(jsonOut)) {
                                json.put(k, jsonOut.get(k));
                            }
                        }
                    }
                    continue;
                } catch (ClassCastException ce) {
                // Ignore
                }
            }
        }
    }
    if (jsonOutput) {
        if (keyJSONObject != null) {
            JSONObject ret = new JSONObject(new LinkedHashMap<>());
            ret.put(keyJSONObject, json);
            return ret;
        }
        return json;
    }
    return null;
}

Example 2 with Explain

use of org.apache.hadoop.hive.ql.plan.Explain in project hive by apache.

the class Vectorizer method setOperatorNotSupported.

private void setOperatorNotSupported(Operator<? extends OperatorDesc> op) {
    OperatorDesc desc = op.getConf();
    Annotation note = AnnotationUtils.getAnnotation(desc.getClass(), Explain.class);
    if (note != null) {
        Explain explainNote = (Explain) note;
        setNodeIssue(explainNote.displayName() + " (" + op.getType() + ") not supported");
    } else {
        setNodeIssue("Operator " + op.getType() + " not supported");
    }
}