Examples with ReduceSinkOperator org.apache.hadoop.hive.ql.exec.ReduceSinkOperator used on opensource projects

Example 6 with ReduceSinkOperator

use of org.apache.hadoop.hive.ql.exec.ReduceSinkOperator in project hive by apache.

the class ReduceSinkMapJoinProc method process.

/* (non-Javadoc)
   * This processor addresses the RS-MJ case that occurs in tez on the small/hash
   * table side of things. The work that RS will be a part of must be connected
   * to the MJ work via be a broadcast edge.
   * We should not walk down the tree when we encounter this pattern because:
   * the type of work (map work or reduce work) needs to be determined
   * on the basis of the big table side because it may be a mapwork (no need for shuffle)
   * or reduce work.
   */
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procContext, Object... nodeOutputs) throws SemanticException {
    GenTezProcContext context = (GenTezProcContext) procContext;
    MapJoinOperator mapJoinOp = (MapJoinOperator) nd;
    // remember the original parent list before we start modifying it.
    if (!context.mapJoinParentMap.containsKey(mapJoinOp)) {
        List<Operator<?>> parents = new ArrayList<Operator<?>>(mapJoinOp.getParentOperators());
        context.mapJoinParentMap.put(mapJoinOp, parents);
    }
    boolean isBigTable = stack.size() < 2 || !(stack.get(stack.size() - 2) instanceof ReduceSinkOperator);
    ReduceSinkOperator parentRS = null;
    if (!isBigTable) {
        parentRS = (ReduceSinkOperator) stack.get(stack.size() - 2);
        // For dynamic partitioned hash join, the big table will also be coming from a ReduceSinkOperator
        // Check for this condition.
        // TODO: use indexOf(), or parentRS.getTag()?
        isBigTable = (mapJoinOp.getParentOperators().indexOf(parentRS) == mapJoinOp.getConf().getPosBigTable());
    }
    if (mapJoinOp.getConf().isDynamicPartitionHashJoin() && !context.mapJoinToUnprocessedSmallTableReduceSinks.containsKey(mapJoinOp)) {
        // Initialize set of unprocessed small tables
        Set<ReduceSinkOperator> rsSet = Sets.newIdentityHashSet();
        for (int pos = 0; pos < mapJoinOp.getParentOperators().size(); ++pos) {
            if (pos == mapJoinOp.getConf().getPosBigTable()) {
                continue;
            }
            rsSet.add((ReduceSinkOperator) mapJoinOp.getParentOperators().get(pos));
        }
        context.mapJoinToUnprocessedSmallTableReduceSinks.put(mapJoinOp, rsSet);
    }
    if (isBigTable) {
        context.currentMapJoinOperators.add(mapJoinOp);
        return null;
    }
    context.preceedingWork = null;
    context.currentRootOperator = null;
    return processReduceSinkToHashJoin(parentRS, mapJoinOp, context);
}

Example 7 with ReduceSinkOperator

use of org.apache.hadoop.hive.ql.exec.ReduceSinkOperator in project hive by apache.

the class MapJoinProcessor method getKeys.

public static ObjectPair<List<ReduceSinkOperator>, Map<Byte, List<ExprNodeDesc>>> getKeys(boolean leftInputJoin, String[] baseSrc, JoinOperator op) {
    // Walk over all the sources (which are guaranteed to be reduce sink
    // operators).
    // The join outputs a concatenation of all the inputs.
    List<ReduceSinkOperator> oldReduceSinkParentOps = new ArrayList<ReduceSinkOperator>(op.getNumParent());
    if (leftInputJoin) {
        // assert mapJoinPos == 0;
        Operator<? extends OperatorDesc> parentOp = op.getParentOperators().get(0);
        assert parentOp.getParentOperators().size() == 1;
        oldReduceSinkParentOps.add((ReduceSinkOperator) parentOp);
    }
    byte pos = 0;
    for (String src : baseSrc) {
        if (src != null) {
            Operator<? extends OperatorDesc> parentOp = op.getParentOperators().get(pos);
            assert parentOp.getParentOperators().size() == 1;
            oldReduceSinkParentOps.add((ReduceSinkOperator) parentOp);
        }
        pos++;
    }
    // get the join keys from old parent ReduceSink operators
    Map<Byte, List<ExprNodeDesc>> keyExprMap = new HashMap<Byte, List<ExprNodeDesc>>();
    for (pos = 0; pos < op.getParentOperators().size(); pos++) {
        ReduceSinkOperator inputRS = oldReduceSinkParentOps.get(pos);
        List<ExprNodeDesc> keyCols = inputRS.getConf().getKeyCols();
        keyExprMap.put(pos, keyCols);
    }
    return new ObjectPair<List<ReduceSinkOperator>, Map<Byte, List<ExprNodeDesc>>>(oldReduceSinkParentOps, keyExprMap);
}

Example 8 with ReduceSinkOperator

use of org.apache.hadoop.hive.ql.exec.ReduceSinkOperator in project hive by apache.

the class QueryPlanTreeTransformation method applyCorrelation.

/**
   * Based on the correlation, we transform the query plan tree (operator tree).
   * In here, we first create DemuxOperator and all bottom ReduceSinkOperators
   * (bottom means near TableScanOperaotr) in the correlation will be be
   * the parents of the DemuxOperaotr. We also reassign tags to those
   * ReduceSinkOperators. Then, we use MuxOperators to replace ReduceSinkOperators
   * which are not bottom ones in this correlation.
   * Example: The original operator tree is ...
   *      JOIN2
   *      /    \
   *     RS4   RS5
   *    /        \
   *   GBY1     JOIN1
   *    |       /    \
   *   RS1     RS2   RS3
   * If GBY1, JOIN1, and JOIN2 can be executed in the same reducer
   * (optimized by Correlation Optimizer).
   * The new operator tree will be ...
   *      JOIN2
   *        |
   *       MUX
   *      /   \
   *    GBY1  JOIN1
   *      \    /
   *       DEMUX
   *      /  |  \
   *     /   |   \
   *    /    |    \
   *   RS1   RS2   RS3
   * @param pCtx
   * @param corrCtx
   * @param correlation
   * @throws SemanticException
   */
protected static void applyCorrelation(ParseContext pCtx, CorrelationNodeProcCtx corrCtx, IntraQueryCorrelation correlation) throws SemanticException {
    final List<ReduceSinkOperator> bottomReduceSinkOperators = correlation.getBottomReduceSinkOperators();
    final int numReducers = correlation.getNumReducers();
    List<Operator<? extends OperatorDesc>> childrenOfDemux = new ArrayList<Operator<? extends OperatorDesc>>();
    List<Operator<? extends OperatorDesc>> parentRSsOfDemux = new ArrayList<Operator<? extends OperatorDesc>>();
    Map<Integer, Integer> childIndexToOriginalNumParents = new HashMap<Integer, Integer>();
    List<TableDesc> keysSerializeInfos = new ArrayList<TableDesc>();
    List<TableDesc> valuessSerializeInfos = new ArrayList<TableDesc>();
    Map<ReduceSinkOperator, Integer> bottomRSToNewTag = new HashMap<ReduceSinkOperator, Integer>();
    int newTag = 0;
    CompilationOpContext opCtx = null;
    for (ReduceSinkOperator rsop : bottomReduceSinkOperators) {
        if (opCtx == null) {
            opCtx = rsop.getCompilationOpContext();
        }
        rsop.getConf().setNumReducers(numReducers);
        bottomRSToNewTag.put(rsop, newTag);
        parentRSsOfDemux.add(rsop);
        keysSerializeInfos.add(rsop.getConf().getKeySerializeInfo());
        valuessSerializeInfos.add(rsop.getConf().getValueSerializeInfo());
        Operator<? extends OperatorDesc> child = CorrelationUtilities.getSingleChild(rsop, true);
        if (!childrenOfDemux.contains(child)) {
            childrenOfDemux.add(child);
            int childIndex = childrenOfDemux.size() - 1;
            childIndexToOriginalNumParents.put(childIndex, child.getNumParent());
        }
        newTag++;
    }
    for (ReduceSinkOperator rsop : bottomReduceSinkOperators) {
        setNewTag(correlation, childrenOfDemux, rsop, bottomRSToNewTag);
    }
    // Create the DemuxOperaotr
    DemuxDesc demuxDesc = new DemuxDesc(correlation.getNewTagToOldTag(), correlation.getNewTagToChildIndex(), childIndexToOriginalNumParents, keysSerializeInfos, valuessSerializeInfos);
    Operator<? extends OperatorDesc> demuxOp = OperatorFactory.get(opCtx, demuxDesc);
    demuxOp.setChildOperators(childrenOfDemux);
    demuxOp.setParentOperators(parentRSsOfDemux);
    for (Operator<? extends OperatorDesc> child : childrenOfDemux) {
        List<Operator<? extends OperatorDesc>> parentsWithMultipleDemux = new ArrayList<Operator<? extends OperatorDesc>>();
        boolean hasBottomReduceSinkOperators = false;
        boolean hasNonBottomReduceSinkOperators = false;
        for (int i = 0; i < child.getParentOperators().size(); i++) {
            Operator<? extends OperatorDesc> p = child.getParentOperators().get(i);
            assert p instanceof ReduceSinkOperator;
            ReduceSinkOperator rsop = (ReduceSinkOperator) p;
            if (bottomReduceSinkOperators.contains(rsop)) {
                hasBottomReduceSinkOperators = true;
                parentsWithMultipleDemux.add(demuxOp);
            } else {
                hasNonBottomReduceSinkOperators = true;
                parentsWithMultipleDemux.add(rsop);
            }
        }
        if (hasBottomReduceSinkOperators && hasNonBottomReduceSinkOperators) {
            child.setParentOperators(parentsWithMultipleDemux);
        } else {
            child.setParentOperators(Utilities.makeList(demuxOp));
        }
    }
    for (Operator<? extends OperatorDesc> parent : parentRSsOfDemux) {
        parent.setChildOperators(Utilities.makeList(demuxOp));
    }
    // replace all ReduceSinkOperators which are not at the bottom of
    // this correlation to MuxOperators
    Set<ReduceSinkOperator> handledRSs = new HashSet<ReduceSinkOperator>();
    for (ReduceSinkOperator rsop : correlation.getAllReduceSinkOperators()) {
        if (!bottomReduceSinkOperators.contains(rsop)) {
            if (handledRSs.contains(rsop)) {
                continue;
            }
            Operator<? extends OperatorDesc> childOP = CorrelationUtilities.getSingleChild(rsop, true);
            if (childOP instanceof GroupByOperator) {
                CorrelationUtilities.removeReduceSinkForGroupBy(rsop, (GroupByOperator) childOP, pCtx, corrCtx);
                List<Operator<? extends OperatorDesc>> parentsOfMux = new ArrayList<Operator<? extends OperatorDesc>>();
                Operator<? extends OperatorDesc> parentOp = CorrelationUtilities.getSingleParent(childOP, true);
                parentsOfMux.add(parentOp);
                Operator<? extends OperatorDesc> mux = OperatorFactory.get(childOP.getCompilationOpContext(), new MuxDesc(parentsOfMux));
                mux.setChildOperators(Utilities.makeList(childOP));
                mux.setParentOperators(parentsOfMux);
                childOP.setParentOperators(Utilities.makeList(mux));
                parentOp.setChildOperators(Utilities.makeList(mux));
            } else {
                List<Operator<? extends OperatorDesc>> parentsOfMux = new ArrayList<Operator<? extends OperatorDesc>>();
                List<Operator<? extends OperatorDesc>> siblingOPs = CorrelationUtilities.findSiblingOperators(rsop);
                for (Operator<? extends OperatorDesc> op : siblingOPs) {
                    if (op instanceof DemuxOperator) {
                        parentsOfMux.add(op);
                    } else if (op instanceof ReduceSinkOperator) {
                        GroupByOperator pGBYm = CorrelationUtilities.getSingleParent(op, GroupByOperator.class);
                        if (pGBYm != null && pGBYm.getConf().getMode() == GroupByDesc.Mode.HASH) {
                            // We get a semi join at here.
                            // This map-side GroupByOperator needs to be removed
                            CorrelationUtilities.removeOperator(pGBYm, op, CorrelationUtilities.getSingleParent(pGBYm, true), pCtx);
                        }
                        handledRSs.add((ReduceSinkOperator) op);
                        parentsOfMux.add(CorrelationUtilities.getSingleParent(op, true));
                    } else {
                        throw new SemanticException("A sibling of ReduceSinkOperator is neither a " + "DemuxOperator nor a ReduceSinkOperator");
                    }
                }
                MuxDesc muxDesc = new MuxDesc(siblingOPs);
                Operator<? extends OperatorDesc> mux = OperatorFactory.get(rsop.getCompilationOpContext(), muxDesc);
                mux.setChildOperators(Utilities.makeList(childOP));
                mux.setParentOperators(parentsOfMux);
                for (Operator<? extends OperatorDesc> op : parentsOfMux) {
                    if (op instanceof DemuxOperator) {
                        // and childOP.
                        if (op.getChildOperators().contains(childOP)) {
                            op.replaceChild(childOP, mux);
                        }
                    } else {
                        // op is not a DemuxOperator, so it should have
                        // a single child.
                        op.setChildOperators(Utilities.makeList(mux));
                    }
                }
                childOP.setParentOperators(Utilities.makeList(mux));
            }
        }
    }
    for (ReduceSinkOperator rsop : handledRSs) {
        rsop.setChildOperators(null);
        rsop.setParentOperators(null);
    }
}

Example 9 with ReduceSinkOperator

use of org.apache.hadoop.hive.ql.exec.ReduceSinkOperator in project hive by apache.

the class HiveOpConverter method genReduceSinkAndBacktrackSelect.

private static SelectOperator genReduceSinkAndBacktrackSelect(Operator<?> input, ExprNodeDesc[] keys, int tag, ArrayList<ExprNodeDesc> partitionCols, String order, String nullOrder, int numReducers, Operation acidOperation, HiveConf hiveConf, List<String> keepColNames) throws SemanticException {
    // 1. Generate RS operator
    // 1.1 Prune the tableNames, only count the tableNames that are not empty strings
    // as empty string in table aliases is only allowed for virtual columns.
    String tableAlias = null;
    Set<String> tableNames = input.getSchema().getTableNames();
    for (String tableName : tableNames) {
        if (tableName != null) {
            if (tableName.length() == 0) {
                if (tableAlias == null) {
                    tableAlias = tableName;
                }
            } else {
                if (tableAlias == null || tableAlias.length() == 0) {
                    tableAlias = tableName;
                } else {
                    if (!tableName.equals(tableAlias)) {
                        throw new SemanticException("In CBO return path, genReduceSinkAndBacktrackSelect is expecting only one tableAlias but there is more than one");
                    }
                }
            }
        }
    }
    if (tableAlias == null) {
        throw new SemanticException("In CBO return path, genReduceSinkAndBacktrackSelect is expecting only one tableAlias but there is none");
    }
    // 1.2 Now generate RS operator
    ReduceSinkOperator rsOp = genReduceSink(input, tableAlias, keys, tag, partitionCols, order, nullOrder, numReducers, acidOperation, hiveConf);
    // 2. Generate backtrack Select operator
    Map<String, ExprNodeDesc> descriptors = buildBacktrackFromReduceSink(keepColNames, rsOp.getConf().getOutputKeyColumnNames(), rsOp.getConf().getOutputValueColumnNames(), rsOp.getValueIndex(), input);
    SelectDesc selectDesc = new SelectDesc(new ArrayList<ExprNodeDesc>(descriptors.values()), new ArrayList<String>(descriptors.keySet()));
    ArrayList<ColumnInfo> cinfoLst = createColInfosSubset(input, keepColNames);
    SelectOperator selectOp = (SelectOperator) OperatorFactory.getAndMakeChild(selectDesc, new RowSchema(cinfoLst), rsOp);
    selectOp.setColumnExprMap(descriptors);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated " + selectOp + " with row schema: [" + selectOp.getSchema() + "]");
    }
    return selectOp;
}

Example 10 with ReduceSinkOperator

use of org.apache.hadoop.hive.ql.exec.ReduceSinkOperator in project hive by apache.

the class HiveOpConverter method genReduceSink.

@SuppressWarnings({ "rawtypes", "unchecked" })
private static ReduceSinkOperator genReduceSink(Operator<?> input, String tableAlias, ExprNodeDesc[] keys, int tag, ArrayList<ExprNodeDesc> partitionCols, String order, String nullOrder, int numReducers, Operation acidOperation, HiveConf hiveConf) throws SemanticException {
    // dummy for backtracking
    Operator dummy = Operator.createDummy();
    dummy.setParentOperators(Arrays.asList(input));
    ArrayList<ExprNodeDesc> reduceKeys = new ArrayList<ExprNodeDesc>();
    ArrayList<ExprNodeDesc> reduceKeysBack = new ArrayList<ExprNodeDesc>();
    // Compute join keys and store in reduceKeys
    for (ExprNodeDesc key : keys) {
        reduceKeys.add(key);
        reduceKeysBack.add(ExprNodeDescUtils.backtrack(key, dummy, input));
    }
    // Walk over the input schema and copy in the output
    ArrayList<ExprNodeDesc> reduceValues = new ArrayList<ExprNodeDesc>();
    ArrayList<ExprNodeDesc> reduceValuesBack = new ArrayList<ExprNodeDesc>();
    Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
    List<ColumnInfo> inputColumns = input.getSchema().getSignature();
    ArrayList<ColumnInfo> outputColumns = new ArrayList<ColumnInfo>();
    List<String> outputColumnNames = new ArrayList<String>();
    int[] index = new int[inputColumns.size()];
    for (int i = 0; i < inputColumns.size(); i++) {
        ColumnInfo colInfo = inputColumns.get(i);
        String outputColName = colInfo.getInternalName();
        ExprNodeColumnDesc expr = new ExprNodeColumnDesc(colInfo);
        // backtrack can be null when input is script operator
        ExprNodeDesc exprBack = ExprNodeDescUtils.backtrack(expr, dummy, input);
        int kindex = exprBack == null ? -1 : ExprNodeDescUtils.indexOf(exprBack, reduceKeysBack);
        if (kindex >= 0) {
            ColumnInfo newColInfo = new ColumnInfo(colInfo);
            newColInfo.setInternalName(Utilities.ReduceField.KEY + ".reducesinkkey" + kindex);
            newColInfo.setAlias(outputColName);
            newColInfo.setTabAlias(tableAlias);
            outputColumns.add(newColInfo);
            index[i] = kindex;
            continue;
        }
        int vindex = exprBack == null ? -1 : ExprNodeDescUtils.indexOf(exprBack, reduceValuesBack);
        if (vindex >= 0) {
            index[i] = -vindex - 1;
            continue;
        }
        index[i] = -reduceValues.size() - 1;
        reduceValues.add(expr);
        reduceValuesBack.add(exprBack);
        ColumnInfo newColInfo = new ColumnInfo(colInfo);
        newColInfo.setInternalName(Utilities.ReduceField.VALUE + "." + outputColName);
        newColInfo.setAlias(outputColName);
        newColInfo.setTabAlias(tableAlias);
        outputColumns.add(newColInfo);
        outputColumnNames.add(outputColName);
    }
    dummy.setParentOperators(null);
    // Use only 1 reducer if no reduce keys
    if (reduceKeys.size() == 0) {
        numReducers = 1;
        // Cartesian product is not supported in strict mode
        String error = StrictChecks.checkCartesian(hiveConf);
        if (error != null)
            throw new SemanticException(error);
    }
    ReduceSinkDesc rsDesc;
    if (order.isEmpty()) {
        rsDesc = PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, outputColumnNames, false, tag, reduceKeys.size(), numReducers, acidOperation);
    } else {
        rsDesc = PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, outputColumnNames, false, tag, partitionCols, order, nullOrder, numReducers, acidOperation);
    }
    ReduceSinkOperator rsOp = (ReduceSinkOperator) OperatorFactory.getAndMakeChild(rsDesc, new RowSchema(outputColumns), input);
    List<String> keyColNames = rsDesc.getOutputKeyColumnNames();
    for (int i = 0; i < keyColNames.size(); i++) {
        colExprMap.put(Utilities.ReduceField.KEY + "." + keyColNames.get(i), reduceKeys.get(i));
    }
    List<String> valColNames = rsDesc.getOutputValueColumnNames();
    for (int i = 0; i < valColNames.size(); i++) {
        colExprMap.put(Utilities.ReduceField.VALUE + "." + valColNames.get(i), reduceValues.get(i));
    }
    rsOp.setValueIndex(index);
    rsOp.setColumnExprMap(colExprMap);
    rsOp.setInputAliases(input.getSchema().getTableNames().toArray(new String[input.getSchema().getTableNames().size()]));
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated " + rsOp + " with row schema: [" + rsOp.getSchema() + "]");
    }
    return rsOp;
}