Examples with GenMRUnionCtx org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMRUnionCtx used on opensource projects

Example 1 with GenMRUnionCtx

use of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMRUnionCtx in project hive by apache.

the class GenMapRedUtils method initUnionPlan.

/*
   * It is a idempotent function to add various intermediate files as the source
   * for the union. The plan has already been created.
   */
public static void initUnionPlan(GenMRProcContext opProcCtx, UnionOperator currUnionOp, Task<? extends Serializable> currTask, boolean local) throws SemanticException {
    // can be traversed more than one
    if (currUnionOp != null) {
        GenMRUnionCtx uCtx = opProcCtx.getUnionTask(currUnionOp);
        assert uCtx != null;
        setUnionPlan(opProcCtx, local, currTask, uCtx, false);
    }
}

Example 2 with GenMRUnionCtx

use of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMRUnionCtx in project hive by apache.

the class GenMapRedUtils method joinUnionPlan.

/*
   * join current union task to old task
   */
public static void joinUnionPlan(GenMRProcContext opProcCtx, UnionOperator currUnionOp, Task<? extends Serializable> currentUnionTask, Task<? extends Serializable> existingTask, boolean local) throws SemanticException {
    assert currUnionOp != null;
    GenMRUnionCtx uCtx = opProcCtx.getUnionTask(currUnionOp);
    assert uCtx != null;
    setUnionPlan(opProcCtx, local, existingTask, uCtx, true);
    List<Task<? extends Serializable>> parTasks = null;
    if (opProcCtx.getRootTasks().contains(currentUnionTask)) {
        opProcCtx.getRootTasks().remove(currentUnionTask);
        if (!opProcCtx.getRootTasks().contains(existingTask) && (existingTask.getParentTasks() == null || existingTask.getParentTasks().isEmpty())) {
            opProcCtx.getRootTasks().add(existingTask);
        }
    }
    if ((currentUnionTask != null) && (currentUnionTask.getParentTasks() != null) && !currentUnionTask.getParentTasks().isEmpty()) {
        parTasks = new ArrayList<Task<? extends Serializable>>();
        parTasks.addAll(currentUnionTask.getParentTasks());
        Object[] parTaskArr = parTasks.toArray();
        for (Object parTask : parTaskArr) {
            ((Task<? extends Serializable>) parTask).removeDependentTask(currentUnionTask);
        }
    }
    if ((currentUnionTask != null) && (parTasks != null)) {
        for (Task<? extends Serializable> parTask : parTasks) {
            parTask.addDependentTask(existingTask);
            if (opProcCtx.getRootTasks().contains(existingTask)) {
                opProcCtx.getRootTasks().remove(existingTask);
            }
        }
    }
    opProcCtx.setCurrTask(existingTask);
}

Example 3 with GenMRUnionCtx

use of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMRUnionCtx in project hive by apache.

the class GenMapRedUtils method joinUnionPlan.

/*
   * join current union task to old task
   */
public static void joinUnionPlan(GenMRProcContext opProcCtx, UnionOperator currUnionOp, Task<?> currentUnionTask, Task<?> existingTask, boolean local) throws SemanticException {
    assert currUnionOp != null;
    GenMRUnionCtx uCtx = opProcCtx.getUnionTask(currUnionOp);
    assert uCtx != null;
    setUnionPlan(opProcCtx, local, existingTask, uCtx, true);
    List<Task<?>> parTasks = null;
    if (opProcCtx.getRootTasks().contains(currentUnionTask)) {
        opProcCtx.getRootTasks().remove(currentUnionTask);
        if (!opProcCtx.getRootTasks().contains(existingTask) && (existingTask.getParentTasks() == null || existingTask.getParentTasks().isEmpty())) {
            opProcCtx.getRootTasks().add(existingTask);
        }
    }
    if ((currentUnionTask != null) && (currentUnionTask.getParentTasks() != null) && !currentUnionTask.getParentTasks().isEmpty()) {
        parTasks = new ArrayList<Task<?>>();
        parTasks.addAll(currentUnionTask.getParentTasks());
        Object[] parTaskArr = parTasks.toArray();
        for (Object parTask : parTaskArr) {
            ((Task<?>) parTask).removeDependentTask(currentUnionTask);
        }
    }
    if ((currentUnionTask != null) && (parTasks != null)) {
        for (Task<?> parTask : parTasks) {
            parTask.addDependentTask(existingTask);
            if (opProcCtx.getRootTasks().contains(existingTask)) {
                opProcCtx.getRootTasks().remove(existingTask);
            }
        }
    }
    opProcCtx.setCurrTask(existingTask);
}

Example 4 with GenMRUnionCtx

use of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMRUnionCtx in project hive by apache.

the class GenMapRedUtils method initUnionPlan.

/*
   * It is a idempotent function to add various intermediate files as the source
   * for the union. The plan has already been created.
   */
public static void initUnionPlan(GenMRProcContext opProcCtx, UnionOperator currUnionOp, Task<?> currTask, boolean local) throws SemanticException {
    // can be traversed more than one
    if (currUnionOp != null) {
        GenMRUnionCtx uCtx = opProcCtx.getUnionTask(currUnionOp);
        assert uCtx != null;
        setUnionPlan(opProcCtx, local, currTask, uCtx, false);
    }
}

Example 5 with GenMRUnionCtx

use of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMRUnionCtx in project hive by apache.

the class GenMRUnion1 method process.

/**
 * Union Operator encountered . Currently, the algorithm is pretty simple: If
 * all the sub-queries are map-only, don't do anything. Otherwise, insert a
 * FileSink on top of all the sub-queries.
 *
 * This can be optimized later on.
 *
 * @param nd
 *          the file sink operator encountered
 * @param opProcCtx
 *          context
 */
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx opProcCtx, Object... nodeOutputs) throws SemanticException {
    UnionOperator union = (UnionOperator) nd;
    GenMRProcContext ctx = (GenMRProcContext) opProcCtx;
    ParseContext parseCtx = ctx.getParseCtx();
    UnionProcContext uCtx = parseCtx.getUCtx();
    // Map-only subqueries can be optimized in future to not write to a file in
    // future
    Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx();
    if (union.getConf().isAllInputsInSameReducer()) {
        // All inputs of this UnionOperator are in the same Reducer.
        // We do not need to break the operator tree.
        mapCurrCtx.put((Operator<? extends OperatorDesc>) nd, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrAliasId()));
        return null;
    }
    UnionParseContext uPrsCtx = uCtx.getUnionParseContext(union);
    ctx.setCurrUnionOp(union);
    // map-reduce job
    if (uPrsCtx.allMapOnlySubQ()) {
        return processMapOnlyUnion(union, stack, ctx, uCtx);
    }
    assert uPrsCtx != null;
    Task<?> currTask = ctx.getCurrTask();
    int pos = UnionProcFactory.getPositionParent(union, stack);
    Task<?> uTask = null;
    MapredWork uPlan = null;
    // union is encountered for the first time
    GenMRUnionCtx uCtxTask = ctx.getUnionTask(union);
    if (uCtxTask == null) {
        uPlan = GenMapRedUtils.getMapRedWork(parseCtx);
        uTask = TaskFactory.get(uPlan);
        uCtxTask = new GenMRUnionCtx(uTask);
        ctx.setUnionTask(union, uCtxTask);
    } else {
        uTask = uCtxTask.getUTask();
    }
    // Copy into the current union task plan if
    if (uPrsCtx.getMapOnlySubq(pos) && uPrsCtx.getRootTask(pos)) {
        processSubQueryUnionMerge(ctx, uCtxTask, union, stack);
        if (ctx.getRootTasks().contains(currTask)) {
            ctx.getRootTasks().remove(currTask);
        }
    } else // If it a map-reduce job, create a temporary file
    {
        // is the current task a root task
        if (shouldBeRootTask(currTask) && !ctx.getRootTasks().contains(currTask) && (currTask.getParentTasks() == null || currTask.getParentTasks().isEmpty())) {
            ctx.getRootTasks().add(currTask);
        }
        processSubQueryUnionCreateIntermediate(union.getParentOperators().get(pos), union, uTask, ctx, uCtxTask);
        // the currAliasId and CurrTopOp is not valid any more
        ctx.setCurrAliasId(null);
        ctx.setCurrTopOp(null);
        ctx.getOpTaskMap().put(null, uTask);
    }
    ctx.setCurrTask(uTask);
    mapCurrCtx.put((Operator<? extends OperatorDesc>) nd, new GenMapRedCtx(ctx.getCurrTask(), null));
    return true;
}