Examples with TopNKeyDesc org.apache.hadoop.hive.ql.plan.TopNKeyDesc used on opensource projects

Example 6 with TopNKeyDesc

use of org.apache.hadoop.hive.ql.plan.TopNKeyDesc in project hive by apache.

the class TopNKeyPushdownProcessor method pushdownThroughSelect.

/**
 * Push through Project if expression(s) in TopNKey can be mapped to expression(s) based on Project input.
 *
 * @param topNKey TopNKey operator to push
 * @throws SemanticException when removeChildAndAdoptItsChildren was not successful in the method pushdown
 */
private void pushdownThroughSelect(TopNKeyOperator topNKey) throws SemanticException {
    final SelectOperator select = (SelectOperator) topNKey.getParentOperators().get(0);
    final TopNKeyDesc topNKeyDesc = topNKey.getConf();
    final List<ExprNodeDesc> mappedColumns = mapColumns(topNKeyDesc.getKeyColumns(), select.getColumnExprMap());
    if (mappedColumns.size() != topNKeyDesc.getKeyColumns().size()) {
        return;
    }
    LOG.debug("Pushing {} through {}", topNKey.getName(), select.getName());
    topNKeyDesc.setKeyColumns(mappedColumns);
    topNKeyDesc.setPartitionKeyColumns(mappedColumns.subList(0, topNKeyDesc.getPartitionKeyColumns().size()));
    moveDown(topNKey);
    pushdown(topNKey);
}

Example 7 with TopNKeyDesc

use of org.apache.hadoop.hive.ql.plan.TopNKeyDesc in project hive by apache.

the class TopNKeyPushdownProcessor method pushdownThroughReduceSink.

/**
 * Push through ReduceSink. If TopNKey expression is same as ReduceSink expression and order is
 * the same, we can push it and remove it from above ReduceSink. If expression in TopNKey shared
 * common prefix with ReduceSink including same order, TopNKey could be pushed through
 * ReduceSink using that prefix and kept above it.
 *
 * @param topNKey TopNKey operator to push
 * @throws SemanticException when removeChildAndAdoptItsChildren was not successful
 */
private void pushdownThroughReduceSink(TopNKeyOperator topNKey) throws SemanticException {
    ReduceSinkOperator reduceSink = (ReduceSinkOperator) topNKey.getParentOperators().get(0);
    final ReduceSinkDesc reduceSinkDesc = reduceSink.getConf();
    final TopNKeyDesc topNKeyDesc = topNKey.getConf();
    CommonKeyPrefix commonKeyPrefix = CommonKeyPrefix.map(topNKeyDesc, reduceSinkDesc);
    if (commonKeyPrefix.isEmpty() || commonKeyPrefix.size() == topNKeyDesc.getPartitionKeyColumns().size()) {
        return;
    }
    LOG.debug("Pushing a copy of {} through {}", topNKey.getName(), reduceSink.getName());
    final TopNKeyDesc newTopNKeyDesc = topNKeyDesc.combine(commonKeyPrefix);
    pushdown((TopNKeyOperator) copyDown(reduceSink, newTopNKeyDesc));
    if (topNKeyDesc.getKeyColumns().size() == commonKeyPrefix.size()) {
        LOG.debug("Removing {} above {}", topNKey.getName(), reduceSink.getName());
        reduceSink.removeChildAndAdoptItsChildren(topNKey);
    }
}

Example 8 with TopNKeyDesc

use of org.apache.hadoop.hive.ql.plan.TopNKeyDesc in project hive by apache.

the class TopNKeyPushdownProcessor method pushdownThroughLeftOuterJoin.

/**
 * Push through LOJ. If TopNKey expression refers fully to expressions from left input, push
 * with rewriting of expressions and remove from top of LOJ. If TopNKey expression has a prefix
 * that refers to expressions from left input, push with rewriting of those expressions and keep
 * on top of LOJ.
 *
 * @param topNKey TopNKey operator to push
 * @throws SemanticException when removeChildAndAdoptItsChildren was not successful
 */
private void pushdownThroughLeftOuterJoin(TopNKeyOperator topNKey) throws SemanticException {
    final TopNKeyDesc topNKeyDesc = topNKey.getConf();
    final CommonJoinOperator<? extends JoinDesc> join = (CommonJoinOperator<? extends JoinDesc>) topNKey.getParentOperators().get(0);
    final List<Operator<? extends OperatorDesc>> joinInputs = join.getParentOperators();
    final ReduceSinkOperator reduceSinkOperator = (ReduceSinkOperator) joinInputs.get(0);
    final ReduceSinkDesc reduceSinkDesc = reduceSinkOperator.getConf();
    CommonKeyPrefix commonKeyPrefix = CommonKeyPrefix.map(mapUntilColumnEquals(topNKeyDesc.getKeyColumns(), join.getColumnExprMap()), topNKeyDesc.getColumnSortOrder(), topNKeyDesc.getNullOrder(), reduceSinkDesc.getKeyCols(), reduceSinkDesc.getColumnExprMap(), reduceSinkDesc.getOrder(), reduceSinkDesc.getNullOrder());
    if (commonKeyPrefix.isEmpty() || commonKeyPrefix.size() == topNKeyDesc.getPartitionKeyColumns().size()) {
        return;
    }
    LOG.debug("Pushing a copy of {} through {} and {}", topNKey.getName(), join.getName(), reduceSinkOperator.getName());
    final TopNKeyDesc newTopNKeyDesc = topNKeyDesc.combine(commonKeyPrefix);
    pushdown((TopNKeyOperator) copyDown(reduceSinkOperator, newTopNKeyDesc));
    if (topNKeyDesc.getKeyColumns().size() == commonKeyPrefix.size()) {
        LOG.debug("Removing {} above {}", topNKey.getName(), join.getName());
        join.removeChildAndAdoptItsChildren(topNKey);
    }
}

Example 9 with TopNKeyDesc

use of org.apache.hadoop.hive.ql.plan.TopNKeyDesc in project hive by apache.

the class TopNKeyPushdownProcessor method pushdownInnerJoin.

/**
 * Tries to push the TopNKeyFilter through an inner join:
 *  requirements:
 *    - being PK-FK join
 *    - PK side is not filtered
 *    - First n TopNKey key columns (Order By) are originated from the FK side.
 * @throws SemanticException
 */
private void pushdownInnerJoin(TopNKeyOperator topNKey, int fkJoinInputIndex, boolean nonFkSideIsFiltered) throws SemanticException {
    TopNKeyDesc topNKeyDesc = topNKey.getConf();
    CommonJoinOperator<? extends JoinDesc> join = (CommonJoinOperator<? extends JoinDesc>) topNKey.getParentOperators().get(0);
    List<Operator<? extends OperatorDesc>> joinInputs = join.getParentOperators();
    ReduceSinkOperator fkJoinInput = (ReduceSinkOperator) joinInputs.get(fkJoinInputIndex);
    if (nonFkSideIsFiltered) {
        LOG.debug("Not pushing {} through {} as non FK side of the join is filtered", topNKey.getName(), join.getName());
        return;
    }
    CommonKeyPrefix commonKeyPrefix = CommonKeyPrefix.map(mapUntilColumnEquals(topNKeyDesc.getKeyColumns(), join.getColumnExprMap()), topNKeyDesc.getColumnSortOrder(), topNKeyDesc.getNullOrder(), fkJoinInput.getConf().getKeyCols(), fkJoinInput.getConf().getColumnExprMap(), fkJoinInput.getConf().getOrder(), fkJoinInput.getConf().getNullOrder());
    if (commonKeyPrefix.isEmpty() || commonKeyPrefix.size() == topNKeyDesc.getPartitionKeyColumns().size()) {
        return;
    }
    LOG.debug("Pushing a copy of {} through {} and {}", topNKey.getName(), join.getName(), fkJoinInput.getName());
    final TopNKeyDesc newTopNKeyDesc = topNKeyDesc.combine(commonKeyPrefix);
    pushdown((TopNKeyOperator) copyDown(fkJoinInput, newTopNKeyDesc));
    if (topNKeyDesc.getKeyColumns().size() == commonKeyPrefix.size()) {
        LOG.debug("Removing {} above {}", topNKey.getName(), join.getName());
        join.removeChildAndAdoptItsChildren(topNKey);
    }
}