use of org.apache.hadoop.hive.ql.parse.WindowingComponentizer in project hive by apache.
the class HiveOpConverter method genPTF.
private OpAttr genPTF(OpAttr inputOpAf, WindowingSpec wSpec) throws SemanticException {
Operator<?> input = inputOpAf.inputs.get(0);
wSpec.validateAndMakeEffective();
WindowingComponentizer groups = new WindowingComponentizer(wSpec);
RowResolver rr = new RowResolver();
for (ColumnInfo ci : input.getSchema().getSignature()) {
rr.put(inputOpAf.tabAlias, ci.getInternalName(), ci);
}
while (groups.hasNext()) {
wSpec = groups.next(hiveConf, semanticAnalyzer, unparseTranslator, rr);
// 1. Create RS and backtrack Select operator on top
ArrayList<ExprNodeDesc> keyCols = new ArrayList<ExprNodeDesc>();
ArrayList<ExprNodeDesc> partCols = new ArrayList<ExprNodeDesc>();
StringBuilder order = new StringBuilder();
StringBuilder nullOrder = new StringBuilder();
for (PartitionExpression partCol : wSpec.getQueryPartitionSpec().getExpressions()) {
ExprNodeDesc partExpr = semanticAnalyzer.genExprNodeDesc(partCol.getExpression(), rr);
if (ExprNodeDescUtils.indexOf(partExpr, partCols) < 0) {
keyCols.add(partExpr);
partCols.add(partExpr);
order.append('+');
nullOrder.append('a');
}
}
if (wSpec.getQueryOrderSpec() != null) {
for (OrderExpression orderCol : wSpec.getQueryOrderSpec().getExpressions()) {
ExprNodeDesc orderExpr = semanticAnalyzer.genExprNodeDesc(orderCol.getExpression(), rr);
char orderChar = orderCol.getOrder() == PTFInvocationSpec.Order.ASC ? '+' : '-';
char nullOrderChar = orderCol.getNullOrder() == PTFInvocationSpec.NullOrder.NULLS_FIRST ? 'a' : 'z';
int index = ExprNodeDescUtils.indexOf(orderExpr, keyCols);
if (index >= 0) {
order.setCharAt(index, orderChar);
nullOrder.setCharAt(index, nullOrderChar);
continue;
}
keyCols.add(orderExpr);
order.append(orderChar);
nullOrder.append(nullOrderChar);
}
}
SelectOperator selectOp = genReduceSinkAndBacktrackSelect(input, keyCols.toArray(new ExprNodeDesc[keyCols.size()]), 0, partCols, order.toString(), nullOrder.toString(), -1, Operation.NOT_ACID, hiveConf);
// 2. Finally create PTF
PTFTranslator translator = new PTFTranslator();
PTFDesc ptfDesc = translator.translate(wSpec, semanticAnalyzer, hiveConf, rr, unparseTranslator);
RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr();
Operator<?> ptfOp = OperatorFactory.getAndMakeChild(ptfDesc, new RowSchema(ptfOpRR.getColumnInfos()), selectOp);
if (LOG.isDebugEnabled()) {
LOG.debug("Generated " + ptfOp + " with row schema: [" + ptfOp.getSchema() + "]");
}
// 3. Prepare for next iteration (if any)
rr = ptfOpRR;
input = ptfOp;
}
return inputOpAf.clone(input);
}
Aggregations