use of org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef in project hive by apache.
the class PTFTranslator method buildRowResolverForWindowing.
protected RowResolver buildRowResolverForWindowing(WindowTableFunctionDef def) throws SemanticException {
RowResolver rr = new RowResolver();
HashMap<String, WindowExpressionSpec> aliasToExprMap = windowingSpec.getAliasToWdwExpr();
/*
* add Window Functions
*/
for (WindowFunctionDef wFnDef : def.getWindowFunctions()) {
ASTNode ast = aliasToExprMap.get(wFnDef.getAlias()).getExpression();
ObjectInspector wFnOI = null;
if (wFnDef.isPivotResult()) {
wFnOI = ((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector();
} else {
wFnOI = wFnDef.getOI();
}
ColumnInfo cInfo = new ColumnInfo(wFnDef.getAlias(), TypeInfoUtils.getTypeInfoFromObjectInspector(wFnOI), null, true, true);
rr.putExpression(ast, cInfo);
}
RowResolver inpRR = def.getRawInputShape().getRr();
/*
* add columns from inpRR
*/
for (ColumnInfo inpCInfo : inputRR.getColumnInfos()) {
ColumnInfo cInfo = new ColumnInfo(inpCInfo);
ASTNode inExpr = PTFTranslator.getASTNode(inpCInfo, inpRR);
if (inExpr != null) {
rr.putExpression(inExpr, cInfo);
} else {
String[] tabColAlias = inputRR.reverseLookup(inpCInfo.getInternalName());
if (tabColAlias != null) {
rr.put(tabColAlias[0], tabColAlias[1], cInfo);
} else {
rr.put(inpCInfo.getTabAlias(), inpCInfo.getAlias(), cInfo);
}
}
String[] altMapping = inputRR.getAlternateMappings(inpCInfo.getInternalName());
if (altMapping != null) {
rr.put(altMapping[0], altMapping[1], cInfo);
}
}
return rr;
}
use of org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef in project hive by apache.
the class PTFDeserializer method initializeWindowing.
public void initializeWindowing(WindowTableFunctionDef def) throws HiveException {
ShapeDetails inpShape = def.getInput().getOutputShape();
/*
* 1. setup resolve, make connections
*/
TableFunctionEvaluator tEval = def.getTFunction();
WindowingTableFunctionResolver tResolver = (WindowingTableFunctionResolver) constructResolver(def.getResolverClassName());
tResolver.initialize(ptfDesc, def, tEval);
/*
* 2. initialize WFns.
*/
for (WindowFunctionDef wFnDef : def.getWindowFunctions()) {
if (wFnDef.getArgs() != null) {
for (PTFExpressionDef arg : wFnDef.getArgs()) {
initialize(arg, inpShape);
}
}
if (wFnDef.getWindowFrame() != null) {
WindowFrameDef wFrmDef = wFnDef.getWindowFrame();
initialize(wFrmDef, inpShape);
}
setupWdwFnEvaluator(wFnDef);
}
ArrayList<String> aliases = new ArrayList<String>();
ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
for (WindowFunctionDef wFnDef : def.getWindowFunctions()) {
aliases.add(wFnDef.getAlias());
if (wFnDef.isPivotResult()) {
fieldOIs.add(((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector());
} else {
fieldOIs.add(wFnDef.getOI());
}
}
PTFDeserializer.addInputColumnsToList(inpShape, aliases, fieldOIs);
StructObjectInspector wdwOutOI = ObjectInspectorFactory.getStandardStructObjectInspector(aliases, fieldOIs);
tResolver.setWdwProcessingOutputOI(wdwOutOI);
initialize(def.getOutputShape(), wdwOutOI);
tResolver.initializeOutputOI();
}
use of org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef in project hive by apache.
the class WindowingTableFunction method execute.
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public void execute(PTFPartitionIterator<Object> pItr, PTFPartition outP) throws HiveException {
ArrayList<List<?>> oColumns = new ArrayList<List<?>>();
PTFPartition iPart = pItr.getPartition();
StructObjectInspector inputOI = iPart.getOutputOI();
WindowTableFunctionDef wTFnDef = (WindowTableFunctionDef) getTableDef();
for (WindowFunctionDef wFn : wTFnDef.getWindowFunctions()) {
boolean processWindow = processWindow(wFn.getWindowFrame());
pItr.reset();
if (!processWindow) {
Object out = evaluateFunctionOnPartition(wFn, iPart);
if (!wFn.isPivotResult()) {
out = new SameList(iPart.size(), out);
}
oColumns.add((List<?>) out);
} else {
oColumns.add(executeFnwithWindow(wFn, iPart));
}
}
for (int i = 0; i < iPart.size(); i++) {
ArrayList oRow = new ArrayList();
Object iRow = iPart.getAt(i);
for (int j = 0; j < oColumns.size(); j++) {
oRow.add(oColumns.get(j).get(i));
}
for (StructField f : inputOI.getAllStructFieldRefs()) {
oRow.add(inputOI.getStructFieldData(iRow, f));
}
outP.append(oRow);
}
}
use of org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef in project hive by apache.
the class WindowingTableFunction method initializeWindowingFunctionInfoHelpers.
private void initializeWindowingFunctionInfoHelpers() throws SemanticException {
// to the object during the map/reduce tasks.
if (windowingFunctionHelpers != null) {
return;
}
windowingFunctionHelpers = new HashMap<String, WindowingFunctionInfoHelper>();
WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef();
for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i);
WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFn.getName());
boolean supportsWindow = wFnInfo.isSupportsWindow();
windowingFunctionHelpers.put(wFn.getName(), new WindowingFunctionInfoHelper(supportsWindow));
}
}
use of org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef in project hive by apache.
the class WindowingTableFunction method processRow.
/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator#processRow(java
* .lang.Object)
*
* - hand row to each Function, provided there are enough rows for Function's
* window. - call getNextObject on each Function. - output as many rows as
* possible, based on minimum sz of Output List
*/
@Override
public List<Object> processRow(Object row) throws HiveException {
/*
* Once enough rows have been output, there is no need to process input rows.
*/
if (streamingState.rankLimitReached()) {
return null;
}
streamingState.rollingPart.append(row);
WindowTableFunctionDef tabDef = (WindowTableFunctionDef) tableDef;
for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i);
GenericUDAFEvaluator fnEval = wFn.getWFnEval();
int a = 0;
if (wFn.getArgs() != null) {
for (PTFExpressionDef arg : wFn.getArgs()) {
streamingState.funcArgs[i][a++] = arg.getExprEvaluator().evaluate(row);
}
}
if (fnEval != null && fnEval instanceof ISupportStreamingModeForWindowing) {
fnEval.aggregate(streamingState.aggBuffers[i], streamingState.funcArgs[i]);
Object out = ((ISupportStreamingModeForWindowing) fnEval).getNextResult(streamingState.aggBuffers[i]);
if (out != null) {
streamingState.fnOutputs[i].add(out == ISupportStreamingModeForWindowing.NULL_RESULT ? null : out);
}
} else {
int rowToProcess = streamingState.rollingPart.rowToProcess(wFn.getWindowFrame());
if (rowToProcess >= 0) {
Object out = evaluateWindowFunction(wFn, rowToProcess, streamingState.rollingPart);
streamingState.fnOutputs[i].add(out);
}
}
}
List<Object> oRows = new ArrayList<Object>();
while (true) {
boolean hasRow = streamingState.hasOutputRow();
if (!hasRow) {
break;
}
oRows.add(streamingState.nextOutputRow());
}
return oRows.size() == 0 ? null : oRows;
}
Aggregations