use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator in project asterixdb by apache.
the class RemoveRedundantListifyRule method appliesForReverseCase.
private boolean appliesForReverseCase(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> varUsedAbove, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
return false;
}
AggregateOperator agg = (AggregateOperator) op1;
if (agg.getVariables().size() > 1 || agg.getVariables().size() <= 0) {
return false;
}
LogicalVariable aggVar = agg.getVariables().get(0);
ILogicalExpression aggFun = agg.getExpressions().get(0).getValue();
AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) aggFun;
if (!BuiltinFunctions.LISTIFY.equals(f.getFunctionIdentifier())) {
return false;
}
if (f.getArguments().size() != 1) {
return false;
}
ILogicalExpression arg0 = f.getArguments().get(0).getValue();
if (((AbstractLogicalExpression) arg0).getExpressionTag() != LogicalExpressionTag.VARIABLE) {
return false;
}
LogicalVariable aggInputVar = ((VariableReferenceExpression) arg0).getVariableReference();
if (varUsedAbove.contains(aggInputVar)) {
return false;
}
if (agg.getInputs().size() == 0) {
return false;
}
AbstractLogicalOperator op2 = (AbstractLogicalOperator) agg.getInputs().get(0).getValue();
if (op2.getOperatorTag() != LogicalOperatorTag.UNNEST) {
return false;
}
UnnestOperator unnest = (UnnestOperator) op2;
if (unnest.getPositionalVariable() != null) {
return false;
}
if (!unnest.getVariable().equals(aggInputVar)) {
return false;
}
ILogicalExpression unnestArg = unnest.getExpressionRef().getValue();
if (unnestArg.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
return false;
}
AbstractFunctionCallExpression scanFunc = (AbstractFunctionCallExpression) unnestArg;
if (scanFunc.getFunctionIdentifier() != BuiltinFunctions.SCAN_COLLECTION) {
return false;
}
if (scanFunc.getArguments().size() != 1) {
return false;
}
List<LogicalVariable> assgnVars = new ArrayList<>(1);
assgnVars.add(aggVar);
AssignOperator assign = new AssignOperator(assgnVars, scanFunc.getArguments());
assign.getInputs().add(unnest.getInputs().get(0));
context.computeAndSetTypeEnvironmentForOperator(assign);
opRef.setValue(assign);
return true;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator in project asterixdb by apache.
the class LangExpressionToPlanTranslator method visit.
@Override
public Pair<ILogicalOperator, LogicalVariable> visit(QuantifiedExpression qe, Mutable<ILogicalOperator> tupSource) throws CompilationException {
Mutable<ILogicalOperator> topOp = tupSource;
ILogicalOperator firstOp = null;
Mutable<ILogicalOperator> lastOp = null;
for (QuantifiedPair qt : qe.getQuantifiedList()) {
Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo1 = langExprToAlgExpression(qt.getExpr(), topOp);
topOp = eo1.second;
LogicalVariable uVar = context.newVarFromExpression(qt.getVarExpr());
ILogicalOperator u = new UnnestOperator(uVar, new MutableObject<>(makeUnnestExpression(eo1.first)));
if (firstOp == null) {
firstOp = u;
}
if (lastOp != null) {
u.getInputs().add(lastOp);
}
lastOp = new MutableObject<>(u);
}
// We make all the unnest correspond. to quantif. vars. sit on top
// in the hope of enabling joins & other optimiz.
firstOp.getInputs().add(topOp);
topOp = lastOp;
Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo2 = langExprToAlgExpression(qe.getSatisfiesExpr(), topOp);
AggregateFunctionCallExpression fAgg;
SelectOperator s;
if (qe.getQuantifier() == Quantifier.SOME) {
s = new SelectOperator(new MutableObject<>(eo2.first), false, null);
s.getInputs().add(eo2.second);
fAgg = BuiltinFunctions.makeAggregateFunctionExpression(BuiltinFunctions.NON_EMPTY_STREAM, new ArrayList<>());
} else {
// EVERY
List<Mutable<ILogicalExpression>> satExprList = new ArrayList<>(1);
satExprList.add(new MutableObject<>(eo2.first));
s = new SelectOperator(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.NOT), satExprList)), false, null);
s.getInputs().add(eo2.second);
fAgg = BuiltinFunctions.makeAggregateFunctionExpression(BuiltinFunctions.EMPTY_STREAM, new ArrayList<>());
}
LogicalVariable qeVar = context.newVar();
AggregateOperator a = new AggregateOperator(mkSingletonArrayList(qeVar), (List) mkSingletonArrayList(new MutableObject<>(fAgg)));
a.getInputs().add(new MutableObject<>(s));
return new Pair<>(a, qeVar);
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator in project asterixdb by apache.
the class SqlppExpressionToPlanTranslator method visit.
@Override
public Pair<ILogicalOperator, LogicalVariable> visit(JoinClause joinClause, Mutable<ILogicalOperator> inputRef) throws CompilationException {
Mutable<ILogicalOperator> leftInputRef = uncorrelatedLeftBranchStack.pop();
if (joinClause.getJoinType() == JoinType.INNER) {
Pair<ILogicalOperator, LogicalVariable> rightBranch = generateUnnestForBinaryCorrelateRightBranch(joinClause, inputRef, true);
// A join operator with condition TRUE.
AbstractBinaryJoinOperator joinOperator = new InnerJoinOperator(new MutableObject<ILogicalExpression>(ConstantExpression.TRUE), leftInputRef, new MutableObject<ILogicalOperator>(rightBranch.first));
Mutable<ILogicalOperator> joinOpRef = new MutableObject<>(joinOperator);
// Add an additional filter operator.
Pair<ILogicalExpression, Mutable<ILogicalOperator>> conditionExprOpPair = langExprToAlgExpression(joinClause.getConditionExpression(), joinOpRef);
SelectOperator filter = new SelectOperator(new MutableObject<ILogicalExpression>(conditionExprOpPair.first), false, null);
filter.getInputs().add(conditionExprOpPair.second);
return new Pair<>(filter, rightBranch.second);
} else {
// Creates a subplan operator.
SubplanOperator subplanOp = new SubplanOperator();
Mutable<ILogicalOperator> ntsRef = new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(subplanOp)));
subplanOp.getInputs().add(leftInputRef);
// Enters the translation for a subplan.
context.enterSubplan();
// Adds an unnest operator to unnest to right expression.
Pair<ILogicalOperator, LogicalVariable> rightBranch = generateUnnestForBinaryCorrelateRightBranch(joinClause, ntsRef, true);
AbstractUnnestNonMapOperator rightUnnestOp = (AbstractUnnestNonMapOperator) rightBranch.first;
// Adds an additional filter operator for the join condition.
Pair<ILogicalExpression, Mutable<ILogicalOperator>> conditionExprOpPair = langExprToAlgExpression(joinClause.getConditionExpression(), new MutableObject<ILogicalOperator>(rightUnnestOp));
SelectOperator filter = new SelectOperator(new MutableObject<ILogicalExpression>(conditionExprOpPair.first), false, null);
filter.getInputs().add(conditionExprOpPair.second);
ILogicalOperator currentTopOp = filter;
LogicalVariable varToListify;
boolean hasRightPosVar = rightUnnestOp.getPositionalVariable() != null;
if (hasRightPosVar) {
// Creates record to get correlation between the two aggregate variables.
ScalarFunctionCallExpression recordCreationFunc = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CLOSED_RECORD_CONSTRUCTOR), // Field name for the listified right unnest var.
new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AString("unnestvar")))), // The listified right unnest var
new MutableObject<ILogicalExpression>(new VariableReferenceExpression(rightUnnestOp.getVariable())), // Field name for the listified right unnest positional var.
new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AString("posvar")))), // The listified right unnest positional var.
new MutableObject<ILogicalExpression>(new VariableReferenceExpression(rightUnnestOp.getPositionalVariable())));
// Assigns the record constructor function to a record variable.
LogicalVariable recordVar = context.newVar();
AssignOperator assignOp = new AssignOperator(recordVar, new MutableObject<ILogicalExpression>(recordCreationFunc));
assignOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTopOp));
// Sets currentTopOp and varToListify for later usages.
currentTopOp = assignOp;
varToListify = recordVar;
} else {
varToListify = rightUnnestOp.getVariable();
}
// Adds an aggregate operator to listfy unnest variables.
AggregateFunctionCallExpression fListify = BuiltinFunctions.makeAggregateFunctionExpression(BuiltinFunctions.LISTIFY, mkSingletonArrayList(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(varToListify))));
LogicalVariable aggVar = context.newSubplanOutputVar();
AggregateOperator aggOp = new AggregateOperator(mkSingletonArrayList(aggVar), mkSingletonArrayList(new MutableObject<ILogicalExpression>(fListify)));
aggOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTopOp));
// Exits the translation of a subplan.
context.exitSubplan();
// Sets the nested subplan of the subplan operator.
ILogicalPlan subplan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(aggOp));
subplanOp.getNestedPlans().add(subplan);
// Outer unnest the aggregated var from the subplan.
LogicalVariable outerUnnestVar = context.newVar();
LeftOuterUnnestOperator outerUnnestOp = new LeftOuterUnnestOperator(outerUnnestVar, new MutableObject<ILogicalExpression>(makeUnnestExpression(new VariableReferenceExpression(aggVar))));
outerUnnestOp.getInputs().add(new MutableObject<ILogicalOperator>(subplanOp));
currentTopOp = outerUnnestOp;
if (hasRightPosVar) {
ScalarFunctionCallExpression fieldAccessForRightUnnestVar = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.FIELD_ACCESS_BY_INDEX), new MutableObject<ILogicalExpression>(new VariableReferenceExpression(outerUnnestVar)), new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(0)))));
ScalarFunctionCallExpression fieldAccessForRightPosVar = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.FIELD_ACCESS_BY_INDEX), new MutableObject<ILogicalExpression>(new VariableReferenceExpression(outerUnnestVar)), new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(1)))));
// Creates variables for assign.
LogicalVariable rightUnnestVar = context.newVar();
LogicalVariable rightPosVar = context.newVar();
// Relate the assigned variables to the variable expression in AST.
context.setVar(joinClause.getRightVariable(), rightUnnestVar);
context.setVar(joinClause.getPositionalVariable(), rightPosVar);
// Varaibles to assign.
List<LogicalVariable> assignVars = new ArrayList<>();
assignVars.add(rightUnnestVar);
assignVars.add(rightPosVar);
// Expressions for assign.
List<Mutable<ILogicalExpression>> assignExprs = new ArrayList<>();
assignExprs.add(new MutableObject<ILogicalExpression>(fieldAccessForRightUnnestVar));
assignExprs.add(new MutableObject<ILogicalExpression>(fieldAccessForRightPosVar));
// Creates the assign operator.
AssignOperator assignOp = new AssignOperator(assignVars, assignExprs);
assignOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTopOp));
currentTopOp = assignOp;
} else {
context.setVar(joinClause.getRightVariable(), outerUnnestVar);
}
return new Pair<>(currentTopOp, null);
}
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator in project asterixdb by apache.
the class LangExpressionToPlanTranslator method visit.
@Override
public Pair<ILogicalOperator, LogicalVariable> visit(GroupbyClause gc, Mutable<ILogicalOperator> tupSource) throws CompilationException {
Mutable<ILogicalOperator> topOp = tupSource;
if (gc.hasGroupVar()) {
List<Pair<Expression, Identifier>> groupFieldList = gc.getGroupFieldList();
List<Mutable<ILogicalExpression>> groupRecordConstructorArgList = new ArrayList<>();
for (Pair<Expression, Identifier> groupField : groupFieldList) {
ILogicalExpression groupFieldNameExpr = langExprToAlgExpression(new LiteralExpr(new StringLiteral(groupField.second.getValue())), topOp).first;
groupRecordConstructorArgList.add(new MutableObject<>(groupFieldNameExpr));
ILogicalExpression groupFieldExpr = langExprToAlgExpression(groupField.first, topOp).first;
groupRecordConstructorArgList.add(new MutableObject<>(groupFieldExpr));
}
LogicalVariable groupVar = context.newVarFromExpression(gc.getGroupVar());
AssignOperator groupVarAssignOp = new AssignOperator(groupVar, new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR), groupRecordConstructorArgList)));
groupVarAssignOp.getInputs().add(topOp);
topOp = new MutableObject<>(groupVarAssignOp);
}
GroupByOperator gOp = new GroupByOperator();
for (GbyVariableExpressionPair ve : gc.getGbyPairList()) {
VariableExpr vexpr = ve.getVar();
LogicalVariable v = vexpr == null ? context.newVar() : context.newVarFromExpression(vexpr);
Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp);
gOp.addGbyExpression(v, eo.first);
topOp = eo.second;
}
for (GbyVariableExpressionPair ve : gc.getDecorPairList()) {
VariableExpr vexpr = ve.getVar();
LogicalVariable v = vexpr == null ? context.newVar() : context.newVarFromExpression(vexpr);
Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp);
gOp.addDecorExpression(v, eo.first);
topOp = eo.second;
}
gOp.getInputs().add(topOp);
for (Entry<Expression, VariableExpr> entry : gc.getWithVarMap().entrySet()) {
Pair<ILogicalExpression, Mutable<ILogicalOperator>> listifyInput = langExprToAlgExpression(entry.getKey(), new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(gOp))));
List<Mutable<ILogicalExpression>> flArgs = new ArrayList<>(1);
flArgs.add(new MutableObject<>(listifyInput.first));
AggregateFunctionCallExpression fListify = BuiltinFunctions.makeAggregateFunctionExpression(BuiltinFunctions.LISTIFY, flArgs);
LogicalVariable aggVar = context.newVar();
AggregateOperator agg = new AggregateOperator(mkSingletonArrayList(aggVar), mkSingletonArrayList(new MutableObject<>(fListify)));
agg.getInputs().add(listifyInput.second);
ILogicalPlan plan = new ALogicalPlanImpl(new MutableObject<>(agg));
gOp.getNestedPlans().add(plan);
// Hide the variable that was part of the "with", replacing it with
// the one bound by the aggregation op.
context.setVar(entry.getValue(), aggVar);
}
gOp.setGroupAll(gc.isGroupAll());
gOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, gc.hasHashGroupByHint());
return new Pair<>(gOp, null);
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator in project asterixdb by apache.
the class SortGroupByPOperator method contributeRuntimeOperator.
@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
List<LogicalVariable> gbyCols = getGbyColumns();
int[] keys = JobGenHelper.variablesToFieldIndexes(gbyCols, inputSchemas[0]);
GroupByOperator gby = (GroupByOperator) op;
int numFds = gby.getDecorList().size();
int[] fdColumns = new int[numFds];
int j = 0;
for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
ILogicalExpression expr = p.second.getValue();
if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
throw new AlgebricksException("Sort group-by expects variable references.");
}
VariableReferenceExpression v = (VariableReferenceExpression) expr;
LogicalVariable decor = v.getVariableReference();
fdColumns[j++] = inputSchemas[0].findVariable(decor);
}
if (gby.getNestedPlans().size() != 1) {
throw new AlgebricksException("Sort group-by currently works only for one nested plan with one root containing" + "an aggregate and a nested-tuple-source.");
}
ILogicalPlan p0 = gby.getNestedPlans().get(0);
if (p0.getRoots().size() != 1) {
throw new AlgebricksException("Sort group-by currently works only for one nested plan with one root containing" + "an aggregate and a nested-tuple-source.");
}
Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
AggregateOperator aggOp = (AggregateOperator) r0.getValue();
IPartialAggregationTypeComputer partialAggregationTypeComputer = context.getPartialAggregationTypeComputer();
List<Object> intermediateTypes = new ArrayList<Object>();
int n = aggOp.getExpressions().size();
IAggregateEvaluatorFactory[] aff = new IAggregateEvaluatorFactory[n];
int i = 0;
IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
IVariableTypeEnvironment aggOpInputEnv = context.getTypeEnvironment(aggOp.getInputs().get(0).getValue());
IVariableTypeEnvironment outputEnv = context.getTypeEnvironment(op);
for (Mutable<ILogicalExpression> exprRef : aggOp.getExpressions()) {
AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) exprRef.getValue();
aff[i++] = expressionRuntimeProvider.createAggregateFunctionFactory(aggFun, aggOpInputEnv, inputSchemas, context);
intermediateTypes.add(partialAggregationTypeComputer.getType(aggFun, aggOpInputEnv, context.getMetadataProvider()));
}
int[] keyAndDecFields = new int[keys.length + fdColumns.length];
for (i = 0; i < keys.length; ++i) {
keyAndDecFields[i] = keys[i];
}
for (i = 0; i < fdColumns.length; i++) {
keyAndDecFields[keys.length + i] = fdColumns[i];
}
List<LogicalVariable> keyAndDecVariables = new ArrayList<LogicalVariable>();
for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getGroupByList()) {
keyAndDecVariables.add(p.first);
}
for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
keyAndDecVariables.add(GroupByOperator.getDecorVariable(p));
}
for (LogicalVariable var : keyAndDecVariables) {
aggOpInputEnv.setVarType(var, outputEnv.getVarType(var));
}
compileSubplans(inputSchemas[0], gby, opSchema, context);
IOperatorDescriptorRegistry spec = builder.getJobSpec();
IBinaryComparatorFactory[] compFactories = new IBinaryComparatorFactory[gbyCols.size()];
IBinaryComparatorFactoryProvider bcfProvider = context.getBinaryComparatorFactoryProvider();
i = 0;
for (LogicalVariable v : gbyCols) {
Object type = aggOpInputEnv.getVarType(v);
if (orderColumns[i].getOrder() == OrderKind.ASC) {
compFactories[i] = bcfProvider.getBinaryComparatorFactory(type, true);
} else {
compFactories[i] = bcfProvider.getBinaryComparatorFactory(type, false);
}
i++;
}
RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
IAggregateEvaluatorFactory[] merges = new IAggregateEvaluatorFactory[n];
List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
IOperatorSchema[] localInputSchemas = new IOperatorSchema[1];
localInputSchemas[0] = new OperatorSchemaImpl();
for (i = 0; i < n; i++) {
AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions().get(i).getValue();
aggFun.getUsedVariables(usedVars);
}
i = 0;
for (Object type : intermediateTypes) {
aggOpInputEnv.setVarType(usedVars.get(i++), type);
}
for (LogicalVariable keyVar : keyAndDecVariables) {
localInputSchemas[0].addVariable(keyVar);
}
for (LogicalVariable usedVar : usedVars) {
localInputSchemas[0].addVariable(usedVar);
}
for (i = 0; i < n; i++) {
AggregateFunctionCallExpression mergeFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions().get(i).getValue();
merges[i] = expressionRuntimeProvider.createAggregateFunctionFactory(mergeFun, aggOpInputEnv, localInputSchemas, context);
}
RecordDescriptor partialAggRecordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), localInputSchemas[0], context);
IAggregatorDescriptorFactory aggregatorFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(aff, keyAndDecFields);
IAggregatorDescriptorFactory mergeFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(merges, keyAndDecFields);
INormalizedKeyComputerFactory normalizedKeyFactory = null;
INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
if (nkcfProvider == null) {
normalizedKeyFactory = null;
}
Object type = aggOpInputEnv.getVarType(gbyCols.get(0));
normalizedKeyFactory = orderColumns[0].getOrder() == OrderKind.ASC ? nkcfProvider.getNormalizedKeyComputerFactory(type, true) : nkcfProvider.getNormalizedKeyComputerFactory(type, false);
SortGroupByOperatorDescriptor gbyOpDesc = new SortGroupByOperatorDescriptor(spec, frameLimit, keys, keyAndDecFields, normalizedKeyFactory, compFactories, aggregatorFactory, mergeFactory, partialAggRecordDescriptor, recordDescriptor, false);
contributeOpDesc(builder, gby, gbyOpDesc);
ILogicalOperator src = op.getInputs().get(0).getValue();
builder.contributeGraphEdge(src, 0, op, 0);
}
Aggregations