use of org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo in project asterixdb by apache.
the class IntroduceGroupByForSubplanRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
return false;
}
SubplanOperator subplan = (SubplanOperator) op0;
Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
ILogicalPlan p = null;
while (plansIter.hasNext()) {
p = plansIter.next();
}
if (p == null) {
return false;
}
if (p.getRoots().size() != 1) {
return false;
}
Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
Mutable<ILogicalOperator> botRef = subplanRoot;
AbstractLogicalOperator op2;
// Project is optional
if (op1.getOperatorTag() != LogicalOperatorTag.PROJECT) {
op2 = op1;
} else {
ProjectOperator project = (ProjectOperator) op1;
botRef = project.getInputs().get(0);
op2 = (AbstractLogicalOperator) botRef.getValue();
}
if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
return false;
}
AggregateOperator aggregate = (AggregateOperator) op2;
Set<LogicalVariable> free = new HashSet<LogicalVariable>();
VariableUtilities.getUsedVariables(aggregate, free);
Mutable<ILogicalOperator> op3Ref = aggregate.getInputs().get(0);
AbstractLogicalOperator op3 = (AbstractLogicalOperator) op3Ref.getValue();
while (op3.getInputs().size() == 1) {
Set<LogicalVariable> prod = new HashSet<LogicalVariable>();
VariableUtilities.getProducedVariables(op3, prod);
free.removeAll(prod);
VariableUtilities.getUsedVariables(op3, free);
botRef = op3Ref;
op3Ref = op3.getInputs().get(0);
op3 = (AbstractLogicalOperator) op3Ref.getValue();
}
if (op3.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op3.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
return false;
}
AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op3;
if (join.getCondition().getValue() == ConstantExpression.TRUE) {
return false;
}
VariableUtilities.getUsedVariables(join, free);
AbstractLogicalOperator b0 = (AbstractLogicalOperator) join.getInputs().get(0).getValue();
// see if there's an NTS at the end of the pipeline
NestedTupleSourceOperator outerNts = getNts(b0);
if (outerNts == null) {
AbstractLogicalOperator b1 = (AbstractLogicalOperator) join.getInputs().get(1).getValue();
outerNts = getNts(b1);
if (outerNts == null) {
return false;
}
}
Set<LogicalVariable> pkVars = computeGbyVars(outerNts, free, context);
if (pkVars == null || pkVars.size() < 1) {
// there is no non-trivial primary key, group-by keys are all live variables
// that were produced by descendant or self
ILogicalOperator subplanInput = subplan.getInputs().get(0).getValue();
pkVars = new HashSet<LogicalVariable>();
//get live variables
VariableUtilities.getLiveVariables(subplanInput, pkVars);
//get produced variables
Set<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
VariableUtilities.getProducedVariablesInDescendantsAndSelf(subplanInput, producedVars);
//retain the intersection
pkVars.retainAll(producedVars);
}
AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Found FD for introducing group-by: " + pkVars);
Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
LogicalVariable testForNull = null;
AbstractLogicalOperator right = (AbstractLogicalOperator) rightRef.getValue();
switch(right.getOperatorTag()) {
case UNNEST:
{
UnnestOperator innerUnnest = (UnnestOperator) right;
// Select [ $y != null ]
testForNull = innerUnnest.getVariable();
break;
}
case RUNNINGAGGREGATE:
{
ILogicalOperator inputToRunningAggregate = right.getInputs().get(0).getValue();
Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
VariableUtilities.getProducedVariables(inputToRunningAggregate, producedVars);
if (!producedVars.isEmpty()) {
// Select [ $y != null ]
testForNull = producedVars.iterator().next();
}
break;
}
case DATASOURCESCAN:
{
DataSourceScanOperator innerScan = (DataSourceScanOperator) right;
// Select [ $y != null ]
if (innerScan.getVariables().size() == 1) {
testForNull = innerScan.getVariables().get(0);
}
break;
}
default:
break;
}
if (testForNull == null) {
testForNull = context.newVar();
AssignOperator tmpAsgn = new AssignOperator(testForNull, new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
tmpAsgn.getInputs().add(new MutableObject<ILogicalOperator>(rightRef.getValue()));
rightRef.setValue(tmpAsgn);
context.computeAndSetTypeEnvironmentForOperator(tmpAsgn);
}
IFunctionInfo finfoEq = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.IS_MISSING);
ILogicalExpression isNullTest = new ScalarFunctionCallExpression(finfoEq, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(testForNull)));
IFunctionInfo finfoNot = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.NOT);
ScalarFunctionCallExpression nonNullTest = new ScalarFunctionCallExpression(finfoNot, new MutableObject<ILogicalExpression>(isNullTest));
SelectOperator selectNonNull = new SelectOperator(new MutableObject<ILogicalExpression>(nonNullTest), false, null);
GroupByOperator g = new GroupByOperator();
Mutable<ILogicalOperator> newSubplanRef = new MutableObject<ILogicalOperator>(subplan);
NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(g));
opRef.setValue(g);
selectNonNull.getInputs().add(new MutableObject<ILogicalOperator>(nts));
List<Mutable<ILogicalOperator>> prodInpList = botRef.getValue().getInputs();
prodInpList.clear();
prodInpList.add(new MutableObject<ILogicalOperator>(selectNonNull));
ILogicalPlan gPlan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(subplanRoot.getValue()));
g.getNestedPlans().add(gPlan);
subplanRoot.setValue(op3Ref.getValue());
g.getInputs().add(newSubplanRef);
HashSet<LogicalVariable> underVars = new HashSet<LogicalVariable>();
VariableUtilities.getLiveVariables(subplan.getInputs().get(0).getValue(), underVars);
underVars.removeAll(pkVars);
Map<LogicalVariable, LogicalVariable> mappedVars = buildVarExprList(pkVars, context, g, g.getGroupByList());
context.updatePrimaryKeys(mappedVars);
for (LogicalVariable uv : underVars) {
g.getDecorList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(null, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(uv))));
}
OperatorPropertiesUtil.typeOpRec(subplanRoot, context);
OperatorPropertiesUtil.typeOpRec(gPlan.getRoots().get(0), context);
context.computeAndSetTypeEnvironmentForOperator(g);
return true;
}
use of org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo in project asterixdb by apache.
the class ListifyUnnestingFunctionRule method listifyUnnestingFunction.
// Performs the actual logical transformation.
private boolean listifyUnnestingFunction(ILogicalOperator op, Mutable<ILogicalExpression> exprRef, AbstractFunctionCallExpression func, IOptimizationContext context) throws AlgebricksException {
IFunctionInfo functionInfo = func.getFunctionInfo();
// Checks if the function is an unnesting function.
if (!BuiltinFunctions.isBuiltinUnnestingFunction(functionInfo.getFunctionIdentifier())) {
return false;
}
// Generates the listified collection in a subplan.
SubplanOperator subplanOperator = new SubplanOperator();
// Creates a nested tuple source operator.
NestedTupleSourceOperator ntsOperator = new NestedTupleSourceOperator(new MutableObject<>(subplanOperator));
// Unnests the dataset.
LogicalVariable unnestVar = context.newVar();
ILogicalExpression unnestExpr = new UnnestingFunctionCallExpression(functionInfo, func.getArguments());
UnnestOperator unnestOperator = new UnnestOperator(unnestVar, new MutableObject<>(unnestExpr));
unnestOperator.getInputs().add(new MutableObject<>(ntsOperator));
// Listify the dataset into one collection.
LogicalVariable aggVar = context.newVar();
Mutable<ILogicalExpression> aggArgExprRef = new MutableObject<>(new VariableReferenceExpression(unnestVar));
ILogicalExpression aggExpr = new AggregateFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.LISTIFY), false, new ArrayList<>(Collections.singletonList(aggArgExprRef)));
AggregateOperator aggregateOperator = new AggregateOperator(new ArrayList<>(Collections.singletonList(aggVar)), new ArrayList<>(Collections.singletonList(new MutableObject<>(aggExpr))));
aggregateOperator.getInputs().add(new MutableObject<>(unnestOperator));
// Adds the aggregate operator as the root of the subplan.
subplanOperator.setRootOp(new MutableObject<>(aggregateOperator));
// Sticks a subplan operator into the query plan.
// Note: given the way we compile JOINs, the unnesting function expression cannot appear in
// any binary operators.
// Example test queries:
// asterixdb/asterix-app/src/test/resources/runtimets/results/list/query-ASTERIXDB-159-2
// asterixdb/asterix-app/src/test/resources/runtimets/results/list/query-ASTERIXDB-159-3
subplanOperator.getInputs().add(op.getInputs().get(0));
op.getInputs().set(0, new MutableObject<>(subplanOperator));
exprRef.setValue(new VariableReferenceExpression(aggVar));
// Computes type environments for new operators.
context.computeAndSetTypeEnvironmentForOperator(ntsOperator);
context.computeAndSetTypeEnvironmentForOperator(unnestOperator);
context.computeAndSetTypeEnvironmentForOperator(aggregateOperator);
context.computeAndSetTypeEnvironmentForOperator(subplanOperator);
return true;
}
use of org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo in project asterixdb by apache.
the class AccessMethodUtils method createExternalDataLookupUnnestMap.
public static UnnestMapOperator createExternalDataLookupUnnestMap(AbstractDataSourceOperator dataSourceOp, Dataset dataset, ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext context, boolean retainInput, boolean retainNull) throws AlgebricksException {
List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset, inputOp);
// add a sort on the RID fields before fetching external data.
OrderOperator order = new OrderOperator();
for (LogicalVariable pkVar : primaryKeyVars) {
Mutable<ILogicalExpression> vRef = new MutableObject<>(new VariableReferenceExpression(pkVar));
order.getOrderExpressions().add(new Pair<>(OrderOperator.ASC_ORDER, vRef));
}
// The secondary-index search feeds into the sort.
order.getInputs().add(new MutableObject<>(inputOp));
order.setExecutionMode(ExecutionMode.LOCAL);
context.computeAndSetTypeEnvironmentForOperator(order);
List<Mutable<ILogicalExpression>> externalLookupArgs = new ArrayList<>();
//Add dataverse to the arguments
AccessMethodUtils.addStringArg(dataset.getDataverseName(), externalLookupArgs);
//Add dataset to the arguments
AccessMethodUtils.addStringArg(dataset.getDatasetName(), externalLookupArgs);
//Add PK vars to the arguments
AccessMethodUtils.writeVarList(primaryKeyVars, externalLookupArgs);
// Variables and types coming out of the external access.
List<LogicalVariable> externalUnnestVars = new ArrayList<>();
List<Object> outputTypes = new ArrayList<>();
// Append output variables/types generated by the data scan (not forwarded from input).
externalUnnestVars.addAll(dataSourceOp.getVariables());
appendExternalRecTypes(dataset, recordType, outputTypes);
IFunctionInfo externalLookup = FunctionUtil.getFunctionInfo(BuiltinFunctions.EXTERNAL_LOOKUP);
AbstractFunctionCallExpression externalLookupFunc = new ScalarFunctionCallExpression(externalLookup, externalLookupArgs);
UnnestMapOperator unnestOp = new UnnestMapOperator(externalUnnestVars, new MutableObject<ILogicalExpression>(externalLookupFunc), outputTypes, retainInput);
// Fed by the order operator or the secondaryIndexUnnestOp.
unnestOp.getInputs().add(new MutableObject<ILogicalOperator>(order));
context.computeAndSetTypeEnvironmentForOperator(unnestOp);
unnestOp.setExecutionMode(ExecutionMode.PARTITIONED);
//set the physical operator
DataSourceId dataSourceId = new DataSourceId(dataset.getDataverseName(), dataset.getDatasetName());
unnestOp.setPhysicalOperator(new ExternalDataLookupPOperator(dataSourceId, dataset, recordType, primaryKeyVars, false, retainInput, retainNull));
return unnestOp;
}
use of org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo in project asterixdb by apache.
the class AccessMethodUtils method createSecondaryIndexUnnestMap.
public static ILogicalOperator createSecondaryIndexUnnestMap(Dataset dataset, ARecordType recordType, ARecordType metaRecordType, Index index, ILogicalOperator inputOp, AccessMethodJobGenParams jobGenParams, IOptimizationContext context, boolean outputPrimaryKeysOnly, boolean retainInput, boolean retainNull) throws AlgebricksException {
// The job gen parameters are transferred to the actual job gen via the UnnestMapOperator's function arguments.
ArrayList<Mutable<ILogicalExpression>> secondaryIndexFuncArgs = new ArrayList<>();
jobGenParams.writeToFuncArgs(secondaryIndexFuncArgs);
// Variables and types coming out of the secondary-index search.
List<LogicalVariable> secondaryIndexUnnestVars = new ArrayList<>();
List<Object> secondaryIndexOutputTypes = new ArrayList<>();
// Append output variables/types generated by the secondary-index search (not forwarded from input).
appendSecondaryIndexOutputVars(dataset, recordType, metaRecordType, index, outputPrimaryKeysOnly, context, secondaryIndexUnnestVars);
appendSecondaryIndexTypes(dataset, recordType, metaRecordType, index, outputPrimaryKeysOnly, secondaryIndexOutputTypes);
// An index search is expressed as an unnest over an index-search function.
IFunctionInfo secondaryIndexSearch = FunctionUtil.getFunctionInfo(BuiltinFunctions.INDEX_SEARCH);
UnnestingFunctionCallExpression secondaryIndexSearchFunc = new UnnestingFunctionCallExpression(secondaryIndexSearch, secondaryIndexFuncArgs);
secondaryIndexSearchFunc.setReturnsUniqueValues(true);
// Then, we use the LEFT-OUTER-UNNEST-MAP operator instead of unnest-map operator.
if (retainNull) {
if (retainInput) {
LeftOuterUnnestMapOperator secondaryIndexLeftOuterUnnestOp = new LeftOuterUnnestMapOperator(secondaryIndexUnnestVars, new MutableObject<ILogicalExpression>(secondaryIndexSearchFunc), secondaryIndexOutputTypes, true);
secondaryIndexLeftOuterUnnestOp.getInputs().add(new MutableObject<>(inputOp));
context.computeAndSetTypeEnvironmentForOperator(secondaryIndexLeftOuterUnnestOp);
secondaryIndexLeftOuterUnnestOp.setExecutionMode(ExecutionMode.PARTITIONED);
return secondaryIndexLeftOuterUnnestOp;
} else {
// Left-outer-join without retainInput doesn't make sense.
throw new AlgebricksException("Left-outer-join should propagate all inputs from the outer branch.");
}
} else {
// If this is not a left-outer-join case, then we use UNNEST-MAP operator.
UnnestMapOperator secondaryIndexUnnestOp = new UnnestMapOperator(secondaryIndexUnnestVars, new MutableObject<ILogicalExpression>(secondaryIndexSearchFunc), secondaryIndexOutputTypes, retainInput);
secondaryIndexUnnestOp.getInputs().add(new MutableObject<>(inputOp));
context.computeAndSetTypeEnvironmentForOperator(secondaryIndexUnnestOp);
secondaryIndexUnnestOp.setExecutionMode(ExecutionMode.PARTITIONED);
return secondaryIndexUnnestOp;
}
}
use of org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo in project asterixdb by apache.
the class BTreeAccessMethod method createSecondaryToPrimaryPlan.
@Override
public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException {
Dataset dataset = indexSubTree.getDataset();
ARecordType recordType = indexSubTree.getRecordType();
ARecordType metaRecordType = indexSubTree.getMetaRecordType();
// we made sure indexSubTree has datasource scan
AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.getDataSourceRef().getValue();
List<Pair<Integer, Integer>> exprAndVarList = analysisCtx.getIndexExprsFromIndexExprsAndVars(chosenIndex);
int numSecondaryKeys = analysisCtx.getNumberOfMatchedKeys(chosenIndex);
// List of function expressions that will be replaced by the secondary-index search.
// These func exprs will be removed from the select condition at the very end of this method.
Set<ILogicalExpression> replacedFuncExprs = new HashSet<>();
// Info on high and low keys for the BTree search predicate.
ILogicalExpression[] lowKeyExprs = new ILogicalExpression[numSecondaryKeys];
ILogicalExpression[] highKeyExprs = new ILogicalExpression[numSecondaryKeys];
LimitType[] lowKeyLimits = new LimitType[numSecondaryKeys];
LimitType[] highKeyLimits = new LimitType[numSecondaryKeys];
boolean[] lowKeyInclusive = new boolean[numSecondaryKeys];
boolean[] highKeyInclusive = new boolean[numSecondaryKeys];
ILogicalExpression[] constantAtRuntimeExpressions = new ILogicalExpression[numSecondaryKeys];
LogicalVariable[] constAtRuntimeExprVars = new LogicalVariable[numSecondaryKeys];
/* TODO: For now we don't do any sophisticated analysis of the func exprs to come up with "the best" range
* predicate. If we can't figure out how to integrate a certain funcExpr into the current predicate,
* we just bail by setting this flag.*/
boolean couldntFigureOut = false;
boolean doneWithExprs = false;
boolean isEqCondition = false;
BitSet setLowKeys = new BitSet(numSecondaryKeys);
BitSet setHighKeys = new BitSet(numSecondaryKeys);
// Go through the func exprs listed as optimizable by the chosen index,
// and formulate a range predicate on the secondary-index keys.
// checks whether a type casting happened from a real (FLOAT, DOUBLE) value to an INT value
// since we have a round issues when dealing with LT(<) OR GT(>) operator.
boolean realTypeConvertedToIntegerType;
for (Pair<Integer, Integer> exprIndex : exprAndVarList) {
// Position of the field of matchedFuncExprs.get(exprIndex) in the chosen index's indexed exprs.
IOptimizableFuncExpr optFuncExpr = analysisCtx.getMatchedFuncExpr(exprIndex.first);
int keyPos = indexOf(optFuncExpr.getFieldName(0), chosenIndex.getKeyFieldNames());
if (keyPos < 0 && optFuncExpr.getNumLogicalVars() > 1) {
// If we are optimizing a join, the matching field may be the second field name.
keyPos = indexOf(optFuncExpr.getFieldName(1), chosenIndex.getKeyFieldNames());
}
if (keyPos < 0) {
throw CompilationException.create(ErrorCode.NO_INDEX_FIELD_NAME_FOR_GIVEN_FUNC_EXPR);
}
Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr = AccessMethodUtils.createSearchKeyExpr(optFuncExpr, indexSubTree, probeSubTree);
ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first;
if (searchKeyExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
constantAtRuntimeExpressions[keyPos] = searchKeyExpr;
constAtRuntimeExprVars[keyPos] = context.newVar();
searchKeyExpr = new VariableReferenceExpression(constAtRuntimeExprVars[keyPos]);
}
realTypeConvertedToIntegerType = returnedSearchKeyExpr.second;
LimitType limit = getLimitType(optFuncExpr, probeSubTree);
//
if (realTypeConvertedToIntegerType) {
if (limit == LimitType.HIGH_EXCLUSIVE) {
limit = LimitType.HIGH_INCLUSIVE;
} else if (limit == LimitType.LOW_EXCLUSIVE) {
limit = LimitType.LOW_INCLUSIVE;
}
}
switch(limit) {
case EQUAL:
{
if (lowKeyLimits[keyPos] == null && highKeyLimits[keyPos] == null) {
lowKeyLimits[keyPos] = highKeyLimits[keyPos] = limit;
lowKeyInclusive[keyPos] = highKeyInclusive[keyPos] = true;
lowKeyExprs[keyPos] = highKeyExprs[keyPos] = searchKeyExpr;
setLowKeys.set(keyPos);
setHighKeys.set(keyPos);
isEqCondition = true;
} else {
// (once from analyzing each side of the join)
if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true && lowKeyExprs[keyPos].equals(searchKeyExpr) && highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true && highKeyExprs[keyPos].equals(searchKeyExpr)) {
isEqCondition = true;
break;
}
couldntFigureOut = true;
}
// If high and low keys are set, we exit for now.
if (setLowKeys.cardinality() == numSecondaryKeys && setHighKeys.cardinality() == numSecondaryKeys) {
doneWithExprs = true;
}
break;
}
case HIGH_EXCLUSIVE:
{
if (highKeyLimits[keyPos] == null || (highKeyLimits[keyPos] != null && highKeyInclusive[keyPos])) {
highKeyLimits[keyPos] = limit;
highKeyExprs[keyPos] = searchKeyExpr;
highKeyInclusive[keyPos] = false;
} else {
// (once from analyzing each side of the join)
if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == false && highKeyExprs[keyPos].equals(searchKeyExpr)) {
break;
}
couldntFigureOut = true;
doneWithExprs = true;
}
break;
}
case HIGH_INCLUSIVE:
{
if (highKeyLimits[keyPos] == null) {
highKeyLimits[keyPos] = limit;
highKeyExprs[keyPos] = searchKeyExpr;
highKeyInclusive[keyPos] = true;
} else {
// (once from analyzing each side of the join)
if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true && highKeyExprs[keyPos].equals(searchKeyExpr)) {
break;
}
couldntFigureOut = true;
doneWithExprs = true;
}
break;
}
case LOW_EXCLUSIVE:
{
if (lowKeyLimits[keyPos] == null || (lowKeyLimits[keyPos] != null && lowKeyInclusive[keyPos])) {
lowKeyLimits[keyPos] = limit;
lowKeyExprs[keyPos] = searchKeyExpr;
lowKeyInclusive[keyPos] = false;
} else {
// (once from analyzing each side of the join)
if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == false && lowKeyExprs[keyPos].equals(searchKeyExpr)) {
break;
}
couldntFigureOut = true;
doneWithExprs = true;
}
break;
}
case LOW_INCLUSIVE:
{
if (lowKeyLimits[keyPos] == null) {
lowKeyLimits[keyPos] = limit;
lowKeyExprs[keyPos] = searchKeyExpr;
lowKeyInclusive[keyPos] = true;
} else {
// (once from analyzing each side of the join)
if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true && lowKeyExprs[keyPos].equals(searchKeyExpr)) {
break;
}
couldntFigureOut = true;
doneWithExprs = true;
}
break;
}
default:
{
throw new IllegalStateException();
}
}
if (!couldntFigureOut) {
// Remember to remove this funcExpr later.
replacedFuncExprs.add(analysisCtx.getMatchedFuncExpr(exprIndex.first).getFuncExpr());
}
if (doneWithExprs) {
break;
}
}
if (couldntFigureOut) {
return null;
}
// If the select condition contains mixed open/closed intervals on multiple keys, then we make all intervals
// closed to obtain a superset of answers and leave the original selection in place.
boolean primaryIndexPostProccessingIsNeeded = false;
for (int i = 1; i < numSecondaryKeys; ++i) {
if (lowKeyInclusive[i] != lowKeyInclusive[0]) {
Arrays.fill(lowKeyInclusive, true);
primaryIndexPostProccessingIsNeeded = true;
break;
}
}
for (int i = 1; i < numSecondaryKeys; ++i) {
if (highKeyInclusive[i] != highKeyInclusive[0]) {
Arrays.fill(highKeyInclusive, true);
primaryIndexPostProccessingIsNeeded = true;
break;
}
}
// determine cases when prefix search could be applied
for (int i = 1; i < lowKeyExprs.length; i++) {
if (lowKeyLimits[0] == null && lowKeyLimits[i] != null || lowKeyLimits[0] != null && lowKeyLimits[i] == null || highKeyLimits[0] == null && highKeyLimits[i] != null || highKeyLimits[0] != null && highKeyLimits[i] == null) {
numSecondaryKeys--;
primaryIndexPostProccessingIsNeeded = true;
}
}
if (lowKeyLimits[0] == null) {
lowKeyInclusive[0] = true;
}
if (highKeyLimits[0] == null) {
highKeyInclusive[0] = true;
}
// Here we generate vars and funcs for assigning the secondary-index keys to be fed into the secondary-index
// search.
// List of variables for the assign.
ArrayList<LogicalVariable> keyVarList = new ArrayList<>();
// List of variables and expressions for the assign.
ArrayList<LogicalVariable> assignKeyVarList = new ArrayList<>();
ArrayList<Mutable<ILogicalExpression>> assignKeyExprList = new ArrayList<>();
int numLowKeys = createKeyVarsAndExprs(numSecondaryKeys, lowKeyLimits, lowKeyExprs, assignKeyVarList, assignKeyExprList, keyVarList, context, constantAtRuntimeExpressions, constAtRuntimeExprVars);
int numHighKeys = createKeyVarsAndExprs(numSecondaryKeys, highKeyLimits, highKeyExprs, assignKeyVarList, assignKeyExprList, keyVarList, context, constantAtRuntimeExpressions, constAtRuntimeExprVars);
BTreeJobGenParams jobGenParams = new BTreeJobGenParams(chosenIndex.getIndexName(), IndexType.BTREE, dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
jobGenParams.setLowKeyInclusive(lowKeyInclusive[0]);
jobGenParams.setHighKeyInclusive(highKeyInclusive[0]);
jobGenParams.setIsEqCondition(isEqCondition);
jobGenParams.setLowKeyVarList(keyVarList, 0, numLowKeys);
jobGenParams.setHighKeyVarList(keyVarList, numLowKeys, numHighKeys);
ILogicalOperator inputOp = null;
if (!assignKeyVarList.isEmpty()) {
// Assign operator that sets the constant secondary-index search-key fields if necessary.
AssignOperator assignConstantSearchKeys = new AssignOperator(assignKeyVarList, assignKeyExprList);
// Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
assignConstantSearchKeys.getInputs().add(new MutableObject<>(OperatorManipulationUtil.deepCopy(dataSourceOp.getInputs().get(0).getValue())));
assignConstantSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
inputOp = assignConstantSearchKeys;
} else if (probeSubTree == null) {
//nonpure case
//Make sure that the nonpure function is unpartitioned
ILogicalOperator checkOp = dataSourceOp.getInputs().get(0).getValue();
while (checkOp.getExecutionMode() != ExecutionMode.UNPARTITIONED) {
if (checkOp.getInputs().size() == 1) {
checkOp = checkOp.getInputs().get(0).getValue();
} else {
return null;
}
}
inputOp = dataSourceOp.getInputs().get(0).getValue();
} else {
// All index search keys are variables.
inputOp = probeSubTree.getRoot();
}
ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType, metaRecordType, chosenIndex, inputOp, jobGenParams, context, false, retainInput, retainNull);
// Generate the rest of the upstream plan which feeds the search results into the primary index.
AbstractUnnestMapOperator primaryIndexUnnestOp = null;
boolean isPrimaryIndex = chosenIndex.isPrimaryIndex();
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
// External dataset
UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, retainInput, retainNull);
indexSubTree.getDataSourceRef().setValue(externalDataAccessOp);
return externalDataAccessOp;
} else if (!isPrimaryIndex) {
primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
// Adds equivalence classes --- one equivalent class between a primary key
// variable and a record field-access expression.
EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp, dataSourceOp.getVariables(), recordType, metaRecordType, dataset, context);
} else {
List<Object> primaryIndexOutputTypes = new ArrayList<>();
AccessMethodUtils.appendPrimaryIndexTypes(dataset, recordType, metaRecordType, primaryIndexOutputTypes);
List<LogicalVariable> scanVariables = dataSourceOp.getVariables();
// If not, we create a new condition based on remaining ones.
if (!primaryIndexPostProccessingIsNeeded) {
List<Mutable<ILogicalExpression>> remainingFuncExprs = new ArrayList<>();
try {
getNewConditionExprs(conditionRef, replacedFuncExprs, remainingFuncExprs);
} catch (CompilationException e) {
return null;
}
// Generate new condition.
if (!remainingFuncExprs.isEmpty()) {
ILogicalExpression pulledCond = createSelectCondition(remainingFuncExprs);
conditionRef.setValue(pulledCond);
} else {
conditionRef.setValue(null);
}
}
// Checks whether LEFT_OUTER_UNNESTMAP operator is required.
boolean leftOuterUnnestMapRequired = false;
if (retainNull && retainInput) {
leftOuterUnnestMapRequired = true;
} else {
leftOuterUnnestMapRequired = false;
}
if (conditionRef.getValue() != null) {
// The job gen parameters are transferred to the actual job gen
// via the UnnestMapOperator's function arguments.
List<Mutable<ILogicalExpression>> primaryIndexFuncArgs = new ArrayList<>();
jobGenParams.writeToFuncArgs(primaryIndexFuncArgs);
// An index search is expressed as an unnest-map over an
// index-search function.
IFunctionInfo primaryIndexSearch = FunctionUtil.getFunctionInfo(BuiltinFunctions.INDEX_SEARCH);
UnnestingFunctionCallExpression primaryIndexSearchFunc = new UnnestingFunctionCallExpression(primaryIndexSearch, primaryIndexFuncArgs);
primaryIndexSearchFunc.setReturnsUniqueValues(true);
if (!leftOuterUnnestMapRequired) {
primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, retainInput);
} else {
primaryIndexUnnestOp = new LeftOuterUnnestMapOperator(scanVariables, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, true);
}
} else {
if (!leftOuterUnnestMapRequired) {
primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, ((UnnestMapOperator) secondaryIndexUnnestOp).getExpressionRef(), primaryIndexOutputTypes, retainInput);
} else {
primaryIndexUnnestOp = new LeftOuterUnnestMapOperator(scanVariables, ((LeftOuterUnnestMapOperator) secondaryIndexUnnestOp).getExpressionRef(), primaryIndexOutputTypes, true);
}
}
primaryIndexUnnestOp.getInputs().add(new MutableObject<>(inputOp));
// Adds equivalence classes --- one equivalent class between a primary key
// variable and a record field-access expression.
EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp, scanVariables, recordType, metaRecordType, dataset, context);
}
return primaryIndexUnnestOp;
}
Aggregations