use of org.apache.asterix.om.types.IAType in project asterixdb by apache.
the class LangExpressionToPlanTranslator method translate.
public ILogicalPlan translate(Query expr, String outputDatasetName, ICompiledDmlStatement stmt, ILogicalOperator baseOp) throws AlgebricksException {
MutableObject<ILogicalOperator> base = new MutableObject<>(new EmptyTupleSourceOperator());
if (baseOp != null) {
base = new MutableObject<>(baseOp);
}
Pair<ILogicalOperator, LogicalVariable> p = expr.accept(this, base);
ArrayList<Mutable<ILogicalOperator>> globalPlanRoots = new ArrayList<>();
ILogicalOperator topOp = p.first;
List<LogicalVariable> liveVars = new ArrayList<>();
VariableUtilities.getLiveVariables(topOp, liveVars);
LogicalVariable unnestVar = liveVars.get(0);
LogicalVariable resVar = unnestVar;
if (outputDatasetName == null) {
FileSplit outputFileSplit = metadataProvider.getOutputFile();
if (outputFileSplit == null) {
outputFileSplit = getDefaultOutputFileLocation(metadataProvider.getApplicationContext());
}
metadataProvider.setOutputFile(outputFileSplit);
List<Mutable<ILogicalExpression>> writeExprList = new ArrayList<>(1);
writeExprList.add(new MutableObject<>(new VariableReferenceExpression(resVar)));
ResultSetSinkId rssId = new ResultSetSinkId(metadataProvider.getResultSetId());
ResultSetDataSink sink = new ResultSetDataSink(rssId, null);
DistributeResultOperator newTop = new DistributeResultOperator(writeExprList, sink);
newTop.getInputs().add(new MutableObject<>(topOp));
topOp = newTop;
// Retrieve the Output RecordType (if any) and store it on
// the DistributeResultOperator
IAType outputRecordType = metadataProvider.findOutputRecordType();
if (outputRecordType != null) {
topOp.getAnnotations().put("output-record-type", outputRecordType);
}
} else {
/**
* add the collection-to-sequence right before the project,
* because dataset only accept non-collection records
*/
LogicalVariable seqVar = context.newVar();
/**
* This assign adds a marker function collection-to-sequence: if the input is a singleton collection, unnest
* it; otherwise do nothing.
*/
AssignOperator assignCollectionToSequence = new AssignOperator(seqVar, new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.COLLECTION_TO_SEQUENCE), new MutableObject<>(new VariableReferenceExpression(resVar)))));
assignCollectionToSequence.getInputs().add(new MutableObject<>(topOp.getInputs().get(0).getValue()));
topOp.getInputs().get(0).setValue(assignCollectionToSequence);
ProjectOperator projectOperator = (ProjectOperator) topOp;
projectOperator.getVariables().set(0, seqVar);
resVar = seqVar;
DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName());
List<Integer> keySourceIndicator = ((InternalDatasetDetails) targetDatasource.getDataset().getDatasetDetails()).getKeySourceIndicator();
ArrayList<LogicalVariable> vars = new ArrayList<>();
ArrayList<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<>();
List<List<String>> partitionKeys = targetDatasource.getDataset().getPrimaryKeys();
int numOfPrimaryKeys = partitionKeys.size();
for (int i = 0; i < numOfPrimaryKeys; i++) {
if (keySourceIndicator == null || keySourceIndicator.get(i).intValue() == 0) {
// record part
PlanTranslationUtil.prepareVarAndExpression(partitionKeys.get(i), resVar, vars, exprs, varRefsForLoading, context);
} else {
// meta part
PlanTranslationUtil.prepareMetaKeyAccessExpression(partitionKeys.get(i), unnestVar, exprs, vars, varRefsForLoading, context);
}
}
AssignOperator assign = new AssignOperator(vars, exprs);
List<String> additionalFilteringField = DatasetUtil.getFilterField(targetDatasource.getDataset());
List<LogicalVariable> additionalFilteringVars;
List<Mutable<ILogicalExpression>> additionalFilteringAssignExpressions;
List<Mutable<ILogicalExpression>> additionalFilteringExpressions = null;
AssignOperator additionalFilteringAssign = null;
if (additionalFilteringField != null) {
additionalFilteringVars = new ArrayList<>();
additionalFilteringAssignExpressions = new ArrayList<>();
additionalFilteringExpressions = new ArrayList<>();
PlanTranslationUtil.prepareVarAndExpression(additionalFilteringField, resVar, additionalFilteringVars, additionalFilteringAssignExpressions, additionalFilteringExpressions, context);
additionalFilteringAssign = new AssignOperator(additionalFilteringVars, additionalFilteringAssignExpressions);
additionalFilteringAssign.getInputs().add(new MutableObject<>(topOp));
assign.getInputs().add(new MutableObject<>(additionalFilteringAssign));
} else {
assign.getInputs().add(new MutableObject<>(topOp));
}
Mutable<ILogicalExpression> varRef = new MutableObject<>(new VariableReferenceExpression(resVar));
ILogicalOperator leafOperator;
switch(stmt.getKind()) {
case Statement.Kind.INSERT:
leafOperator = translateInsert(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign, stmt);
break;
case Statement.Kind.UPSERT:
leafOperator = translateUpsert(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign, additionalFilteringField, unnestVar, topOp, exprs, resVar, additionalFilteringAssign, stmt);
break;
case Statement.Kind.DELETE:
leafOperator = translateDelete(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign);
break;
case Statement.Kind.CONNECT_FEED:
leafOperator = translateConnectFeed(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign);
break;
case Statement.Kind.SUBSCRIBE_FEED:
leafOperator = translateSubscribeFeed((CompiledSubscribeFeedStatement) stmt, targetDatasource, unnestVar, topOp, exprs, resVar, varRefsForLoading, varRef, assign, additionalFilteringField, additionalFilteringAssign, additionalFilteringExpressions);
break;
default:
throw new AlgebricksException("Unsupported statement kind " + stmt.getKind());
}
topOp = leafOperator;
}
globalPlanRoots.add(new MutableObject<>(topOp));
ILogicalPlan plan = new ALogicalPlanImpl(globalPlanRoots);
eliminateSharedOperatorReferenceForPlan(plan);
return plan;
}
use of org.apache.asterix.om.types.IAType in project asterixdb by apache.
the class LangExpressionToPlanTranslator method validateDatasetInfo.
private DatasetDataSource validateDatasetInfo(MetadataProvider metadataProvider, String dataverseName, String datasetName) throws AlgebricksException {
Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
if (dataset == null) {
throw new AlgebricksException("Cannot find dataset " + datasetName + " in dataverse " + dataverseName);
}
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
throw new AlgebricksException("Cannot write output to an external dataset.");
}
DataSourceId sourceId = new DataSourceId(dataverseName, datasetName);
IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
IAType metaItemType = metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
INodeDomain domain = metadataProvider.findNodeDomain(dataset.getNodeGroupName());
return new DatasetDataSource(sourceId, dataset, itemType, metaItemType, DataSource.Type.INTERNAL_DATASET, dataset.getDatasetDetails(), domain);
}
use of org.apache.asterix.om.types.IAType in project asterixdb by apache.
the class LangExpressionToPlanTranslator method translateLoad.
@Override
public ILogicalPlan translateLoad(ICompiledDmlStatement stmt) throws AlgebricksException {
CompiledLoadFromFileStatement clffs = (CompiledLoadFromFileStatement) stmt;
Dataset dataset = metadataProvider.findDataset(clffs.getDataverseName(), clffs.getDatasetName());
if (dataset == null) {
// This would never happen since we check for this in AqlTranslator
throw new AlgebricksException("Unable to load dataset " + clffs.getDatasetName() + " since it does not exist");
}
IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
IAType metaItemType = metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName());
List<List<String>> partitionKeys = targetDatasource.getDataset().getPrimaryKeys();
if (dataset.hasMetaPart()) {
throw new AlgebricksException(dataset.getDatasetName() + ": load dataset is not supported on Datasets with Meta records");
}
LoadableDataSource lds;
try {
lds = new LoadableDataSource(dataset, itemType, metaItemType, clffs.getAdapter(), clffs.getProperties());
} catch (IOException e) {
throw new AlgebricksException(e);
}
// etsOp is a dummy input operator used to keep the compiler happy. it
// could be removed but would result in
// the need to fix many rewrite rules that assume that datasourcescan
// operators always have input.
ILogicalOperator etsOp = new EmptyTupleSourceOperator();
// Add a logical variable for the record.
List<LogicalVariable> payloadVars = new ArrayList<>();
payloadVars.add(context.newVar());
// Create a scan operator and make the empty tuple source its input
DataSourceScanOperator dssOp = new DataSourceScanOperator(payloadVars, lds);
dssOp.getInputs().add(new MutableObject<>(etsOp));
ILogicalExpression payloadExpr = new VariableReferenceExpression(payloadVars.get(0));
Mutable<ILogicalExpression> payloadRef = new MutableObject<>(payloadExpr);
// Creating the assign to extract the PK out of the record
ArrayList<LogicalVariable> pkVars = new ArrayList<>();
ArrayList<Mutable<ILogicalExpression>> pkExprs = new ArrayList<>();
List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<>();
LogicalVariable payloadVar = payloadVars.get(0);
for (List<String> keyFieldName : partitionKeys) {
PlanTranslationUtil.prepareVarAndExpression(keyFieldName, payloadVar, pkVars, pkExprs, varRefsForLoading, context);
}
AssignOperator assign = new AssignOperator(pkVars, pkExprs);
assign.getInputs().add(new MutableObject<>(dssOp));
// If the input is pre-sorted, we set the ordering property explicitly in the assign
if (clffs.alreadySorted()) {
List<OrderColumn> orderColumns = new ArrayList<>();
for (int i = 0; i < pkVars.size(); ++i) {
orderColumns.add(new OrderColumn(pkVars.get(i), OrderKind.ASC));
}
assign.setExplicitOrderingProperty(new LocalOrderProperty(orderColumns));
}
List<String> additionalFilteringField = DatasetUtil.getFilterField(targetDatasource.getDataset());
List<LogicalVariable> additionalFilteringVars;
List<Mutable<ILogicalExpression>> additionalFilteringAssignExpressions;
List<Mutable<ILogicalExpression>> additionalFilteringExpressions = null;
AssignOperator additionalFilteringAssign = null;
if (additionalFilteringField != null) {
additionalFilteringVars = new ArrayList<>();
additionalFilteringAssignExpressions = new ArrayList<>();
additionalFilteringExpressions = new ArrayList<>();
PlanTranslationUtil.prepareVarAndExpression(additionalFilteringField, payloadVar, additionalFilteringVars, additionalFilteringAssignExpressions, additionalFilteringExpressions, context);
additionalFilteringAssign = new AssignOperator(additionalFilteringVars, additionalFilteringAssignExpressions);
}
InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, payloadRef, varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, true);
insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
if (additionalFilteringAssign != null) {
additionalFilteringAssign.getInputs().add(new MutableObject<>(assign));
insertOp.getInputs().add(new MutableObject<>(additionalFilteringAssign));
} else {
insertOp.getInputs().add(new MutableObject<>(assign));
}
ILogicalOperator leafOperator = new SinkOperator();
leafOperator.getInputs().add(new MutableObject<>(insertOp));
return new ALogicalPlanImpl(new MutableObject<>(leafOperator));
}
use of org.apache.asterix.om.types.IAType in project asterixdb by apache.
the class StaticTypeCastUtil method injectCastToRelaxType.
private static boolean injectCastToRelaxType(Mutable<ILogicalExpression> expRef, IAType inputFieldType, IVariableTypeEnvironment env) throws AlgebricksException {
ILogicalExpression argExpr = expRef.getValue();
List<LogicalVariable> parameterVars = new ArrayList<LogicalVariable>();
argExpr.getUsedVariables(parameterVars);
// we need to handle open fields recursively by their default
// types
// for list, their item type is any
// for record, their
boolean castInjected = false;
if (argExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL || argExpr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
IAType reqFieldType = inputFieldType;
// do not enforce nested type in the case of no-used variables
switch(inputFieldType.getTypeTag()) {
case OBJECT:
reqFieldType = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
break;
case ARRAY:
reqFieldType = DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE;
break;
case MULTISET:
reqFieldType = DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE;
break;
default:
break;
}
// do not enforce nested type in the case of no-used variables
if (!inputFieldType.equals(reqFieldType) && !parameterVars.isEmpty()) {
//inject dynamic type casting
injectCastFunction(FunctionUtil.getFunctionInfo(BuiltinFunctions.CAST_TYPE), reqFieldType, inputFieldType, expRef, argExpr);
castInjected = true;
}
//recursively rewrite function arguments
if (argExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL && TypeCastUtils.getRequiredType((AbstractFunctionCallExpression) argExpr) == null && reqFieldType != null) {
if (castInjected) {
//rewrite the arg expression inside the dynamic cast
ScalarFunctionCallExpression argFunc = (ScalarFunctionCallExpression) argExpr;
rewriteFuncExpr(argFunc, inputFieldType, inputFieldType, env);
} else {
//rewrite arg
ScalarFunctionCallExpression argFunc = (ScalarFunctionCallExpression) argExpr;
rewriteFuncExpr(argFunc, reqFieldType, inputFieldType, env);
}
}
}
return castInjected;
}
use of org.apache.asterix.om.types.IAType in project asterixdb by apache.
the class StaticTypeCastUtil method compatible.
/**
* Determine if two types are compatible
*
* @param reqType
* the required type
* @param inputType
* the input type
* @return true if the two types are compatible; false otherwise
*/
public static boolean compatible(IAType reqType, IAType inputType) {
if (reqType.getTypeTag() == ATypeTag.ANY || inputType.getTypeTag() == ATypeTag.ANY) {
return true;
}
if (reqType.getTypeTag() != ATypeTag.UNION && inputType.getTypeTag() != ATypeTag.UNION) {
if (reqType.equals(inputType)) {
return true;
} else {
return false;
}
}
Set<IAType> reqTypePossible = new HashSet<IAType>();
Set<IAType> inputTypePossible = new HashSet<IAType>();
if (reqType.getTypeTag() == ATypeTag.UNION) {
AUnionType unionType = (AUnionType) reqType;
reqTypePossible.addAll(unionType.getUnionList());
} else {
reqTypePossible.add(reqType);
}
if (inputType.getTypeTag() == ATypeTag.UNION) {
AUnionType unionType = (AUnionType) inputType;
inputTypePossible.addAll(unionType.getUnionList());
} else {
inputTypePossible.add(inputType);
}
return reqTypePossible.equals(inputTypePossible);
}
Aggregations