use of org.apache.asterix.om.base.AString in project asterixdb by apache.
the class PushFieldAccessRule method propagateFieldAccessRec.
@SuppressWarnings("unchecked")
private boolean propagateFieldAccessRec(Mutable<ILogicalOperator> opRef, IOptimizationContext context, String finalAnnot) throws AlgebricksException {
AssignOperator access = (AssignOperator) opRef.getValue();
Mutable<ILogicalOperator> opRef2 = access.getInputs().get(0);
AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
// rewritten into index search.
if (op2.getOperatorTag() == LogicalOperatorTag.PROJECT || context.checkAndAddToAlreadyCompared(access, op2) && !(op2.getOperatorTag() == LogicalOperatorTag.SELECT && isAccessToIndexedField(access, context))) {
return false;
}
Object annotation = op2.getAnnotations().get(IS_MOVABLE);
if (annotation != null && !((Boolean) annotation)) {
return false;
}
if (tryingToPushThroughSelectionWithSameDataSource(access, op2)) {
return false;
}
if (testAndModifyRedundantOp(access, op2)) {
propagateFieldAccessRec(opRef2, context, finalAnnot);
return true;
}
List<LogicalVariable> usedInAccess = new LinkedList<>();
VariableUtilities.getUsedVariables(access, usedInAccess);
List<LogicalVariable> produced2 = new LinkedList<>();
if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
VariableUtilities.getLiveVariables(op2, produced2);
} else {
VariableUtilities.getProducedVariables(op2, produced2);
}
boolean pushItDown = false;
List<LogicalVariable> inter = new ArrayList<>(usedInAccess);
if (inter.isEmpty()) {
// ground value
return false;
}
inter.retainAll(produced2);
if (inter.isEmpty()) {
pushItDown = true;
} else if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
GroupByOperator g = (GroupByOperator) op2;
List<Pair<LogicalVariable, LogicalVariable>> varMappings = new ArrayList<>();
for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getDecorList()) {
ILogicalExpression e = p.second.getValue();
if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
if (inter.contains(decorVar)) {
inter.remove(decorVar);
LogicalVariable v1 = ((VariableReferenceExpression) e).getVariableReference();
varMappings.add(new Pair<>(decorVar, v1));
}
}
}
if (inter.isEmpty()) {
boolean changed = false;
for (Pair<LogicalVariable, LogicalVariable> m : varMappings) {
LogicalVariable v2 = context.newVar();
LogicalVariable oldVar = access.getVariables().get(0);
g.getDecorList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(oldVar, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(v2))));
changed = true;
access.getVariables().set(0, v2);
VariableUtilities.substituteVariables(access, m.first, m.second, context);
}
if (changed) {
context.computeAndSetTypeEnvironmentForOperator(g);
}
usedInAccess.clear();
VariableUtilities.getUsedVariables(access, usedInAccess);
pushItDown = true;
}
}
if (pushItDown) {
if (op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
Mutable<ILogicalOperator> childOfSubplan = ((NestedTupleSourceOperator) op2).getDataSourceReference().getValue().getInputs().get(0);
pushAccessDown(opRef, op2, childOfSubplan, context, finalAnnot);
return true;
}
if (op2.getInputs().size() == 1 && !op2.hasNestedPlans()) {
pushAccessDown(opRef, op2, op2.getInputs().get(0), context, finalAnnot);
return true;
} else {
for (Mutable<ILogicalOperator> inp : op2.getInputs()) {
HashSet<LogicalVariable> v2 = new HashSet<>();
VariableUtilities.getLiveVariables(inp.getValue(), v2);
if (v2.containsAll(usedInAccess)) {
pushAccessDown(opRef, op2, inp, context, finalAnnot);
return true;
}
}
}
if (op2.hasNestedPlans()) {
AbstractOperatorWithNestedPlans nestedOp = (AbstractOperatorWithNestedPlans) op2;
for (ILogicalPlan plan : nestedOp.getNestedPlans()) {
for (Mutable<ILogicalOperator> root : plan.getRoots()) {
HashSet<LogicalVariable> v2 = new HashSet<>();
VariableUtilities.getLiveVariables(root.getValue(), v2);
if (v2.containsAll(usedInAccess)) {
pushAccessDown(opRef, op2, root, context, finalAnnot);
return true;
}
}
}
}
throw new AlgebricksException("Field access " + access.getExpressions().get(0).getValue() + " does not correspond to any input of operator " + op2);
} else {
// fields. If yes, we can equate the two variables.
if (op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
DataSourceScanOperator scan = (DataSourceScanOperator) op2;
int n = scan.getVariables().size();
LogicalVariable scanRecordVar = scan.getVariables().get(n - 1);
AbstractFunctionCallExpression accessFun = (AbstractFunctionCallExpression) access.getExpressions().get(0).getValue();
ILogicalExpression e0 = accessFun.getArguments().get(0).getValue();
LogicalExpressionTag tag = e0.getExpressionTag();
if (tag == LogicalExpressionTag.VARIABLE) {
VariableReferenceExpression varRef = (VariableReferenceExpression) e0;
if (varRef.getVariableReference() == scanRecordVar) {
ILogicalExpression e1 = accessFun.getArguments().get(1).getValue();
if (e1.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
IDataSource<DataSourceId> dataSource = (IDataSource<DataSourceId>) scan.getDataSource();
byte dsType = ((DataSource) dataSource).getDatasourceType();
if (dsType == DataSource.Type.FEED || dsType == DataSource.Type.LOADABLE) {
return false;
}
DataSourceId asid = dataSource.getId();
MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
Dataset dataset = mp.findDataset(asid.getDataverseName(), asid.getDatasourceName());
if (dataset == null) {
throw new AlgebricksException("Dataset " + asid.getDatasourceName() + " not found.");
}
if (dataset.getDatasetType() != DatasetType.INTERNAL) {
setAsFinal(access, context, finalAnnot);
return false;
}
ConstantExpression ce = (ConstantExpression) e1;
IAObject obj = ((AsterixConstantValue) ce.getValue()).getObject();
String fldName;
if (obj.getType().getTypeTag() == ATypeTag.STRING) {
fldName = ((AString) obj).getStringValue();
} else {
int pos = ((AInt32) obj).getIntegerValue();
String tName = dataset.getItemTypeName();
IAType t = mp.findType(dataset.getItemTypeDataverseName(), tName);
if (t.getTypeTag() != ATypeTag.OBJECT) {
return false;
}
ARecordType rt = (ARecordType) t;
if (pos >= rt.getFieldNames().length) {
setAsFinal(access, context, finalAnnot);
return false;
}
fldName = rt.getFieldNames()[pos];
}
int p = DatasetUtil.getPositionOfPartitioningKeyField(dataset, fldName);
if (p < 0) {
// not one of the partitioning fields
setAsFinal(access, context, finalAnnot);
return false;
}
LogicalVariable keyVar = scan.getVariables().get(p);
access.getExpressions().get(0).setValue(new VariableReferenceExpression(keyVar));
return true;
}
}
}
}
setAsFinal(access, context, finalAnnot);
return false;
}
}
use of org.apache.asterix.om.base.AString in project asterixdb by apache.
the class UnnestToDataScanRule method handleFunction.
protected boolean handleFunction(Mutable<ILogicalOperator> opRef, IOptimizationContext context, UnnestOperator unnest, AbstractFunctionCallExpression f) throws AlgebricksException {
FunctionIdentifier fid = f.getFunctionIdentifier();
if (fid.equals(BuiltinFunctions.DATASET)) {
if (unnest.getPositionalVariable() != null) {
// TODO remove this after enabling the support of positional variables in data scan
throw new AlgebricksException("No positional variables are allowed over datasets.");
}
ILogicalExpression expr = f.getArguments().get(0).getValue();
if (expr.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
return false;
}
ConstantExpression ce = (ConstantExpression) expr;
IAlgebricksConstantValue acv = ce.getValue();
if (!(acv instanceof AsterixConstantValue)) {
return false;
}
AsterixConstantValue acv2 = (AsterixConstantValue) acv;
if (acv2.getObject().getType().getTypeTag() != ATypeTag.STRING) {
return false;
}
String datasetArg = ((AString) acv2.getObject()).getStringValue();
MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
Pair<String, String> datasetReference = parseDatasetReference(metadataProvider, datasetArg);
String dataverseName = datasetReference.first;
String datasetName = datasetReference.second;
Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
if (dataset == null) {
throw new AlgebricksException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
}
DataSourceId asid = new DataSourceId(dataverseName, datasetName);
List<LogicalVariable> variables = new ArrayList<>();
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
int numPrimaryKeys = dataset.getPrimaryKeys().size();
for (int i = 0; i < numPrimaryKeys; i++) {
variables.add(context.newVar());
}
}
variables.add(unnest.getVariable());
DataSource dataSource = metadataProvider.findDataSource(asid);
boolean hasMeta = dataSource.hasMeta();
if (hasMeta) {
variables.add(context.newVar());
}
DataSourceScanOperator scan = new DataSourceScanOperator(variables, dataSource);
List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
scanInpList.addAll(unnest.getInputs());
opRef.setValue(scan);
addPrimaryKey(variables, dataSource, context);
context.computeAndSetTypeEnvironmentForOperator(scan);
// Adds equivalence classes --- one equivalent class between a primary key
// variable and a record field-access expression.
IAType[] schemaTypes = dataSource.getSchemaTypes();
ARecordType recordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 2] : schemaTypes[schemaTypes.length - 1]);
ARecordType metaRecordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 1] : null);
EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(scan, variables, recordType, metaRecordType, dataset, context);
return true;
} else if (fid.equals(BuiltinFunctions.FEED_COLLECT)) {
if (unnest.getPositionalVariable() != null) {
throw new AlgebricksException("No positional variables are allowed over feeds.");
}
String dataverse = ConstantExpressionUtil.getStringArgument(f, 0);
String sourceFeedName = ConstantExpressionUtil.getStringArgument(f, 1);
String getTargetFeed = ConstantExpressionUtil.getStringArgument(f, 2);
String subscriptionLocation = ConstantExpressionUtil.getStringArgument(f, 3);
String targetDataset = ConstantExpressionUtil.getStringArgument(f, 4);
String outputType = ConstantExpressionUtil.getStringArgument(f, 5);
MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
DataSourceId asid = new DataSourceId(dataverse, getTargetFeed);
String policyName = metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME);
FeedPolicyEntity policy = metadataProvider.findFeedPolicy(dataverse, policyName);
if (policy == null) {
policy = BuiltinFeedPolicies.getFeedPolicy(policyName);
if (policy == null) {
throw new AlgebricksException("Unknown feed policy:" + policyName);
}
}
ArrayList<LogicalVariable> feedDataScanOutputVariables = new ArrayList<>();
String csLocations = metadataProvider.getConfig().get(FeedActivityDetails.COLLECT_LOCATIONS);
List<LogicalVariable> pkVars = new ArrayList<>();
FeedDataSource ds = createFeedDataSource(asid, targetDataset, sourceFeedName, subscriptionLocation, metadataProvider, policy, outputType, csLocations, unnest.getVariable(), context, pkVars);
// The order for feeds is <Record-Meta-PK>
feedDataScanOutputVariables.add(unnest.getVariable());
// Does it produce meta?
if (ds.hasMeta()) {
feedDataScanOutputVariables.add(context.newVar());
}
// Does it produce pk?
if (ds.isChange()) {
feedDataScanOutputVariables.addAll(pkVars);
}
DataSourceScanOperator scan = new DataSourceScanOperator(feedDataScanOutputVariables, ds);
List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
scanInpList.addAll(unnest.getInputs());
opRef.setValue(scan);
context.computeAndSetTypeEnvironmentForOperator(scan);
return true;
}
return false;
}
use of org.apache.asterix.om.base.AString in project asterixdb by apache.
the class RecordRemoveFieldsTypeComputer method setFieldNameSet.
private boolean setFieldNameSet(ILogicalExpression expr, Set<String> fieldNameSet) {
if (expr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
AOrderedList orderedList = (AOrderedList) (((AsterixConstantValue) ((ConstantExpression) expr).getValue()).getObject());
for (int i = 0; i < orderedList.size(); i++) {
AString as = (AString) orderedList.getItem(i);
fieldNameSet.add(as.getStringValue());
}
// Success
return true;
}
return false;
}
use of org.apache.asterix.om.base.AString in project asterixdb by apache.
the class RecordRemoveFieldsTypeComputer method getPathFromConstantExpression.
private void getPathFromConstantExpression(String funcName, ILogicalExpression expression, Set<String> fieldNameSet, List<List<String>> pathList) throws AlgebricksException {
ConstantExpression ce = (ConstantExpression) expression;
if (!(ce.getValue() instanceof AsterixConstantValue)) {
throw new InvalidExpressionException(funcName, 1, ce, LogicalExpressionTag.CONSTANT);
}
IAObject item = ((AsterixConstantValue) ce.getValue()).getObject();
ATypeTag type = item.getType().getTypeTag();
switch(type) {
case STRING:
String fn = ((AString) item).getStringValue();
fieldNameSet.add(fn);
break;
case ARRAY:
AOrderedList pathOrdereList = (AOrderedList) item;
String fieldName = ((AString) pathOrdereList.getItem(0)).getStringValue();
fieldNameSet.add(fieldName);
List<String> path = new ArrayList<>();
for (int i = 0; i < pathOrdereList.size(); i++) {
path.add(((AString) pathOrdereList.getItem(i)).getStringValue());
}
pathList.add(path);
break;
default:
throw new UnsupportedTypeException(funcName, type);
}
}
use of org.apache.asterix.om.base.AString in project asterixdb by apache.
the class IntroduceAutogenerateIDRule method createPrimaryKeyRecordExpression.
private AbstractFunctionCallExpression createPrimaryKeyRecordExpression(List<String> pkFieldName) {
//Create lowest level of nested uuid
AbstractFunctionCallExpression uuidFn = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CREATE_UUID));
List<Mutable<ILogicalExpression>> openRecordConsArgs = new ArrayList<>();
Mutable<ILogicalExpression> pkFieldNameExpression = new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AString(pkFieldName.get(pkFieldName.size() - 1)))));
openRecordConsArgs.add(pkFieldNameExpression);
Mutable<ILogicalExpression> pkFieldValueExpression = new MutableObject<ILogicalExpression>(uuidFn);
openRecordConsArgs.add(pkFieldValueExpression);
AbstractFunctionCallExpression openRecFn = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR), openRecordConsArgs);
//Create higher levels
for (int i = pkFieldName.size() - 2; i > -1; i--) {
AString fieldName = new AString(pkFieldName.get(i));
openRecordConsArgs = new ArrayList<>();
openRecordConsArgs.add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(fieldName))));
openRecordConsArgs.add(new MutableObject<ILogicalExpression>(openRecFn));
openRecFn = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR), openRecordConsArgs);
}
return openRecFn;
}
Aggregations