use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator in project asterixdb by apache.
the class LangExpressionToPlanTranslator method translateSubscribeFeed.
private ILogicalOperator translateSubscribeFeed(CompiledSubscribeFeedStatement sfs, DatasetDataSource targetDatasource, LogicalVariable unnestVar, ILogicalOperator topOp, ArrayList<Mutable<ILogicalExpression>> exprs, LogicalVariable resVar, List<Mutable<ILogicalExpression>> varRefsForLoading, Mutable<ILogicalExpression> varRef, ILogicalOperator assign, List<String> additionalFilteringField, AssignOperator additionalFilteringAssign, List<Mutable<ILogicalExpression>> additionalFilteringExpressions) throws AlgebricksException {
// if the feed is a change feed (i.e, performs different operations), we need to project op variable
InsertDeleteUpsertOperator feedModificationOp;
AssignOperator metaAndKeysAssign;
List<LogicalVariable> metaAndKeysVars = null;
List<Mutable<ILogicalExpression>> metaAndKeysExprs = null;
List<Mutable<ILogicalExpression>> metaExpSingletonList = null;
Feed feed = metadataProvider.findFeed(sfs.getDataverseName(), sfs.getFeedName());
boolean isChangeFeed = ExternalDataUtils.isChangeFeed(feed.getAdapterConfiguration());
boolean isUpsertFeed = ExternalDataUtils.isUpsertFeed(feed.getAdapterConfiguration());
ProjectOperator project = (ProjectOperator) topOp;
if (targetDatasource.getDataset().hasMetaPart() || isChangeFeed) {
metaAndKeysVars = new ArrayList<>();
metaAndKeysExprs = new ArrayList<>();
if (targetDatasource.getDataset().hasMetaPart()) {
// add the meta function
IFunctionInfo finfoMeta = FunctionUtil.getFunctionInfo(BuiltinFunctions.META);
ScalarFunctionCallExpression metaFunction = new ScalarFunctionCallExpression(finfoMeta, new MutableObject<>(new VariableReferenceExpression(unnestVar)));
// create assign for the meta part
LogicalVariable metaVar = context.newVar();
metaExpSingletonList = new ArrayList<>(1);
metaExpSingletonList.add(new MutableObject<>(new VariableReferenceExpression(metaVar)));
metaAndKeysVars.add(metaVar);
metaAndKeysExprs.add(new MutableObject<>(metaFunction));
project.getVariables().add(metaVar);
}
}
if (isChangeFeed) {
varRefsForLoading.clear();
for (Mutable<ILogicalExpression> assignExpr : exprs) {
if (assignExpr.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
AbstractFunctionCallExpression funcCall = (AbstractFunctionCallExpression) assignExpr.getValue();
funcCall.substituteVar(resVar, unnestVar);
LogicalVariable pkVar = context.newVar();
metaAndKeysVars.add(pkVar);
metaAndKeysExprs.add(new MutableObject<>(assignExpr.getValue()));
project.getVariables().add(pkVar);
varRefsForLoading.add(new MutableObject<>(new VariableReferenceExpression(pkVar)));
}
}
// A change feed, we don't need the assign to access PKs
feedModificationOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, metaExpSingletonList, InsertDeleteUpsertOperator.Kind.UPSERT, false);
// Create and add a new variable used for representing the original record
feedModificationOp.setPrevRecordVar(context.newVar());
feedModificationOp.setPrevRecordType(targetDatasource.getItemType());
if (targetDatasource.getDataset().hasMetaPart()) {
List<LogicalVariable> metaVars = new ArrayList<>();
metaVars.add(context.newVar());
feedModificationOp.setPrevAdditionalNonFilteringVars(metaVars);
List<Object> metaTypes = new ArrayList<>();
metaTypes.add(targetDatasource.getMetaItemType());
feedModificationOp.setPrevAdditionalNonFilteringTypes(metaTypes);
}
if (additionalFilteringField != null) {
feedModificationOp.setPrevFilterVar(context.newVar());
feedModificationOp.setPrevFilterType(((ARecordType) targetDatasource.getItemType()).getFieldType(additionalFilteringField.get(0)));
additionalFilteringAssign.getInputs().clear();
additionalFilteringAssign.getInputs().add(assign.getInputs().get(0));
feedModificationOp.getInputs().add(new MutableObject<>(additionalFilteringAssign));
} else {
feedModificationOp.getInputs().add(assign.getInputs().get(0));
}
} else {
final InsertDeleteUpsertOperator.Kind opKind = isUpsertFeed ? InsertDeleteUpsertOperator.Kind.UPSERT : InsertDeleteUpsertOperator.Kind.INSERT;
feedModificationOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, metaExpSingletonList, opKind, false);
if (isUpsertFeed) {
feedModificationOp.setPrevRecordVar(context.newVar());
feedModificationOp.setPrevRecordType(targetDatasource.getItemType());
}
feedModificationOp.getInputs().add(new MutableObject<>(assign));
}
if (targetDatasource.getDataset().hasMetaPart() || isChangeFeed) {
metaAndKeysAssign = new AssignOperator(metaAndKeysVars, metaAndKeysExprs);
metaAndKeysAssign.getInputs().add(topOp.getInputs().get(0));
topOp.getInputs().set(0, new MutableObject<>(metaAndKeysAssign));
}
feedModificationOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
ILogicalOperator leafOperator = new DelegateOperator(new CommitOperator(true));
leafOperator.getInputs().add(new MutableObject<>(feedModificationOp));
return leafOperator;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator in project asterixdb by apache.
the class LangExpressionToPlanTranslator method translateLoad.
@Override
public ILogicalPlan translateLoad(ICompiledDmlStatement stmt) throws AlgebricksException {
CompiledLoadFromFileStatement clffs = (CompiledLoadFromFileStatement) stmt;
Dataset dataset = metadataProvider.findDataset(clffs.getDataverseName(), clffs.getDatasetName());
if (dataset == null) {
// This would never happen since we check for this in AqlTranslator
throw new AlgebricksException("Unable to load dataset " + clffs.getDatasetName() + " since it does not exist");
}
IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
IAType metaItemType = metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName());
List<List<String>> partitionKeys = targetDatasource.getDataset().getPrimaryKeys();
if (dataset.hasMetaPart()) {
throw new AlgebricksException(dataset.getDatasetName() + ": load dataset is not supported on Datasets with Meta records");
}
LoadableDataSource lds;
try {
lds = new LoadableDataSource(dataset, itemType, metaItemType, clffs.getAdapter(), clffs.getProperties());
} catch (IOException e) {
throw new AlgebricksException(e);
}
// etsOp is a dummy input operator used to keep the compiler happy. it
// could be removed but would result in
// the need to fix many rewrite rules that assume that datasourcescan
// operators always have input.
ILogicalOperator etsOp = new EmptyTupleSourceOperator();
// Add a logical variable for the record.
List<LogicalVariable> payloadVars = new ArrayList<>();
payloadVars.add(context.newVar());
// Create a scan operator and make the empty tuple source its input
DataSourceScanOperator dssOp = new DataSourceScanOperator(payloadVars, lds);
dssOp.getInputs().add(new MutableObject<>(etsOp));
ILogicalExpression payloadExpr = new VariableReferenceExpression(payloadVars.get(0));
Mutable<ILogicalExpression> payloadRef = new MutableObject<>(payloadExpr);
// Creating the assign to extract the PK out of the record
ArrayList<LogicalVariable> pkVars = new ArrayList<>();
ArrayList<Mutable<ILogicalExpression>> pkExprs = new ArrayList<>();
List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<>();
LogicalVariable payloadVar = payloadVars.get(0);
for (List<String> keyFieldName : partitionKeys) {
PlanTranslationUtil.prepareVarAndExpression(keyFieldName, payloadVar, pkVars, pkExprs, varRefsForLoading, context);
}
AssignOperator assign = new AssignOperator(pkVars, pkExprs);
assign.getInputs().add(new MutableObject<>(dssOp));
// If the input is pre-sorted, we set the ordering property explicitly in the assign
if (clffs.alreadySorted()) {
List<OrderColumn> orderColumns = new ArrayList<>();
for (int i = 0; i < pkVars.size(); ++i) {
orderColumns.add(new OrderColumn(pkVars.get(i), OrderKind.ASC));
}
assign.setExplicitOrderingProperty(new LocalOrderProperty(orderColumns));
}
List<String> additionalFilteringField = DatasetUtil.getFilterField(targetDatasource.getDataset());
List<LogicalVariable> additionalFilteringVars;
List<Mutable<ILogicalExpression>> additionalFilteringAssignExpressions;
List<Mutable<ILogicalExpression>> additionalFilteringExpressions = null;
AssignOperator additionalFilteringAssign = null;
if (additionalFilteringField != null) {
additionalFilteringVars = new ArrayList<>();
additionalFilteringAssignExpressions = new ArrayList<>();
additionalFilteringExpressions = new ArrayList<>();
PlanTranslationUtil.prepareVarAndExpression(additionalFilteringField, payloadVar, additionalFilteringVars, additionalFilteringAssignExpressions, additionalFilteringExpressions, context);
additionalFilteringAssign = new AssignOperator(additionalFilteringVars, additionalFilteringAssignExpressions);
}
InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, payloadRef, varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, true);
insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
if (additionalFilteringAssign != null) {
additionalFilteringAssign.getInputs().add(new MutableObject<>(assign));
insertOp.getInputs().add(new MutableObject<>(additionalFilteringAssign));
} else {
insertOp.getInputs().add(new MutableObject<>(assign));
}
ILogicalOperator leafOperator = new SinkOperator();
leafOperator.getInputs().add(new MutableObject<>(insertOp));
return new ALogicalPlanImpl(new MutableObject<>(leafOperator));
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator in project asterixdb by apache.
the class IntroduceDynamicTypeCastRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
// Depending on the operator type, we need to extract the following pieces of information.
AbstractLogicalOperator op;
ARecordType requiredRecordType;
LogicalVariable recordVar;
// We identify INSERT and DISTRIBUTE_RESULT operators.
AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
switch(op1.getOperatorTag()) {
case SINK:
case DELEGATE_OPERATOR:
{
/**
* pattern match: commit insert assign
* resulting plan: commit-insert-project-assign
*/
if (op1.getOperatorTag() == LogicalOperatorTag.DELEGATE_OPERATOR) {
DelegateOperator eOp = (DelegateOperator) op1;
if (!(eOp.getDelegate() instanceof CommitOperator)) {
return false;
}
}
AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
if (op2.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE_UPSERT) {
InsertDeleteUpsertOperator insertDeleteOp = (InsertDeleteUpsertOperator) op2;
if (insertDeleteOp.getOperation() == InsertDeleteUpsertOperator.Kind.DELETE) {
return false;
}
// Remember this is the operator we need to modify
op = insertDeleteOp;
// Derive the required ARecordType based on the schema of the DataSource
InsertDeleteUpsertOperator insertDeleteOperator = (InsertDeleteUpsertOperator) op2;
DataSource dataSource = (DataSource) insertDeleteOperator.getDataSource();
requiredRecordType = (ARecordType) dataSource.getItemType();
// Derive the Variable which we will potentially wrap with cast/null functions
ILogicalExpression expr = insertDeleteOperator.getPayloadExpression().getValue();
List<LogicalVariable> payloadVars = new ArrayList<>();
expr.getUsedVariables(payloadVars);
recordVar = payloadVars.get(0);
} else {
return false;
}
break;
}
case DISTRIBUTE_RESULT:
{
// First, see if there was an output-record-type specified
requiredRecordType = (ARecordType) op1.getAnnotations().get("output-record-type");
if (requiredRecordType == null) {
return false;
}
// Remember this is the operator we need to modify
op = op1;
recordVar = ((VariableReferenceExpression) ((DistributeResultOperator) op).getExpressions().get(0).getValue()).getVariableReference();
break;
}
default:
{
return false;
}
}
// Derive the statically-computed type of the record
IVariableTypeEnvironment env = op.computeOutputTypeEnvironment(context);
IAType inputRecordType = (IAType) env.getVarType(recordVar);
/** the input record type can be an union type -- for the case when it comes from a subplan or left-outer join */
boolean checkUnknown = false;
while (NonTaggedFormatUtil.isOptional(inputRecordType)) {
/** while-loop for the case there is a nested multi-level union */
inputRecordType = ((AUnionType) inputRecordType).getActualType();
checkUnknown = true;
}
/** see whether the input record type needs to be casted */
boolean cast = !compatible(requiredRecordType, inputRecordType);
if (checkUnknown) {
recordVar = addWrapperFunction(requiredRecordType, recordVar, op, context, BuiltinFunctions.CHECK_UNKNOWN);
}
if (cast) {
addWrapperFunction(requiredRecordType, recordVar, op, context, BuiltinFunctions.CAST_TYPE);
}
return cast || checkUnknown;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator in project asterixdb by apache.
the class OperatorDeepCopyVisitor method visitInsertDeleteUpsertOperator.
@Override
public ILogicalOperator visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Void arg) throws AlgebricksException {
List<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<>();
deepCopyExpressionRefs(newKeyExpressions, op.getPrimaryKeyExpressions());
List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<>();
deepCopyExpressionRefs(newKeyExpressions, op.getAdditionalFilteringExpressions());
InsertDeleteUpsertOperator insertDeleteOp = new InsertDeleteUpsertOperator(op.getDataSource(), deepCopyExpressionRef(op.getPayloadExpression()), newKeyExpressions, op.getOperation(), op.isBulkload());
insertDeleteOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions);
return insertDeleteOp;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator in project asterixdb by apache.
the class IntroduceSecondaryIndexInsertDeleteRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
if (op0.getOperatorTag() != LogicalOperatorTag.DELEGATE_OPERATOR && op0.getOperatorTag() != LogicalOperatorTag.SINK) {
return false;
}
if (op0.getOperatorTag() == LogicalOperatorTag.DELEGATE_OPERATOR) {
DelegateOperator eOp = (DelegateOperator) op0;
if (!(eOp.getDelegate() instanceof CommitOperator)) {
return false;
}
}
AbstractLogicalOperator op1 = (AbstractLogicalOperator) op0.getInputs().get(0).getValue();
if (op1.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE_UPSERT) {
return false;
}
/** find the record variable */
InsertDeleteUpsertOperator primaryIndexModificationOp = (InsertDeleteUpsertOperator) op0.getInputs().get(0).getValue();
boolean isBulkload = primaryIndexModificationOp.isBulkload();
ILogicalExpression newRecordExpr = primaryIndexModificationOp.getPayloadExpression().getValue();
List<Mutable<ILogicalExpression>> newMetaExprs = primaryIndexModificationOp.getAdditionalNonFilteringExpressions();
LogicalVariable newRecordVar;
LogicalVariable newMetaVar = null;
/**
* inputOp is the assign operator which extracts primary keys from the input
* variables (record or meta)
*/
AbstractLogicalOperator inputOp = (AbstractLogicalOperator) primaryIndexModificationOp.getInputs().get(0).getValue();
newRecordVar = getRecordVar(context, inputOp, newRecordExpr, 0);
if (newMetaExprs != null && !newMetaExprs.isEmpty()) {
if (newMetaExprs.size() > 1) {
throw new AlgebricksException("Number of meta records can't be more than 1. Number of meta records found = " + newMetaExprs.size());
}
newMetaVar = getRecordVar(context, inputOp, newMetaExprs.get(0).getValue(), 1);
}
/*
* At this point, we have the record variable and the insert/delete/upsert operator
* Note: We have two operators:
* 1. An InsertDeleteOperator (primary)
* 2. An IndexInsertDeleteOperator (secondary)
* The current primaryIndexModificationOp is of the first type
*/
DataSource datasetSource = (DataSource) primaryIndexModificationOp.getDataSource();
MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
String dataverseName = datasetSource.getId().getDataverseName();
String datasetName = datasetSource.getId().getDatasourceName();
Dataset dataset = mp.findDataset(dataverseName, datasetName);
if (dataset == null) {
throw new AlgebricksException("Unknown dataset " + datasetName + " in dataverse " + dataverseName);
}
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
return false;
}
// Create operators for secondary index insert / delete.
String itemTypeName = dataset.getItemTypeName();
IAType itemType = mp.findType(dataset.getItemTypeDataverseName(), itemTypeName);
if (itemType.getTypeTag() != ATypeTag.OBJECT) {
throw new AlgebricksException("Only record types can be indexed.");
}
ARecordType recType = (ARecordType) itemType;
// meta type
ARecordType metaType = null;
if (dataset.hasMetaPart()) {
metaType = (ARecordType) mp.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
}
List<Index> indexes = mp.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
// Set the top operator pointer to the primary IndexInsertDeleteOperator
ILogicalOperator currentTop = primaryIndexModificationOp;
boolean hasSecondaryIndex = false;
// Put an n-gram or a keyword index in the later stage of index-update,
// since TokenizeOperator needs to be involved.
Collections.sort(indexes, (o1, o2) -> o1.getIndexType().ordinal() - o2.getIndexType().ordinal());
// At this point, we have the data type info, and the indexes info as well
int secondaryIndexTotalCnt = indexes.size() - 1;
if (secondaryIndexTotalCnt > 0) {
op0.getInputs().clear();
} else {
return false;
}
// Initialize inputs to the SINK operator Op0 (The SINK) is now without input
// Prepare filtering field information (This is the filter created using the "filter with" key word in the
// create dataset ddl)
List<String> filteringFields = ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterField();
List<LogicalVariable> filteringVars;
List<Mutable<ILogicalExpression>> filteringExpressions = null;
if (filteringFields != null) {
// The filter field var already exists. we can simply get it from the insert op
filteringVars = new ArrayList<>();
filteringExpressions = new ArrayList<>();
for (Mutable<ILogicalExpression> filteringExpression : primaryIndexModificationOp.getAdditionalFilteringExpressions()) {
filteringExpression.getValue().getUsedVariables(filteringVars);
for (LogicalVariable var : filteringVars) {
filteringExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(var)));
}
}
}
// Replicate Operator is applied only when doing the bulk-load.
ReplicateOperator replicateOp = null;
if (secondaryIndexTotalCnt > 1 && primaryIndexModificationOp.isBulkload()) {
// Split the logical plan into "each secondary index update branch"
// to replicate each <PK,OBJECT> pair.
replicateOp = new ReplicateOperator(secondaryIndexTotalCnt);
replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
replicateOp.setExecutionMode(ExecutionMode.PARTITIONED);
context.computeAndSetTypeEnvironmentForOperator(replicateOp);
currentTop = replicateOp;
}
/*
* The two maps are used to store variables to which [casted] field access is assigned.
* One for the beforeOp record and the other for the new record.
* There are two uses for these maps:
* 1. used for shared fields in indexes with overlapping keys.
* 2. used for setting variables of secondary keys for each secondary index operator.
*/
Map<IndexFieldId, LogicalVariable> fieldVarsForBeforeOperation = new HashMap<>();
Map<IndexFieldId, LogicalVariable> fieldVarsForNewRecord = new HashMap<>();
/*
* if the index is enforcing field types (For open indexes), We add a cast
* operator to ensure type safety
*/
try {
if (primaryIndexModificationOp.getOperation() == Kind.INSERT || primaryIndexModificationOp.getOperation() == Kind.UPSERT || /* Actually, delete should not be here but it is now until issue
* https://issues.apache.org/jira/browse/ASTERIXDB-1507
* is solved
*/
primaryIndexModificationOp.getOperation() == Kind.DELETE) {
injectFieldAccessesForIndexes(context, dataset, indexes, fieldVarsForNewRecord, recType, metaType, newRecordVar, newMetaVar, primaryIndexModificationOp, false);
if (replicateOp != null) {
context.computeAndSetTypeEnvironmentForOperator(replicateOp);
}
}
if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) /* Actually, delete should be here but it is not until issue
* https://issues.apache.org/jira/browse/ASTERIXDB-1507
* is solved
*/
{
List<LogicalVariable> beforeOpMetaVars = primaryIndexModificationOp.getBeforeOpAdditionalNonFilteringVars();
LogicalVariable beforeOpMetaVar = beforeOpMetaVars == null ? null : beforeOpMetaVars.get(0);
currentTop = injectFieldAccessesForIndexes(context, dataset, indexes, fieldVarsForBeforeOperation, recType, metaType, primaryIndexModificationOp.getBeforeOpRecordVar(), beforeOpMetaVar, currentTop, true);
}
} catch (AsterixException e) {
throw new AlgebricksException(e);
}
// At first, op1 is the index insert op insertOp
for (Index index : indexes) {
if (!index.isSecondaryIndex()) {
continue;
}
hasSecondaryIndex = true;
// Get the secondary fields names and types
List<List<String>> secondaryKeyFields = index.getKeyFieldNames();
List<LogicalVariable> secondaryKeyVars = new ArrayList<>();
List<Mutable<ILogicalExpression>> secondaryExpressions = new ArrayList<>();
List<Mutable<ILogicalExpression>> beforeOpSecondaryExpressions = new ArrayList<>();
ILogicalOperator replicateOutput;
for (int i = 0; i < secondaryKeyFields.size(); i++) {
IndexFieldId indexFieldId = new IndexFieldId(index.getKeyFieldSourceIndicators().get(i), secondaryKeyFields.get(i));
LogicalVariable skVar = fieldVarsForNewRecord.get(indexFieldId);
secondaryKeyVars.add(skVar);
secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(skVar)));
if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
beforeOpSecondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(fieldVarsForBeforeOperation.get(indexFieldId))));
}
}
IndexInsertDeleteUpsertOperator indexUpdate;
if (index.getIndexType() != IndexType.RTREE) {
// Create an expression per key
Mutable<ILogicalExpression> filterExpression = (primaryIndexModificationOp.getOperation() == Kind.UPSERT) ? null : createFilterExpression(secondaryKeyVars, context.getOutputTypeEnvironment(currentTop), index.isEnforcingKeyFileds());
DataSourceIndex dataSourceIndex = new DataSourceIndex(index, dataverseName, datasetName, mp);
// and index type is keyword or n-gram.
if (index.getIndexType() != IndexType.BTREE && primaryIndexModificationOp.isBulkload()) {
// Note: Bulk load case, we don't need to take care of it for upsert operation
// Check whether the index is length-partitioned or not.
// If partitioned, [input variables to TokenizeOperator,
// token, number of token] pairs will be generated and
// fed into the IndexInsertDeleteOperator.
// If not, [input variables, token] pairs will be generated
// and fed into the IndexInsertDeleteOperator.
// Input variables are passed since TokenizeOperator is not an
// filtering operator.
boolean isPartitioned = index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
// Create a new logical variable - token
List<LogicalVariable> tokenizeKeyVars = new ArrayList<>();
List<Mutable<ILogicalExpression>> tokenizeKeyExprs = new ArrayList<>();
LogicalVariable tokenVar = context.newVar();
tokenizeKeyVars.add(tokenVar);
tokenizeKeyExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(tokenVar)));
// Check the field type of the secondary key.
IAType secondaryKeyType;
Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), secondaryKeyFields.get(0), recType);
secondaryKeyType = keyPairType.first;
List<Object> varTypes = new ArrayList<>();
varTypes.add(NonTaggedFormatUtil.getTokenType(secondaryKeyType));
// The type is short, and this does not contain type info.
if (isPartitioned) {
LogicalVariable lengthVar = context.newVar();
tokenizeKeyVars.add(lengthVar);
tokenizeKeyExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(lengthVar)));
varTypes.add(BuiltinType.SHORTWITHOUTTYPEINFO);
}
// TokenizeOperator to tokenize [SK, PK] pairs
TokenizeOperator tokenUpdate = new TokenizeOperator(dataSourceIndex, primaryIndexModificationOp.getPrimaryKeyExpressions(), secondaryExpressions, tokenizeKeyVars, filterExpression, primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(), isPartitioned, varTypes);
tokenUpdate.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
context.computeAndSetTypeEnvironmentForOperator(tokenUpdate);
replicateOutput = tokenUpdate;
indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex, primaryIndexModificationOp.getPrimaryKeyExpressions(), tokenizeKeyExprs, filterExpression, primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(), primaryIndexModificationOp.getAdditionalNonFilteringExpressions() == null ? 0 : primaryIndexModificationOp.getAdditionalNonFilteringExpressions().size());
indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(tokenUpdate));
} else {
// When TokenizeOperator is not needed
indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex, primaryIndexModificationOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression, primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(), primaryIndexModificationOp.getAdditionalNonFilteringExpressions() == null ? 0 : primaryIndexModificationOp.getAdditionalNonFilteringExpressions().size());
indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
replicateOutput = indexUpdate;
// We add the necessary expressions for upsert
if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
indexUpdate.setBeforeOpSecondaryKeyExprs(beforeOpSecondaryExpressions);
if (filteringFields != null) {
indexUpdate.setBeforeOpAdditionalFilteringExpression(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(primaryIndexModificationOp.getBeforeOpFilterVar())));
}
}
indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
}
} else {
// Get type, dimensions and number of keys
Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), secondaryKeyFields.get(0), recType);
IAType spatialType = keyPairType.first;
boolean isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT || spatialType.getTypeTag() == ATypeTag.POINT3D;
int dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
int numKeys = (isPointMBR && isBulkload) ? dimension : dimension * 2;
// Get variables and expressions
List<LogicalVariable> keyVarList = new ArrayList<>();
List<Mutable<ILogicalExpression>> keyExprList = new ArrayList<>();
for (int i = 0; i < numKeys; i++) {
LogicalVariable keyVar = context.newVar();
keyVarList.add(keyVar);
AbstractFunctionCallExpression createMBR = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CREATE_MBR));
createMBR.getArguments().add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVars.get(0))));
createMBR.getArguments().add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(dimension)))));
createMBR.getArguments().add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(i)))));
keyExprList.add(new MutableObject<ILogicalExpression>(createMBR));
}
secondaryExpressions.clear();
for (LogicalVariable secondaryKeyVar : keyVarList) {
secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
}
if (isPointMBR && isBulkload) {
//createFieldPermutationForBulkLoadOp(int) for more details.
for (LogicalVariable secondaryKeyVar : keyVarList) {
secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
}
}
AssignOperator assignCoordinates = new AssignOperator(keyVarList, keyExprList);
assignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
context.computeAndSetTypeEnvironmentForOperator(assignCoordinates);
replicateOutput = assignCoordinates;
Mutable<ILogicalExpression> filterExpression = null;
AssignOperator originalAssignCoordinates = null;
// We do something similar for beforeOp key if the operation is an upsert
if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
List<LogicalVariable> originalKeyVarList = new ArrayList<>();
List<Mutable<ILogicalExpression>> originalKeyExprList = new ArrayList<>();
// we don't do any filtering since nulls are expected here and there
for (int i = 0; i < numKeys; i++) {
LogicalVariable keyVar = context.newVar();
originalKeyVarList.add(keyVar);
AbstractFunctionCallExpression createMBR = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CREATE_MBR));
createMBR.getArguments().add(beforeOpSecondaryExpressions.get(0));
createMBR.getArguments().add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(dimension)))));
createMBR.getArguments().add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(i)))));
originalKeyExprList.add(new MutableObject<ILogicalExpression>(createMBR));
}
beforeOpSecondaryExpressions.clear();
for (LogicalVariable secondaryKeyVar : originalKeyVarList) {
beforeOpSecondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
}
originalAssignCoordinates = new AssignOperator(originalKeyVarList, originalKeyExprList);
originalAssignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(assignCoordinates));
context.computeAndSetTypeEnvironmentForOperator(originalAssignCoordinates);
} else {
// We must enforce the filter if the originating spatial type is
// nullable.
boolean forceFilter = keyPairType.second;
filterExpression = createFilterExpression(keyVarList, context.getOutputTypeEnvironment(assignCoordinates), forceFilter);
}
DataSourceIndex dataSourceIndex = new DataSourceIndex(index, dataverseName, datasetName, mp);
indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex, primaryIndexModificationOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression, primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(), primaryIndexModificationOp.getAdditionalNonFilteringExpressions() == null ? 0 : primaryIndexModificationOp.getAdditionalNonFilteringExpressions().size());
indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
// set before op secondary key expressions
if (filteringFields != null) {
indexUpdate.setBeforeOpAdditionalFilteringExpression(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(primaryIndexModificationOp.getBeforeOpFilterVar())));
}
// set filtering expressions
indexUpdate.setBeforeOpSecondaryKeyExprs(beforeOpSecondaryExpressions);
// assign --> assign beforeOp values --> secondary index upsert
indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(originalAssignCoordinates));
} else {
indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assignCoordinates));
}
}
context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
if (!primaryIndexModificationOp.isBulkload() || secondaryIndexTotalCnt == 1) {
currentTop = indexUpdate;
} else {
replicateOp.getOutputs().add(new MutableObject<>(replicateOutput));
}
if (primaryIndexModificationOp.isBulkload()) {
// For bulk load, we connect all fanned out insert operator to a single SINK operator
op0.getInputs().add(new MutableObject<ILogicalOperator>(indexUpdate));
}
}
if (!hasSecondaryIndex) {
return false;
}
if (!primaryIndexModificationOp.isBulkload()) {
// If this is an upsert, we need to
// Remove the current input to the SINK operator (It is actually already removed above)
op0.getInputs().clear();
// Connect the last index update to the SINK
op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
}
return true;
}
Aggregations