use of io.siddhi.core.util.collection.operator.CompiledSelection in project siddhi by wso2.
the class TestStoreContainingInMemoryTable method compileSelection.
@Override
protected CompiledSelection compileSelection(List<SelectAttributeBuilder> selectAttributeBuilders, List<ExpressionBuilder> groupByExpressionBuilder, ExpressionBuilder havingExpressionBuilder, List<OrderByAttributeBuilder> orderByAttributeBuilders, Long limit, Long offset) {
selectAttributeBuilders.forEach((selectAttributeBuilder -> {
TestStoreConditionVisitor testStoreConditionVisitor = new TestStoreConditionVisitor("");
selectAttributeBuilder.getExpressionBuilder().build(testStoreConditionVisitor);
if (testStoreConditionVisitor.getStreamVarCount() > 0) {
throw new SiddhiAppCreationException("testStoreContainingInMemoryTable does not support " + "lookup with stream variables");
}
}));
CompiledSelectionWithCache compiledSelectionWithCache;
MetaStateEvent metaStateEvent = matchingMetaInfoHolderForTestOnDemandQuery.getMetaStateEvent().clone();
ReturnStream returnStream = new ReturnStream(OutputStream.OutputEventType.CURRENT_EVENTS);
int metaPosition = SiddhiConstants.UNKNOWN_STATE;
List<VariableExpressionExecutor> variableExpressionExecutorsForQuerySelector = new ArrayList<>();
if (metaStateEvent.getOutputDataAttributes().size() == 0) {
// MetaStateEvent metaStateEventWithOutputData = new MetaStateEvent(metaStateEvent.getStreamEventCount());
for (Attribute outputAttribute : metaStateEvent.getMetaStreamEvents()[0].getOnAfterWindowData()) {
metaStateEvent.getMetaStreamEvents()[0].addOutputData(outputAttribute);
}
}
if (metaStateEvent.getOutputDataAttributes().size() > 0) {
while (metaStateEvent.getMetaStreamEvent(0).getOutputData().size() > 0) {
metaStateEvent.getMetaStreamEvent(0).getOutputData().remove(0);
}
}
QuerySelector querySelector = SelectorParser.parse(selectorForTestOnDemandQuery, returnStream, metaStateEvent, tableMap, variableExpressionExecutorsForQuerySelector, metaPosition, ProcessingMode.BATCH, false, siddhiQueryContextForTestOnDemandQuery);
QueryParserHelper.updateVariablePosition(metaStateEvent, variableExpressionExecutorsForQuerySelector);
querySelector.setEventPopulator(StateEventPopulatorFactory.constructEventPopulator(metaStateEvent));
compiledSelectionWithCache = new CompiledSelectionWithCache(null, querySelector, metaStateEvent, 0, null);
return compiledSelectionWithCache;
}
use of io.siddhi.core.util.collection.operator.CompiledSelection in project siddhi by wso2.
the class TestStoreForCacheMiss method compileSelection.
@Override
protected CompiledSelection compileSelection(List<SelectAttributeBuilder> selectAttributeBuilders, List<ExpressionBuilder> groupByExpressionBuilder, ExpressionBuilder havingExpressionBuilder, List<OrderByAttributeBuilder> orderByAttributeBuilders, Long limit, Long offset) {
selectAttributeBuilders.forEach((selectAttributeBuilder -> {
TestStoreConditionVisitor testStoreConditionVisitor = new TestStoreConditionVisitor("");
selectAttributeBuilder.getExpressionBuilder().build(testStoreConditionVisitor);
if (testStoreConditionVisitor.getStreamVarCount() > 0) {
throw new SiddhiAppCreationException("testStoreContainingInMemoryTable does not support " + "lookup with stream variables");
}
}));
CompiledSelectionWithCache compiledSelectionWithCache;
MetaStateEvent metaStateEvent = matchingMetaInfoHolderForTestOnDemandQuery.getMetaStateEvent().clone();
ReturnStream returnStream = new ReturnStream(OutputStream.OutputEventType.CURRENT_EVENTS);
int metaPosition = SiddhiConstants.UNKNOWN_STATE;
List<VariableExpressionExecutor> variableExpressionExecutorsForQuerySelector = new ArrayList<>();
if (metaStateEvent.getOutputDataAttributes().size() == 0) {
for (Attribute outputAttribute : metaStateEvent.getMetaStreamEvents()[0].getOnAfterWindowData()) {
metaStateEvent.getMetaStreamEvents()[0].addOutputData(outputAttribute);
}
}
QuerySelector querySelector = SelectorParser.parse(selectorForTestOnDemandQuery, returnStream, metaStateEvent, tableMap, variableExpressionExecutorsForQuerySelector, metaPosition, ProcessingMode.BATCH, false, siddhiQueryContextForTestOnDemandQuery);
QueryParserHelper.updateVariablePosition(metaStateEvent, variableExpressionExecutorsForQuerySelector);
querySelector.setEventPopulator(StateEventPopulatorFactory.constructEventPopulator(metaStateEvent));
compiledSelectionWithCache = new CompiledSelectionWithCache(null, querySelector, metaStateEvent, 0, null);
return compiledSelectionWithCache;
}
use of io.siddhi.core.util.collection.operator.CompiledSelection in project siddhi by wso2.
the class AggregationRuntime method compileExpression.
public CompiledCondition compileExpression(Expression expression, Within within, Expression per, List<Variable> queryGroupByList, MatchingMetaInfoHolder matchingMetaInfoHolder, List<VariableExpressionExecutor> variableExpressionExecutors, Map<String, Table> tableMap, SiddhiQueryContext siddhiQueryContext) {
String aggregationName = aggregationDefinition.getId();
boolean isOptimisedTableLookup = isOptimisedLookup;
Map<TimePeriod.Duration, CompiledCondition> withinTableCompiledConditions = new HashMap<>();
CompiledCondition withinInMemoryCompileCondition;
CompiledCondition onCompiledCondition;
List<Attribute> additionalAttributes = new ArrayList<>();
// Define additional attribute list
additionalAttributes.add(new Attribute("_START", Attribute.Type.LONG));
additionalAttributes.add(new Attribute("_END", Attribute.Type.LONG));
int lowerGranularitySize = this.activeIncrementalDurations.size() - 1;
List<String> lowerGranularityAttributes = new ArrayList<>();
if (isDistributed) {
// for values calculated in in-memory in the shards
for (int i = 0; i < lowerGranularitySize; i++) {
String attributeName = "_AGG_TIMESTAMP_FILTER_" + i;
additionalAttributes.add(new Attribute(attributeName, Attribute.Type.LONG));
lowerGranularityAttributes.add(attributeName);
}
}
// Get table definition. Table definitions for all the tables used to persist aggregates are similar.
// Therefore it's enough to get the definition from one table.
AbstractDefinition tableDefinition = aggregationTables.get(activeIncrementalDurations.get(0)).getTableDefinition();
boolean isOnDemandQuery = matchingMetaInfoHolder.getMetaStateEvent().getMetaStreamEvents().length == 1;
// Alter existing meta stream event or create new one if a meta stream doesn't exist
// After calling this method the original MatchingMetaInfoHolder's meta stream event would be altered
// Alter meta info holder to contain stream event and aggregate both when it's a on-demand query
MetaStreamEvent metaStreamEventForTableLookups;
if (isOnDemandQuery) {
metaStreamEventForTableLookups = alterMetaStreamEvent(true, new MetaStreamEvent(), additionalAttributes);
matchingMetaInfoHolder = alterMetaInfoHolderForOnDemandQuery(metaStreamEventForTableLookups, matchingMetaInfoHolder);
} else {
metaStreamEventForTableLookups = alterMetaStreamEvent(false, matchingMetaInfoHolder.getMetaStateEvent().getMetaStreamEvent(0), additionalAttributes);
}
// Create new MatchingMetaInfoHolder containing newMetaStreamEventWithStartEnd and table meta event
String aggReferenceId = matchingMetaInfoHolder.getMetaStateEvent().getMetaStreamEvent(1).getInputReferenceId();
String referenceName = aggReferenceId == null ? aggregationName : aggReferenceId;
MetaStreamEvent metaStoreEventForTableLookups = createMetaStoreEvent(tableDefinition, referenceName);
// Create new MatchingMetaInfoHolder containing metaStreamEventForTableLookups and table meta event
MatchingMetaInfoHolder metaInfoHolderForTableLookups = createNewStreamTableMetaInfoHolder(metaStreamEventForTableLookups, metaStoreEventForTableLookups);
// Create per expression executor
ExpressionExecutor perExpressionExecutor;
if (per != null) {
perExpressionExecutor = ExpressionParser.parseExpression(per, matchingMetaInfoHolder.getMetaStateEvent(), matchingMetaInfoHolder.getCurrentState(), tableMap, variableExpressionExecutors, false, 0, ProcessingMode.BATCH, false, siddhiQueryContext);
if (perExpressionExecutor.getReturnType() != Attribute.Type.STRING) {
throw new SiddhiAppCreationException("Query " + siddhiQueryContext.getName() + "'s per value expected a string but found " + perExpressionExecutor.getReturnType(), per.getQueryContextStartIndex(), per.getQueryContextEndIndex());
}
// Additional Per time function verification at compile time if it is a constant
if (perExpressionExecutor instanceof ConstantExpressionExecutor) {
String perValue = ((ConstantExpressionExecutor) perExpressionExecutor).getValue().toString();
try {
normalizeDuration(perValue);
} catch (SiddhiAppValidationException e) {
throw new SiddhiAppValidationException("Aggregation Query's per value is expected to be of a valid time function of the " + "following " + TimePeriod.Duration.SECONDS + ", " + TimePeriod.Duration.MINUTES + ", " + TimePeriod.Duration.HOURS + ", " + TimePeriod.Duration.DAYS + ", " + TimePeriod.Duration.MONTHS + ", " + TimePeriod.Duration.YEARS + ".");
}
}
} else {
throw new SiddhiAppCreationException("Syntax Error: Aggregation join query must contain a `per` " + "definition for granularity");
}
// Create start and end time expression
Expression startEndTimeExpression;
ExpressionExecutor startTimeEndTimeExpressionExecutor;
if (within != null) {
if (within.getTimeRange().size() == 1) {
startEndTimeExpression = new AttributeFunction("incrementalAggregator", "startTimeEndTime", within.getTimeRange().get(0));
} else {
// within.getTimeRange().size() == 2
startEndTimeExpression = new AttributeFunction("incrementalAggregator", "startTimeEndTime", within.getTimeRange().get(0), within.getTimeRange().get(1));
}
startTimeEndTimeExpressionExecutor = ExpressionParser.parseExpression(startEndTimeExpression, matchingMetaInfoHolder.getMetaStateEvent(), matchingMetaInfoHolder.getCurrentState(), tableMap, variableExpressionExecutors, false, 0, ProcessingMode.BATCH, false, siddhiQueryContext);
} else {
throw new SiddhiAppCreationException("Syntax Error : Aggregation read query must contain a `within` " + "definition for filtering of aggregation data.");
}
// Create within expression
Expression timeFilterExpression;
if (isProcessingOnExternalTime) {
timeFilterExpression = Expression.variable(AGG_EXTERNAL_TIMESTAMP_COL);
} else {
timeFilterExpression = Expression.variable(AGG_START_TIMESTAMP_COL);
}
Expression withinExpression;
Expression start = Expression.variable(additionalAttributes.get(0).getName());
Expression end = Expression.variable(additionalAttributes.get(1).getName());
Expression compareWithStartTime = Compare.compare(start, Compare.Operator.LESS_THAN_EQUAL, timeFilterExpression);
Expression compareWithEndTime = Compare.compare(timeFilterExpression, Compare.Operator.LESS_THAN, end);
withinExpression = Expression.and(compareWithStartTime, compareWithEndTime);
List<ExpressionExecutor> timestampFilterExecutors = new ArrayList<>();
if (isDistributed) {
for (int i = 0; i < lowerGranularitySize; i++) {
Expression[] expressionArray = new Expression[] { new AttributeFunction("", "currentTimeMillis", null), Expression.value(this.activeIncrementalDurations.get(i + 1).toString()) };
Expression filterExpression = new AttributeFunction("incrementalAggregator", "getAggregationStartTime", expressionArray);
timestampFilterExecutors.add(ExpressionParser.parseExpression(filterExpression, matchingMetaInfoHolder.getMetaStateEvent(), matchingMetaInfoHolder.getCurrentState(), tableMap, variableExpressionExecutors, false, 0, ProcessingMode.BATCH, false, siddhiQueryContext));
}
}
// Create compile condition per each table used to persist aggregates.
// These compile conditions are used to check whether the aggregates in tables are within the given duration.
// Combine with and on condition for table query
boolean shouldApplyReducedCondition = false;
Expression reducedExpression = null;
// Check if there is no on conditions
if (!(expression instanceof BoolConstant)) {
// For abstract queryable table
AggregationExpressionBuilder aggregationExpressionBuilder = new AggregationExpressionBuilder(expression);
AggregationExpressionVisitor expressionVisitor = new AggregationExpressionVisitor(metaStreamEventForTableLookups.getInputReferenceId(), metaStreamEventForTableLookups.getLastInputDefinition().getAttributeList(), this.tableAttributesNameList);
aggregationExpressionBuilder.build(expressionVisitor);
shouldApplyReducedCondition = expressionVisitor.applyReducedExpression();
reducedExpression = expressionVisitor.getReducedExpression();
}
Expression withinExpressionTable;
if (shouldApplyReducedCondition) {
withinExpressionTable = Expression.and(withinExpression, reducedExpression);
} else {
withinExpressionTable = withinExpression;
}
List<Variable> queryGroupByListCopy = new ArrayList<>(queryGroupByList);
Variable timestampVariable = new Variable(AGG_START_TIMESTAMP_COL);
List<String> queryGroupByNamesList = queryGroupByListCopy.stream().map(Variable::getAttributeName).collect(Collectors.toList());
boolean queryGroupByContainsTimestamp = queryGroupByNamesList.remove(AGG_START_TIMESTAMP_COL);
boolean isQueryGroupBySameAsAggGroupBy = queryGroupByListCopy.isEmpty() || (queryGroupByListCopy.contains(timestampVariable) && queryGroupByNamesList.equals(groupByVariablesList));
List<VariableExpressionExecutor> variableExpExecutorsForTableLookups = new ArrayList<>();
Map<TimePeriod.Duration, CompiledSelection> withinTableCompiledSelection = new HashMap<>();
if (isOptimisedTableLookup) {
Selector selector = Selector.selector();
List<Variable> groupByList = new ArrayList<>();
if (!isQueryGroupBySameAsAggGroupBy) {
if (queryGroupByContainsTimestamp) {
if (isProcessingOnExternalTime) {
groupByList.add(new Variable(AGG_EXTERNAL_TIMESTAMP_COL));
} else {
groupByList.add(new Variable(AGG_START_TIMESTAMP_COL));
}
// Remove timestamp to process the rest
queryGroupByListCopy.remove(timestampVariable);
}
for (Variable queryGroupBy : queryGroupByListCopy) {
String referenceId = queryGroupBy.getStreamId();
if (referenceId == null) {
if (tableAttributesNameList.contains(queryGroupBy.getAttributeName())) {
groupByList.add(queryGroupBy);
}
} else if (referenceId.equalsIgnoreCase(referenceName)) {
groupByList.add(queryGroupBy);
}
}
// If query group bys are based on joining stream
if (groupByList.isEmpty()) {
isQueryGroupBySameAsAggGroupBy = true;
}
}
groupByList.forEach((groupBy) -> groupBy.setStreamId(referenceName));
selector.addGroupByList(groupByList);
List<OutputAttribute> selectorList;
if (!isQueryGroupBySameAsAggGroupBy) {
selectorList = constructSelectorList(isProcessingOnExternalTime, isDistributed, isLatestEventColAdded, baseAggregatorBeginIndex, groupByVariablesList.size(), finalBaseExpressionsList, tableDefinition, groupByList);
} else {
selectorList = defaultSelectorList;
}
for (OutputAttribute outputAttribute : selectorList) {
if (outputAttribute.getExpression() instanceof Variable) {
((Variable) outputAttribute.getExpression()).setStreamId(referenceName);
} else {
for (Expression parameter : ((AttributeFunction) outputAttribute.getExpression()).getParameters()) {
((Variable) parameter).setStreamId(referenceName);
}
}
}
selector.addSelectionList(selectorList);
try {
aggregationTables.entrySet().forEach((durationTableEntry -> {
CompiledSelection compiledSelection = ((QueryableProcessor) durationTableEntry.getValue()).compileSelection(selector, tableDefinition.getAttributeList(), metaInfoHolderForTableLookups, variableExpExecutorsForTableLookups, tableMap, siddhiQueryContext);
withinTableCompiledSelection.put(durationTableEntry.getKey(), compiledSelection);
}));
} catch (SiddhiAppCreationException | SiddhiAppValidationException | QueryableRecordTableException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Aggregation Query optimization failed for aggregation: '" + aggregationName + "'. " + "Creating table lookup query in normal mode. Reason for failure: " + e.getMessage(), e);
}
isOptimisedTableLookup = false;
}
}
for (Map.Entry<TimePeriod.Duration, Table> entry : aggregationTables.entrySet()) {
CompiledCondition withinTableCompileCondition = entry.getValue().compileCondition(withinExpressionTable, metaInfoHolderForTableLookups, variableExpExecutorsForTableLookups, tableMap, siddhiQueryContext);
withinTableCompiledConditions.put(entry.getKey(), withinTableCompileCondition);
}
// Create compile condition for in-memory data.
// This compile condition is used to check whether the running aggregates (in-memory data)
// are within given duration
withinInMemoryCompileCondition = OperatorParser.constructOperator(new ComplexEventChunk<>(), withinExpression, metaInfoHolderForTableLookups, variableExpExecutorsForTableLookups, tableMap, siddhiQueryContext);
// Create compile condition for in-memory data, in case of distributed
// Look at the lower level granularities
Map<TimePeriod.Duration, CompiledCondition> withinTableLowerGranularityCompileCondition = new HashMap<>();
Expression lowerGranularity;
if (isDistributed) {
for (int i = 0; i < lowerGranularitySize; i++) {
if (isProcessingOnExternalTime) {
lowerGranularity = Expression.and(Expression.compare(Expression.variable("AGG_TIMESTAMP"), Compare.Operator.GREATER_THAN_EQUAL, Expression.variable(lowerGranularityAttributes.get(i))), withinExpressionTable);
} else {
if (shouldApplyReducedCondition) {
lowerGranularity = Expression.and(Expression.compare(Expression.variable("AGG_TIMESTAMP"), Compare.Operator.GREATER_THAN_EQUAL, Expression.variable(lowerGranularityAttributes.get(i))), reducedExpression);
} else {
lowerGranularity = Expression.compare(Expression.variable("AGG_TIMESTAMP"), Compare.Operator.GREATER_THAN_EQUAL, Expression.variable(lowerGranularityAttributes.get(i)));
}
}
TimePeriod.Duration duration = this.activeIncrementalDurations.get(i);
String tableName = aggregationName + "_" + duration.toString();
CompiledCondition compiledCondition = tableMap.get(tableName).compileCondition(lowerGranularity, metaInfoHolderForTableLookups, variableExpExecutorsForTableLookups, tableMap, siddhiQueryContext);
withinTableLowerGranularityCompileCondition.put(duration, compiledCondition);
}
}
QueryParserHelper.reduceMetaComplexEvent(metaInfoHolderForTableLookups.getMetaStateEvent());
// On compile condition.
// After finding all the aggregates belonging to within duration, the final on condition (given as
// "on stream1.name == aggregator.nickName ..." in the join query) must be executed on that data.
// This condition is used for that purpose.
onCompiledCondition = OperatorParser.constructOperator(new ComplexEventChunk<>(), expression, matchingMetaInfoHolder, variableExpressionExecutors, tableMap, siddhiQueryContext);
return new IncrementalAggregateCompileCondition(isOnDemandQuery, aggregationName, isProcessingOnExternalTime, isDistributed, activeIncrementalDurations, aggregationTables, outputExpressionExecutors, isOptimisedTableLookup, withinTableCompiledSelection, withinTableCompiledConditions, withinInMemoryCompileCondition, withinTableLowerGranularityCompileCondition, onCompiledCondition, additionalAttributes, perExpressionExecutor, startTimeEndTimeExpressionExecutor, timestampFilterExecutors, aggregateMetaSteamEvent, matchingMetaInfoHolder, metaInfoHolderForTableLookups, variableExpExecutorsForTableLookups);
}
use of io.siddhi.core.util.collection.operator.CompiledSelection in project siddhi by wso2.
the class QueryParser method parse.
/**
* Parse a query and return corresponding QueryRuntime.
*
* @param query query to be parsed.
* @param siddhiAppContext associated Siddhi app context.
* @param streamDefinitionMap keyvalue containing user given stream definitions.
* @param tableDefinitionMap keyvalue containing table definitions.
* @param windowDefinitionMap keyvalue containing window definition map.
* @param aggregationDefinitionMap keyvalue containing aggregation definition map.
* @param tableMap keyvalue containing event tables.
* @param aggregationMap keyvalue containing aggrigation runtimes.
* @param windowMap keyvalue containing event window map.
* @param lockSynchronizer Lock synchronizer for sync the lock across queries.
* @param queryIndex query index to identify unknown query by number
* @param partitioned is the query partitioned
* @param partitionId The ID of the partition
* @return queryRuntime
*/
public static QueryRuntimeImpl parse(Query query, SiddhiAppContext siddhiAppContext, Map<String, AbstractDefinition> streamDefinitionMap, Map<String, AbstractDefinition> tableDefinitionMap, Map<String, AbstractDefinition> windowDefinitionMap, Map<String, AbstractDefinition> aggregationDefinitionMap, Map<String, Table> tableMap, Map<String, AggregationRuntime> aggregationMap, Map<String, Window> windowMap, LockSynchronizer lockSynchronizer, String queryIndex, boolean partitioned, String partitionId) {
List<VariableExpressionExecutor> executors = new ArrayList<>();
QueryRuntimeImpl queryRuntime;
Element nameElement = null;
LatencyTracker latencyTracker = null;
LockWrapper lockWrapper = null;
try {
nameElement = AnnotationHelper.getAnnotationElement("info", "name", query.getAnnotations());
String queryName;
if (nameElement != null) {
queryName = nameElement.getValue();
} else {
queryName = "query_" + queryIndex;
}
SiddhiQueryContext siddhiQueryContext = new SiddhiQueryContext(siddhiAppContext, queryName, partitionId);
siddhiQueryContext.setPartitioned(partitioned);
latencyTracker = QueryParserHelper.createLatencyTracker(siddhiAppContext, siddhiQueryContext.getName(), SiddhiConstants.METRIC_INFIX_QUERIES, null);
siddhiQueryContext.setLatencyTracker(latencyTracker);
OutputStream.OutputEventType outputEventType = query.getOutputStream().getOutputEventType();
if (query.getOutputRate() != null && query.getOutputRate() instanceof SnapshotOutputRate) {
if (outputEventType != OutputStream.OutputEventType.ALL_EVENTS) {
throw new SiddhiAppCreationException("As query '" + siddhiQueryContext.getName() + "' is performing snapshot rate limiting, it can only insert '" + OutputStream.OutputEventType.ALL_EVENTS + "' but it is inserting '" + outputEventType + "'!", query.getOutputStream().getQueryContextStartIndex(), query.getOutputStream().getQueryContextEndIndex());
}
}
siddhiQueryContext.setOutputEventType(outputEventType);
boolean outputExpectsExpiredEvents = false;
if (outputEventType != OutputStream.OutputEventType.CURRENT_EVENTS) {
outputExpectsExpiredEvents = true;
}
StreamRuntime streamRuntime = InputStreamParser.parse(query.getInputStream(), query, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, tableMap, windowMap, aggregationMap, executors, outputExpectsExpiredEvents, siddhiQueryContext);
QuerySelector selector;
if (streamRuntime.getQuerySelector() != null) {
selector = streamRuntime.getQuerySelector();
} else {
selector = SelectorParser.parse(query.getSelector(), query.getOutputStream(), streamRuntime.getMetaComplexEvent(), tableMap, executors, SiddhiConstants.UNKNOWN_STATE, streamRuntime.getProcessingMode(), outputExpectsExpiredEvents, siddhiQueryContext);
}
boolean isWindow = query.getInputStream() instanceof JoinInputStream;
if (!isWindow && query.getInputStream() instanceof SingleInputStream) {
for (StreamHandler streamHandler : ((SingleInputStream) query.getInputStream()).getStreamHandlers()) {
if (streamHandler instanceof io.siddhi.query.api.execution.query.input.handler.Window) {
isWindow = true;
break;
}
}
}
Element synchronizedElement = AnnotationHelper.getAnnotationElement("synchronized", null, query.getAnnotations());
if (synchronizedElement != null) {
if (!("false".equalsIgnoreCase(synchronizedElement.getValue()))) {
// Query LockWrapper does not need a unique
lockWrapper = new LockWrapper("");
// id since it will
// not be passed to the LockSynchronizer.
// LockWrapper does not have a default lock
lockWrapper.setLock(new ReentrantLock());
}
} else {
if (isWindow || !(streamRuntime instanceof SingleStreamRuntime)) {
if (streamRuntime instanceof JoinStreamRuntime) {
// If at least one Window is involved in the join, use the LockWrapper of that window
// for the query as well.
// If join is between two EventWindows, sync the locks of the LockWrapper of those windows
// and use either of them for query.
MetaStateEvent metaStateEvent = (MetaStateEvent) streamRuntime.getMetaComplexEvent();
MetaStreamEvent[] metaStreamEvents = metaStateEvent.getMetaStreamEvents();
if (metaStreamEvents[0].getEventType() == EventType.WINDOW && metaStreamEvents[1].getEventType() == EventType.WINDOW) {
LockWrapper leftLockWrapper = windowMap.get(metaStreamEvents[0].getLastInputDefinition().getId()).getLock();
LockWrapper rightLockWrapper = windowMap.get(metaStreamEvents[1].getLastInputDefinition().getId()).getLock();
if (!leftLockWrapper.equals(rightLockWrapper)) {
// Sync the lock across both wrappers
lockSynchronizer.sync(leftLockWrapper, rightLockWrapper);
}
// Can use either leftLockWrapper or rightLockWrapper since both of them will hold the
// same lock internally
// If either of their lock is updated later, the other lock also will be update by the
// LockSynchronizer.
lockWrapper = leftLockWrapper;
} else if (metaStreamEvents[0].getEventType() == EventType.WINDOW) {
// Share the same wrapper as the query lock wrapper
lockWrapper = windowMap.get(metaStreamEvents[0].getLastInputDefinition().getId()).getLock();
} else if (metaStreamEvents[1].getEventType() == EventType.WINDOW) {
// Share the same wrapper as the query lock wrapper
lockWrapper = windowMap.get(metaStreamEvents[1].getLastInputDefinition().getId()).getLock();
} else {
// Join does not contain any Window
// Query LockWrapper does not need a unique
lockWrapper = new LockWrapper("");
// id since
// it will not be passed to the LockSynchronizer.
// LockWrapper does not have a default lock
lockWrapper.setLock(new ReentrantLock());
}
} else {
lockWrapper = new LockWrapper("");
lockWrapper.setLock(new ReentrantLock());
}
}
}
OutputRateLimiter outputRateLimiter = OutputParser.constructOutputRateLimiter(query.getOutputStream().getId(), query.getOutputRate(), query.getSelector().getGroupByList().size() != 0, isWindow, siddhiQueryContext);
if (outputRateLimiter instanceof WrappedSnapshotOutputRateLimiter) {
selector.setBatchingEnabled(false);
}
boolean groupBy = !query.getSelector().getGroupByList().isEmpty();
OutputCallback outputCallback = OutputParser.constructOutputCallback(query.getOutputStream(), streamRuntime.getMetaComplexEvent().getOutputStreamDefinition(), tableMap, windowMap, !(streamRuntime instanceof SingleStreamRuntime) || groupBy, siddhiQueryContext);
QueryParserHelper.reduceMetaComplexEvent(streamRuntime.getMetaComplexEvent());
QueryParserHelper.updateVariablePosition(streamRuntime.getMetaComplexEvent(), executors);
QueryParserHelper.initStreamRuntime(streamRuntime, streamRuntime.getMetaComplexEvent(), lockWrapper, siddhiQueryContext.getName());
// Update cache compile selection variable expression executors
if (streamRuntime instanceof JoinStreamRuntime) {
streamRuntime.getSingleStreamRuntimes().forEach((singleStreamRuntime -> {
Processor processorChain = singleStreamRuntime.getProcessorChain();
if (processorChain instanceof JoinProcessor) {
CompiledSelection compiledSelection = ((JoinProcessor) processorChain).getCompiledSelection();
if (compiledSelection instanceof AbstractQueryableRecordTable.CompiledSelectionWithCache) {
List<VariableExpressionExecutor> variableExpressionExecutors = ((AbstractQueryableRecordTable.CompiledSelectionWithCache) compiledSelection).getVariableExpressionExecutorsForQuerySelector();
QueryParserHelper.updateVariablePosition(streamRuntime.getMetaComplexEvent(), variableExpressionExecutors);
}
}
}));
}
selector.setEventPopulator(StateEventPopulatorFactory.constructEventPopulator(streamRuntime.getMetaComplexEvent()));
queryRuntime = new QueryRuntimeImpl(query, streamRuntime, selector, outputRateLimiter, outputCallback, streamRuntime.getMetaComplexEvent(), siddhiQueryContext);
if (outputRateLimiter instanceof WrappedSnapshotOutputRateLimiter) {
selector.setBatchingEnabled(false);
((WrappedSnapshotOutputRateLimiter) outputRateLimiter).init(streamRuntime.getMetaComplexEvent().getOutputStreamDefinition().getAttributeList().size(), selector.getAttributeProcessorList(), streamRuntime.getMetaComplexEvent());
}
outputRateLimiter.init(lockWrapper, groupBy, siddhiQueryContext);
} catch (DuplicateDefinitionException e) {
if (nameElement != null) {
throw new DuplicateDefinitionException(e.getMessageWithOutContext() + ", when creating query " + nameElement.getValue(), e, e.getQueryContextStartIndex(), e.getQueryContextEndIndex(), siddhiAppContext.getName(), siddhiAppContext.getSiddhiAppString());
} else {
throw new DuplicateDefinitionException(e.getMessage(), e, e.getQueryContextStartIndex(), e.getQueryContextEndIndex(), siddhiAppContext.getName(), siddhiAppContext.getSiddhiAppString());
}
} catch (Throwable t) {
ExceptionUtil.populateQueryContext(t, query, siddhiAppContext);
throw t;
}
return queryRuntime;
}
use of io.siddhi.core.util.collection.operator.CompiledSelection in project siddhi by wso2.
the class JoinInputStreamParser method parseInputStream.
public static StreamRuntime parseInputStream(JoinInputStream joinInputStream, Query query, Map<String, AbstractDefinition> streamDefinitionMap, Map<String, AbstractDefinition> tableDefinitionMap, Map<String, AbstractDefinition> windowDefinitionMap, Map<String, AbstractDefinition> aggregationDefinitionMap, Map<String, Table> tableMap, Map<String, Window> windowMap, Map<String, AggregationRuntime> aggregationMap, List<VariableExpressionExecutor> executors, boolean outputExpectsExpiredEvents, SiddhiQueryContext siddhiQueryContext) {
try {
ProcessStreamReceiver leftProcessStreamReceiver;
ProcessStreamReceiver rightProcessStreamReceiver;
MetaStreamEvent leftMetaStreamEvent = new MetaStreamEvent();
MetaStreamEvent rightMetaStreamEvent = new MetaStreamEvent();
String leftInputStreamId = ((SingleInputStream) joinInputStream.getLeftInputStream()).getStreamId();
String rightInputStreamId = ((SingleInputStream) joinInputStream.getRightInputStream()).getStreamId();
boolean leftOuterJoinProcessor = false;
boolean rightOuterJoinProcessor = false;
if (joinInputStream.getAllStreamIds().size() == 2) {
setEventType(streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, leftMetaStreamEvent, leftInputStreamId);
setEventType(streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, rightMetaStreamEvent, rightInputStreamId);
leftProcessStreamReceiver = new ProcessStreamReceiver(leftInputStreamId, siddhiQueryContext);
rightProcessStreamReceiver = new ProcessStreamReceiver(rightInputStreamId, siddhiQueryContext);
if ((leftMetaStreamEvent.getEventType() == TABLE || leftMetaStreamEvent.getEventType() == AGGREGATE) && (rightMetaStreamEvent.getEventType() == TABLE || rightMetaStreamEvent.getEventType() == AGGREGATE)) {
throw new SiddhiAppCreationException("Both inputs of join " + leftInputStreamId + " and " + rightInputStreamId + " are from static sources");
}
if (leftMetaStreamEvent.getEventType() != AGGREGATE && rightMetaStreamEvent.getEventType() != AGGREGATE) {
if (joinInputStream.getPer() != null) {
throw new SiddhiAppCreationException("When joining " + leftInputStreamId + " and " + rightInputStreamId + " 'per' cannot be used as neither of them is an aggregation ");
} else if (joinInputStream.getWithin() != null) {
throw new SiddhiAppCreationException("When joining " + leftInputStreamId + " and " + rightInputStreamId + " 'within' cannot be used as neither of them is an aggregation ");
}
}
} else {
if (windowDefinitionMap.containsKey(joinInputStream.getAllStreamIds().get(0))) {
leftMetaStreamEvent.setEventType(WINDOW);
rightMetaStreamEvent.setEventType(WINDOW);
rightProcessStreamReceiver = new MultiProcessStreamReceiver(joinInputStream.getAllStreamIds().get(0), 1, new Object(), siddhiQueryContext);
leftProcessStreamReceiver = rightProcessStreamReceiver;
} else if (streamDefinitionMap.containsKey(joinInputStream.getAllStreamIds().get(0))) {
rightProcessStreamReceiver = new MultiProcessStreamReceiver(joinInputStream.getAllStreamIds().get(0), 2, new Object(), siddhiQueryContext);
leftProcessStreamReceiver = rightProcessStreamReceiver;
} else {
throw new SiddhiAppCreationException("Input of join is from static source " + leftInputStreamId + " and " + rightInputStreamId);
}
}
SingleStreamRuntime leftStreamRuntime = SingleInputStreamParser.parseInputStream((SingleInputStream) joinInputStream.getLeftInputStream(), executors, streamDefinitionMap, leftMetaStreamEvent.getEventType() != TABLE ? null : tableDefinitionMap, leftMetaStreamEvent.getEventType() != WINDOW ? null : windowDefinitionMap, leftMetaStreamEvent.getEventType() != AGGREGATE ? null : aggregationDefinitionMap, tableMap, leftMetaStreamEvent, leftProcessStreamReceiver, true, outputExpectsExpiredEvents, true, false, siddhiQueryContext);
for (VariableExpressionExecutor variableExpressionExecutor : executors) {
variableExpressionExecutor.getPosition()[SiddhiConstants.STREAM_EVENT_CHAIN_INDEX] = 0;
}
int size = executors.size();
SingleStreamRuntime rightStreamRuntime = SingleInputStreamParser.parseInputStream((SingleInputStream) joinInputStream.getRightInputStream(), executors, streamDefinitionMap, rightMetaStreamEvent.getEventType() != TABLE ? null : tableDefinitionMap, rightMetaStreamEvent.getEventType() != WINDOW ? null : windowDefinitionMap, rightMetaStreamEvent.getEventType() != AGGREGATE ? null : aggregationDefinitionMap, tableMap, rightMetaStreamEvent, rightProcessStreamReceiver, true, outputExpectsExpiredEvents, true, false, siddhiQueryContext);
for (int i = size; i < executors.size(); i++) {
VariableExpressionExecutor variableExpressionExecutor = executors.get(i);
variableExpressionExecutor.getPosition()[SiddhiConstants.STREAM_EVENT_CHAIN_INDEX] = 1;
}
setStreamRuntimeProcessorChain(leftMetaStreamEvent, leftStreamRuntime, leftInputStreamId, tableMap, windowMap, aggregationMap, executors, outputExpectsExpiredEvents, joinInputStream.getWithin(), joinInputStream.getPer(), query.getSelector().getGroupByList(), siddhiQueryContext, joinInputStream.getLeftInputStream());
setStreamRuntimeProcessorChain(rightMetaStreamEvent, rightStreamRuntime, rightInputStreamId, tableMap, windowMap, aggregationMap, executors, outputExpectsExpiredEvents, joinInputStream.getWithin(), joinInputStream.getPer(), query.getSelector().getGroupByList(), siddhiQueryContext, joinInputStream.getRightInputStream());
MetaStateEvent metaStateEvent = new MetaStateEvent(2);
metaStateEvent.addEvent(leftMetaStreamEvent);
metaStateEvent.addEvent(rightMetaStreamEvent);
switch(joinInputStream.getType()) {
case FULL_OUTER_JOIN:
leftOuterJoinProcessor = true;
rightOuterJoinProcessor = true;
break;
case RIGHT_OUTER_JOIN:
rightOuterJoinProcessor = true;
break;
case LEFT_OUTER_JOIN:
leftOuterJoinProcessor = true;
break;
}
JoinProcessor leftPreJoinProcessor = new JoinProcessor(true, true, leftOuterJoinProcessor, 0, siddhiQueryContext.getSiddhiAppContext().getName(), siddhiQueryContext.getName());
JoinProcessor leftPostJoinProcessor = new JoinProcessor(true, false, leftOuterJoinProcessor, 0, siddhiQueryContext.getSiddhiAppContext().getName(), siddhiQueryContext.getName());
FindableProcessor leftFindableProcessor = insertJoinProcessorsAndGetFindable(leftPreJoinProcessor, leftPostJoinProcessor, leftStreamRuntime, outputExpectsExpiredEvents, joinInputStream.getLeftInputStream(), siddhiQueryContext);
JoinProcessor rightPreJoinProcessor = new JoinProcessor(false, true, rightOuterJoinProcessor, 1, siddhiQueryContext.getSiddhiAppContext().getName(), siddhiQueryContext.getName());
JoinProcessor rightPostJoinProcessor = new JoinProcessor(false, false, rightOuterJoinProcessor, 1, siddhiQueryContext.getSiddhiAppContext().getName(), siddhiQueryContext.getName());
FindableProcessor rightFindableProcessor = insertJoinProcessorsAndGetFindable(rightPreJoinProcessor, rightPostJoinProcessor, rightStreamRuntime, outputExpectsExpiredEvents, joinInputStream.getRightInputStream(), siddhiQueryContext);
leftPreJoinProcessor.setFindableProcessor(rightFindableProcessor);
leftPostJoinProcessor.setFindableProcessor(rightFindableProcessor);
rightPreJoinProcessor.setFindableProcessor(leftFindableProcessor);
rightPostJoinProcessor.setFindableProcessor(leftFindableProcessor);
Expression compareCondition = joinInputStream.getOnCompare();
if (compareCondition == null) {
compareCondition = Expression.value(true);
}
QuerySelector querySelector = null;
if (!(rightFindableProcessor instanceof TableWindowProcessor || rightFindableProcessor instanceof AggregateWindowProcessor) && (joinInputStream.getTrigger() != JoinInputStream.EventTrigger.LEFT)) {
MatchingMetaInfoHolder leftMatchingMetaInfoHolder = MatcherParser.constructMatchingMetaStateHolder(metaStateEvent, 1, leftMetaStreamEvent.getLastInputDefinition(), SiddhiConstants.UNKNOWN_STATE);
CompiledCondition rightCompiledCondition = leftFindableProcessor.compileCondition(compareCondition, leftMatchingMetaInfoHolder, executors, tableMap, siddhiQueryContext);
List<Attribute> expectedOutputAttributes = new ArrayList<>();
CompiledSelection rightCompiledSelection = null;
if (leftFindableProcessor instanceof TableWindowProcessor && ((TableWindowProcessor) leftFindableProcessor).isOptimisableLookup()) {
querySelector = SelectorParser.parse(query.getSelector(), query.getOutputStream(), metaStateEvent, tableMap, executors, SiddhiConstants.UNKNOWN_STATE, ProcessingMode.BATCH, outputExpectsExpiredEvents, siddhiQueryContext);
expectedOutputAttributes = metaStateEvent.getOutputStreamDefinition().getAttributeList();
try {
rightCompiledSelection = ((QueryableProcessor) leftFindableProcessor).compileSelection(query.getSelector(), expectedOutputAttributes, leftMatchingMetaInfoHolder, executors, tableMap, siddhiQueryContext);
} catch (SiddhiAppCreationException | SiddhiAppValidationException | QueryableRecordTableException e) {
if (log.isDebugEnabled()) {
log.debug("Performing select clause in databases failed for query: '" + siddhiQueryContext.getName() + "' within Siddhi app '" + siddhiQueryContext.getSiddhiAppContext().getName() + "' hence reverting back to " + "querying only with where clause. Reason for failure: " + e.getMessage(), e);
}
// Nothing to override
}
}
populateJoinProcessors(rightMetaStreamEvent, rightInputStreamId, rightPreJoinProcessor, rightPostJoinProcessor, rightCompiledCondition, rightCompiledSelection, expectedOutputAttributes);
}
if (!(leftFindableProcessor instanceof TableWindowProcessor || leftFindableProcessor instanceof AggregateWindowProcessor) && (joinInputStream.getTrigger() != JoinInputStream.EventTrigger.RIGHT)) {
MatchingMetaInfoHolder rightMatchingMetaInfoHolder = MatcherParser.constructMatchingMetaStateHolder(metaStateEvent, 0, rightMetaStreamEvent.getLastInputDefinition(), SiddhiConstants.UNKNOWN_STATE);
CompiledCondition leftCompiledCondition = rightFindableProcessor.compileCondition(compareCondition, rightMatchingMetaInfoHolder, executors, tableMap, siddhiQueryContext);
List<Attribute> expectedOutputAttributes = new ArrayList<>();
CompiledSelection leftCompiledSelection = null;
if (rightFindableProcessor instanceof TableWindowProcessor && ((TableWindowProcessor) rightFindableProcessor).isOptimisableLookup()) {
querySelector = SelectorParser.parse(query.getSelector(), query.getOutputStream(), metaStateEvent, tableMap, executors, SiddhiConstants.UNKNOWN_STATE, ProcessingMode.BATCH, outputExpectsExpiredEvents, siddhiQueryContext);
expectedOutputAttributes = metaStateEvent.getOutputStreamDefinition().getAttributeList();
try {
leftCompiledSelection = ((QueryableProcessor) rightFindableProcessor).compileSelection(query.getSelector(), expectedOutputAttributes, rightMatchingMetaInfoHolder, executors, tableMap, siddhiQueryContext);
} catch (SiddhiAppCreationException | SiddhiAppValidationException | QueryableRecordTableException e) {
if (log.isDebugEnabled()) {
log.debug("Performing select clause in databases failed for query: '" + siddhiQueryContext.getName() + "' within Siddhi app '" + siddhiQueryContext.getSiddhiAppContext().getName() + "' hence reverting back to " + "querying only with where clause. Reason for failure: " + e.getMessage(), e);
}
// Nothing to override
}
}
populateJoinProcessors(leftMetaStreamEvent, leftInputStreamId, leftPreJoinProcessor, leftPostJoinProcessor, leftCompiledCondition, leftCompiledSelection, expectedOutputAttributes);
}
JoinStreamRuntime joinStreamRuntime = new JoinStreamRuntime(siddhiQueryContext, metaStateEvent);
joinStreamRuntime.addRuntime(leftStreamRuntime);
joinStreamRuntime.addRuntime(rightStreamRuntime);
joinStreamRuntime.setQuerySelector(querySelector);
return joinStreamRuntime;
} catch (Throwable t) {
ExceptionUtil.populateQueryContext(t, joinInputStream, siddhiQueryContext.getSiddhiAppContext(), siddhiQueryContext);
throw t;
}
}
Aggregations