use of io.siddhi.core.query.input.ProcessStreamReceiver in project siddhi by wso2.
the class JoinInputStreamParser method parseInputStream.
public static StreamRuntime parseInputStream(JoinInputStream joinInputStream, Query query, Map<String, AbstractDefinition> streamDefinitionMap, Map<String, AbstractDefinition> tableDefinitionMap, Map<String, AbstractDefinition> windowDefinitionMap, Map<String, AbstractDefinition> aggregationDefinitionMap, Map<String, Table> tableMap, Map<String, Window> windowMap, Map<String, AggregationRuntime> aggregationMap, List<VariableExpressionExecutor> executors, boolean outputExpectsExpiredEvents, SiddhiQueryContext siddhiQueryContext) {
try {
ProcessStreamReceiver leftProcessStreamReceiver;
ProcessStreamReceiver rightProcessStreamReceiver;
MetaStreamEvent leftMetaStreamEvent = new MetaStreamEvent();
MetaStreamEvent rightMetaStreamEvent = new MetaStreamEvent();
String leftInputStreamId = ((SingleInputStream) joinInputStream.getLeftInputStream()).getStreamId();
String rightInputStreamId = ((SingleInputStream) joinInputStream.getRightInputStream()).getStreamId();
boolean leftOuterJoinProcessor = false;
boolean rightOuterJoinProcessor = false;
if (joinInputStream.getAllStreamIds().size() == 2) {
setEventType(streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, leftMetaStreamEvent, leftInputStreamId);
setEventType(streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, rightMetaStreamEvent, rightInputStreamId);
leftProcessStreamReceiver = new ProcessStreamReceiver(leftInputStreamId, siddhiQueryContext);
rightProcessStreamReceiver = new ProcessStreamReceiver(rightInputStreamId, siddhiQueryContext);
if ((leftMetaStreamEvent.getEventType() == TABLE || leftMetaStreamEvent.getEventType() == AGGREGATE) && (rightMetaStreamEvent.getEventType() == TABLE || rightMetaStreamEvent.getEventType() == AGGREGATE)) {
throw new SiddhiAppCreationException("Both inputs of join " + leftInputStreamId + " and " + rightInputStreamId + " are from static sources");
}
if (leftMetaStreamEvent.getEventType() != AGGREGATE && rightMetaStreamEvent.getEventType() != AGGREGATE) {
if (joinInputStream.getPer() != null) {
throw new SiddhiAppCreationException("When joining " + leftInputStreamId + " and " + rightInputStreamId + " 'per' cannot be used as neither of them is an aggregation ");
} else if (joinInputStream.getWithin() != null) {
throw new SiddhiAppCreationException("When joining " + leftInputStreamId + " and " + rightInputStreamId + " 'within' cannot be used as neither of them is an aggregation ");
}
}
} else {
if (windowDefinitionMap.containsKey(joinInputStream.getAllStreamIds().get(0))) {
leftMetaStreamEvent.setEventType(WINDOW);
rightMetaStreamEvent.setEventType(WINDOW);
rightProcessStreamReceiver = new MultiProcessStreamReceiver(joinInputStream.getAllStreamIds().get(0), 1, new Object(), siddhiQueryContext);
leftProcessStreamReceiver = rightProcessStreamReceiver;
} else if (streamDefinitionMap.containsKey(joinInputStream.getAllStreamIds().get(0))) {
rightProcessStreamReceiver = new MultiProcessStreamReceiver(joinInputStream.getAllStreamIds().get(0), 2, new Object(), siddhiQueryContext);
leftProcessStreamReceiver = rightProcessStreamReceiver;
} else {
throw new SiddhiAppCreationException("Input of join is from static source " + leftInputStreamId + " and " + rightInputStreamId);
}
}
SingleStreamRuntime leftStreamRuntime = SingleInputStreamParser.parseInputStream((SingleInputStream) joinInputStream.getLeftInputStream(), executors, streamDefinitionMap, leftMetaStreamEvent.getEventType() != TABLE ? null : tableDefinitionMap, leftMetaStreamEvent.getEventType() != WINDOW ? null : windowDefinitionMap, leftMetaStreamEvent.getEventType() != AGGREGATE ? null : aggregationDefinitionMap, tableMap, leftMetaStreamEvent, leftProcessStreamReceiver, true, outputExpectsExpiredEvents, true, false, siddhiQueryContext);
for (VariableExpressionExecutor variableExpressionExecutor : executors) {
variableExpressionExecutor.getPosition()[SiddhiConstants.STREAM_EVENT_CHAIN_INDEX] = 0;
}
int size = executors.size();
SingleStreamRuntime rightStreamRuntime = SingleInputStreamParser.parseInputStream((SingleInputStream) joinInputStream.getRightInputStream(), executors, streamDefinitionMap, rightMetaStreamEvent.getEventType() != TABLE ? null : tableDefinitionMap, rightMetaStreamEvent.getEventType() != WINDOW ? null : windowDefinitionMap, rightMetaStreamEvent.getEventType() != AGGREGATE ? null : aggregationDefinitionMap, tableMap, rightMetaStreamEvent, rightProcessStreamReceiver, true, outputExpectsExpiredEvents, true, false, siddhiQueryContext);
for (int i = size; i < executors.size(); i++) {
VariableExpressionExecutor variableExpressionExecutor = executors.get(i);
variableExpressionExecutor.getPosition()[SiddhiConstants.STREAM_EVENT_CHAIN_INDEX] = 1;
}
setStreamRuntimeProcessorChain(leftMetaStreamEvent, leftStreamRuntime, leftInputStreamId, tableMap, windowMap, aggregationMap, executors, outputExpectsExpiredEvents, joinInputStream.getWithin(), joinInputStream.getPer(), query.getSelector().getGroupByList(), siddhiQueryContext, joinInputStream.getLeftInputStream());
setStreamRuntimeProcessorChain(rightMetaStreamEvent, rightStreamRuntime, rightInputStreamId, tableMap, windowMap, aggregationMap, executors, outputExpectsExpiredEvents, joinInputStream.getWithin(), joinInputStream.getPer(), query.getSelector().getGroupByList(), siddhiQueryContext, joinInputStream.getRightInputStream());
MetaStateEvent metaStateEvent = new MetaStateEvent(2);
metaStateEvent.addEvent(leftMetaStreamEvent);
metaStateEvent.addEvent(rightMetaStreamEvent);
switch(joinInputStream.getType()) {
case FULL_OUTER_JOIN:
leftOuterJoinProcessor = true;
rightOuterJoinProcessor = true;
break;
case RIGHT_OUTER_JOIN:
rightOuterJoinProcessor = true;
break;
case LEFT_OUTER_JOIN:
leftOuterJoinProcessor = true;
break;
}
JoinProcessor leftPreJoinProcessor = new JoinProcessor(true, true, leftOuterJoinProcessor, 0, siddhiQueryContext.getSiddhiAppContext().getName(), siddhiQueryContext.getName());
JoinProcessor leftPostJoinProcessor = new JoinProcessor(true, false, leftOuterJoinProcessor, 0, siddhiQueryContext.getSiddhiAppContext().getName(), siddhiQueryContext.getName());
FindableProcessor leftFindableProcessor = insertJoinProcessorsAndGetFindable(leftPreJoinProcessor, leftPostJoinProcessor, leftStreamRuntime, outputExpectsExpiredEvents, joinInputStream.getLeftInputStream(), siddhiQueryContext);
JoinProcessor rightPreJoinProcessor = new JoinProcessor(false, true, rightOuterJoinProcessor, 1, siddhiQueryContext.getSiddhiAppContext().getName(), siddhiQueryContext.getName());
JoinProcessor rightPostJoinProcessor = new JoinProcessor(false, false, rightOuterJoinProcessor, 1, siddhiQueryContext.getSiddhiAppContext().getName(), siddhiQueryContext.getName());
FindableProcessor rightFindableProcessor = insertJoinProcessorsAndGetFindable(rightPreJoinProcessor, rightPostJoinProcessor, rightStreamRuntime, outputExpectsExpiredEvents, joinInputStream.getRightInputStream(), siddhiQueryContext);
leftPreJoinProcessor.setFindableProcessor(rightFindableProcessor);
leftPostJoinProcessor.setFindableProcessor(rightFindableProcessor);
rightPreJoinProcessor.setFindableProcessor(leftFindableProcessor);
rightPostJoinProcessor.setFindableProcessor(leftFindableProcessor);
Expression compareCondition = joinInputStream.getOnCompare();
if (compareCondition == null) {
compareCondition = Expression.value(true);
}
QuerySelector querySelector = null;
if (!(rightFindableProcessor instanceof TableWindowProcessor || rightFindableProcessor instanceof AggregateWindowProcessor) && (joinInputStream.getTrigger() != JoinInputStream.EventTrigger.LEFT)) {
MatchingMetaInfoHolder leftMatchingMetaInfoHolder = MatcherParser.constructMatchingMetaStateHolder(metaStateEvent, 1, leftMetaStreamEvent.getLastInputDefinition(), SiddhiConstants.UNKNOWN_STATE);
CompiledCondition rightCompiledCondition = leftFindableProcessor.compileCondition(compareCondition, leftMatchingMetaInfoHolder, executors, tableMap, siddhiQueryContext);
List<Attribute> expectedOutputAttributes = new ArrayList<>();
CompiledSelection rightCompiledSelection = null;
if (leftFindableProcessor instanceof TableWindowProcessor && ((TableWindowProcessor) leftFindableProcessor).isOptimisableLookup()) {
querySelector = SelectorParser.parse(query.getSelector(), query.getOutputStream(), metaStateEvent, tableMap, executors, SiddhiConstants.UNKNOWN_STATE, ProcessingMode.BATCH, outputExpectsExpiredEvents, siddhiQueryContext);
expectedOutputAttributes = metaStateEvent.getOutputStreamDefinition().getAttributeList();
try {
rightCompiledSelection = ((QueryableProcessor) leftFindableProcessor).compileSelection(query.getSelector(), expectedOutputAttributes, leftMatchingMetaInfoHolder, executors, tableMap, siddhiQueryContext);
} catch (SiddhiAppCreationException | SiddhiAppValidationException | QueryableRecordTableException e) {
if (log.isDebugEnabled()) {
log.debug("Performing select clause in databases failed for query: '" + siddhiQueryContext.getName() + "' within Siddhi app '" + siddhiQueryContext.getSiddhiAppContext().getName() + "' hence reverting back to " + "querying only with where clause. Reason for failure: " + e.getMessage(), e);
}
// Nothing to override
}
}
populateJoinProcessors(rightMetaStreamEvent, rightInputStreamId, rightPreJoinProcessor, rightPostJoinProcessor, rightCompiledCondition, rightCompiledSelection, expectedOutputAttributes);
}
if (!(leftFindableProcessor instanceof TableWindowProcessor || leftFindableProcessor instanceof AggregateWindowProcessor) && (joinInputStream.getTrigger() != JoinInputStream.EventTrigger.RIGHT)) {
MatchingMetaInfoHolder rightMatchingMetaInfoHolder = MatcherParser.constructMatchingMetaStateHolder(metaStateEvent, 0, rightMetaStreamEvent.getLastInputDefinition(), SiddhiConstants.UNKNOWN_STATE);
CompiledCondition leftCompiledCondition = rightFindableProcessor.compileCondition(compareCondition, rightMatchingMetaInfoHolder, executors, tableMap, siddhiQueryContext);
List<Attribute> expectedOutputAttributes = new ArrayList<>();
CompiledSelection leftCompiledSelection = null;
if (rightFindableProcessor instanceof TableWindowProcessor && ((TableWindowProcessor) rightFindableProcessor).isOptimisableLookup()) {
querySelector = SelectorParser.parse(query.getSelector(), query.getOutputStream(), metaStateEvent, tableMap, executors, SiddhiConstants.UNKNOWN_STATE, ProcessingMode.BATCH, outputExpectsExpiredEvents, siddhiQueryContext);
expectedOutputAttributes = metaStateEvent.getOutputStreamDefinition().getAttributeList();
try {
leftCompiledSelection = ((QueryableProcessor) rightFindableProcessor).compileSelection(query.getSelector(), expectedOutputAttributes, rightMatchingMetaInfoHolder, executors, tableMap, siddhiQueryContext);
} catch (SiddhiAppCreationException | SiddhiAppValidationException | QueryableRecordTableException e) {
if (log.isDebugEnabled()) {
log.debug("Performing select clause in databases failed for query: '" + siddhiQueryContext.getName() + "' within Siddhi app '" + siddhiQueryContext.getSiddhiAppContext().getName() + "' hence reverting back to " + "querying only with where clause. Reason for failure: " + e.getMessage(), e);
}
// Nothing to override
}
}
populateJoinProcessors(leftMetaStreamEvent, leftInputStreamId, leftPreJoinProcessor, leftPostJoinProcessor, leftCompiledCondition, leftCompiledSelection, expectedOutputAttributes);
}
JoinStreamRuntime joinStreamRuntime = new JoinStreamRuntime(siddhiQueryContext, metaStateEvent);
joinStreamRuntime.addRuntime(leftStreamRuntime);
joinStreamRuntime.addRuntime(rightStreamRuntime);
joinStreamRuntime.setQuerySelector(querySelector);
return joinStreamRuntime;
} catch (Throwable t) {
ExceptionUtil.populateQueryContext(t, joinInputStream, siddhiQueryContext.getSiddhiAppContext(), siddhiQueryContext);
throw t;
}
}
use of io.siddhi.core.query.input.ProcessStreamReceiver in project siddhi by wso2.
the class StateInputStreamParser method parseInputStream.
public static StateStreamRuntime parseInputStream(StateInputStream stateInputStream, MetaStateEvent metaStateEvent, Map<String, AbstractDefinition> streamDefinitionMap, Map<String, AbstractDefinition> tableDefinitionMap, Map<String, AbstractDefinition> windowDefinitionMap, Map<String, AbstractDefinition> aggregationDefinitionMap, Map<String, Table> tableMap, List<VariableExpressionExecutor> variableExpressionExecutors, SiddhiQueryContext siddhiQueryContext) {
Map<String, ProcessStreamReceiver> processStreamReceiverMap = new HashMap<String, ProcessStreamReceiver>();
StateStreamRuntime stateStreamRuntime = new StateStreamRuntime(siddhiQueryContext, metaStateEvent);
Object patternSyncObject = new Object();
for (String streamId : stateInputStream.getAllStreamIds()) {
int streamCount = stateInputStream.getStreamCount(streamId);
if (streamCount == 1) {
if (stateInputStream.getStateType() == StateInputStream.Type.SEQUENCE) {
processStreamReceiverMap.put(streamId, new SequenceSingleProcessStreamReceiver(streamId, stateStreamRuntime, patternSyncObject, siddhiQueryContext));
} else {
processStreamReceiverMap.put(streamId, new PatternSingleProcessStreamReceiver(streamId, patternSyncObject, siddhiQueryContext));
}
} else {
if (stateInputStream.getStateType() == StateInputStream.Type.SEQUENCE) {
processStreamReceiverMap.put(streamId, new SequenceMultiProcessStreamReceiver(streamId, streamCount, stateStreamRuntime, patternSyncObject, siddhiQueryContext));
} else {
processStreamReceiverMap.put(streamId, new PatternMultiProcessStreamReceiver(streamId, streamCount, patternSyncObject, siddhiQueryContext));
}
}
}
StateElement stateElement = stateInputStream.getStateElement();
List<PreStateProcessor> preStateProcessors = new ArrayList<>();
List<PreStateProcessor> startupPreStateProcessors = new ArrayList<>();
InnerStateRuntime innerStateRuntime = parse(stateElement, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, tableMap, metaStateEvent, variableExpressionExecutors, processStreamReceiverMap, null, null, stateInputStream.getStateType(), false, preStateProcessors, true, startupPreStateProcessors, siddhiQueryContext);
stateStreamRuntime.setInnerStateRuntime(innerStateRuntime);
stateStreamRuntime.setStartupPreStateProcessors(startupPreStateProcessors);
for (ProcessStreamReceiver processStreamReceiver : processStreamReceiverMap.values()) {
processStreamReceiver.setAllStatefulProcessors(preStateProcessors);
}
if (stateInputStream.getWithinTime() != null) {
List<Integer> startStateIdList = new ArrayList<>();
for (PreStateProcessor preStateProcessor : preStateProcessors) {
if (preStateProcessor.isStartState()) {
startStateIdList.add(preStateProcessor.getStateId());
}
}
int[] startStateIds = Ints.toArray(startStateIdList);
for (PreStateProcessor preStateProcessor : preStateProcessors) {
preStateProcessor.setStartStateIds(startStateIds);
preStateProcessor.setWithinTime(stateInputStream.getWithinTime().value());
}
}
((StreamPreStateProcessor) innerStateRuntime.getFirstProcessor()).setThisLastProcessor((StreamPostStateProcessor) innerStateRuntime.getLastProcessor());
return stateStreamRuntime;
}
use of io.siddhi.core.query.input.ProcessStreamReceiver in project siddhi by wso2.
the class SiddhiAppRuntimeBuilder method defineAggregation.
public void defineAggregation(AggregationDefinition aggregationDefinition) {
AggregationRuntime aggregationRuntime = AggregationParser.parse(aggregationDefinition, siddhiAppContext, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, tableMap, windowMap, aggregationMap, this);
DefinitionParserHelper.validateDefinition(aggregationDefinition, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap);
aggregationDefinitionMap.putIfAbsent(aggregationDefinition.getId(), aggregationDefinition);
ProcessStreamReceiver processStreamReceiver = aggregationRuntime.getSingleStreamRuntime().getProcessStreamReceiver();
streamJunctionMap.get(processStreamReceiver.getStreamId()).subscribe(processStreamReceiver);
aggregationMap.putIfAbsent(aggregationDefinition.getId(), aggregationRuntime);
}
use of io.siddhi.core.query.input.ProcessStreamReceiver in project siddhi by wso2.
the class SiddhiAppRuntimeBuilder method addQuery.
public String addQuery(QueryRuntimeImpl queryRuntime) {
QueryRuntime oldQueryRuntime = queryProcessorMap.put(queryRuntime.getQueryId(), queryRuntime);
if (oldQueryRuntime != null) {
throw new SiddhiAppCreationException("Multiple queries with name '" + queryRuntime.getQueryId() + "' defined in Siddhi App '" + siddhiAppContext.getName() + "'", queryRuntime.getQuery().getQueryContextStartIndex(), queryRuntime.getQuery().getQueryContextEndIndex());
}
StreamRuntime streamRuntime = queryRuntime.getStreamRuntime();
for (SingleStreamRuntime singleStreamRuntime : streamRuntime.getSingleStreamRuntimes()) {
ProcessStreamReceiver processStreamReceiver = singleStreamRuntime.getProcessStreamReceiver();
if (processStreamReceiver.toStream()) {
StreamJunction streamJunction = streamJunctionMap.get(processStreamReceiver.getStreamId());
if (streamJunction != null) {
streamJunction.subscribe(processStreamReceiver);
} else {
throw new SiddhiAppCreationException("Expecting a stream, but provided '" + processStreamReceiver.getStreamId() + "' is not a stream");
}
}
}
OutputCallback outputCallback = queryRuntime.getOutputCallback();
if (outputCallback != null && outputCallback instanceof InsertIntoStreamCallback) {
InsertIntoStreamCallback insertIntoStreamCallback = (InsertIntoStreamCallback) outputCallback;
StreamDefinition streamDefinition = insertIntoStreamCallback.getOutputStreamDefinition();
streamDefinitionMap.putIfAbsent(streamDefinition.getId(), streamDefinition);
DefinitionParserHelper.validateOutputStream(streamDefinition, streamDefinitionMap.get(streamDefinition.getId()));
StreamJunction outputStreamJunction = streamJunctionMap.get(streamDefinition.getId());
if (outputStreamJunction == null) {
outputStreamJunction = new StreamJunction(streamDefinition, siddhiAppContext.getExecutorService(), siddhiAppContext.getBufferSize(), null, siddhiAppContext);
streamJunctionMap.putIfAbsent(streamDefinition.getId(), outputStreamJunction);
}
insertIntoStreamCallback.init(streamJunctionMap.get(insertIntoStreamCallback.getOutputStreamDefinition().getId()));
} else if (outputCallback != null && outputCallback instanceof InsertIntoWindowCallback) {
InsertIntoWindowCallback insertIntoWindowCallback = (InsertIntoWindowCallback) outputCallback;
StreamDefinition streamDefinition = insertIntoWindowCallback.getOutputStreamDefinition();
windowDefinitionMap.putIfAbsent(streamDefinition.getId(), streamDefinition);
DefinitionParserHelper.validateOutputStream(streamDefinition, windowDefinitionMap.get(streamDefinition.getId()));
StreamJunction outputStreamJunction = streamJunctionMap.get(streamDefinition.getId());
if (outputStreamJunction == null) {
outputStreamJunction = new StreamJunction(streamDefinition, siddhiAppContext.getExecutorService(), siddhiAppContext.getBufferSize(), null, siddhiAppContext);
streamJunctionMap.putIfAbsent(streamDefinition.getId(), outputStreamJunction);
}
insertIntoWindowCallback.getWindow().setPublisher(streamJunctionMap.get(insertIntoWindowCallback.getOutputStreamDefinition().getId()).constructPublisher());
}
return queryRuntime.getQueryId();
}
use of io.siddhi.core.query.input.ProcessStreamReceiver in project siddhi by wso2.
the class InputStreamParser method parse.
/**
* Parse an InputStream returning corresponding StreamRuntime
*
* @param inputStream input stream to be parsed
* @param streamDefinitionMap map containing user given stream definitions
* @param tableDefinitionMap table definition map
* @param windowDefinitionMap window definition map
* @param aggregationDefinitionMap aggregation definition map
* @param tableMap Table Map
* @param windowMap event window map
* @param aggregationMap aggregator map
* @param executors List to hold VariableExpressionExecutors to update after query parsing
* @param outputExpectsExpiredEvents is expired events sent as output
* @param siddhiQueryContext Siddhi query context.
* @return StreamRuntime
*/
public static StreamRuntime parse(InputStream inputStream, Query query, Map<String, AbstractDefinition> streamDefinitionMap, Map<String, AbstractDefinition> tableDefinitionMap, Map<String, AbstractDefinition> windowDefinitionMap, Map<String, AbstractDefinition> aggregationDefinitionMap, Map<String, Table> tableMap, Map<String, Window> windowMap, Map<String, AggregationRuntime> aggregationMap, List<VariableExpressionExecutor> executors, boolean outputExpectsExpiredEvents, SiddhiQueryContext siddhiQueryContext) {
if (inputStream instanceof BasicSingleInputStream || inputStream instanceof SingleInputStream) {
SingleInputStream singleInputStream = (SingleInputStream) inputStream;
ProcessStreamReceiver processStreamReceiver = new ProcessStreamReceiver(singleInputStream.getStreamId(), siddhiQueryContext);
return SingleInputStreamParser.parseInputStream((SingleInputStream) inputStream, executors, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, tableMap, new MetaStreamEvent(), processStreamReceiver, true, outputExpectsExpiredEvents, false, false, siddhiQueryContext);
} else if (inputStream instanceof JoinInputStream) {
return JoinInputStreamParser.parseInputStream(((JoinInputStream) inputStream), query, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, tableMap, windowMap, aggregationMap, executors, outputExpectsExpiredEvents, siddhiQueryContext);
} else if (inputStream instanceof StateInputStream) {
MetaStateEvent metaStateEvent = new MetaStateEvent(inputStream.getAllStreamIds().size());
return StateInputStreamParser.parseInputStream(((StateInputStream) inputStream), metaStateEvent, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, tableMap, executors, siddhiQueryContext);
} else {
throw new OperationNotSupportedException();
}
}
Aggregations