use of org.ballerinalang.siddhi.core.executor.VariableExpressionExecutor in project ballerina by ballerina-lang.
the class StreamPartitioner method createJoinInputStreamExecutors.
private void createJoinInputStreamExecutors(JoinInputStream inputStream, Partition partition, MetaStateEvent metaEvent, List<VariableExpressionExecutor> executors, SiddhiAppContext siddhiAppContext, String queryName) {
createExecutors(inputStream.getLeftInputStream(), partition, metaEvent.getMetaStreamEvent(0), executors, siddhiAppContext, queryName);
int size = executors.size();
for (VariableExpressionExecutor variableExpressionExecutor : executors) {
variableExpressionExecutor.getPosition()[SiddhiConstants.STREAM_EVENT_CHAIN_INDEX] = 0;
}
createExecutors(inputStream.getRightInputStream(), partition, metaEvent.getMetaStreamEvent(1), executors, siddhiAppContext, queryName);
for (int i = size; i < executors.size(); i++) {
executors.get(i).getPosition()[SiddhiConstants.STREAM_EVENT_CHAIN_INDEX] = 1;
}
}
use of org.ballerinalang.siddhi.core.executor.VariableExpressionExecutor in project ballerina by ballerina-lang.
the class PartitionParser method parse.
public static PartitionRuntime parse(SiddhiAppRuntimeBuilder siddhiAppRuntimeBuilder, Partition partition, SiddhiAppContext siddhiAppContext, ConcurrentMap<String, AbstractDefinition> streamDefinitionMap, int queryIndex) {
PartitionRuntime partitionRuntime = new PartitionRuntime(siddhiAppRuntimeBuilder.getStreamDefinitionMap(), siddhiAppRuntimeBuilder.getStreamJunctions(), partition, siddhiAppContext);
validateStreamPartitions(partition.getPartitionTypeMap(), streamDefinitionMap);
for (Query query : partition.getQueryList()) {
List<VariableExpressionExecutor> executors = new ArrayList<VariableExpressionExecutor>();
ConcurrentMap<String, AbstractDefinition> combinedStreamMap = new ConcurrentHashMap<String, AbstractDefinition>();
combinedStreamMap.putAll(streamDefinitionMap);
combinedStreamMap.putAll(partitionRuntime.getLocalStreamDefinitionMap());
QueryRuntime queryRuntime = QueryParser.parse(query, siddhiAppContext, combinedStreamMap, siddhiAppRuntimeBuilder.getTableDefinitionMap(), siddhiAppRuntimeBuilder.getWindowDefinitionMap(), siddhiAppRuntimeBuilder.getAggregationDefinitionMap(), siddhiAppRuntimeBuilder.getTableMap(), siddhiAppRuntimeBuilder.getAggregationMap(), siddhiAppRuntimeBuilder.getWindowMap(), siddhiAppRuntimeBuilder.getLockSynchronizer(), String.valueOf(queryIndex));
queryIndex++;
MetaStateEvent metaStateEvent = createMetaEventForPartitioner(queryRuntime.getMetaComplexEvent());
partitionRuntime.addQuery(queryRuntime);
partitionRuntime.addPartitionReceiver(queryRuntime, executors, metaStateEvent);
QueryParserHelper.reduceMetaComplexEvent(metaStateEvent);
if (queryRuntime.getMetaComplexEvent() instanceof MetaStateEvent) {
QueryParserHelper.updateVariablePosition(metaStateEvent, executors);
} else {
QueryParserHelper.updateVariablePosition(metaStateEvent.getMetaStreamEvent(0), executors);
}
partitionRuntime.init();
}
return partitionRuntime;
}
use of org.ballerinalang.siddhi.core.executor.VariableExpressionExecutor in project ballerina by ballerina-lang.
the class QueryParser method parse.
/**
* Parse a query and return corresponding QueryRuntime.
*
* @param query query to be parsed.
* @param siddhiAppContext associated Siddhi app context.
* @param streamDefinitionMap keyvalue containing user given stream definitions.
* @param tableDefinitionMap keyvalue containing table definitions.
* @param windowDefinitionMap keyvalue containing window definition map.
* @param aggregationDefinitionMap keyvalue containing aggregation definition map.
* @param tableMap keyvalue containing event tables.
* @param aggregationMap keyvalue containing aggrigation runtimes.
* @param windowMap keyvalue containing event window map.
* @param lockSynchronizer Lock synchronizer for sync the lock across queries.
* @param queryIndex query index to identify unknown query by number
* @return queryRuntime
*/
public static QueryRuntime parse(Query query, SiddhiAppContext siddhiAppContext, Map<String, AbstractDefinition> streamDefinitionMap, Map<String, AbstractDefinition> tableDefinitionMap, Map<String, AbstractDefinition> windowDefinitionMap, Map<String, AbstractDefinition> aggregationDefinitionMap, Map<String, Table> tableMap, Map<String, AggregationRuntime> aggregationMap, Map<String, Window> windowMap, LockSynchronizer lockSynchronizer, String queryIndex) {
List<VariableExpressionExecutor> executors = new ArrayList<VariableExpressionExecutor>();
QueryRuntime queryRuntime;
Element nameElement = null;
LatencyTracker latencyTracker = null;
LockWrapper lockWrapper = null;
try {
nameElement = AnnotationHelper.getAnnotationElement("info", "name", query.getAnnotations());
String queryName = null;
if (nameElement != null) {
queryName = nameElement.getValue();
} else {
queryName = "query_" + queryIndex + "_" + UUID.randomUUID().toString();
}
latencyTracker = QueryParserHelper.createLatencyTracker(siddhiAppContext, queryName, SiddhiConstants.METRIC_INFIX_QUERIES, null);
OutputStream.OutputEventType outputEventType = query.getOutputStream().getOutputEventType();
boolean outputExpectsExpiredEvents = false;
if (outputEventType != OutputStream.OutputEventType.CURRENT_EVENTS) {
outputExpectsExpiredEvents = true;
}
StreamRuntime streamRuntime = InputStreamParser.parse(query.getInputStream(), siddhiAppContext, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, tableMap, windowMap, aggregationMap, executors, latencyTracker, outputExpectsExpiredEvents, queryName);
QuerySelector selector = SelectorParser.parse(query.getSelector(), query.getOutputStream(), siddhiAppContext, streamRuntime.getMetaComplexEvent(), tableMap, executors, queryName, SiddhiConstants.UNKNOWN_STATE);
boolean isWindow = query.getInputStream() instanceof JoinInputStream;
if (!isWindow && query.getInputStream() instanceof SingleInputStream) {
for (StreamHandler streamHandler : ((SingleInputStream) query.getInputStream()).getStreamHandlers()) {
if (streamHandler instanceof org.ballerinalang.siddhi.query.api.execution.query.input.handler.Window) {
isWindow = true;
break;
}
}
}
Element synchronizedElement = AnnotationHelper.getAnnotationElement("synchronized", null, query.getAnnotations());
if (synchronizedElement != null) {
if (!("false".equalsIgnoreCase(synchronizedElement.getValue()))) {
// Query LockWrapper does not need a unique
lockWrapper = new LockWrapper("");
// id since it will
// not be passed to the LockSynchronizer.
// LockWrapper does not have a default lock
lockWrapper.setLock(new ReentrantLock());
}
} else {
if (isWindow || !(streamRuntime instanceof SingleStreamRuntime)) {
if (streamRuntime instanceof JoinStreamRuntime) {
// If at least one Window is involved in the join, use the LockWrapper of that window
// for the query as well.
// If join is between two EventWindows, sync the locks of the LockWrapper of those windows
// and use either of them for query.
MetaStateEvent metaStateEvent = (MetaStateEvent) streamRuntime.getMetaComplexEvent();
MetaStreamEvent[] metaStreamEvents = metaStateEvent.getMetaStreamEvents();
if (metaStreamEvents[0].getEventType() == EventType.WINDOW && metaStreamEvents[1].getEventType() == EventType.WINDOW) {
LockWrapper leftLockWrapper = windowMap.get(metaStreamEvents[0].getLastInputDefinition().getId()).getLock();
LockWrapper rightLockWrapper = windowMap.get(metaStreamEvents[1].getLastInputDefinition().getId()).getLock();
if (!leftLockWrapper.equals(rightLockWrapper)) {
// Sync the lock across both wrappers
lockSynchronizer.sync(leftLockWrapper, rightLockWrapper);
}
// Can use either leftLockWrapper or rightLockWrapper since both of them will hold the
// same lock internally
// If either of their lock is updated later, the other lock also will be update by the
// LockSynchronizer.
lockWrapper = leftLockWrapper;
} else if (metaStreamEvents[0].getEventType() == EventType.WINDOW) {
// Share the same wrapper as the query lock wrapper
lockWrapper = windowMap.get(metaStreamEvents[0].getLastInputDefinition().getId()).getLock();
} else if (metaStreamEvents[1].getEventType() == EventType.WINDOW) {
// Share the same wrapper as the query lock wrapper
lockWrapper = windowMap.get(metaStreamEvents[1].getLastInputDefinition().getId()).getLock();
} else {
// Join does not contain any Window
// Query LockWrapper does not need a unique
lockWrapper = new LockWrapper("");
// id since
// it will not be passed to the LockSynchronizer.
// LockWrapper does not have a default lock
lockWrapper.setLock(new ReentrantLock());
}
} else {
lockWrapper = new LockWrapper("");
lockWrapper.setLock(new ReentrantLock());
}
}
}
OutputRateLimiter outputRateLimiter = OutputParser.constructOutputRateLimiter(query.getOutputStream().getId(), query.getOutputRate(), query.getSelector().getGroupByList().size() != 0, isWindow, siddhiAppContext.getScheduledExecutorService(), siddhiAppContext, queryName);
if (outputRateLimiter instanceof WrappedSnapshotOutputRateLimiter) {
selector.setBatchingEnabled(false);
}
siddhiAppContext.addEternalReferencedHolder(outputRateLimiter);
OutputCallback outputCallback = OutputParser.constructOutputCallback(query.getOutputStream(), streamRuntime.getMetaComplexEvent().getOutputStreamDefinition(), tableMap, windowMap, siddhiAppContext, !(streamRuntime instanceof SingleStreamRuntime), queryName);
QueryParserHelper.reduceMetaComplexEvent(streamRuntime.getMetaComplexEvent());
QueryParserHelper.updateVariablePosition(streamRuntime.getMetaComplexEvent(), executors);
QueryParserHelper.initStreamRuntime(streamRuntime, streamRuntime.getMetaComplexEvent(), lockWrapper, queryName);
selector.setEventPopulator(StateEventPopulatorFactory.constructEventPopulator(streamRuntime.getMetaComplexEvent()));
queryRuntime = new QueryRuntime(query, siddhiAppContext, streamRuntime, selector, outputRateLimiter, outputCallback, streamRuntime.getMetaComplexEvent(), lockWrapper != null, queryName);
if (outputRateLimiter instanceof WrappedSnapshotOutputRateLimiter) {
selector.setBatchingEnabled(false);
((WrappedSnapshotOutputRateLimiter) outputRateLimiter).init(streamRuntime.getMetaComplexEvent().getOutputStreamDefinition().getAttributeList().size(), selector.getAttributeProcessorList(), streamRuntime.getMetaComplexEvent());
}
outputRateLimiter.init(siddhiAppContext, lockWrapper, queryName);
} catch (DuplicateDefinitionException e) {
if (nameElement != null) {
throw new DuplicateDefinitionException(e.getMessageWithOutContext() + ", when creating query " + nameElement.getValue(), e, e.getQueryContextStartIndex(), e.getQueryContextEndIndex(), siddhiAppContext.getName(), siddhiAppContext.getSiddhiAppString());
} else {
throw new DuplicateDefinitionException(e.getMessage(), e, e.getQueryContextStartIndex(), e.getQueryContextEndIndex(), siddhiAppContext.getName(), siddhiAppContext.getSiddhiAppString());
}
} catch (Throwable t) {
ExceptionUtil.populateQueryContext(t, query, siddhiAppContext);
throw t;
}
return queryRuntime;
}
use of org.ballerinalang.siddhi.core.executor.VariableExpressionExecutor in project ballerina by ballerina-lang.
the class SelectorParser method getAttributeProcessors.
/**
* Method to construct AttributeProcessor list for the selector.
*
* @param selector Selector
* @param id stream id
* @param siddhiAppContext siddhi app context
* @param metaComplexEvent meta ComplexEvent
* @param tableMap Table Map
* @param variableExpressionExecutors list of VariableExpressionExecutors
* @param outputStream
* @return list of AttributeProcessors
*/
private static List<AttributeProcessor> getAttributeProcessors(Selector selector, String id, SiddhiAppContext siddhiAppContext, MetaComplexEvent metaComplexEvent, Map<String, Table> tableMap, List<VariableExpressionExecutor> variableExpressionExecutors, OutputStream outputStream, String queryName, int metaPosition) {
List<AttributeProcessor> attributeProcessorList = new ArrayList<>();
StreamDefinition outputDefinition = StreamDefinition.id(id);
outputDefinition.setQueryContextStartIndex(outputStream.getQueryContextStartIndex());
outputDefinition.setQueryContextEndIndex(outputStream.getQueryContextEndIndex());
List<OutputAttribute> outputAttributes = selector.getSelectionList();
if (selector.getSelectionList().size() == 0) {
if (metaComplexEvent instanceof MetaStreamEvent) {
List<Attribute> attributeList = ((MetaStreamEvent) metaComplexEvent).getLastInputDefinition().getAttributeList();
for (Attribute attribute : attributeList) {
outputAttributes.add(new OutputAttribute(new Variable(attribute.getName())));
}
} else {
int position = 0;
for (MetaStreamEvent metaStreamEvent : ((MetaStateEvent) metaComplexEvent).getMetaStreamEvents()) {
if (metaPosition == SiddhiConstants.UNKNOWN_STATE || metaPosition == position) {
List<Attribute> attributeList = metaStreamEvent.getLastInputDefinition().getAttributeList();
for (Attribute attribute : attributeList) {
OutputAttribute outputAttribute = new OutputAttribute(new Variable(attribute.getName()));
if (!outputAttributes.contains(outputAttribute)) {
outputAttributes.add(outputAttribute);
} else {
List<AbstractDefinition> definitions = new ArrayList<>();
for (MetaStreamEvent aMetaStreamEvent : ((MetaStateEvent) metaComplexEvent).getMetaStreamEvents()) {
definitions.add(aMetaStreamEvent.getLastInputDefinition());
}
throw new DuplicateAttributeException("Duplicate attribute exist in streams " + definitions, outputStream.getQueryContextStartIndex(), outputStream.getQueryContextEndIndex());
}
}
}
++position;
}
}
}
int i = 0;
for (OutputAttribute outputAttribute : outputAttributes) {
ExpressionExecutor expressionExecutor = ExpressionParser.parseExpression(outputAttribute.getExpression(), metaComplexEvent, SiddhiConstants.UNKNOWN_STATE, tableMap, variableExpressionExecutors, siddhiAppContext, !(selector.getGroupByList().isEmpty()), 0, queryName);
if (expressionExecutor instanceof VariableExpressionExecutor) {
// for variables we will directly put
// value at conversion stage
VariableExpressionExecutor executor = ((VariableExpressionExecutor) expressionExecutor);
if (metaComplexEvent instanceof MetaStateEvent) {
((MetaStateEvent) metaComplexEvent).addOutputDataAllowingDuplicate(new MetaStateEventAttribute(executor.getAttribute(), executor.getPosition()));
} else {
((MetaStreamEvent) metaComplexEvent).addOutputDataAllowingDuplicate(executor.getAttribute());
}
outputDefinition.attribute(outputAttribute.getRename(), ((VariableExpressionExecutor) expressionExecutor).getAttribute().getType());
} else {
// To maintain output variable positions
if (metaComplexEvent instanceof MetaStateEvent) {
((MetaStateEvent) metaComplexEvent).addOutputDataAllowingDuplicate(null);
} else {
((MetaStreamEvent) metaComplexEvent).addOutputDataAllowingDuplicate(null);
}
AttributeProcessor attributeProcessor = new AttributeProcessor(expressionExecutor);
attributeProcessor.setOutputPosition(i);
attributeProcessorList.add(attributeProcessor);
outputDefinition.attribute(outputAttribute.getRename(), attributeProcessor.getOutputType());
}
i++;
}
metaComplexEvent.setOutputDefinition(outputDefinition);
return attributeProcessorList;
}
use of org.ballerinalang.siddhi.core.executor.VariableExpressionExecutor in project ballerina by ballerina-lang.
the class AggregationParser method populateIncomingAggregatorsAndExecutors.
private static void populateIncomingAggregatorsAndExecutors(AggregationDefinition aggregationDefinition, SiddhiAppContext siddhiAppContext, Map<String, Table> tableMap, List<VariableExpressionExecutor> incomingVariableExpressionExecutors, String aggregatorName, MetaStreamEvent incomingMetaStreamEvent, List<ExpressionExecutor> incomingExpressionExecutors, List<IncrementalAttributeAggregator> incrementalAttributeAggregators, List<Variable> groupByVariableList, List<Expression> outputExpressions) {
ExpressionExecutor[] timeStampTimeZoneExecutors = setTimeStampTimeZoneExecutors(aggregationDefinition, siddhiAppContext, tableMap, incomingVariableExpressionExecutors, aggregatorName, incomingMetaStreamEvent);
ExpressionExecutor timestampExecutor = timeStampTimeZoneExecutors[0];
ExpressionExecutor timeZoneExecutor = timeStampTimeZoneExecutors[1];
Attribute timestampAttribute = new Attribute("AGG_TIMESTAMP", Attribute.Type.LONG);
incomingMetaStreamEvent.addOutputData(timestampAttribute);
incomingExpressionExecutors.add(timestampExecutor);
incomingMetaStreamEvent.addOutputData(new Attribute("AGG_TIMEZONE", Attribute.Type.STRING));
incomingExpressionExecutors.add(timeZoneExecutor);
AbstractDefinition incomingLastInputStreamDefinition = incomingMetaStreamEvent.getLastInputDefinition();
for (Variable groupByVariable : groupByVariableList) {
incomingMetaStreamEvent.addOutputData(incomingLastInputStreamDefinition.getAttributeList().get(incomingLastInputStreamDefinition.getAttributePosition(groupByVariable.getAttributeName())));
incomingExpressionExecutors.add(ExpressionParser.parseExpression(groupByVariable, incomingMetaStreamEvent, 0, tableMap, incomingVariableExpressionExecutors, siddhiAppContext, false, 0, aggregatorName));
}
// Add AGG_TIMESTAMP to output as well
outputExpressions.add(Expression.variable("AGG_TIMESTAMP"));
aggregationDefinition.getAttributeList().add(timestampAttribute);
for (OutputAttribute outputAttribute : aggregationDefinition.getSelector().getSelectionList()) {
Expression expression = outputAttribute.getExpression();
if (expression instanceof AttributeFunction) {
IncrementalAttributeAggregator incrementalAggregator = null;
try {
incrementalAggregator = (IncrementalAttributeAggregator) SiddhiClassLoader.loadExtensionImplementation(new AttributeFunction("incrementalAggregator", ((AttributeFunction) expression).getName(), ((AttributeFunction) expression).getParameters()), IncrementalAttributeAggregatorExtensionHolder.getInstance(siddhiAppContext));
} catch (SiddhiAppCreationException ex) {
try {
SiddhiClassLoader.loadExtensionImplementation((AttributeFunction) expression, FunctionExecutorExtensionHolder.getInstance(siddhiAppContext));
ExpressionExecutor expressionExecutor = ExpressionParser.parseExpression(expression, incomingMetaStreamEvent, 0, tableMap, incomingVariableExpressionExecutors, siddhiAppContext, false, 0, aggregatorName);
incomingExpressionExecutors.add(expressionExecutor);
incomingMetaStreamEvent.addOutputData(new Attribute(outputAttribute.getRename(), expressionExecutor.getReturnType()));
aggregationDefinition.getAttributeList().add(new Attribute(outputAttribute.getRename(), expressionExecutor.getReturnType()));
outputExpressions.add(Expression.variable(outputAttribute.getRename()));
} catch (SiddhiAppCreationException e) {
throw new SiddhiAppCreationException("'" + ((AttributeFunction) expression).getName() + "' is neither a incremental attribute aggregator extension or a function" + " extension", expression.getQueryContextStartIndex(), expression.getQueryContextEndIndex());
}
}
if (incrementalAggregator != null) {
initIncrementalAttributeAggregator(incomingLastInputStreamDefinition, (AttributeFunction) expression, incrementalAggregator);
incrementalAttributeAggregators.add(incrementalAggregator);
aggregationDefinition.getAttributeList().add(new Attribute(outputAttribute.getRename(), incrementalAggregator.getReturnType()));
outputExpressions.add(incrementalAggregator.aggregate());
}
} else {
if (expression instanceof Variable && groupByVariableList.contains(expression)) {
Attribute groupByAttribute = null;
for (Attribute attribute : incomingMetaStreamEvent.getOutputData()) {
if (attribute.getName().equals(((Variable) expression).getAttributeName())) {
groupByAttribute = attribute;
break;
}
}
if (groupByAttribute == null) {
throw new SiddhiAppCreationException("Expected GroupBy attribute '" + ((Variable) expression).getAttributeName() + "' not used in aggregation '" + aggregatorName + "' processing.", expression.getQueryContextStartIndex(), expression.getQueryContextEndIndex());
}
aggregationDefinition.getAttributeList().add(new Attribute(outputAttribute.getRename(), groupByAttribute.getType()));
outputExpressions.add(Expression.variable(groupByAttribute.getName()));
} else {
ExpressionExecutor expressionExecutor = ExpressionParser.parseExpression(expression, incomingMetaStreamEvent, 0, tableMap, incomingVariableExpressionExecutors, siddhiAppContext, false, 0, aggregatorName);
incomingExpressionExecutors.add(expressionExecutor);
incomingMetaStreamEvent.addOutputData(new Attribute(outputAttribute.getRename(), expressionExecutor.getReturnType()));
aggregationDefinition.getAttributeList().add(new Attribute(outputAttribute.getRename(), expressionExecutor.getReturnType()));
outputExpressions.add(Expression.variable(outputAttribute.getRename()));
}
}
}
}
Aggregations