use of io.siddhi.query.api.expression.Variable in project siddhi by wso2.
the class SelectorParser method getAttributeProcessors.
/**
* Method to construct AttributeProcessor list for the selector.
*
* @param selector Selector
* @param id stream id
* @param metaComplexEvent meta ComplexEvent
* @param tableMap Table Map
* @param variableExpressionExecutors list of VariableExpressionExecutors
* @param outputStream output stream
* @param processingMode processing mode of the query
* @param outputExpectsExpiredEvents is expired events sent as output
* @param groupBy is Attributes groupBy
* @param siddhiQueryContext current siddhi query context @return list of AttributeProcessors
*/
private static List<AttributeProcessor> getAttributeProcessors(Selector selector, String id, MetaComplexEvent metaComplexEvent, Map<String, Table> tableMap, List<VariableExpressionExecutor> variableExpressionExecutors, OutputStream outputStream, int metaPosition, ProcessingMode processingMode, boolean outputExpectsExpiredEvents, boolean groupBy, SiddhiQueryContext siddhiQueryContext) {
List<AttributeProcessor> attributeProcessorList = new ArrayList<>();
StreamDefinition outputDefinition = StreamDefinition.id(id);
outputDefinition.setQueryContextStartIndex(outputStream.getQueryContextStartIndex());
outputDefinition.setQueryContextEndIndex(outputStream.getQueryContextEndIndex());
List<OutputAttribute> outputAttributes = selector.getSelectionList();
if (selector.getSelectionList().size() == 0) {
if (metaComplexEvent instanceof MetaStreamEvent) {
List<Attribute> attributeList = ((MetaStreamEvent) metaComplexEvent).getLastInputDefinition().getAttributeList();
for (Attribute attribute : attributeList) {
Variable variable = new Variable(attribute.getName());
variable.setQueryContextStartIndex(selector.getQueryContextStartIndex());
variable.setQueryContextEndIndex(selector.getQueryContextEndIndex());
OutputAttribute outputAttribute = new OutputAttribute(variable);
outputAttribute.setQueryContextStartIndex(selector.getQueryContextStartIndex());
outputAttribute.setQueryContextEndIndex(selector.getQueryContextEndIndex());
outputAttributes.add(outputAttribute);
}
} else {
int position = 0;
for (MetaStreamEvent metaStreamEvent : ((MetaStateEvent) metaComplexEvent).getMetaStreamEvents()) {
if (metaPosition == SiddhiConstants.UNKNOWN_STATE || metaPosition == position) {
List<Attribute> attributeList = metaStreamEvent.getLastInputDefinition().getAttributeList();
for (Attribute attribute : attributeList) {
Variable variable = new Variable(attribute.getName());
variable.setQueryContextStartIndex(selector.getQueryContextStartIndex());
variable.setQueryContextEndIndex(selector.getQueryContextEndIndex());
OutputAttribute outputAttribute = new OutputAttribute(variable);
outputAttribute.setQueryContextStartIndex(selector.getQueryContextStartIndex());
outputAttribute.setQueryContextEndIndex(selector.getQueryContextEndIndex());
if (!outputAttributes.contains(outputAttribute)) {
outputAttributes.add(outputAttribute);
} else {
List<AbstractDefinition> definitions = new ArrayList<>();
for (MetaStreamEvent aMetaStreamEvent : ((MetaStateEvent) metaComplexEvent).getMetaStreamEvents()) {
definitions.add(aMetaStreamEvent.getLastInputDefinition());
}
throw new DuplicateAttributeException("Duplicate attribute exist in streams " + definitions, outputStream.getQueryContextStartIndex(), outputStream.getQueryContextEndIndex());
}
}
}
++position;
}
}
}
int i = 0;
for (OutputAttribute outputAttribute : outputAttributes) {
ExpressionExecutor expressionExecutor = ExpressionParser.parseExpression(outputAttribute.getExpression(), metaComplexEvent, SiddhiConstants.UNKNOWN_STATE, tableMap, variableExpressionExecutors, groupBy, 0, processingMode, outputExpectsExpiredEvents, siddhiQueryContext);
if (expressionExecutor instanceof VariableExpressionExecutor) {
// for variables we will directly put
// value at conversion stage
VariableExpressionExecutor executor = ((VariableExpressionExecutor) expressionExecutor);
if (metaComplexEvent instanceof MetaStateEvent) {
((MetaStateEvent) metaComplexEvent).addOutputDataAllowingDuplicate(new MetaStateEventAttribute(executor.getAttribute(), executor.getPosition()));
} else {
((MetaStreamEvent) metaComplexEvent).addOutputDataAllowingDuplicate(executor.getAttribute());
}
outputDefinition.attribute(outputAttribute.getRename(), ((VariableExpressionExecutor) expressionExecutor).getAttribute().getType());
} else {
// To maintain output variable positions
if (metaComplexEvent instanceof MetaStateEvent) {
((MetaStateEvent) metaComplexEvent).addOutputDataAllowingDuplicate(null);
} else {
((MetaStreamEvent) metaComplexEvent).addOutputDataAllowingDuplicate(null);
}
AttributeProcessor attributeProcessor = new AttributeProcessor(expressionExecutor);
attributeProcessor.setOutputPosition(i);
attributeProcessorList.add(attributeProcessor);
outputDefinition.attribute(outputAttribute.getRename(), attributeProcessor.getOutputType());
}
i++;
}
metaComplexEvent.setOutputDefinition(outputDefinition);
return attributeProcessorList;
}
use of io.siddhi.query.api.expression.Variable in project siddhi by wso2.
the class SelectorParser method parse.
/**
* Parse Selector portion of a query and return corresponding QuerySelector.
*
* @param selector selector to be parsed
* @param outputStream output stream
* @param metaComplexEvent Meta event used to collect execution info of stream associated with query
* @param tableMap Table Map
* @param variableExpressionExecutors variable expression executors
* @param metaPosition helps to identify the meta position of aggregates
* @param processingMode processing mode of the query
* @param outputExpectsExpiredEvents is expired events sent as output
* @param siddhiQueryContext current siddhi query context
* @return QuerySelector
*/
public static QuerySelector parse(Selector selector, OutputStream outputStream, MetaComplexEvent metaComplexEvent, Map<String, Table> tableMap, List<VariableExpressionExecutor> variableExpressionExecutors, int metaPosition, ProcessingMode processingMode, boolean outputExpectsExpiredEvents, SiddhiQueryContext siddhiQueryContext) {
boolean currentOn = false;
boolean expiredOn = false;
String id = null;
if (outputStream.getOutputEventType() == OutputStream.OutputEventType.CURRENT_EVENTS || outputStream.getOutputEventType() == OutputStream.OutputEventType.ALL_EVENTS) {
currentOn = true;
}
if (outputStream.getOutputEventType() == OutputStream.OutputEventType.EXPIRED_EVENTS || outputStream.getOutputEventType() == OutputStream.OutputEventType.ALL_EVENTS) {
expiredOn = true;
}
boolean groupBy = !selector.getGroupByList().isEmpty();
id = outputStream.getId();
containsAggregatorThreadLocal.remove();
QuerySelector querySelector = new QuerySelector(id, selector, currentOn, expiredOn, siddhiQueryContext);
List<AttributeProcessor> attributeProcessors = getAttributeProcessors(selector, id, metaComplexEvent, tableMap, variableExpressionExecutors, outputStream, metaPosition, processingMode, outputExpectsExpiredEvents, groupBy, siddhiQueryContext);
querySelector.setAttributeProcessorList(attributeProcessors, "true".equals(containsAggregatorThreadLocal.get()));
containsAggregatorThreadLocal.remove();
ConditionExpressionExecutor havingCondition = generateHavingExecutor(selector.getHavingExpression(), metaComplexEvent, tableMap, variableExpressionExecutors, siddhiQueryContext);
querySelector.setHavingConditionExecutor(havingCondition, "true".equals(containsAggregatorThreadLocal.get()));
containsAggregatorThreadLocal.remove();
if (!selector.getGroupByList().isEmpty()) {
List<Expression> groupByExpressionList = selector.getGroupByList().stream().map(groupByVariable -> (Expression) groupByVariable).collect(Collectors.toList());
querySelector.setGroupByKeyGenerator(new GroupByKeyGenerator(groupByExpressionList, metaComplexEvent, SiddhiConstants.UNKNOWN_STATE, null, variableExpressionExecutors, siddhiQueryContext));
}
if (!selector.getOrderByList().isEmpty()) {
querySelector.setOrderByEventComparator(new OrderByEventComparator(selector.getOrderByList(), metaComplexEvent, SiddhiConstants.HAVING_STATE, null, variableExpressionExecutors, siddhiQueryContext));
}
if (selector.getLimit() != null) {
ExpressionExecutor expressionExecutor = ExpressionParser.parseExpression((Expression) selector.getLimit(), metaComplexEvent, SiddhiConstants.HAVING_STATE, tableMap, variableExpressionExecutors, false, 0, ProcessingMode.BATCH, false, siddhiQueryContext);
containsAggregatorThreadLocal.remove();
querySelector.setLimit(((Number) (((ConstantExpressionExecutor) expressionExecutor).getValue())).longValue());
}
if (selector.getOffset() != null) {
ExpressionExecutor expressionExecutor = ExpressionParser.parseExpression((Expression) selector.getOffset(), metaComplexEvent, SiddhiConstants.HAVING_STATE, tableMap, variableExpressionExecutors, false, 0, ProcessingMode.BATCH, false, siddhiQueryContext);
containsAggregatorThreadLocal.remove();
querySelector.setOffset(((Number) (((ConstantExpressionExecutor) expressionExecutor).getValue())).longValue());
}
return querySelector;
}
use of io.siddhi.query.api.expression.Variable in project siddhi by wso2.
the class SingleInputStreamParser method generateProcessor.
public static Processor generateProcessor(StreamHandler streamHandler, MetaComplexEvent metaEvent, List<VariableExpressionExecutor> variableExpressionExecutors, Map<String, Table> tableMap, boolean supportsBatchProcessing, boolean outputExpectsExpiredEvents, boolean findToBeExecuted, SiddhiQueryContext siddhiQueryContext) {
Expression[] parameters = streamHandler.getParameters();
MetaStreamEvent metaStreamEvent;
int stateIndex = SiddhiConstants.UNKNOWN_STATE;
if (metaEvent instanceof MetaStateEvent) {
stateIndex = ((MetaStateEvent) metaEvent).getStreamEventCount() - 1;
metaStreamEvent = ((MetaStateEvent) metaEvent).getMetaStreamEvent(stateIndex);
} else {
metaStreamEvent = (MetaStreamEvent) metaEvent;
}
if (streamHandler instanceof Window) {
metaStreamEvent.initializeOnAfterWindowData();
}
ExpressionExecutor[] attributeExpressionExecutors;
if (parameters != null) {
if (parameters.length > 0) {
attributeExpressionExecutors = new ExpressionExecutor[parameters.length];
for (int i = 0, parametersLength = parameters.length; i < parametersLength; i++) {
attributeExpressionExecutors[i] = ExpressionParser.parseExpression(parameters[i], metaEvent, stateIndex, tableMap, variableExpressionExecutors, false, SiddhiConstants.CURRENT, ProcessingMode.BATCH, false, siddhiQueryContext);
}
} else {
List<Attribute> attributeList = metaStreamEvent.getLastInputDefinition().getAttributeList();
int parameterSize = attributeList.size();
attributeExpressionExecutors = new ExpressionExecutor[parameterSize];
for (int i = 0; i < parameterSize; i++) {
attributeExpressionExecutors[i] = ExpressionParser.parseExpression(new Variable(attributeList.get(i).getName()), metaEvent, stateIndex, tableMap, variableExpressionExecutors, false, SiddhiConstants.CURRENT, ProcessingMode.BATCH, false, siddhiQueryContext);
}
}
} else {
attributeExpressionExecutors = new ExpressionExecutor[0];
}
ConfigReader configReader;
if (streamHandler instanceof Filter) {
return new FilterProcessor(attributeExpressionExecutors[0]);
} else if (streamHandler instanceof Window) {
WindowProcessor windowProcessor = (WindowProcessor) SiddhiClassLoader.loadExtensionImplementation((Extension) streamHandler, WindowProcessorExtensionHolder.getInstance(siddhiQueryContext.getSiddhiAppContext()));
configReader = siddhiQueryContext.getSiddhiContext().getConfigManager().generateConfigReader(((Window) streamHandler).getNamespace(), ((Window) streamHandler).getName());
windowProcessor.initProcessor(metaStreamEvent, attributeExpressionExecutors, configReader, outputExpectsExpiredEvents, findToBeExecuted, false, streamHandler, siddhiQueryContext);
return windowProcessor;
} else if (streamHandler instanceof StreamFunction) {
AbstractStreamProcessor abstractStreamProcessor;
configReader = siddhiQueryContext.getSiddhiContext().getConfigManager().generateConfigReader(((StreamFunction) streamHandler).getNamespace(), ((StreamFunction) streamHandler).getName());
if (supportsBatchProcessing) {
try {
abstractStreamProcessor = (StreamProcessor) SiddhiClassLoader.loadExtensionImplementation((Extension) streamHandler, StreamProcessorExtensionHolder.getInstance(siddhiQueryContext.getSiddhiAppContext()));
abstractStreamProcessor.initProcessor(metaStreamEvent, attributeExpressionExecutors, configReader, outputExpectsExpiredEvents, false, false, streamHandler, siddhiQueryContext);
return abstractStreamProcessor;
} catch (SiddhiAppCreationException e) {
if (!e.isClassLoadingIssue()) {
ExceptionUtil.populateQueryContext(e, streamHandler, siddhiQueryContext.getSiddhiAppContext(), siddhiQueryContext);
throw e;
}
}
}
abstractStreamProcessor = (StreamFunctionProcessor) SiddhiClassLoader.loadExtensionImplementation((Extension) streamHandler, StreamFunctionProcessorExtensionHolder.getInstance(siddhiQueryContext.getSiddhiAppContext()));
abstractStreamProcessor.initProcessor(metaStreamEvent, attributeExpressionExecutors, configReader, outputExpectsExpiredEvents, false, false, streamHandler, siddhiQueryContext);
return abstractStreamProcessor;
} else {
throw new SiddhiAppCreationException(streamHandler.getClass().getName() + " is not supported", streamHandler.getQueryContextStartIndex(), streamHandler.getQueryContextEndIndex());
}
}
use of io.siddhi.query.api.expression.Variable in project siddhi by wso2.
the class AggregationParser method initAggregateQueryExecutor.
private static Map<TimePeriod.Duration, Processor> initAggregateQueryExecutor(List<TimePeriod.Duration> aggregationDurations, Map<TimePeriod.Duration, List<ExpressionExecutor>> processExpressionExecutorsMap, StreamDefinition incomingOutputStreamDefinition, boolean isDistributed, String shardID, boolean isProcessingOnExternalTime, SiddhiQueryContext siddhiQueryContext, AggregationDefinition aggregationDefinition, ConfigManager configManager, Map<TimePeriod.Duration, Table> aggregationTables, List<Variable> groupByVariableList) {
Map<TimePeriod.Duration, Processor> cudProcessors = new LinkedHashMap<>();
String datasourceName = AnnotationHelper.getAnnotationElement(SiddhiConstants.NAMESPACE_STORE, "datasource", aggregationDefinition.getAnnotations()).getValue();
if (datasourceName == null || datasourceName.isEmpty()) {
throw new SiddhiAppCreationException("Datasource configuration must be provided inorder to use persisted " + "aggregation mode");
}
Database databaseType = getDatabaseType(configManager, datasourceName);
if (log.isDebugEnabled()) {
log.debug("Database type " + databaseType);
}
SiddhiAppContext cudSiddhiAppContext = new SiddhiAppContext();
SiddhiContext context = new SiddhiContext();
context.setConfigManager(configManager);
cudSiddhiAppContext.setSiddhiContext(context);
StringConstant datasource = new StringConstant(datasourceName);
ConstantExpressionExecutor datasourceExecutor = new ConstantExpressionExecutor(datasource.getValue(), Attribute.Type.STRING);
Expression[] streamHandler;
ExpressionExecutor[] cudStreamProcessorInputVariables;
if (isProcessingOnExternalTime) {
streamHandler = new Expression[7];
} else {
streamHandler = new Expression[5];
}
try {
DBAggregationQueryConfigurationEntry dbAggregationQueryConfigurationEntry = DBAggregationQueryUtil.lookupCurrentQueryConfigurationEntry(databaseType);
if (log.isDebugEnabled()) {
log.debug("CUD queries for aggregation " + aggregationDefinition.getId());
}
for (int i = aggregationDurations.size() - 1; i > 0; i--) {
if (aggregationDurations.get(i).ordinal() >= 3) {
if (log.isDebugEnabled()) {
log.debug(" Initializing cudProcessors for duration " + aggregationDurations.get(i));
}
String databaseSelectQuery = generateDatabaseQuery(processExpressionExecutorsMap.get(aggregationDurations.get(i)), dbAggregationQueryConfigurationEntry, incomingOutputStreamDefinition, isDistributed, shardID, isProcessingOnExternalTime, aggregationTables.get(aggregationDurations.get(i)), aggregationTables.get(aggregationDurations.get(i - 1)), groupByVariableList, aggregationDurations.get(i));
StringConstant selectQuery = new StringConstant(databaseSelectQuery);
if (log.isDebugEnabled()) {
log.debug(selectQuery);
}
ConstantExpressionExecutor selectExecutor = new ConstantExpressionExecutor(selectQuery.getValue(), Attribute.Type.STRING);
Map<Attribute, int[]> cudInputStreamAttributesMap = generateCUDInputStreamAttributes(isProcessingOnExternalTime);
if (isProcessingOnExternalTime) {
cudStreamProcessorInputVariables = new ExpressionExecutor[7];
} else {
cudStreamProcessorInputVariables = new ExpressionExecutor[5];
}
cudStreamProcessorInputVariables[0] = datasourceExecutor;
cudStreamProcessorInputVariables[1] = selectExecutor;
streamHandler[0] = datasource;
streamHandler[1] = selectQuery;
MetaStreamEvent metaStreamEvent = generateCUDMetaStreamEvent(isProcessingOnExternalTime);
StreamDefinition outputStream = new StreamDefinition();
VariableExpressionExecutor variableExpressionExecutor;
int j = 0;
for (Map.Entry<Attribute, int[]> entry : cudInputStreamAttributesMap.entrySet()) {
Attribute attribute = entry.getKey();
Variable timestampVariable = new Variable(attribute.getName());
for (int position : entry.getValue()) {
streamHandler[position + 2] = timestampVariable;
variableExpressionExecutor = new VariableExpressionExecutor(attribute, 0, 0);
variableExpressionExecutor.setPosition(new int[] { 2, j });
cudStreamProcessorInputVariables[position + 2] = variableExpressionExecutor;
}
outputStream.attribute(attribute.getName(), attribute.getType());
j++;
}
StreamFunction cudStreamFunction = new StreamFunction(NAMESPACE_RDBMS, FUNCTION_NAME_CUD, streamHandler);
cudProcessors.put(aggregationDurations.get(i), getCudProcessor(cudStreamFunction, siddhiQueryContext, metaStreamEvent, cudStreamProcessorInputVariables, aggregationDurations.get(i)));
}
}
return cudProcessors;
} catch (CannotLoadConfigurationException e) {
throw new SiddhiAppCreationException("Error occurred while initializing the persisted incremental " + "aggregation. Could not load the db quires for database type " + databaseType);
}
}
use of io.siddhi.query.api.expression.Variable in project siddhi by wso2.
the class AggregationParser method initDefaultTables.
private static HashMap<TimePeriod.Duration, Table> initDefaultTables(String aggregatorName, List<TimePeriod.Duration> aggregationDurations, StreamDefinition streamDefinition, SiddhiAppRuntimeBuilder siddhiAppRuntimeBuilder, List<Annotation> annotations, List<Variable> groupByVariableList, boolean isProcessingOnExternalTime, boolean enablePartitioning) {
HashMap<TimePeriod.Duration, Table> aggregationTableMap = new HashMap<>();
// Create annotations for primary key
Annotation primaryKeyAnnotation = new Annotation(SiddhiConstants.ANNOTATION_PRIMARY_KEY);
primaryKeyAnnotation.element(null, AGG_START_TIMESTAMP_COL);
if (enablePartitioning) {
primaryKeyAnnotation.element(null, AGG_SHARD_ID_COL);
}
if (isProcessingOnExternalTime) {
primaryKeyAnnotation.element(null, AGG_EXTERNAL_TIMESTAMP_COL);
}
for (Variable groupByVariable : groupByVariableList) {
primaryKeyAnnotation.element(null, groupByVariable.getAttributeName());
}
if (streamDefinition.getAttributeList().contains(new Attribute(AGG_LAST_TIMESTAMP_COL, Attribute.Type.LONG))) {
Annotation indexAnnotation = new Annotation(SiddhiConstants.ANNOTATION_INDEX);
indexAnnotation.element(null, AGG_LAST_TIMESTAMP_COL);
annotations.add(indexAnnotation);
}
annotations.add(primaryKeyAnnotation);
for (TimePeriod.Duration duration : aggregationDurations) {
String tableId = aggregatorName + "_" + duration.toString();
TableDefinition tableDefinition = TableDefinition.id(tableId);
for (Attribute attribute : streamDefinition.getAttributeList()) {
tableDefinition.attribute(attribute.getName(), attribute.getType());
}
annotations.forEach(tableDefinition::annotation);
siddhiAppRuntimeBuilder.defineTable(tableDefinition);
aggregationTableMap.put(duration, siddhiAppRuntimeBuilder.getTableMap().get(tableId));
}
return aggregationTableMap;
}
Aggregations