use of io.siddhi.core.table.Table in project siddhi by wso2.
the class QueryParser method parse.
/**
* Parse a query and return corresponding QueryRuntime.
*
* @param query query to be parsed.
* @param siddhiAppContext associated Siddhi app context.
* @param streamDefinitionMap keyvalue containing user given stream definitions.
* @param tableDefinitionMap keyvalue containing table definitions.
* @param windowDefinitionMap keyvalue containing window definition map.
* @param aggregationDefinitionMap keyvalue containing aggregation definition map.
* @param tableMap keyvalue containing event tables.
* @param aggregationMap keyvalue containing aggrigation runtimes.
* @param windowMap keyvalue containing event window map.
* @param lockSynchronizer Lock synchronizer for sync the lock across queries.
* @param queryIndex query index to identify unknown query by number
* @param partitioned is the query partitioned
* @param partitionId The ID of the partition
* @return queryRuntime
*/
public static QueryRuntimeImpl parse(Query query, SiddhiAppContext siddhiAppContext, Map<String, AbstractDefinition> streamDefinitionMap, Map<String, AbstractDefinition> tableDefinitionMap, Map<String, AbstractDefinition> windowDefinitionMap, Map<String, AbstractDefinition> aggregationDefinitionMap, Map<String, Table> tableMap, Map<String, AggregationRuntime> aggregationMap, Map<String, Window> windowMap, LockSynchronizer lockSynchronizer, String queryIndex, boolean partitioned, String partitionId) {
List<VariableExpressionExecutor> executors = new ArrayList<>();
QueryRuntimeImpl queryRuntime;
Element nameElement = null;
LatencyTracker latencyTracker = null;
LockWrapper lockWrapper = null;
try {
nameElement = AnnotationHelper.getAnnotationElement("info", "name", query.getAnnotations());
String queryName;
if (nameElement != null) {
queryName = nameElement.getValue();
} else {
queryName = "query_" + queryIndex;
}
SiddhiQueryContext siddhiQueryContext = new SiddhiQueryContext(siddhiAppContext, queryName, partitionId);
siddhiQueryContext.setPartitioned(partitioned);
latencyTracker = QueryParserHelper.createLatencyTracker(siddhiAppContext, siddhiQueryContext.getName(), SiddhiConstants.METRIC_INFIX_QUERIES, null);
siddhiQueryContext.setLatencyTracker(latencyTracker);
OutputStream.OutputEventType outputEventType = query.getOutputStream().getOutputEventType();
if (query.getOutputRate() != null && query.getOutputRate() instanceof SnapshotOutputRate) {
if (outputEventType != OutputStream.OutputEventType.ALL_EVENTS) {
throw new SiddhiAppCreationException("As query '" + siddhiQueryContext.getName() + "' is performing snapshot rate limiting, it can only insert '" + OutputStream.OutputEventType.ALL_EVENTS + "' but it is inserting '" + outputEventType + "'!", query.getOutputStream().getQueryContextStartIndex(), query.getOutputStream().getQueryContextEndIndex());
}
}
siddhiQueryContext.setOutputEventType(outputEventType);
boolean outputExpectsExpiredEvents = false;
if (outputEventType != OutputStream.OutputEventType.CURRENT_EVENTS) {
outputExpectsExpiredEvents = true;
}
StreamRuntime streamRuntime = InputStreamParser.parse(query.getInputStream(), query, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, tableMap, windowMap, aggregationMap, executors, outputExpectsExpiredEvents, siddhiQueryContext);
QuerySelector selector;
if (streamRuntime.getQuerySelector() != null) {
selector = streamRuntime.getQuerySelector();
} else {
selector = SelectorParser.parse(query.getSelector(), query.getOutputStream(), streamRuntime.getMetaComplexEvent(), tableMap, executors, SiddhiConstants.UNKNOWN_STATE, streamRuntime.getProcessingMode(), outputExpectsExpiredEvents, siddhiQueryContext);
}
boolean isWindow = query.getInputStream() instanceof JoinInputStream;
if (!isWindow && query.getInputStream() instanceof SingleInputStream) {
for (StreamHandler streamHandler : ((SingleInputStream) query.getInputStream()).getStreamHandlers()) {
if (streamHandler instanceof io.siddhi.query.api.execution.query.input.handler.Window) {
isWindow = true;
break;
}
}
}
Element synchronizedElement = AnnotationHelper.getAnnotationElement("synchronized", null, query.getAnnotations());
if (synchronizedElement != null) {
if (!("false".equalsIgnoreCase(synchronizedElement.getValue()))) {
// Query LockWrapper does not need a unique
lockWrapper = new LockWrapper("");
// id since it will
// not be passed to the LockSynchronizer.
// LockWrapper does not have a default lock
lockWrapper.setLock(new ReentrantLock());
}
} else {
if (isWindow || !(streamRuntime instanceof SingleStreamRuntime)) {
if (streamRuntime instanceof JoinStreamRuntime) {
// If at least one Window is involved in the join, use the LockWrapper of that window
// for the query as well.
// If join is between two EventWindows, sync the locks of the LockWrapper of those windows
// and use either of them for query.
MetaStateEvent metaStateEvent = (MetaStateEvent) streamRuntime.getMetaComplexEvent();
MetaStreamEvent[] metaStreamEvents = metaStateEvent.getMetaStreamEvents();
if (metaStreamEvents[0].getEventType() == EventType.WINDOW && metaStreamEvents[1].getEventType() == EventType.WINDOW) {
LockWrapper leftLockWrapper = windowMap.get(metaStreamEvents[0].getLastInputDefinition().getId()).getLock();
LockWrapper rightLockWrapper = windowMap.get(metaStreamEvents[1].getLastInputDefinition().getId()).getLock();
if (!leftLockWrapper.equals(rightLockWrapper)) {
// Sync the lock across both wrappers
lockSynchronizer.sync(leftLockWrapper, rightLockWrapper);
}
// Can use either leftLockWrapper or rightLockWrapper since both of them will hold the
// same lock internally
// If either of their lock is updated later, the other lock also will be update by the
// LockSynchronizer.
lockWrapper = leftLockWrapper;
} else if (metaStreamEvents[0].getEventType() == EventType.WINDOW) {
// Share the same wrapper as the query lock wrapper
lockWrapper = windowMap.get(metaStreamEvents[0].getLastInputDefinition().getId()).getLock();
} else if (metaStreamEvents[1].getEventType() == EventType.WINDOW) {
// Share the same wrapper as the query lock wrapper
lockWrapper = windowMap.get(metaStreamEvents[1].getLastInputDefinition().getId()).getLock();
} else {
// Join does not contain any Window
// Query LockWrapper does not need a unique
lockWrapper = new LockWrapper("");
// id since
// it will not be passed to the LockSynchronizer.
// LockWrapper does not have a default lock
lockWrapper.setLock(new ReentrantLock());
}
} else {
lockWrapper = new LockWrapper("");
lockWrapper.setLock(new ReentrantLock());
}
}
}
OutputRateLimiter outputRateLimiter = OutputParser.constructOutputRateLimiter(query.getOutputStream().getId(), query.getOutputRate(), query.getSelector().getGroupByList().size() != 0, isWindow, siddhiQueryContext);
if (outputRateLimiter instanceof WrappedSnapshotOutputRateLimiter) {
selector.setBatchingEnabled(false);
}
boolean groupBy = !query.getSelector().getGroupByList().isEmpty();
OutputCallback outputCallback = OutputParser.constructOutputCallback(query.getOutputStream(), streamRuntime.getMetaComplexEvent().getOutputStreamDefinition(), tableMap, windowMap, !(streamRuntime instanceof SingleStreamRuntime) || groupBy, siddhiQueryContext);
QueryParserHelper.reduceMetaComplexEvent(streamRuntime.getMetaComplexEvent());
QueryParserHelper.updateVariablePosition(streamRuntime.getMetaComplexEvent(), executors);
QueryParserHelper.initStreamRuntime(streamRuntime, streamRuntime.getMetaComplexEvent(), lockWrapper, siddhiQueryContext.getName());
// Update cache compile selection variable expression executors
if (streamRuntime instanceof JoinStreamRuntime) {
streamRuntime.getSingleStreamRuntimes().forEach((singleStreamRuntime -> {
Processor processorChain = singleStreamRuntime.getProcessorChain();
if (processorChain instanceof JoinProcessor) {
CompiledSelection compiledSelection = ((JoinProcessor) processorChain).getCompiledSelection();
if (compiledSelection instanceof AbstractQueryableRecordTable.CompiledSelectionWithCache) {
List<VariableExpressionExecutor> variableExpressionExecutors = ((AbstractQueryableRecordTable.CompiledSelectionWithCache) compiledSelection).getVariableExpressionExecutorsForQuerySelector();
QueryParserHelper.updateVariablePosition(streamRuntime.getMetaComplexEvent(), variableExpressionExecutors);
}
}
}));
}
selector.setEventPopulator(StateEventPopulatorFactory.constructEventPopulator(streamRuntime.getMetaComplexEvent()));
queryRuntime = new QueryRuntimeImpl(query, streamRuntime, selector, outputRateLimiter, outputCallback, streamRuntime.getMetaComplexEvent(), siddhiQueryContext);
if (outputRateLimiter instanceof WrappedSnapshotOutputRateLimiter) {
selector.setBatchingEnabled(false);
((WrappedSnapshotOutputRateLimiter) outputRateLimiter).init(streamRuntime.getMetaComplexEvent().getOutputStreamDefinition().getAttributeList().size(), selector.getAttributeProcessorList(), streamRuntime.getMetaComplexEvent());
}
outputRateLimiter.init(lockWrapper, groupBy, siddhiQueryContext);
} catch (DuplicateDefinitionException e) {
if (nameElement != null) {
throw new DuplicateDefinitionException(e.getMessageWithOutContext() + ", when creating query " + nameElement.getValue(), e, e.getQueryContextStartIndex(), e.getQueryContextEndIndex(), siddhiAppContext.getName(), siddhiAppContext.getSiddhiAppString());
} else {
throw new DuplicateDefinitionException(e.getMessage(), e, e.getQueryContextStartIndex(), e.getQueryContextEndIndex(), siddhiAppContext.getName(), siddhiAppContext.getSiddhiAppString());
}
} catch (Throwable t) {
ExceptionUtil.populateQueryContext(t, query, siddhiAppContext);
throw t;
}
return queryRuntime;
}
use of io.siddhi.core.table.Table in project siddhi by wso2.
the class SelectorParser method parse.
/**
* Parse Selector portion of a query and return corresponding QuerySelector.
*
* @param selector selector to be parsed
* @param outputStream output stream
* @param metaComplexEvent Meta event used to collect execution info of stream associated with query
* @param tableMap Table Map
* @param variableExpressionExecutors variable expression executors
* @param metaPosition helps to identify the meta position of aggregates
* @param processingMode processing mode of the query
* @param outputExpectsExpiredEvents is expired events sent as output
* @param siddhiQueryContext current siddhi query context
* @return QuerySelector
*/
public static QuerySelector parse(Selector selector, OutputStream outputStream, MetaComplexEvent metaComplexEvent, Map<String, Table> tableMap, List<VariableExpressionExecutor> variableExpressionExecutors, int metaPosition, ProcessingMode processingMode, boolean outputExpectsExpiredEvents, SiddhiQueryContext siddhiQueryContext) {
boolean currentOn = false;
boolean expiredOn = false;
String id = null;
if (outputStream.getOutputEventType() == OutputStream.OutputEventType.CURRENT_EVENTS || outputStream.getOutputEventType() == OutputStream.OutputEventType.ALL_EVENTS) {
currentOn = true;
}
if (outputStream.getOutputEventType() == OutputStream.OutputEventType.EXPIRED_EVENTS || outputStream.getOutputEventType() == OutputStream.OutputEventType.ALL_EVENTS) {
expiredOn = true;
}
boolean groupBy = !selector.getGroupByList().isEmpty();
id = outputStream.getId();
containsAggregatorThreadLocal.remove();
QuerySelector querySelector = new QuerySelector(id, selector, currentOn, expiredOn, siddhiQueryContext);
List<AttributeProcessor> attributeProcessors = getAttributeProcessors(selector, id, metaComplexEvent, tableMap, variableExpressionExecutors, outputStream, metaPosition, processingMode, outputExpectsExpiredEvents, groupBy, siddhiQueryContext);
querySelector.setAttributeProcessorList(attributeProcessors, "true".equals(containsAggregatorThreadLocal.get()));
containsAggregatorThreadLocal.remove();
ConditionExpressionExecutor havingCondition = generateHavingExecutor(selector.getHavingExpression(), metaComplexEvent, tableMap, variableExpressionExecutors, siddhiQueryContext);
querySelector.setHavingConditionExecutor(havingCondition, "true".equals(containsAggregatorThreadLocal.get()));
containsAggregatorThreadLocal.remove();
if (!selector.getGroupByList().isEmpty()) {
List<Expression> groupByExpressionList = selector.getGroupByList().stream().map(groupByVariable -> (Expression) groupByVariable).collect(Collectors.toList());
querySelector.setGroupByKeyGenerator(new GroupByKeyGenerator(groupByExpressionList, metaComplexEvent, SiddhiConstants.UNKNOWN_STATE, null, variableExpressionExecutors, siddhiQueryContext));
}
if (!selector.getOrderByList().isEmpty()) {
querySelector.setOrderByEventComparator(new OrderByEventComparator(selector.getOrderByList(), metaComplexEvent, SiddhiConstants.HAVING_STATE, null, variableExpressionExecutors, siddhiQueryContext));
}
if (selector.getLimit() != null) {
ExpressionExecutor expressionExecutor = ExpressionParser.parseExpression((Expression) selector.getLimit(), metaComplexEvent, SiddhiConstants.HAVING_STATE, tableMap, variableExpressionExecutors, false, 0, ProcessingMode.BATCH, false, siddhiQueryContext);
containsAggregatorThreadLocal.remove();
querySelector.setLimit(((Number) (((ConstantExpressionExecutor) expressionExecutor).getValue())).longValue());
}
if (selector.getOffset() != null) {
ExpressionExecutor expressionExecutor = ExpressionParser.parseExpression((Expression) selector.getOffset(), metaComplexEvent, SiddhiConstants.HAVING_STATE, tableMap, variableExpressionExecutors, false, 0, ProcessingMode.BATCH, false, siddhiQueryContext);
containsAggregatorThreadLocal.remove();
querySelector.setOffset(((Number) (((ConstantExpressionExecutor) expressionExecutor).getValue())).longValue());
}
return querySelector;
}
use of io.siddhi.core.table.Table in project siddhi by wso2.
the class AggregationParser method initDefaultTables.
private static HashMap<TimePeriod.Duration, Table> initDefaultTables(String aggregatorName, List<TimePeriod.Duration> aggregationDurations, StreamDefinition streamDefinition, SiddhiAppRuntimeBuilder siddhiAppRuntimeBuilder, List<Annotation> annotations, List<Variable> groupByVariableList, boolean isProcessingOnExternalTime, boolean enablePartitioning) {
HashMap<TimePeriod.Duration, Table> aggregationTableMap = new HashMap<>();
// Create annotations for primary key
Annotation primaryKeyAnnotation = new Annotation(SiddhiConstants.ANNOTATION_PRIMARY_KEY);
primaryKeyAnnotation.element(null, AGG_START_TIMESTAMP_COL);
if (enablePartitioning) {
primaryKeyAnnotation.element(null, AGG_SHARD_ID_COL);
}
if (isProcessingOnExternalTime) {
primaryKeyAnnotation.element(null, AGG_EXTERNAL_TIMESTAMP_COL);
}
for (Variable groupByVariable : groupByVariableList) {
primaryKeyAnnotation.element(null, groupByVariable.getAttributeName());
}
if (streamDefinition.getAttributeList().contains(new Attribute(AGG_LAST_TIMESTAMP_COL, Attribute.Type.LONG))) {
Annotation indexAnnotation = new Annotation(SiddhiConstants.ANNOTATION_INDEX);
indexAnnotation.element(null, AGG_LAST_TIMESTAMP_COL);
annotations.add(indexAnnotation);
}
annotations.add(primaryKeyAnnotation);
for (TimePeriod.Duration duration : aggregationDurations) {
String tableId = aggregatorName + "_" + duration.toString();
TableDefinition tableDefinition = TableDefinition.id(tableId);
for (Attribute attribute : streamDefinition.getAttributeList()) {
tableDefinition.attribute(attribute.getName(), attribute.getType());
}
annotations.forEach(tableDefinition::annotation);
siddhiAppRuntimeBuilder.defineTable(tableDefinition);
aggregationTableMap.put(duration, siddhiAppRuntimeBuilder.getTableMap().get(tableId));
}
return aggregationTableMap;
}
use of io.siddhi.core.table.Table in project siddhi by wso2.
the class IncrementalAggregateCompileCondition method find.
public StreamEvent find(StateEvent matchingEvent, Map<TimePeriod.Duration, Executor> incrementalExecutorMap, Map<TimePeriod.Duration, List<ExpressionExecutor>> aggregateProcessingExecutorsMap, Map<TimePeriod.Duration, GroupByKeyGenerator> groupByKeyGeneratorMap, ExpressionExecutor shouldUpdateTimestamp, String timeZone) {
ComplexEventChunk<StreamEvent> complexEventChunkToHoldWithinMatches = new ComplexEventChunk<>();
// Create matching event if it is on-demand query
int additionTimestampAttributesSize = this.timestampFilterExecutors.size() + 2;
Long[] timestampFilters = new Long[additionTimestampAttributesSize];
if (matchingEvent.getStreamEvent(0) == null) {
StreamEvent streamEvent = new StreamEvent(0, additionTimestampAttributesSize, 0);
matchingEvent.addEvent(0, streamEvent);
}
Long[] startTimeEndTime = (Long[]) startTimeEndTimeExpressionExecutor.execute(matchingEvent);
if (startTimeEndTime == null) {
throw new SiddhiAppRuntimeException("Start and end times for within duration cannot be retrieved");
}
timestampFilters[0] = startTimeEndTime[0];
timestampFilters[1] = startTimeEndTime[1];
if (isDistributed) {
for (int i = 0; i < additionTimestampAttributesSize - 2; i++) {
timestampFilters[i + 2] = ((Long) this.timestampFilterExecutors.get(i).execute(matchingEvent));
}
}
complexEventPopulater.populateComplexEvent(matchingEvent.getStreamEvent(0), timestampFilters);
// Get all the aggregates within the given duration, from table corresponding to "per" duration
// Retrieve per value
String perValueAsString = perExpressionExecutor.execute(matchingEvent).toString();
TimePeriod.Duration perValue;
try {
// Per time function verification
perValue = normalizeDuration(perValueAsString);
} catch (SiddhiAppValidationException e) {
throw new SiddhiAppRuntimeException("Aggregation Query's per value is expected to be of a valid time function of the " + "following " + TimePeriod.Duration.SECONDS + ", " + TimePeriod.Duration.MINUTES + ", " + TimePeriod.Duration.HOURS + ", " + TimePeriod.Duration.DAYS + ", " + TimePeriod.Duration.MONTHS + ", " + TimePeriod.Duration.YEARS + ".");
}
if (!incrementalExecutorMap.keySet().contains(perValue)) {
throw new SiddhiAppRuntimeException("The aggregate values for " + perValue.toString() + " granularity cannot be provided since aggregation definition " + aggregationName + " does not contain " + perValue.toString() + " duration");
}
Table tableForPerDuration = aggregationTableMap.get(perValue);
StreamEvent withinMatchFromPersistedEvents;
if (isOptimisedLookup) {
withinMatchFromPersistedEvents = query(tableForPerDuration, matchingEvent, withinTableCompiledConditions.get(perValue), withinTableCompiledSelection.get(perValue), tableMetaStreamEvent.getLastInputDefinition().getAttributeList().toArray(new Attribute[0]));
} else {
withinMatchFromPersistedEvents = tableForPerDuration.find(matchingEvent, withinTableCompiledConditions.get(perValue));
}
complexEventChunkToHoldWithinMatches.add(withinMatchFromPersistedEvents);
// Optimization step.
long oldestInMemoryEventTimestamp = getOldestInMemoryEventTimestamp(incrementalExecutorMap, activeIncrementalDurations, perValue);
// If processing on external time, the in-memory data also needs to be queried
if (isProcessingOnExternalTime || requiresAggregatingInMemoryData(oldestInMemoryEventTimestamp, startTimeEndTime)) {
if (isDistributed) {
int perValueIndex = this.activeIncrementalDurations.indexOf(perValue);
if (perValueIndex != 0) {
Map<TimePeriod.Duration, CompiledCondition> lowerGranularityLookups = new HashMap<>();
for (int i = 0; i < perValueIndex; i++) {
TimePeriod.Duration key = this.activeIncrementalDurations.get(i);
lowerGranularityLookups.put(key, withinTableLowerGranularityCompileCondition.get(key));
}
List<StreamEvent> eventChunks = lowerGranularityLookups.entrySet().stream().map((entry) -> {
Table table = aggregationTableMap.get(entry.getKey());
if (isOptimisedLookup) {
return query(table, matchingEvent, entry.getValue(), withinTableCompiledSelection.get(entry.getKey()), tableMetaStreamEvent.getLastInputDefinition().getAttributeList().toArray(new Attribute[0]));
} else {
return table.find(matchingEvent, entry.getValue());
}
}).filter(Objects::nonNull).collect(Collectors.toList());
eventChunks.forEach(complexEventChunkToHoldWithinMatches::add);
}
} else {
TimePeriod.Duration rootDuration = activeIncrementalDurations.get(0);
IncrementalDataAggregator incrementalDataAggregator = new IncrementalDataAggregator(activeIncrementalDurations, perValue, oldestInMemoryEventTimestamp, aggregateProcessingExecutorsMap.get(rootDuration), shouldUpdateTimestamp, groupByKeyGeneratorMap.get(rootDuration) != null, tableMetaStreamEvent, timeZone);
ComplexEventChunk<StreamEvent> aggregatedInMemoryEventChunk;
// Aggregate in-memory data and create an event chunk out of it
aggregatedInMemoryEventChunk = incrementalDataAggregator.aggregateInMemoryData(incrementalExecutorMap);
// Get the in-memory aggregate data, which is within given duration
StreamEvent withinMatchFromInMemory = ((Operator) inMemoryStoreCompileCondition).find(matchingEvent, aggregatedInMemoryEventChunk, tableEventCloner);
complexEventChunkToHoldWithinMatches.add(withinMatchFromInMemory);
}
}
ComplexEventChunk<StreamEvent> processedEvents;
if (isDistributed || isProcessingOnExternalTime) {
List<ExpressionExecutor> expressionExecutors = aggregateProcessingExecutorsMap.get(perValue);
GroupByKeyGenerator groupByKeyGenerator = groupByKeyGeneratorMap.get(perValue);
OutOfOrderEventsDataAggregator outOfOrderEventsDataAggregator = new OutOfOrderEventsDataAggregator(expressionExecutors, shouldUpdateTimestamp, groupByKeyGenerator, tableMetaStreamEvent);
processedEvents = outOfOrderEventsDataAggregator.aggregateData(complexEventChunkToHoldWithinMatches);
} else {
processedEvents = complexEventChunkToHoldWithinMatches;
}
// Get the final event chunk from the data which is within given duration. This event chunk contains the values
// in the select clause of an aggregate definition.
ComplexEventChunk<StreamEvent> aggregateSelectionComplexEventChunk = createAggregateSelectionEventChunk(processedEvents, outputExpressionExecutors);
// Execute the on compile condition
return ((Operator) onCompiledCondition).find(matchingEvent, aggregateSelectionComplexEventChunk, aggregateEventCloner);
}
use of io.siddhi.core.table.Table in project siddhi by wso2.
the class InputManager method constructTableInputHandler.
public synchronized TableInputHandler constructTableInputHandler(String tableId) {
Table correspondingTable = tableMap.get(tableId);
if (correspondingTable == null) {
throw new DefinitionNotExistException("Table with table ID " + tableId + " has not been defined");
}
TableInputHandler tableInputHandler = new TableInputHandler(correspondingTable, siddhiAppContext);
tableInputHandlerMap.put(tableId, tableInputHandler);
return tableInputHandler;
}
Aggregations