use of io.siddhi.core.event.stream.MetaStreamEvent in project siddhi by wso2.
the class ExpressionWindowProcessor method process.
@Override
protected void process(ComplexEventChunk<StreamEvent> streamEventChunk, Processor nextProcessor, StreamEventCloner streamEventCloner, WindowState state) {
if (expressionStringExecutor == null && !init) {
MetaStateEvent metaStateEvent = new MetaStateEvent(new MetaStreamEvent[] { metaStreamEvent, metaStreamEvent, metaStreamEvent });
QueryParserHelper.updateVariablePosition(metaStateEvent, variableExpressionExecutors);
init = true;
}
synchronized (state) {
long currentTime = siddhiQueryContext.getSiddhiAppContext().getTimestampGenerator().currentTime();
while (streamEventChunk.hasNext()) {
StreamEvent streamEvent = streamEventChunk.next();
if (expressionStringExecutor != null) {
String expressionStringNew = (String) expressionStringExecutor.execute(streamEvent);
if (!expressionStringNew.equals(expressionString)) {
expressionString = expressionStringNew;
processAllExpiredEvents(streamEventChunk, state, currentTime);
}
}
StreamEvent clonedEvent = streamEventCloner.copyStreamEvent(streamEvent);
clonedEvent.setType(StreamEvent.Type.EXPIRED);
processStreamEvent(streamEventChunk, state, currentTime, clonedEvent);
}
}
nextProcessor.process(streamEventChunk);
}
use of io.siddhi.core.event.stream.MetaStreamEvent in project siddhi by wso2.
the class SnapshotableEventQueueTestCase method incrementalPersistenceTest4.
@Test
public void incrementalPersistenceTest4() throws InterruptedException, IOException, ClassNotFoundException {
MetaStreamEvent metaStreamEvent = new MetaStreamEvent();
metaStreamEvent.addOutputData(new Attribute("symbol", Attribute.Type.STRING));
metaStreamEvent.addOutputData(new Attribute("price", Attribute.Type.FLOAT));
metaStreamEvent.addOutputData(new Attribute("volume", Attribute.Type.LONG));
StreamEventCloner streamEventCloner = new StreamEventCloner(metaStreamEvent, new StreamEventFactory(metaStreamEvent));
SnapshotableStreamEventQueue snapshotableStreamEventQueue = new SnapshotableStreamEventQueue(new StreamEventClonerHolder(streamEventCloner));
StreamEvent streamEvent = new StreamEvent(metaStreamEvent.getBeforeWindowData().size(), metaStreamEvent.getOnAfterWindowData().size(), metaStreamEvent.getOutputData().size());
streamEvent.setOutputData(new Object[] { "IBM", 500.6f, 1 });
for (int i = 0; i < 10; i++) {
streamEvent.getOutputData()[2] = i;
snapshotableStreamEventQueue.add(streamEventCloner.copyStreamEvent(streamEvent));
}
HashMap<Long, String> snapshots = new HashMap<>();
Snapshot snapshot1 = snapshotableStreamEventQueue.getSnapshot();
StreamEvent streamEvents = (StreamEvent) snapshot1.getState();
Assert.assertTrue(streamEvents != null);
snapshots.put(3L, toString(snapshot1));
snapshotableStreamEventQueue.poll();
snapshotableStreamEventQueue.poll();
snapshotableStreamEventQueue.next();
snapshotableStreamEventQueue.next();
snapshotableStreamEventQueue.next();
for (int i = 7; i < 10; i++) {
snapshotableStreamEventQueue.poll();
}
Snapshot snapshot2 = snapshotableStreamEventQueue.getSnapshot();
ArrayList<Operation> operationLog = (ArrayList<Operation>) snapshot2.getState();
Assert.assertTrue(operationLog != null);
snapshots.put(4L, toString(snapshot2));
for (int i = 10; i < 15; i++) {
streamEvent.getOutputData()[2] = i;
snapshotableStreamEventQueue.add(streamEventCloner.copyStreamEvent(streamEvent));
}
Snapshot snapshot3 = snapshotableStreamEventQueue.getSnapshot();
operationLog = (ArrayList<Operation>) snapshot3.getState();
Assert.assertTrue(operationLog != null);
snapshots.put(5L, toString(snapshot3));
SnapshotableStreamEventQueue snapshotableStreamEventQueue2 = new SnapshotableStreamEventQueue(new StreamEventClonerHolder(streamEventCloner));
SnapshotStateList snapshotStateList = new SnapshotStateList();
for (Map.Entry<Long, String> entry : snapshots.entrySet()) {
snapshotStateList.putSnapshotState(entry.getKey(), (Snapshot) fromString(entry.getValue()));
}
snapshotableStreamEventQueue2.restore(snapshotStateList);
Assert.assertEquals(snapshotableStreamEventQueue, snapshotableStreamEventQueue2);
}
use of io.siddhi.core.event.stream.MetaStreamEvent in project siddhi by wso2.
the class SnapshotableEventQueueTestCase method incrementalPersistenceTest1.
@Test
public void incrementalPersistenceTest1() throws InterruptedException, IOException, ClassNotFoundException {
MetaStreamEvent metaStreamEvent = new MetaStreamEvent();
metaStreamEvent.addOutputData(new Attribute("symbol", Attribute.Type.STRING));
metaStreamEvent.addOutputData(new Attribute("price", Attribute.Type.FLOAT));
metaStreamEvent.addOutputData(new Attribute("volume", Attribute.Type.LONG));
StreamEventCloner streamEventCloner = new StreamEventCloner(metaStreamEvent, new StreamEventFactory(metaStreamEvent));
SnapshotableStreamEventQueue snapshotableStreamEventQueue = new SnapshotableStreamEventQueue(new StreamEventClonerHolder(streamEventCloner));
StreamEvent streamEvent = new StreamEvent(metaStreamEvent.getBeforeWindowData().size(), metaStreamEvent.getOnAfterWindowData().size(), metaStreamEvent.getOutputData().size());
streamEvent.setOutputData(new Object[] { "IBM", 500.6f, 1 });
for (int i = 0; i < 20; i++) {
streamEvent.getOutputData()[2] = i;
snapshotableStreamEventQueue.add(streamEventCloner.copyStreamEvent(streamEvent));
}
HashMap<Long, String> snapshots = new HashMap<>();
Snapshot snapshot1 = snapshotableStreamEventQueue.getSnapshot();
StreamEvent streamEvents = (StreamEvent) snapshot1.getState();
Assert.assertTrue(streamEvents != null);
snapshots.put(3L, toString(snapshot1));
for (int i = 20; i < 40; i++) {
streamEvent.getOutputData()[2] = i;
snapshotableStreamEventQueue.add(streamEventCloner.copyStreamEvent(streamEvent));
}
Snapshot snapshot2 = snapshotableStreamEventQueue.getSnapshot();
ArrayList<Operation> operationLog = (ArrayList<Operation>) snapshot2.getState();
Assert.assertTrue(operationLog != null);
snapshots.put(4L, toString(snapshot2));
for (int i = 40; i < 80; i++) {
streamEvent.getOutputData()[2] = i;
snapshotableStreamEventQueue.add(streamEventCloner.copyStreamEvent(streamEvent));
}
Snapshot snapshot3 = snapshotableStreamEventQueue.getSnapshot();
operationLog = (ArrayList<Operation>) snapshot3.getState();
Assert.assertTrue(operationLog != null);
snapshots.put(5L, toString(snapshot3));
SnapshotableStreamEventQueue snapshotableStreamEventQueue2 = new SnapshotableStreamEventQueue(new StreamEventClonerHolder(streamEventCloner));
SnapshotStateList snapshotStateList = new SnapshotStateList();
for (Map.Entry<Long, String> entry : snapshots.entrySet()) {
snapshotStateList.putSnapshotState(entry.getKey(), (Snapshot) fromString(entry.getValue()));
}
snapshotableStreamEventQueue2.restore(snapshotStateList);
Assert.assertEquals(snapshotableStreamEventQueue, snapshotableStreamEventQueue2);
for (int i = 80; i < 130; i++) {
streamEvent.getOutputData()[2] = i;
snapshotableStreamEventQueue.add(streamEventCloner.copyStreamEvent(streamEvent));
}
Snapshot snapshot4 = snapshotableStreamEventQueue.getSnapshot();
streamEvents = (StreamEvent) snapshot4.getState();
Assert.assertTrue(streamEvents != null);
snapshots = new HashMap<>();
snapshots.put(6L, toString(snapshot4));
SnapshotableStreamEventQueue snapshotableStreamEventQueue3 = new SnapshotableStreamEventQueue(new StreamEventClonerHolder(streamEventCloner));
snapshotStateList = new SnapshotStateList();
for (Map.Entry<Long, String> entry : snapshots.entrySet()) {
snapshotStateList.putSnapshotState(entry.getKey(), (Snapshot) fromString(entry.getValue()));
}
snapshotableStreamEventQueue3.restore(snapshotStateList);
snapshotableStreamEventQueue.reset();
Assert.assertEquals(snapshotableStreamEventQueue, snapshotableStreamEventQueue3);
}
use of io.siddhi.core.event.stream.MetaStreamEvent in project siddhi by wso2.
the class OutputParser method constructOutputCallback.
public static OutputCallback constructOutputCallback(final OutputStream outStream, StreamDefinition outputStreamDefinition, Map<String, Table> tableMap, Map<String, Window> eventWindowMap, boolean convertToStreamEvent, SiddhiQueryContext siddhiQueryContext) {
String id = outStream.getId();
Table table = null;
Window window = null;
if (id != null) {
table = tableMap.get(id);
window = eventWindowMap.get(id);
}
StreamEventFactory streamEventFactory = null;
StreamEventConverter streamEventConverter = null;
MetaStreamEvent tableMetaStreamEvent = null;
if (table != null) {
tableMetaStreamEvent = new MetaStreamEvent();
tableMetaStreamEvent.setEventType(MetaStreamEvent.EventType.TABLE);
TableDefinition matchingTableDefinition = TableDefinition.id("");
for (Attribute attribute : outputStreamDefinition.getAttributeList()) {
tableMetaStreamEvent.addOutputData(attribute);
matchingTableDefinition.attribute(attribute.getName(), attribute.getType());
}
matchingTableDefinition.setQueryContextStartIndex(outStream.getQueryContextStartIndex());
matchingTableDefinition.setQueryContextEndIndex(outStream.getQueryContextEndIndex());
tableMetaStreamEvent.addInputDefinition(matchingTableDefinition);
streamEventFactory = new StreamEventFactory(tableMetaStreamEvent);
streamEventConverter = new ZeroStreamEventConverter();
}
// Construct CallBack
if (outStream instanceof InsertIntoStream || outStream instanceof ReturnStream) {
if (window != null) {
if (!siddhiQueryContext.isPartitioned()) {
return new InsertIntoWindowCallback(window, outputStreamDefinition, siddhiQueryContext.getName());
} else {
return new InsertIntoWindowEndPartitionCallback(window, outputStreamDefinition, siddhiQueryContext.getName());
}
} else if (table != null) {
DefinitionParserHelper.validateOutputStream(outputStreamDefinition, table.getTableDefinition());
return new InsertIntoTableCallback(table, outputStreamDefinition, convertToStreamEvent, streamEventFactory, streamEventConverter, siddhiQueryContext.getName());
} else {
if (!siddhiQueryContext.isPartitioned() || outputStreamDefinition.getId().startsWith("#")) {
return new InsertIntoStreamCallback(outputStreamDefinition, siddhiQueryContext.getName());
} else {
return new InsertIntoStreamEndPartitionCallback(outputStreamDefinition, siddhiQueryContext.getName());
}
}
} else if (outStream instanceof DeleteStream || outStream instanceof UpdateStream || outStream instanceof UpdateOrInsertStream) {
if (table != null) {
if (outStream instanceof UpdateStream) {
if (((UpdateStream) outStream).getUpdateSet() == null) {
TableDefinition tableDefinition = table.getTableDefinition();
for (Attribute attribute : outputStreamDefinition.getAttributeList()) {
if (!tableDefinition.getAttributeList().contains(attribute)) {
throw new SiddhiAppCreationException("Attribute " + attribute + " does not exist on " + "Event Table " + tableDefinition, outStream.getQueryContextStartIndex(), outStream.getQueryContextEndIndex());
}
}
}
}
if (outStream instanceof UpdateOrInsertStream) {
TableDefinition tableDefinition = table.getTableDefinition();
for (Attribute attribute : outputStreamDefinition.getAttributeList()) {
if (!tableDefinition.getAttributeList().contains(attribute)) {
throw new SiddhiAppCreationException("Attribute " + attribute + " does not exist on " + "Event Table " + tableDefinition, outStream.getQueryContextStartIndex(), outStream.getQueryContextEndIndex());
}
}
}
if (outStream instanceof DeleteStream) {
try {
MatchingMetaInfoHolder matchingMetaInfoHolder = MatcherParser.constructMatchingMetaStateHolder(tableMetaStreamEvent, 0, table.getTableDefinition(), 0);
CompiledCondition compiledCondition = table.compileCondition((((DeleteStream) outStream).getOnDeleteExpression()), matchingMetaInfoHolder, null, tableMap, siddhiQueryContext);
StateEventFactory stateEventFactory = new StateEventFactory(matchingMetaInfoHolder.getMetaStateEvent());
return new DeleteTableCallback(table, compiledCondition, matchingMetaInfoHolder.getMatchingStreamEventIndex(), convertToStreamEvent, stateEventFactory, streamEventFactory, streamEventConverter, siddhiQueryContext.getName());
} catch (SiddhiAppValidationException e) {
throw new SiddhiAppCreationException("Cannot create delete for table '" + outStream.getId() + "', " + e.getMessageWithOutContext(), e, e.getQueryContextStartIndex(), e.getQueryContextEndIndex(), siddhiQueryContext.getName(), siddhiQueryContext.getSiddhiAppContext().getSiddhiAppString());
}
} else if (outStream instanceof UpdateStream) {
try {
MatchingMetaInfoHolder matchingMetaInfoHolder = MatcherParser.constructMatchingMetaStateHolder(tableMetaStreamEvent, 0, table.getTableDefinition(), 0);
CompiledCondition compiledCondition = table.compileCondition((((UpdateStream) outStream).getOnUpdateExpression()), matchingMetaInfoHolder, null, tableMap, siddhiQueryContext);
UpdateSet updateSet = ((UpdateStream) outStream).getUpdateSet();
if (updateSet == null) {
updateSet = new UpdateSet();
for (Attribute attribute : matchingMetaInfoHolder.getMatchingStreamDefinition().getAttributeList()) {
updateSet.set(new Variable(attribute.getName()), new Variable(attribute.getName()));
}
}
CompiledUpdateSet compiledUpdateSet = table.compileUpdateSet(updateSet, matchingMetaInfoHolder, null, tableMap, siddhiQueryContext);
StateEventFactory stateEventFactory = new StateEventFactory(matchingMetaInfoHolder.getMetaStateEvent());
return new UpdateTableCallback(table, compiledCondition, compiledUpdateSet, matchingMetaInfoHolder.getMatchingStreamEventIndex(), convertToStreamEvent, stateEventFactory, streamEventFactory, streamEventConverter, siddhiQueryContext.getName());
} catch (SiddhiAppValidationException e) {
throw new SiddhiAppCreationException("Cannot create update for table '" + outStream.getId() + "', " + e.getMessageWithOutContext(), e, e.getQueryContextStartIndex(), e.getQueryContextEndIndex(), siddhiQueryContext.getSiddhiAppContext());
}
} else {
DefinitionParserHelper.validateOutputStream(outputStreamDefinition, table.getTableDefinition());
try {
MatchingMetaInfoHolder matchingMetaInfoHolder = MatcherParser.constructMatchingMetaStateHolder(tableMetaStreamEvent, 0, table.getTableDefinition(), 0);
CompiledCondition compiledCondition = table.compileCondition((((UpdateOrInsertStream) outStream).getOnUpdateExpression()), matchingMetaInfoHolder, null, tableMap, siddhiQueryContext);
UpdateSet updateSet = ((UpdateOrInsertStream) outStream).getUpdateSet();
if (updateSet == null) {
updateSet = new UpdateSet();
for (Attribute attribute : matchingMetaInfoHolder.getMatchingStreamDefinition().getAttributeList()) {
updateSet.set(new Variable(attribute.getName()), new Variable(attribute.getName()));
}
}
CompiledUpdateSet compiledUpdateSet = table.compileUpdateSet(updateSet, matchingMetaInfoHolder, null, tableMap, siddhiQueryContext);
StateEventFactory stateEventFactory = new StateEventFactory(matchingMetaInfoHolder.getMetaStateEvent());
return new UpdateOrInsertTableCallback(table, compiledCondition, compiledUpdateSet, matchingMetaInfoHolder.getMatchingStreamEventIndex(), convertToStreamEvent, stateEventFactory, streamEventFactory, streamEventConverter, siddhiQueryContext.getName());
} catch (SiddhiAppValidationException e) {
throw new SiddhiAppCreationException("Cannot create update or insert into for table '" + outStream.getId() + "', " + e.getMessageWithOutContext(), e, e.getQueryContextStartIndex(), e.getQueryContextEndIndex(), siddhiQueryContext.getSiddhiAppContext());
}
}
} else {
throw new SiddhiAppCreationException("Event table with id :" + id + " does not exist", outStream.getQueryContextStartIndex(), outStream.getQueryContextEndIndex());
}
} else {
throw new SiddhiAppCreationException(outStream.getClass().getName() + " not supported", outStream.getQueryContextStartIndex(), outStream.getQueryContextEndIndex());
}
}
use of io.siddhi.core.event.stream.MetaStreamEvent in project siddhi by wso2.
the class SingleInputStreamParser method parseInputStream.
/**
* Parse single InputStream and return SingleStreamRuntime
*
* @param inputStream single input stream to be parsed
* @param variableExpressionExecutors List to hold VariableExpressionExecutors to update after query parsing
* @param streamDefinitionMap Stream Definition Map
* @param tableDefinitionMap Table Definition Map
* @param windowDefinitionMap window definition map
* @param aggregationDefinitionMap aggregation definition map
* @param tableMap Table Map
* @param metaComplexEvent MetaComplexEvent
* @param processStreamReceiver ProcessStreamReceiver
* @param supportsBatchProcessing supports batch processing
* @param outputExpectsExpiredEvents is expired events sent as output
* @param findToBeExecuted find will be executed in the steam stores
* @param multiValue event has the possibility to produce multiple values
* @param siddhiQueryContext @return SingleStreamRuntime
*/
public static SingleStreamRuntime parseInputStream(SingleInputStream inputStream, List<VariableExpressionExecutor> variableExpressionExecutors, Map<String, AbstractDefinition> streamDefinitionMap, Map<String, AbstractDefinition> tableDefinitionMap, Map<String, AbstractDefinition> windowDefinitionMap, Map<String, AbstractDefinition> aggregationDefinitionMap, Map<String, Table> tableMap, MetaComplexEvent metaComplexEvent, ProcessStreamReceiver processStreamReceiver, boolean supportsBatchProcessing, boolean outputExpectsExpiredEvents, boolean findToBeExecuted, boolean multiValue, SiddhiQueryContext siddhiQueryContext) {
Processor processor = null;
EntryValveProcessor entryValveProcessor = null;
ProcessingMode processingMode = ProcessingMode.BATCH;
boolean first = true;
MetaStreamEvent metaStreamEvent;
if (metaComplexEvent instanceof MetaStateEvent) {
metaStreamEvent = new MetaStreamEvent();
((MetaStateEvent) metaComplexEvent).addEvent(metaStreamEvent);
initMetaStreamEvent(inputStream, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, multiValue, metaStreamEvent);
} else {
metaStreamEvent = (MetaStreamEvent) metaComplexEvent;
initMetaStreamEvent(inputStream, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, multiValue, metaStreamEvent);
}
// A window cannot be defined for a window stream
if (!inputStream.getStreamHandlers().isEmpty() && windowDefinitionMap != null && windowDefinitionMap.containsKey(inputStream.getStreamId())) {
for (StreamHandler handler : inputStream.getStreamHandlers()) {
if (handler instanceof Window) {
throw new OperationNotSupportedException("Cannot create " + ((Window) handler).getName() + " " + "window for the window stream " + inputStream.getStreamId());
}
}
}
if (!inputStream.getStreamHandlers().isEmpty()) {
for (StreamHandler handler : inputStream.getStreamHandlers()) {
Processor currentProcessor = generateProcessor(handler, metaComplexEvent, variableExpressionExecutors, tableMap, supportsBatchProcessing, outputExpectsExpiredEvents, findToBeExecuted, siddhiQueryContext);
if (currentProcessor instanceof SchedulingProcessor) {
if (entryValveProcessor == null) {
entryValveProcessor = new EntryValveProcessor(siddhiQueryContext.getSiddhiAppContext());
if (first) {
processor = entryValveProcessor;
first = false;
} else {
processor.setToLast(entryValveProcessor);
}
}
Scheduler scheduler = SchedulerParser.parse(entryValveProcessor, siddhiQueryContext);
((SchedulingProcessor) currentProcessor).setScheduler(scheduler);
}
if (currentProcessor instanceof AbstractStreamProcessor) {
processingMode = ProcessingMode.findUpdatedProcessingMode(processingMode, ((AbstractStreamProcessor) currentProcessor).getProcessingMode());
}
if (first) {
processor = currentProcessor;
first = false;
} else {
processor.setToLast(currentProcessor);
}
}
}
metaStreamEvent.initializeOnAfterWindowData();
return new SingleStreamRuntime(processStreamReceiver, processor, processingMode, metaComplexEvent);
}
Aggregations