use of org.wso2.siddhi.core.stream.input.source.Source in project siddhi by wso2.
the class InputEventHandler method sendEvents.
public void sendEvents(Event[] events) throws InterruptedException {
try {
if (latencyTracker != null && siddhiAppContext.isStatsEnabled()) {
latencyTracker.markOut();
}
String[] transportProperties = trpProperties.get();
trpProperties.remove();
long currentTimestamp = timestampGenerator.currentTime();
for (Event event : events) {
if (event.getTimestamp() == -1) {
event.setTimestamp(currentTimestamp);
}
for (int i = 0; i < transportMapping.size(); i++) {
AttributeMapping attributeMapping = transportMapping.get(i);
event.getData()[attributeMapping.getPosition()] = transportProperties[i];
}
}
inputEventHandlerCallback.sendEvents(events);
} catch (RuntimeException e) {
LOG.error(ExceptionUtil.getMessageWithContext(e, siddhiAppContext) + " Error in applying transport property mapping for '" + sourceType + "' source at '" + inputHandler.getStreamId() + "' stream.", e);
} finally {
trpProperties.remove();
}
}
use of org.wso2.siddhi.core.stream.input.source.Source in project siddhi by wso2.
the class JoinInputStreamParser method parseInputStream.
public static StreamRuntime parseInputStream(JoinInputStream joinInputStream, SiddhiAppContext siddhiAppContext, Map<String, AbstractDefinition> streamDefinitionMap, Map<String, AbstractDefinition> tableDefinitionMap, Map<String, AbstractDefinition> windowDefinitionMap, Map<String, AbstractDefinition> aggregationDefinitionMap, Map<String, Table> tableMap, Map<String, Window> windowMap, Map<String, AggregationRuntime> aggregationMap, List<VariableExpressionExecutor> executors, LatencyTracker latencyTracker, boolean outputExpectsExpiredEvents, String queryName) {
try {
ProcessStreamReceiver leftProcessStreamReceiver;
ProcessStreamReceiver rightProcessStreamReceiver;
MetaStreamEvent leftMetaStreamEvent = new MetaStreamEvent();
MetaStreamEvent rightMetaStreamEvent = new MetaStreamEvent();
String leftInputStreamId = ((SingleInputStream) joinInputStream.getLeftInputStream()).getStreamId();
String rightInputStreamId = ((SingleInputStream) joinInputStream.getRightInputStream()).getStreamId();
boolean leftOuterJoinProcessor = false;
boolean rightOuterJoinProcessor = false;
if (joinInputStream.getAllStreamIds().size() == 2) {
setEventType(streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, leftMetaStreamEvent, leftInputStreamId);
setEventType(streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, rightMetaStreamEvent, rightInputStreamId);
leftProcessStreamReceiver = new ProcessStreamReceiver(leftInputStreamId, latencyTracker, queryName, siddhiAppContext);
leftProcessStreamReceiver.setBatchProcessingAllowed(leftMetaStreamEvent.getEventType() == WINDOW);
rightProcessStreamReceiver = new ProcessStreamReceiver(rightInputStreamId, latencyTracker, queryName, siddhiAppContext);
rightProcessStreamReceiver.setBatchProcessingAllowed(rightMetaStreamEvent.getEventType() == WINDOW);
if ((leftMetaStreamEvent.getEventType() == TABLE || leftMetaStreamEvent.getEventType() == AGGREGATE) && (rightMetaStreamEvent.getEventType() == TABLE || rightMetaStreamEvent.getEventType() == AGGREGATE)) {
throw new SiddhiAppCreationException("Both inputs of join " + leftInputStreamId + " and " + rightInputStreamId + " are from static sources");
}
if (leftMetaStreamEvent.getEventType() != AGGREGATE && rightMetaStreamEvent.getEventType() != AGGREGATE) {
if (joinInputStream.getPer() != null) {
throw new SiddhiAppCreationException("When joining " + leftInputStreamId + " and " + rightInputStreamId + " 'per' cannot be used as neither of them is an aggregation ");
} else if (joinInputStream.getWithin() != null) {
throw new SiddhiAppCreationException("When joining " + leftInputStreamId + " and " + rightInputStreamId + " 'within' cannot be used as neither of them is an aggregation ");
}
}
} else {
if (windowDefinitionMap.containsKey(joinInputStream.getAllStreamIds().get(0))) {
leftMetaStreamEvent.setEventType(WINDOW);
rightMetaStreamEvent.setEventType(WINDOW);
rightProcessStreamReceiver = new MultiProcessStreamReceiver(joinInputStream.getAllStreamIds().get(0), 1, latencyTracker, queryName, siddhiAppContext);
rightProcessStreamReceiver.setBatchProcessingAllowed(true);
leftProcessStreamReceiver = rightProcessStreamReceiver;
} else if (streamDefinitionMap.containsKey(joinInputStream.getAllStreamIds().get(0))) {
rightProcessStreamReceiver = new MultiProcessStreamReceiver(joinInputStream.getAllStreamIds().get(0), 2, latencyTracker, queryName, siddhiAppContext);
leftProcessStreamReceiver = rightProcessStreamReceiver;
} else {
throw new SiddhiAppCreationException("Input of join is from static source " + leftInputStreamId + " and " + rightInputStreamId);
}
}
SingleStreamRuntime leftStreamRuntime = SingleInputStreamParser.parseInputStream((SingleInputStream) joinInputStream.getLeftInputStream(), siddhiAppContext, executors, streamDefinitionMap, leftMetaStreamEvent.getEventType() != TABLE ? null : tableDefinitionMap, leftMetaStreamEvent.getEventType() != WINDOW ? null : windowDefinitionMap, leftMetaStreamEvent.getEventType() != AGGREGATE ? null : aggregationDefinitionMap, tableMap, leftMetaStreamEvent, leftProcessStreamReceiver, true, outputExpectsExpiredEvents, queryName);
for (VariableExpressionExecutor variableExpressionExecutor : executors) {
variableExpressionExecutor.getPosition()[SiddhiConstants.STREAM_EVENT_CHAIN_INDEX] = 0;
}
int size = executors.size();
SingleStreamRuntime rightStreamRuntime = SingleInputStreamParser.parseInputStream((SingleInputStream) joinInputStream.getRightInputStream(), siddhiAppContext, executors, streamDefinitionMap, rightMetaStreamEvent.getEventType() != TABLE ? null : tableDefinitionMap, rightMetaStreamEvent.getEventType() != WINDOW ? null : windowDefinitionMap, rightMetaStreamEvent.getEventType() != AGGREGATE ? null : aggregationDefinitionMap, tableMap, rightMetaStreamEvent, rightProcessStreamReceiver, true, outputExpectsExpiredEvents, queryName);
for (int i = size; i < executors.size(); i++) {
VariableExpressionExecutor variableExpressionExecutor = executors.get(i);
variableExpressionExecutor.getPosition()[SiddhiConstants.STREAM_EVENT_CHAIN_INDEX] = 1;
}
setStreamRuntimeProcessorChain(leftMetaStreamEvent, leftStreamRuntime, leftInputStreamId, tableMap, windowMap, aggregationMap, executors, outputExpectsExpiredEvents, queryName, joinInputStream.getWithin(), joinInputStream.getPer(), siddhiAppContext, joinInputStream.getLeftInputStream());
setStreamRuntimeProcessorChain(rightMetaStreamEvent, rightStreamRuntime, rightInputStreamId, tableMap, windowMap, aggregationMap, executors, outputExpectsExpiredEvents, queryName, joinInputStream.getWithin(), joinInputStream.getPer(), siddhiAppContext, joinInputStream.getRightInputStream());
MetaStateEvent metaStateEvent = new MetaStateEvent(2);
metaStateEvent.addEvent(leftMetaStreamEvent);
metaStateEvent.addEvent(rightMetaStreamEvent);
switch(joinInputStream.getType()) {
case FULL_OUTER_JOIN:
leftOuterJoinProcessor = true;
rightOuterJoinProcessor = true;
break;
case RIGHT_OUTER_JOIN:
rightOuterJoinProcessor = true;
break;
case LEFT_OUTER_JOIN:
leftOuterJoinProcessor = true;
break;
}
JoinProcessor leftPreJoinProcessor = new JoinProcessor(true, true, leftOuterJoinProcessor, 0);
JoinProcessor leftPostJoinProcessor = new JoinProcessor(true, false, leftOuterJoinProcessor, 0);
FindableProcessor leftFindableProcessor = insertJoinProcessorsAndGetFindable(leftPreJoinProcessor, leftPostJoinProcessor, leftStreamRuntime, siddhiAppContext, outputExpectsExpiredEvents, queryName, joinInputStream.getLeftInputStream());
JoinProcessor rightPreJoinProcessor = new JoinProcessor(false, true, rightOuterJoinProcessor, 1);
JoinProcessor rightPostJoinProcessor = new JoinProcessor(false, false, rightOuterJoinProcessor, 1);
FindableProcessor rightFindableProcessor = insertJoinProcessorsAndGetFindable(rightPreJoinProcessor, rightPostJoinProcessor, rightStreamRuntime, siddhiAppContext, outputExpectsExpiredEvents, queryName, joinInputStream.getRightInputStream());
leftPreJoinProcessor.setFindableProcessor(rightFindableProcessor);
leftPostJoinProcessor.setFindableProcessor(rightFindableProcessor);
rightPreJoinProcessor.setFindableProcessor(leftFindableProcessor);
rightPostJoinProcessor.setFindableProcessor(leftFindableProcessor);
Expression compareCondition = joinInputStream.getOnCompare();
if (compareCondition == null) {
compareCondition = Expression.value(true);
}
MatchingMetaInfoHolder rightMatchingMetaInfoHolder = MatcherParser.constructMatchingMetaStateHolder(metaStateEvent, 0, rightMetaStreamEvent.getLastInputDefinition(), UNKNOWN_STATE);
CompiledCondition leftCompiledCondition = rightFindableProcessor.compileCondition(compareCondition, rightMatchingMetaInfoHolder, siddhiAppContext, executors, tableMap, queryName);
MatchingMetaInfoHolder leftMatchingMetaInfoHolder = MatcherParser.constructMatchingMetaStateHolder(metaStateEvent, 1, leftMetaStreamEvent.getLastInputDefinition(), UNKNOWN_STATE);
CompiledCondition rightCompiledCondition = leftFindableProcessor.compileCondition(compareCondition, leftMatchingMetaInfoHolder, siddhiAppContext, executors, tableMap, queryName);
if (joinInputStream.getTrigger() != JoinInputStream.EventTrigger.LEFT) {
populateJoinProcessors(rightMetaStreamEvent, rightInputStreamId, rightPreJoinProcessor, rightPostJoinProcessor, rightCompiledCondition);
}
if (joinInputStream.getTrigger() != JoinInputStream.EventTrigger.RIGHT) {
populateJoinProcessors(leftMetaStreamEvent, leftInputStreamId, leftPreJoinProcessor, leftPostJoinProcessor, leftCompiledCondition);
}
JoinStreamRuntime joinStreamRuntime = new JoinStreamRuntime(siddhiAppContext, metaStateEvent);
joinStreamRuntime.addRuntime(leftStreamRuntime);
joinStreamRuntime.addRuntime(rightStreamRuntime);
return joinStreamRuntime;
} catch (Throwable t) {
ExceptionUtil.populateQueryContext(t, joinInputStream, siddhiAppContext);
throw t;
}
}
use of org.wso2.siddhi.core.stream.input.source.Source in project siddhi by wso2.
the class EventTestCase method testQueryParser.
@Test
public void testQueryParser() {
StreamDefinition streamDefinition = StreamDefinition.id("cseEventStream").attribute("symbol", Attribute.Type.STRING).attribute("price", Attribute.Type.FLOAT).attribute("volume", Attribute.Type.INT);
StreamDefinition outStreamDefinition = StreamDefinition.id("outputStream").attribute("symbol", Attribute.Type.STRING).attribute("price", Attribute.Type.FLOAT);
Query query = new Query();
query.annotation(Annotation.annotation("info").element("name", "query1"));
query.from(InputStream.stream("cseEventStream").filter(Expression.compare(Expression.variable("volume"), Compare.Operator.NOT_EQUAL, Expression.value(50))));
query.select(Selector.selector().select("symbol", Expression.variable("symbol")).select("price", Expression.variable("price")));
query.insertInto("outputStream");
Map<String, AbstractDefinition> tableDefinitionMap = new HashMap<>();
Map<String, AbstractDefinition> windowDefinitionMap = new HashMap<>();
Map<String, AbstractDefinition> aggregationDefinitionMap = new HashMap<>();
Map<String, Table> tableMap = new HashMap<String, Table>();
Map<String, Window> eventWindowMap = new HashMap<String, Window>();
Map<String, AggregationRuntime> aggregationMap = new HashMap<String, AggregationRuntime>();
Map<String, List<Source>> eventSourceMap = new HashMap<String, List<Source>>();
Map<String, List<Sink>> eventSinkMap = new HashMap<String, List<Sink>>();
Map<String, AbstractDefinition> streamDefinitionMap = new HashMap<String, AbstractDefinition>();
LockSynchronizer lockSynchronizer = new LockSynchronizer();
streamDefinitionMap.put("cseEventStream", streamDefinition);
streamDefinitionMap.put("outputStream", outStreamDefinition);
SiddhiContext siddhicontext = new SiddhiContext();
SiddhiAppContext context = new SiddhiAppContext();
context.setSiddhiContext(siddhicontext);
context.setElementIdGenerator(new ElementIdGenerator(context.getName()));
context.setSnapshotService(new SnapshotService(context));
QueryRuntime runtime = QueryParser.parse(query, context, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, tableMap, aggregationMap, eventWindowMap, lockSynchronizer, "1");
AssertJUnit.assertNotNull(runtime);
AssertJUnit.assertTrue(runtime.getStreamRuntime() instanceof SingleStreamRuntime);
AssertJUnit.assertNotNull(runtime.getSelector());
AssertJUnit.assertTrue(runtime.getMetaComplexEvent() instanceof MetaStreamEvent);
}
use of org.wso2.siddhi.core.stream.input.source.Source in project siddhi by wso2.
the class InMemoryTransportTestCase method inMemoryTestCase4.
@Test(expectedExceptions = SiddhiAppCreationException.class)
public void inMemoryTestCase4() throws InterruptedException {
log.info("Test inMemory 4");
String streams = "" + "@app:name('TestSiddhiApp')" + "@source(type='inMemory', topic='Foo', @map(type='passThrough', @attributes('symbol','price'))) " + "define stream FooStream (symbol string, price float, volume long); " + "@sink(type='inMemory', topic='{{symbol}}', @map(type='passThrough')) " + "define stream BarStream (symbol string, price float, volume long); ";
String query = "" + "from FooStream " + "select * " + "insert into BarStream; ";
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(streams + query);
}
use of org.wso2.siddhi.core.stream.input.source.Source in project siddhi by wso2.
the class InMemoryTransportTestCase method inMemorySourceSinkAndEventMappingWithSiddhiQL2.
@Test
public void inMemorySourceSinkAndEventMappingWithSiddhiQL2() throws InterruptedException {
log.info("Test inMemory Source Sink And EventMapping With SiddhiQL2");
InMemoryBroker.Subscriber subscriptionWSO2 = new InMemoryBroker.Subscriber() {
@Override
public void onMessage(Object msg) {
wso2Count.incrementAndGet();
}
@Override
public String getTopic() {
return "WSO2";
}
};
InMemoryBroker.Subscriber subscriptionIBM = new InMemoryBroker.Subscriber() {
@Override
public void onMessage(Object msg) {
ibmCount.incrementAndGet();
}
@Override
public String getTopic() {
return "IBM";
}
};
// subscribe to "inMemory" broker per topic
InMemoryBroker.subscribe(subscriptionWSO2);
InMemoryBroker.subscribe(subscriptionIBM);
String streams = "" + "@app:name('TestSiddhiApp')" + "@source(type='inMemory', topic='Foo', @map(type='passThrough')) " + "define stream FooStream (symbol string, price float, volume long); " + "@sink(type='inMemory', topic='{{symbol}}', @map(type='passThrough')) " + "define stream BarStream (symbol string, price float, volume long); ";
String query = "" + "from FooStream " + "select * " + "insert into FooBarStream; " + "" + "from FooBarStream " + "select * " + "insert into BarStream; " + "";
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(streams + query);
siddhiAppRuntime.start();
InMemoryBroker.publish("WSO2", new Event(System.currentTimeMillis(), new Object[] { "WSO2", 55.6f, 100L }));
InMemoryBroker.publish("IBM", new Event(System.currentTimeMillis(), new Object[] { "IBM", 75.6f, 100L }));
InMemoryBroker.publish("WSO2", new Event(System.currentTimeMillis(), new Object[] { "WSO2", 57.6f, 100L }));
Thread.sleep(100);
// assert event count
AssertJUnit.assertEquals("Number of WSO2 events", 2, wso2Count.get());
AssertJUnit.assertEquals("Number of IBM events", 1, ibmCount.get());
siddhiAppRuntime.shutdown();
// unsubscribe from "inMemory" broker per topic
InMemoryBroker.unsubscribe(subscriptionWSO2);
InMemoryBroker.unsubscribe(subscriptionIBM);
}
Aggregations