use of io.siddhi.core.stream.input.source.Source in project siddhi by wso2.
the class DefinitionParserHelper method addEventSource.
public static void addEventSource(StreamDefinition streamDefinition, ConcurrentMap<String, List<Source>> eventSourceMap, SiddhiAppContext siddhiAppContext) {
for (Annotation sourceAnnotation : streamDefinition.getAnnotations()) {
if (SiddhiConstants.ANNOTATION_SOURCE.equalsIgnoreCase(sourceAnnotation.getName())) {
try {
sourceAnnotation = updateAnnotationRef(sourceAnnotation, SiddhiConstants.NAMESPACE_SOURCE, siddhiAppContext);
Annotation mapAnnotation = AnnotationHelper.getAnnotation(SiddhiConstants.ANNOTATION_MAP, sourceAnnotation.getAnnotations());
if (mapAnnotation == null) {
mapAnnotation = Annotation.annotation(SiddhiConstants.ANNOTATION_MAP).element(SiddhiConstants.ANNOTATION_ELEMENT_TYPE, "passThrough");
}
final String sourceType = sourceAnnotation.getElement(SiddhiConstants.ANNOTATION_ELEMENT_TYPE);
if (sourceType == null) {
throw new SiddhiAppCreationException("Attribute 'type' does not exist for annotation '" + sourceAnnotation + "'", sourceAnnotation, siddhiAppContext);
}
final String mapType = mapAnnotation.getElement(SiddhiConstants.ANNOTATION_ELEMENT_TYPE);
if (mapType == null) {
throw new SiddhiAppCreationException("Attribute 'type' does not exist for annotation '" + mapAnnotation + "'", mapAnnotation, siddhiAppContext);
}
SourceHandlerManager sourceHandlerManager = siddhiAppContext.getSiddhiContext().getSourceHandlerManager();
SourceHandler sourceHandler = null;
if (sourceHandlerManager != null) {
sourceHandler = sourceHandlerManager.generateSourceHandler(sourceType);
}
// load input transport extension
Extension sourceExtension = constructExtension(streamDefinition, SiddhiConstants.ANNOTATION_SOURCE, sourceType, sourceAnnotation, SiddhiConstants.NAMESPACE_SOURCE);
Source source = (Source) SiddhiClassLoader.loadExtensionImplementation(sourceExtension, SourceExecutorExtensionHolder.getInstance(siddhiAppContext));
ConfigReader configReader = siddhiAppContext.getSiddhiContext().getConfigManager().generateConfigReader(sourceExtension.getNamespace(), sourceExtension.getName());
// load input mapper extension
Extension mapperExtension = constructExtension(streamDefinition, SiddhiConstants.ANNOTATION_MAP, mapType, sourceAnnotation, SiddhiConstants.NAMESPACE_SOURCE_MAPPER);
SourceMapper sourceMapper = (SourceMapper) SiddhiClassLoader.loadExtensionImplementation(mapperExtension, SourceMapperExecutorExtensionHolder.getInstance(siddhiAppContext));
ConfigReader mapperConfigReader = siddhiAppContext.getSiddhiContext().getConfigManager().generateConfigReader(mapperExtension.getNamespace(), mapperExtension.getName());
validateSourceMapperCompatibility(streamDefinition, sourceType, mapType, source, sourceMapper, sourceAnnotation);
io.siddhi.annotation.Extension sourceExt = source.getClass().getAnnotation(io.siddhi.annotation.Extension.class);
OptionHolder sourceOptionHolder = constructOptionHolder(streamDefinition, sourceAnnotation, sourceExt, null, true);
Map<String, String> deploymentProperties = createDeploymentProperties(sourceAnnotation, sourceExt);
OptionHolder mapOptionHolder = constructOptionHolder(streamDefinition, mapAnnotation, sourceMapper.getClass().getAnnotation(io.siddhi.annotation.Extension.class), null, false);
AttributesHolder attributesHolder = getAttributeMappings(mapAnnotation, mapType, streamDefinition);
String[] transportPropertyNames = getTransportPropertyNames(attributesHolder);
source.init(sourceType, sourceOptionHolder, sourceMapper, transportPropertyNames, configReader, mapType, mapOptionHolder, attributesHolder.payloadMappings, attributesHolder.transportMappings, mapperConfigReader, sourceHandler, streamDefinition, deploymentProperties, siddhiAppContext);
if (sourceHandlerManager != null) {
sourceHandlerManager.registerSourceHandler(sourceHandler.getId(), sourceHandler);
}
List<Source> eventSources = eventSourceMap.get(streamDefinition.getId());
if (eventSources == null) {
eventSources = new ArrayList<>();
eventSources.add(source);
eventSourceMap.put(streamDefinition.getId(), eventSources);
} else {
eventSources.add(source);
}
} catch (Throwable t) {
ExceptionUtil.populateQueryContext(t, sourceAnnotation, siddhiAppContext);
throw t;
}
}
}
}
use of io.siddhi.core.stream.input.source.Source in project siddhi by wso2.
the class SandboxTestCase method sandboxTest2.
@Test
public void sandboxTest2() throws InterruptedException {
log.info("sandbox test2");
SiddhiManager siddhiManager = new SiddhiManager();
String app = "" + "@source(type='foo')" + "@source(type='foo1')" + "@sink(type='foo1')" + "@source(type='inMemory', topic='myTopic')" + "define stream StockStream (symbol string, price float, vol long);\n" + "" + "@sink(type='foo1')" + "@sink(type='inMemory', topic='myTopic1')" + "define stream DeleteStockStream (symbol string, price float, vol long);\n" + "" + "@store(type='rdbms')" + "define table StockTable (symbol string, price float, volume long);\n" + "" + "define stream CountStockStream (symbol string);\n" + "" + "@info(name = 'query1') " + "from StockStream " + "select symbol, price, vol as volume " + "insert into StockTable ;" + "" + "@info(name = 'query2') " + "from DeleteStockStream[vol>=100] " + "delete StockTable " + " on StockTable.symbol==symbol ;" + "" + "@info(name = 'query3') " + "from CountStockStream#window.length(0) join StockTable" + " on CountStockStream.symbol==StockTable.symbol " + "select CountStockStream.symbol as symbol " + "insert into CountResultsStream ;";
SiddhiApp siddhiApp = SiddhiCompiler.parse(app);
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSandboxSiddhiAppRuntime(siddhiApp);
Assert.assertEquals(siddhiAppRuntime.getSources().size(), 1);
Assert.assertEquals(siddhiAppRuntime.getSinks().size(), 1);
Assert.assertEquals(siddhiAppRuntime.getTables().size(), 1);
for (List<Source> sources : siddhiAppRuntime.getSources()) {
for (Source source : sources) {
Assert.assertTrue(source.getType().equalsIgnoreCase("inMemory"));
}
}
for (List<Sink> sinks : siddhiAppRuntime.getSinks()) {
for (Sink sink : sinks) {
Assert.assertTrue(sink.getType().equalsIgnoreCase("inMemory"));
}
}
for (Table table : siddhiAppRuntime.getTables()) {
Assert.assertTrue(table instanceof InMemoryTable);
}
InputHandler stockStream = siddhiAppRuntime.getInputHandler("StockStream");
InputHandler deleteStockStream = siddhiAppRuntime.getInputHandler("DeleteStockStream");
InputHandler countStockStream = siddhiAppRuntime.getInputHandler("CountStockStream");
siddhiAppRuntime.addCallback("CountResultsStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
EventPrinter.print(events);
count.addAndGet(events.length);
}
});
siddhiAppRuntime.start();
stockStream.send(new Object[] { "WSO2", 55.6f, 100L });
stockStream.send(new Object[] { "IBM", 75.6f, 100L });
stockStream.send(new Object[] { "WSO2", 57.6f, 100L });
deleteStockStream.send(new Object[] { "IBM", 57.6f, 100L });
countStockStream.send(new Object[] { "WSO2" });
Thread.sleep(500);
Assert.assertEquals(count.get(), 2);
siddhiAppRuntime.shutdown();
}
use of io.siddhi.core.stream.input.source.Source in project siddhi by wso2.
the class EventTestCase method testQueryParser.
@Test
public void testQueryParser() {
StreamDefinition streamDefinition = StreamDefinition.id("cseEventStream").attribute("symbol", Attribute.Type.STRING).attribute("price", Attribute.Type.FLOAT).attribute("volume", Attribute.Type.INT);
StreamDefinition outStreamDefinition = StreamDefinition.id("outputStream").attribute("symbol", Attribute.Type.STRING).attribute("price", Attribute.Type.FLOAT);
Query query = new Query();
query.annotation(Annotation.annotation("info").element("name", "query1"));
query.from(InputStream.stream("cseEventStream").filter(Expression.compare(Expression.variable("volume"), Compare.Operator.NOT_EQUAL, Expression.value(50))));
query.select(Selector.selector().select("symbol", Expression.variable("symbol")).select("price", Expression.variable("price")));
query.insertInto("outputStream");
Map<String, AbstractDefinition> tableDefinitionMap = new HashMap<>();
Map<String, AbstractDefinition> windowDefinitionMap = new HashMap<>();
Map<String, AbstractDefinition> aggregationDefinitionMap = new HashMap<>();
Map<String, Table> tableMap = new HashMap<String, Table>();
Map<String, Window> eventWindowMap = new HashMap<String, Window>();
Map<String, AggregationRuntime> aggregationMap = new HashMap<String, AggregationRuntime>();
Map<String, List<Source>> eventSourceMap = new HashMap<String, List<Source>>();
Map<String, List<Sink>> eventSinkMap = new HashMap<String, List<Sink>>();
Map<String, AbstractDefinition> streamDefinitionMap = new HashMap<String, AbstractDefinition>();
LockSynchronizer lockSynchronizer = new LockSynchronizer();
streamDefinitionMap.put("cseEventStream", streamDefinition);
streamDefinitionMap.put("outputStream", outStreamDefinition);
SiddhiContext siddhicontext = new SiddhiContext();
SiddhiAppContext context = new SiddhiAppContext();
context.setSiddhiContext(siddhicontext);
context.setIdGenerator(new IdGenerator());
context.setSnapshotService(new SnapshotService(context));
QueryRuntimeImpl runtime = QueryParser.parse(query, context, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, tableMap, aggregationMap, eventWindowMap, lockSynchronizer, "1", false, SiddhiConstants.PARTITION_ID_DEFAULT);
AssertJUnit.assertNotNull(runtime);
AssertJUnit.assertTrue(runtime.getStreamRuntime() instanceof SingleStreamRuntime);
AssertJUnit.assertNotNull(runtime.getSelector());
AssertJUnit.assertTrue(runtime.getMetaComplexEvent() instanceof MetaStreamEvent);
}
use of io.siddhi.core.stream.input.source.Source in project siddhi by wso2.
the class SiddhiAppRuntimeImpl method startSources.
public void startSources() {
if (running) {
log.warn("Error calling startSources() for Siddhi App '" + siddhiAppContext.getName() + "', " + "SiddhiApp already started with the sources.");
return;
}
if (!runningWithoutSources) {
throw new SiddhiAppRuntimeException("Cannot call startSources() without calling startWithoutSources() " + "for Siddhi App '" + siddhiAppContext.getName() + "'");
} else {
try {
for (List<Source> sources : sourceMap.values()) {
for (Source source : sources) {
source.connectWithRetry();
}
}
running = true;
runningWithoutSources = false;
} catch (Throwable t) {
log.error("Error starting Siddhi App '" + siddhiAppContext.getName() + "', " + "triggering shutdown process. " + t.getMessage());
try {
shutdown();
} catch (Throwable t1) {
log.error("Error shutting down partially started Siddhi App '" + siddhiAppContext.getName() + "', " + t1.getMessage());
}
}
}
}
use of io.siddhi.core.stream.input.source.Source in project siddhi by wso2.
the class SiddhiAppRuntimeImpl method shutdown.
public synchronized void shutdown() {
SourceHandlerManager sourceHandlerManager = siddhiAppContext.getSiddhiContext().getSourceHandlerManager();
for (List<Source> sources : sourceMap.values()) {
for (Source source : sources) {
try {
if (sourceHandlerManager != null) {
sourceHandlerManager.unregisterSourceHandler(source.getMapper().getHandler().getId());
}
source.shutdown();
} catch (Throwable t) {
log.error(StringUtil.removeCRLFCharacters(ExceptionUtil.getMessageWithContext(t, siddhiAppContext)) + " Error in shutting down source '" + StringUtil.removeCRLFCharacters(source.getType()) + "' at '" + StringUtil.removeCRLFCharacters(source.getStreamDefinition().getId()) + "' on Siddhi App '" + siddhiAppContext.getName() + "'.", t);
}
}
}
for (Table table : tableMap.values()) {
try {
table.shutdown();
} catch (Throwable t) {
log.error(StringUtil.removeCRLFCharacters(ExceptionUtil.getMessageWithContext(t, siddhiAppContext)) + " Error in shutting down table '" + StringUtil.removeCRLFCharacters(table.getTableDefinition().getId()) + "' on Siddhi App '" + StringUtil.removeCRLFCharacters(siddhiAppContext.getName()) + "'.", t);
}
}
SinkHandlerManager sinkHandlerManager = siddhiAppContext.getSiddhiContext().getSinkHandlerManager();
for (List<Sink> sinks : sinkMap.values()) {
for (Sink sink : sinks) {
try {
if (sinkHandlerManager != null) {
sinkHandlerManager.unregisterSinkHandler(sink.getHandler().getId());
}
sink.shutdown();
} catch (Throwable t) {
log.error(StringUtil.removeCRLFCharacters(ExceptionUtil.getMessageWithContext(t, siddhiAppContext)) + " Error in shutting down sink '" + StringUtil.removeCRLFCharacters(sink.getType()) + "' at '" + StringUtil.removeCRLFCharacters(sink.getStreamDefinition().getId()) + "' on Siddhi App '" + StringUtil.removeCRLFCharacters(siddhiAppContext.getName()) + "'.", t);
}
}
}
for (Table table : tableMap.values()) {
RecordTableHandlerManager recordTableHandlerManager = siddhiAppContext.getSiddhiContext().getRecordTableHandlerManager();
if (recordTableHandlerManager != null) {
String elementId = null;
RecordTableHandler recordTableHandler = table.getHandler();
if (recordTableHandler != null) {
elementId = recordTableHandler.getId();
}
if (elementId != null) {
recordTableHandlerManager.unregisterRecordTableHandler(elementId);
}
}
table.shutdown();
}
for (ExternalReferencedHolder externalReferencedHolder : siddhiAppContext.getExternalReferencedHolders()) {
try {
externalReferencedHolder.stop();
} catch (Throwable t) {
log.error(StringUtil.removeCRLFCharacters(ExceptionUtil.getMessageWithContext(t, siddhiAppContext)) + " Error while stopping ExternalReferencedHolder '" + StringUtil.removeCRLFCharacters(externalReferencedHolder.toString()) + "' down Siddhi app '" + StringUtil.removeCRLFCharacters(siddhiAppContext.getName()) + "'.", t);
}
}
inputManager.disconnect();
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
for (StreamJunction streamJunction : streamJunctionMap.values()) {
streamJunction.stopProcessing();
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
siddhiAppContext.getScheduledExecutorService().shutdownNow();
siddhiAppContext.getExecutorService().shutdownNow();
}
}, "Siddhi-SiddhiApp-" + siddhiAppContext.getName() + "-Shutdown-Cleaner");
thread.start();
if (siddhiAppRuntimeMap != null) {
siddhiAppRuntimeMap.remove(siddhiAppContext.getName());
}
if (siddhiAppContext.getStatisticsManager() != null) {
if (siddhiAppContext.getRootMetricsLevel().compareTo(Level.OFF) != 0) {
siddhiAppContext.getStatisticsManager().stopReporting();
}
siddhiAppContext.getStatisticsManager().cleanup();
}
running = false;
runningWithoutSources = false;
}
Aggregations