use of org.ballerinalang.siddhi.core.stream.output.sink.Sink in project ballerina by ballerina-lang.
the class MultiClientDistributedSink method initTransport.
@Override
public void initTransport(OptionHolder sinkOptionHolder, List<OptionHolder> destinationOptionHolders, Annotation sinkAnnotation, ConfigReader sinkConfigReader, SiddhiAppContext siddhiAppContext) {
String transportType = sinkOptionHolder.validateAndGetStaticValue(SiddhiConstants.ANNOTATION_ELEMENT_TYPE);
Extension sinkExtension = DefinitionParserHelper.constructExtension(streamDefinition, SiddhiConstants.ANNOTATION_SINK, transportType, sinkAnnotation, SiddhiConstants.NAMESPACE_SINK);
destinationOptionHolders.forEach(destinationOption -> {
Sink sink = (Sink) SiddhiClassLoader.loadExtensionImplementation(sinkExtension, SinkExecutorExtensionHolder.getInstance(siddhiAppContext));
destinationOption.merge(sinkOptionHolder);
sink.initOnlyTransport(streamDefinition, destinationOption, sinkConfigReader, siddhiAppContext);
transports.add(sink);
});
}
use of org.ballerinalang.siddhi.core.stream.output.sink.Sink in project ballerina by ballerina-lang.
the class EventTestCase method testQueryParser.
@Test
public void testQueryParser() {
StreamDefinition streamDefinition = StreamDefinition.id("cseEventStream").attribute("symbol", Attribute.Type.STRING).attribute("price", Attribute.Type.FLOAT).attribute("volume", Attribute.Type.INT);
StreamDefinition outStreamDefinition = StreamDefinition.id("outputStream").attribute("symbol", Attribute.Type.STRING).attribute("price", Attribute.Type.FLOAT);
Query query = new Query();
query.annotation(Annotation.annotation("info").element("name", "query1"));
query.from(InputStream.stream("cseEventStream").filter(Expression.compare(Expression.variable("volume"), Compare.Operator.NOT_EQUAL, Expression.value(50))));
query.select(Selector.selector().select("symbol", Expression.variable("symbol")).select("price", Expression.variable("price")));
query.insertInto("outputStream");
Map<String, AbstractDefinition> tableDefinitionMap = new HashMap<>();
Map<String, AbstractDefinition> windowDefinitionMap = new HashMap<>();
Map<String, AbstractDefinition> aggregationDefinitionMap = new HashMap<>();
Map<String, Table> tableMap = new HashMap<String, Table>();
Map<String, Window> eventWindowMap = new HashMap<String, Window>();
Map<String, AggregationRuntime> aggregationMap = new HashMap<String, AggregationRuntime>();
Map<String, List<Source>> eventSourceMap = new HashMap<String, List<Source>>();
Map<String, List<Sink>> eventSinkMap = new HashMap<String, List<Sink>>();
Map<String, AbstractDefinition> streamDefinitionMap = new HashMap<String, AbstractDefinition>();
LockSynchronizer lockSynchronizer = new LockSynchronizer();
streamDefinitionMap.put("cseEventStream", streamDefinition);
streamDefinitionMap.put("outputStream", outStreamDefinition);
SiddhiContext siddhicontext = new SiddhiContext();
SiddhiAppContext context = new SiddhiAppContext();
context.setSiddhiContext(siddhicontext);
context.setElementIdGenerator(new ElementIdGenerator(context.getName()));
context.setSnapshotService(new SnapshotService(context));
QueryRuntime runtime = QueryParser.parse(query, context, streamDefinitionMap, tableDefinitionMap, windowDefinitionMap, aggregationDefinitionMap, tableMap, aggregationMap, eventWindowMap, lockSynchronizer, "1");
AssertJUnit.assertNotNull(runtime);
AssertJUnit.assertTrue(runtime.getStreamRuntime() instanceof SingleStreamRuntime);
AssertJUnit.assertNotNull(runtime.getSelector());
AssertJUnit.assertTrue(runtime.getMetaComplexEvent() instanceof MetaStreamEvent);
}
use of org.ballerinalang.siddhi.core.stream.output.sink.Sink in project ballerina by ballerina-lang.
the class SiddhiAppRuntime method shutdown.
public synchronized void shutdown() {
SourceHandlerManager sourceHandlerManager = siddhiAppContext.getSiddhiContext().getSourceHandlerManager();
for (List<Source> sources : sourceMap.values()) {
for (Source source : sources) {
try {
if (sourceHandlerManager != null) {
sourceHandlerManager.unregisterSourceHandler(source.getMapper().getHandler().getElementId());
}
source.shutdown();
} catch (Throwable t) {
log.error(StringUtil.removeCRLFCharacters(ExceptionUtil.getMessageWithContext(t, siddhiAppContext)) + " Error in shutting down source '" + StringUtil.removeCRLFCharacters(source.getType()) + "' at '" + StringUtil.removeCRLFCharacters(source.getStreamDefinition().getId()) + "' on Siddhi App '" + siddhiAppContext.getName() + "'.", t);
}
}
}
for (Table table : tableMap.values()) {
try {
table.shutdown();
} catch (Throwable t) {
log.error(StringUtil.removeCRLFCharacters(ExceptionUtil.getMessageWithContext(t, siddhiAppContext)) + " Error in shutting down table '" + StringUtil.removeCRLFCharacters(table.getTableDefinition().getId()) + "' on Siddhi App '" + StringUtil.removeCRLFCharacters(siddhiAppContext.getName()) + "'.", t);
}
}
SinkHandlerManager sinkHandlerManager = siddhiAppContext.getSiddhiContext().getSinkHandlerManager();
for (List<Sink> sinks : sinkMap.values()) {
for (Sink sink : sinks) {
try {
if (sinkHandlerManager != null) {
sinkHandlerManager.unregisterSinkHandler(sink.getHandler().getElementId());
}
sink.shutdown();
} catch (Throwable t) {
log.error(StringUtil.removeCRLFCharacters(ExceptionUtil.getMessageWithContext(t, siddhiAppContext)) + " Error in shutting down sink '" + StringUtil.removeCRLFCharacters(sink.getType()) + "' at '" + StringUtil.removeCRLFCharacters(sink.getStreamDefinition().getId()) + "' on Siddhi App '" + StringUtil.removeCRLFCharacters(siddhiAppContext.getName()) + "'.", t);
}
}
}
for (Table table : tableMap.values()) {
RecordTableHandlerManager recordTableHandlerManager = siddhiAppContext.getSiddhiContext().getRecordTableHandlerManager();
if (recordTableHandlerManager != null) {
String elementId = null;
RecordTableHandler recordTableHandler = table.getHandler();
if (recordTableHandler != null) {
elementId = recordTableHandler.getElementId();
}
if (elementId != null) {
recordTableHandlerManager.unregisterRecordTableHandler(elementId);
}
}
table.shutdown();
}
for (EternalReferencedHolder eternalReferencedHolder : siddhiAppContext.getEternalReferencedHolders()) {
try {
eternalReferencedHolder.stop();
} catch (Throwable t) {
log.error(StringUtil.removeCRLFCharacters(ExceptionUtil.getMessageWithContext(t, siddhiAppContext)) + " Error while stopping EternalReferencedHolder '" + StringUtil.removeCRLFCharacters(eternalReferencedHolder.toString()) + "' down Siddhi app '" + StringUtil.removeCRLFCharacters(siddhiAppContext.getName()) + "'.", t);
}
}
inputManager.disconnect();
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
for (StreamJunction streamJunction : streamJunctionMap.values()) {
streamJunction.stopProcessing();
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
siddhiAppContext.getScheduledExecutorService().shutdownNow();
siddhiAppContext.getExecutorService().shutdownNow();
}
}, "Siddhi-SiddhiApp-" + siddhiAppContext.getName() + "-Shutdown-Cleaner");
thread.start();
if (siddhiAppRuntimeMap != null) {
siddhiAppRuntimeMap.remove(siddhiAppContext.getName());
}
if (siddhiAppContext.getStatisticsManager() != null) {
if (siddhiAppContext.isStatsEnabled()) {
siddhiAppContext.getStatisticsManager().stopReporting();
}
siddhiAppContext.getStatisticsManager().cleanup();
}
running = false;
}
use of org.ballerinalang.siddhi.core.stream.output.sink.Sink in project ballerina by ballerina-lang.
the class DefinitionParserHelper method addEventSink.
public static void addEventSink(StreamDefinition streamDefinition, ConcurrentMap<String, List<Sink>> eventSinkMap, SiddhiAppContext siddhiAppContext) {
for (Annotation sinkAnnotation : streamDefinition.getAnnotations()) {
if (SiddhiConstants.ANNOTATION_SINK.equalsIgnoreCase(sinkAnnotation.getName())) {
sinkAnnotation = updateAnnotationRef(sinkAnnotation, SiddhiConstants.NAMESPACE_SINK, siddhiAppContext);
Annotation mapAnnotation = AnnotationHelper.getAnnotation(SiddhiConstants.ANNOTATION_MAP, sinkAnnotation.getAnnotations());
if (mapAnnotation == null) {
mapAnnotation = Annotation.annotation(SiddhiConstants.ANNOTATION_MAP).element(SiddhiConstants.ANNOTATION_ELEMENT_TYPE, "passThrough");
}
Annotation distributionAnnotation = AnnotationHelper.getAnnotation(SiddhiConstants.ANNOTATION_DISTRIBUTION, sinkAnnotation.getAnnotations());
if (mapAnnotation != null) {
String[] supportedDynamicOptions = null;
List<OptionHolder> destinationOptHolders = new ArrayList<>();
String sinkType = sinkAnnotation.getElement(SiddhiConstants.ANNOTATION_ELEMENT_TYPE);
Extension sinkExtension = constructExtension(streamDefinition, SiddhiConstants.ANNOTATION_SINK, sinkType, sinkAnnotation, SiddhiConstants.NAMESPACE_SINK);
ConfigReader sinkConfigReader = siddhiAppContext.getSiddhiContext().getConfigManager().generateConfigReader(sinkExtension.getNamespace(), sinkExtension.getName());
final boolean isDistributedTransport = (distributionAnnotation != null);
boolean isMultiClient = false;
if (isDistributedTransport) {
Sink sink = createSink(sinkExtension, siddhiAppContext);
isMultiClient = isMultiClientDistributedTransport(sink, streamDefinition, distributionAnnotation, siddhiAppContext);
supportedDynamicOptions = sink.getSupportedDynamicOptions();
destinationOptHolders = createDestinationOptionHolders(distributionAnnotation, streamDefinition, sink, siddhiAppContext);
}
final String mapType = mapAnnotation.getElement(SiddhiConstants.ANNOTATION_ELEMENT_TYPE);
if (mapType != null) {
Sink sink;
if (isDistributedTransport) {
sink = (isMultiClient) ? new MultiClientDistributedSink() : new SingleClientDistributedSink();
} else {
sink = createSink(sinkExtension, siddhiAppContext);
}
if (supportedDynamicOptions == null) {
supportedDynamicOptions = sink.getSupportedDynamicOptions();
}
// load output mapper extension
Extension mapperExtension = constructExtension(streamDefinition, SiddhiConstants.ANNOTATION_MAP, mapType, sinkAnnotation, SiddhiConstants.NAMESPACE_SINK_MAPPER);
ConfigReader mapperConfigReader = siddhiAppContext.getSiddhiContext().getConfigManager().generateConfigReader(sinkExtension.getNamespace(), sinkExtension.getName());
SinkMapper sinkMapper = (SinkMapper) SiddhiClassLoader.loadExtensionImplementation(mapperExtension, SinkMapperExecutorExtensionHolder.getInstance(siddhiAppContext));
org.ballerinalang.siddhi.annotation.Extension sinkExt = sink.getClass().getAnnotation(org.ballerinalang.siddhi.annotation.Extension.class);
OptionHolder transportOptionHolder = constructOptionHolder(streamDefinition, sinkAnnotation, sinkExt, supportedDynamicOptions);
OptionHolder mapOptionHolder = constructOptionHolder(streamDefinition, mapAnnotation, sinkMapper.getClass().getAnnotation(org.ballerinalang.siddhi.annotation.Extension.class), sinkMapper.getSupportedDynamicOptions());
List<Element> payloadElementList = getPayload(mapAnnotation);
OptionHolder distributionOptHolder = null;
SinkHandlerManager sinkHandlerManager = siddhiAppContext.getSiddhiContext().getSinkHandlerManager();
SinkHandler sinkHandler = null;
if (sinkHandlerManager != null) {
sinkHandler = sinkHandlerManager.generateSinkHandler();
}
if (isDistributedTransport) {
distributionOptHolder = constructOptionHolder(streamDefinition, distributionAnnotation, sinkExt, supportedDynamicOptions);
String strategyType = distributionOptHolder.validateAndGetStaticValue(SiddhiConstants.DISTRIBUTION_STRATEGY_KEY);
Extension strategyExtension = constructExtension(streamDefinition, SiddhiConstants.ANNOTATION_SINK, strategyType, sinkAnnotation, SiddhiConstants.NAMESPACE_DISTRIBUTION_STRATEGY);
ConfigReader configReader = siddhiAppContext.getSiddhiContext().getConfigManager().generateConfigReader(strategyExtension.getNamespace(), strategyExtension.getName());
DistributionStrategy distributionStrategy = (DistributionStrategy) SiddhiClassLoader.loadExtensionImplementation(strategyExtension, DistributionStrategyExtensionHolder.getInstance(siddhiAppContext));
try {
distributionStrategy.init(streamDefinition, transportOptionHolder, distributionOptHolder, destinationOptHolders, configReader);
((DistributedTransport) sink).init(streamDefinition, sinkType, transportOptionHolder, sinkConfigReader, sinkMapper, mapType, mapOptionHolder, sinkHandler, payloadElementList, mapperConfigReader, siddhiAppContext, destinationOptHolders, sinkAnnotation, distributionStrategy, supportedDynamicOptions);
} catch (Throwable t) {
ExceptionUtil.populateQueryContext(t, sinkAnnotation, siddhiAppContext);
throw t;
}
} else {
try {
sink.init(streamDefinition, sinkType, transportOptionHolder, sinkConfigReader, sinkMapper, mapType, mapOptionHolder, sinkHandler, payloadElementList, mapperConfigReader, siddhiAppContext);
} catch (Throwable t) {
ExceptionUtil.populateQueryContext(t, sinkAnnotation, siddhiAppContext);
throw t;
}
}
if (sinkHandlerManager != null) {
sinkHandlerManager.registerSinkHandler(sinkHandler.getElementId(), sinkHandler);
siddhiAppContext.getSnapshotService().addSnapshotable(streamDefinition.getId(), sinkHandler);
}
validateSinkMapperCompatibility(streamDefinition, sinkType, mapType, sink, sinkMapper, sinkAnnotation);
// Setting the output group determiner
OutputGroupDeterminer groupDeterminer = constructOutputGroupDeterminer(transportOptionHolder, distributionOptHolder, streamDefinition, destinationOptHolders.size());
if (groupDeterminer != null) {
sink.getMapper().setGroupDeterminer(groupDeterminer);
}
siddhiAppContext.getSnapshotService().addSnapshotable(sink.getStreamDefinition().getId(), sink);
List<Sink> eventSinks = eventSinkMap.get(streamDefinition.getId());
if (eventSinks == null) {
eventSinks = new ArrayList<>();
eventSinks.add(sink);
eventSinkMap.put(streamDefinition.getId(), eventSinks);
} else {
eventSinks.add(sink);
}
}
} else {
throw new SiddhiAppCreationException("Both @sink(type=) and @map(type=) are required.", sinkAnnotation.getQueryContextStartIndex(), sinkAnnotation.getQueryContextEndIndex());
}
}
}
}
use of org.ballerinalang.siddhi.core.stream.output.sink.Sink in project ballerina by ballerina-lang.
the class MultiClientDistributedSink method connect.
/**
* Will be called to connect to the backend before events are published.
*
* @throws ConnectionUnavailableException if it cannot connect to the backend
*/
@Override
public void connect() throws ConnectionUnavailableException {
StringBuilder errorMessages = null;
int errorCount = 0;
for (int i = 0; i < transports.size(); i++) {
try {
Sink transport = transports.get(i);
if (!transport.isConnected()) {
transport.connect();
transport.setConnected(true);
strategy.destinationAvailable(i);
log.info("Connected to destination Id " + i);
}
} catch (ConnectionUnavailableException e) {
errorCount++;
if (errorMessages == null) {
errorMessages = new StringBuilder();
}
errorMessages.append("[Destination").append(i).append("]:").append(e.getMessage());
log.warn(ExceptionUtil.getMessageWithContext(e, siddhiAppContext) + " Failed to Connect to destination ID " + i);
}
}
if (errorCount > 0) {
throw new ConnectionUnavailableException("Error on '" + siddhiAppContext.getName() + "'. " + errorCount + "/" + transports.size() + " connections failed while trying to connect with following error " + "messages:" + errorMessages.toString());
}
}
Aggregations