Examples with Sink org.wso2.siddhi.core.stream.output.sink.Sink used on opensource projects

Example 1 with Sink

use of org.wso2.siddhi.core.stream.output.sink.Sink in project siddhi by wso2.

the class PartitionedDistributionStrategy method init.

/**
 * Initialize the Distribution strategy with the information it will require to make decisions.
 *  @param streamDefinition         The stream attached to the sink this DistributionStrategy is used in
 * @param transportOptionHolder    Sink options of the sink which uses this DistributionStrategy
 * @param destinationOptionHolders The list of options under @destination of the relevant sink.
 * @param configReader This hold the {@link PartitionedDistributionStrategy} configuration reader.
 */
@Override
public void init(StreamDefinition streamDefinition, OptionHolder transportOptionHolder, OptionHolder distributionOptionHolder, List<OptionHolder> destinationOptionHolders, ConfigReader configReader) {
    totalDestinationCount = destinationOptionHolders.size();
    String partitionKey = distributionOptionHolder.validateAndGetStaticValue(SiddhiConstants.PARTITION_KEY_FIELD_KEY);
    if (partitionKey == null || partitionKey.isEmpty()) {
        throw new SiddhiAppValidationException("PartitionKey is required for partitioned distribution " + "strategy.");
    }
    try {
        int partitionKeyFieldPosition = streamDefinition.getAttributePosition(partitionKey);
        partitionOption = new Option(partitionKeyFieldPosition);
    } catch (AttributeNotExistException e) {
        throw new SiddhiAppValidationException("Could not find partition key attribute", e);
    }
}

Example 2 with Sink

use of org.wso2.siddhi.core.stream.output.sink.Sink in project siddhi by wso2.

the class DefinitionParserHelper method isMultiClientDistributedTransport.

private static boolean isMultiClientDistributedTransport(Sink clientTransport, StreamDefinition streamDefinition, Annotation distributionAnnotation, SiddhiAppContext siddhiAppContext) {
    // fetch the @distribution annotations from the @sink annotation
    List<OptionHolder> destinationOptHolders = createDestinationOptionHolders(distributionAnnotation, streamDefinition, clientTransport, siddhiAppContext);
    List<String> dynamicOptions = Arrays.asList(clientTransport.getSupportedDynamicOptions());
    // If at least one of the @destination contains a static option then multi client transport should be used
    for (OptionHolder optionHolder : destinationOptHolders) {
        for (String key : optionHolder.getDynamicOptionsKeys()) {
            if (!dynamicOptions.contains(key)) {
                return true;
            }
        }
        for (String key : optionHolder.getStaticOptionsKeys()) {
            if (!dynamicOptions.contains(key)) {
                return true;
            }
        }
    }
    return false;
}

Example 3 with Sink

use of org.wso2.siddhi.core.stream.output.sink.Sink in project siddhi by wso2.

the class DefinitionParserHelper method addEventSink.

public static void addEventSink(StreamDefinition streamDefinition, ConcurrentMap<String, List<Sink>> eventSinkMap, SiddhiAppContext siddhiAppContext) {
    for (Annotation sinkAnnotation : streamDefinition.getAnnotations()) {
        if (SiddhiConstants.ANNOTATION_SINK.equalsIgnoreCase(sinkAnnotation.getName())) {
            sinkAnnotation = updateAnnotationRef(sinkAnnotation, SiddhiConstants.NAMESPACE_SINK, siddhiAppContext);
            Annotation mapAnnotation = AnnotationHelper.getAnnotation(SiddhiConstants.ANNOTATION_MAP, sinkAnnotation.getAnnotations());
            if (mapAnnotation == null) {
                mapAnnotation = Annotation.annotation(SiddhiConstants.ANNOTATION_MAP).element(SiddhiConstants.ANNOTATION_ELEMENT_TYPE, "passThrough");
            }
            Annotation distributionAnnotation = AnnotationHelper.getAnnotation(SiddhiConstants.ANNOTATION_DISTRIBUTION, sinkAnnotation.getAnnotations());
            if (mapAnnotation != null) {
                String[] supportedDynamicOptions = null;
                List<OptionHolder> destinationOptHolders = new ArrayList<>();
                String sinkType = sinkAnnotation.getElement(SiddhiConstants.ANNOTATION_ELEMENT_TYPE);
                Extension sinkExtension = constructExtension(streamDefinition, SiddhiConstants.ANNOTATION_SINK, sinkType, sinkAnnotation, SiddhiConstants.NAMESPACE_SINK);
                ConfigReader sinkConfigReader = siddhiAppContext.getSiddhiContext().getConfigManager().generateConfigReader(sinkExtension.getNamespace(), sinkExtension.getName());
                final boolean isDistributedTransport = (distributionAnnotation != null);
                boolean isMultiClient = false;
                if (isDistributedTransport) {
                    Sink sink = createSink(sinkExtension, siddhiAppContext);
                    isMultiClient = isMultiClientDistributedTransport(sink, streamDefinition, distributionAnnotation, siddhiAppContext);
                    supportedDynamicOptions = sink.getSupportedDynamicOptions();
                    destinationOptHolders = createDestinationOptionHolders(distributionAnnotation, streamDefinition, sink, siddhiAppContext);
                }
                final String mapType = mapAnnotation.getElement(SiddhiConstants.ANNOTATION_ELEMENT_TYPE);
                if (mapType != null) {
                    Sink sink;
                    if (isDistributedTransport) {
                        sink = (isMultiClient) ? new MultiClientDistributedSink() : new SingleClientDistributedSink();
                    } else {
                        sink = createSink(sinkExtension, siddhiAppContext);
                    }
                    if (supportedDynamicOptions == null) {
                        supportedDynamicOptions = sink.getSupportedDynamicOptions();
                    }
                    // load output mapper extension
                    Extension mapperExtension = constructExtension(streamDefinition, SiddhiConstants.ANNOTATION_MAP, mapType, sinkAnnotation, SiddhiConstants.NAMESPACE_SINK_MAPPER);
                    ConfigReader mapperConfigReader = siddhiAppContext.getSiddhiContext().getConfigManager().generateConfigReader(sinkExtension.getNamespace(), sinkExtension.getName());
                    SinkMapper sinkMapper = (SinkMapper) SiddhiClassLoader.loadExtensionImplementation(mapperExtension, SinkMapperExecutorExtensionHolder.getInstance(siddhiAppContext));
                    org.wso2.siddhi.annotation.Extension sinkExt = sink.getClass().getAnnotation(org.wso2.siddhi.annotation.Extension.class);
                    OptionHolder transportOptionHolder = constructOptionHolder(streamDefinition, sinkAnnotation, sinkExt, supportedDynamicOptions);
                    OptionHolder mapOptionHolder = constructOptionHolder(streamDefinition, mapAnnotation, sinkMapper.getClass().getAnnotation(org.wso2.siddhi.annotation.Extension.class), sinkMapper.getSupportedDynamicOptions());
                    List<Element> payloadElementList = getPayload(mapAnnotation);
                    OptionHolder distributionOptHolder = null;
                    SinkHandlerManager sinkHandlerManager = siddhiAppContext.getSiddhiContext().getSinkHandlerManager();
                    SinkHandler sinkHandler = null;
                    if (sinkHandlerManager != null) {
                        sinkHandler = sinkHandlerManager.generateSinkHandler();
                    }
                    if (isDistributedTransport) {
                        distributionOptHolder = constructOptionHolder(streamDefinition, distributionAnnotation, sinkExt, supportedDynamicOptions);
                        String strategyType = distributionOptHolder.validateAndGetStaticValue(SiddhiConstants.DISTRIBUTION_STRATEGY_KEY);
                        Extension strategyExtension = constructExtension(streamDefinition, SiddhiConstants.ANNOTATION_SINK, strategyType, sinkAnnotation, SiddhiConstants.NAMESPACE_DISTRIBUTION_STRATEGY);
                        ConfigReader configReader = siddhiAppContext.getSiddhiContext().getConfigManager().generateConfigReader(strategyExtension.getNamespace(), strategyExtension.getName());
                        DistributionStrategy distributionStrategy = (DistributionStrategy) SiddhiClassLoader.loadExtensionImplementation(strategyExtension, DistributionStrategyExtensionHolder.getInstance(siddhiAppContext));
                        try {
                            distributionStrategy.init(streamDefinition, transportOptionHolder, distributionOptHolder, destinationOptHolders, configReader);
                            ((DistributedTransport) sink).init(streamDefinition, sinkType, transportOptionHolder, sinkConfigReader, sinkMapper, mapType, mapOptionHolder, sinkHandler, payloadElementList, mapperConfigReader, siddhiAppContext, destinationOptHolders, sinkAnnotation, distributionStrategy, supportedDynamicOptions);
                        } catch (Throwable t) {
                            ExceptionUtil.populateQueryContext(t, sinkAnnotation, siddhiAppContext);
                            throw t;
                        }
                    } else {
                        try {
                            sink.init(streamDefinition, sinkType, transportOptionHolder, sinkConfigReader, sinkMapper, mapType, mapOptionHolder, sinkHandler, payloadElementList, mapperConfigReader, siddhiAppContext);
                        } catch (Throwable t) {
                            ExceptionUtil.populateQueryContext(t, sinkAnnotation, siddhiAppContext);
                            throw t;
                        }
                    }
                    if (sinkHandlerManager != null) {
                        sinkHandlerManager.registerSinkHandler(sinkHandler.getElementId(), sinkHandler);
                        siddhiAppContext.getSnapshotService().addSnapshotable(streamDefinition.getId(), sinkHandler);
                    }
                    validateSinkMapperCompatibility(streamDefinition, sinkType, mapType, sink, sinkMapper, sinkAnnotation);
                    // Setting the output group determiner
                    OutputGroupDeterminer groupDeterminer = constructOutputGroupDeterminer(transportOptionHolder, distributionOptHolder, streamDefinition, destinationOptHolders.size());
                    if (groupDeterminer != null) {
                        sink.getMapper().setGroupDeterminer(groupDeterminer);
                    }
                    siddhiAppContext.getSnapshotService().addSnapshotable(sink.getStreamDefinition().getId(), sink);
                    List<Sink> eventSinks = eventSinkMap.get(streamDefinition.getId());
                    if (eventSinks == null) {
                        eventSinks = new ArrayList<>();
                        eventSinks.add(sink);
                        eventSinkMap.put(streamDefinition.getId(), eventSinks);
                    } else {
                        eventSinks.add(sink);
                    }
                }
            } else {
                throw new SiddhiAppCreationException("Both @sink(type=) and @map(type=) are required.", sinkAnnotation.getQueryContextStartIndex(), sinkAnnotation.getQueryContextEndIndex());
            }
        }
    }
}

Example 4 with Sink

use of org.wso2.siddhi.core.stream.output.sink.Sink in project siddhi by wso2.

the class MultiClientDistributedSink method connect.

/**
 * Will be called to connect to the backend before events are published
 *
 * @throws ConnectionUnavailableException if it cannot connect to the backend
 */
@Override
public void connect() throws ConnectionUnavailableException {
    StringBuilder errorMessages = null;
    int errorCount = 0;
    for (int i = 0; i < transports.size(); i++) {
        try {
            Sink transport = transports.get(i);
            if (!transport.isConnected()) {
                transport.connect();
                transport.setConnected(true);
                strategy.destinationAvailable(i);
                log.info("Connected to destination Id " + i);
            }
        } catch (ConnectionUnavailableException e) {
            errorCount++;
            if (errorMessages == null) {
                errorMessages = new StringBuilder();
            }
            errorMessages.append("[Destination").append(i).append("]:").append(e.getMessage());
            log.warn(ExceptionUtil.getMessageWithContext(e, siddhiAppContext) + " Failed to Connect to destination ID " + i);
        }
    }
    if (errorCount > 0) {
        throw new ConnectionUnavailableException("Error on '" + siddhiAppContext.getName() + "'. " + errorCount + "/" + transports.size() + " connections failed while trying to connect with following error " + "messages:" + errorMessages.toString());
    }
}

Example 5 with Sink

use of org.wso2.siddhi.core.stream.output.sink.Sink in project siddhi by wso2.

the class SingleClientDistributedSink method initTransport.

@Override
public void initTransport(OptionHolder sinkOptionHolder, List<OptionHolder> destinationOptionHolders, Annotation sinkAnnotation, ConfigReader sinkConfigReader, SiddhiAppContext siddhiAppContext) {
    final String transportType = sinkOptionHolder.validateAndGetStaticValue(SiddhiConstants.ANNOTATION_ELEMENT_TYPE);
    Extension sinkExtension = DefinitionParserHelper.constructExtension(streamDefinition, SiddhiConstants.ANNOTATION_SINK, transportType, sinkAnnotation, SiddhiConstants.NAMESPACE_SINK);
    Set<String> allDynamicOptionKeys = findAllDynamicOptions(destinationOptionHolders);
    destinationOptionHolders.forEach(optionHolder -> {
        optionHolder.merge(sinkOptionHolder);
        allDynamicOptionKeys.forEach(optionKey -> {
            String optionValue = optionHolder.getOrCreateOption(optionKey, null).getValue();
            if (optionValue == null || optionValue.isEmpty()) {
                throw new SiddhiAppValidationException("Destination properties can only contain " + "non-empty static values.");
            }
            Option sinkOption = sinkOptionHolder.getOrAddStaticOption(optionKey, optionValue);
            sinkOption.addVariableValue(optionValue);
            destinationCount++;
        });
    });
    this.sink = (Sink) SiddhiClassLoader.loadExtensionImplementation(sinkExtension, SinkExecutorExtensionHolder.getInstance(siddhiAppContext));
    this.sink.initOnlyTransport(streamDefinition, sinkOptionHolder, sinkConfigReader, siddhiAppContext);
}