Examples with BatchTimeoutHelper org.apache.metron.writer.bolt.BatchTimeoutHelper used on opensource projects

Example 1 with BatchTimeoutHelper

use of org.apache.metron.writer.bolt.BatchTimeoutHelper in project metron by apache.

the class ParserBolt method prepare.

@SuppressWarnings("unchecked")
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    super.prepare(stormConf, context, collector);
    messageGetStrategy = MessageGetters.DEFAULT_BYTES_FROM_POSITION.get();
    this.collector = collector;
    this.parserRunner.init(this::getConfigurations, initializeStellar());
    ackTuplesPolicy = new AckTuplesPolicy(collector, messageGetStrategy);
    // Need to prep all sensors
    for (Map.Entry<String, WriterHandler> entry : sensorToWriterMap.entrySet()) {
        String sensor = entry.getKey();
        SensorParserConfig config = getSensorParserConfig(sensor);
        if (config != null) {
            config.init();
            topicToSensorMap.put(config.getSensorTopic(), sensor);
        } else {
            throw new IllegalStateException("Unable to retrieve a parser config for " + sensor);
        }
        WriterHandler writer = sensorToWriterMap.get(sensor);
        if (maxBatchTimeout == 0) {
            // This means getComponentConfiguration was never called to initialize maxBatchTimeout,
            // probably because we are in a unit test scenario.  So calculate it here.
            WriterConfiguration writerConfig = getConfigurationStrategy().createWriterConfig(writer.getBulkMessageWriter(), getConfigurations());
            BatchTimeoutHelper timeoutHelper = new BatchTimeoutHelper(writerConfig::getAllConfiguredTimeouts, batchTimeoutDivisor);
            maxBatchTimeout = timeoutHelper.getMaxBatchTimeout();
        }
        writer.init(stormConf, context, collector, getConfigurations(), ackTuplesPolicy, maxBatchTimeout);
    }
}

Example 2 with BatchTimeoutHelper

use of org.apache.metron.writer.bolt.BatchTimeoutHelper in project metron by apache.

the class ParserBolt method getComponentConfiguration.

/**
 * This method is called by TopologyBuilder.createTopology() to obtain topology and
 * bolt specific configuration parameters.  We use it primarily to configure how often
 * a tick tuple will be sent to our bolt.
 * @return conf topology and bolt specific configuration parameters
 */
@Override
public Map<String, Object> getComponentConfiguration() {
    // This is called long before prepare(), so do some of the same stuff as prepare() does,
    // to get the valid WriterConfiguration.  But don't store any non-serializable objects,
    // else Storm will throw a runtime error.
    Function<WriterConfiguration, WriterConfiguration> configurationXform;
    WriterHandler writer = sensorToWriterMap.entrySet().iterator().next().getValue();
    if (writer.isWriterToBulkWriter()) {
        configurationXform = WriterToBulkWriter.TRANSFORMATION;
    } else {
        configurationXform = x -> x;
    }
    WriterConfiguration writerconf = configurationXform.apply(getConfigurationStrategy().createWriterConfig(writer.getBulkMessageWriter(), getConfigurations()));
    BatchTimeoutHelper timeoutHelper = new BatchTimeoutHelper(writerconf::getAllConfiguredTimeouts, batchTimeoutDivisor);
    this.requestedTickFreqSecs = timeoutHelper.getRecommendedTickInterval();
    // And while we've got BatchTimeoutHelper handy, capture the maxBatchTimeout for writerComponent.
    this.maxBatchTimeout = timeoutHelper.getMaxBatchTimeout();
    Map<String, Object> conf = super.getComponentConfiguration();
    if (conf == null) {
        conf = new HashMap<>();
    }
    conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, requestedTickFreqSecs);
    LOG.info("Requesting " + Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS + " set to " + Integer.toString(requestedTickFreqSecs));
    return conf;
}