Examples with AckTuplesPolicy org.apache.metron.writer.AckTuplesPolicy used on opensource projects

Example 1 with AckTuplesPolicy

use of org.apache.metron.writer.AckTuplesPolicy in project metron by apache.

the class ParserBolt method prepare.

@SuppressWarnings("unchecked")
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    super.prepare(stormConf, context, collector);
    messageGetStrategy = MessageGetters.DEFAULT_BYTES_FROM_POSITION.get();
    this.collector = collector;
    this.parserRunner.init(this::getConfigurations, initializeStellar());
    ackTuplesPolicy = new AckTuplesPolicy(collector, messageGetStrategy);
    // Need to prep all sensors
    for (Map.Entry<String, WriterHandler> entry : sensorToWriterMap.entrySet()) {
        String sensor = entry.getKey();
        SensorParserConfig config = getSensorParserConfig(sensor);
        if (config != null) {
            config.init();
            topicToSensorMap.put(config.getSensorTopic(), sensor);
        } else {
            throw new IllegalStateException("Unable to retrieve a parser config for " + sensor);
        }
        WriterHandler writer = sensorToWriterMap.get(sensor);
        if (maxBatchTimeout == 0) {
            // This means getComponentConfiguration was never called to initialize maxBatchTimeout,
            // probably because we are in a unit test scenario.  So calculate it here.
            WriterConfiguration writerConfig = getConfigurationStrategy().createWriterConfig(writer.getBulkMessageWriter(), getConfigurations());
            BatchTimeoutHelper timeoutHelper = new BatchTimeoutHelper(writerConfig::getAllConfiguredTimeouts, batchTimeoutDivisor);
            maxBatchTimeout = timeoutHelper.getMaxBatchTimeout();
        }
        writer.init(stormConf, context, collector, getConfigurations(), ackTuplesPolicy, maxBatchTimeout);
    }
}

Example 2 with AckTuplesPolicy

use of org.apache.metron.writer.AckTuplesPolicy in project metron by apache.

the class BulkMessageWriterBolt method prepare.

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    super.prepare(stormConf, context, collector);
    if (messageGetField != null) {
        messageGetStrategy = MessageGetters.valueOf(messageGetStrategyType).get(messageGetField);
    } else {
        messageGetStrategy = MessageGetters.valueOf(messageGetStrategyType).get();
    }
    if (bulkMessageWriter instanceof WriterToBulkWriter) {
        configurationTransformation = WriterToBulkWriter.TRANSFORMATION;
    } else {
        configurationTransformation = x -> x;
    }
    ackTuplesPolicy = new AckTuplesPolicy(collector, messageGetStrategy);
    try {
        WriterConfiguration writerconf = configurationTransformation.apply(getConfigurationStrategy().createWriterConfig(bulkMessageWriter, getConfigurations()));
        if (maxBatchTimeout == 0) {
            // This means getComponentConfiguration was never called to initialize maxBatchTimeout,
            // probably because we are in a unit test scenario.  So calculate it here.
            BatchTimeoutHelper timeoutHelper = new BatchTimeoutHelper(writerconf::getAllConfiguredTimeouts, batchTimeoutDivisor);
            maxBatchTimeout = timeoutHelper.getMaxBatchTimeout();
        }
        BulkWriterComponent<JSONObject> bulkWriterComponent = new BulkWriterComponent<>(maxBatchTimeout);
        bulkWriterComponent.addFlushPolicy(ackTuplesPolicy);
        setWriterComponent(bulkWriterComponent);
        bulkMessageWriter.init(stormConf, writerconf);
        if (bulkMessageWriter instanceof HdfsWriter) {
            ((HdfsWriter) bulkMessageWriter).initFileNameFormat(context);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 3 with AckTuplesPolicy

use of org.apache.metron.writer.AckTuplesPolicy in project metron by apache.

the class WriterBolt method prepare.

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    messageGetStrategy = MessageGetters.DEFAULT_JSON_FROM_FIELD.get();
    ackTuplesPolicy = new AckTuplesPolicy(collector, messageGetStrategy);
    handler.init(stormConf, context, collector, configuration, ackTuplesPolicy, UNINITIALIZED_MAX_BATCH_TIMEOUT);
}