Examples with WriterToBulkWriter org.apache.metron.writer.WriterToBulkWriter used on opensource projects

Example 1 with WriterToBulkWriter

use of org.apache.metron.writer.WriterToBulkWriter in project metron by apache.

the class BulkMessageWriterBolt method prepare.

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.writerComponent = new BulkWriterComponent<>(collector);
    this.collector = collector;
    super.prepare(stormConf, context, collector);
    if (messageGetField != null) {
        messageGetStrategy = MessageGetters.valueOf(messageGetStrategyType).get(messageGetField);
    } else {
        messageGetStrategy = MessageGetters.valueOf(messageGetStrategyType).get();
    }
    if (bulkMessageWriter instanceof WriterToBulkWriter) {
        configurationTransformation = WriterToBulkWriter.TRANSFORMATION;
    } else {
        configurationTransformation = x -> x;
    }
    try {
        WriterConfiguration writerconf = configurationTransformation.apply(new IndexingWriterConfiguration(bulkMessageWriter.getName(), getConfigurations()));
        if (defaultBatchTimeout == 0) {
            // This means getComponentConfiguration was never called to initialize defaultBatchTimeout,
            // probably because we are in a unit test scenario.  So calculate it here.
            BatchTimeoutHelper timeoutHelper = new BatchTimeoutHelper(writerconf::getAllConfiguredTimeouts, batchTimeoutDivisor);
            defaultBatchTimeout = timeoutHelper.getDefaultBatchTimeout();
        }
        writerComponent.setDefaultBatchTimeout(defaultBatchTimeout);
        bulkMessageWriter.init(stormConf, context, writerconf);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 2 with WriterToBulkWriter

use of org.apache.metron.writer.WriterToBulkWriter in project metron by apache.

the class BulkMessageWriterBolt method getComponentConfiguration.

/**
 * This method is called by TopologyBuilder.createTopology() to obtain topology and
 * bolt specific configuration parameters.  We use it primarily to configure how often
 * a tick tuple will be sent to our bolt.
 * @return conf topology and bolt specific configuration parameters
 */
@Override
public Map<String, Object> getComponentConfiguration() {
    // This is called long before prepare(), so do some of the same stuff as prepare() does,
    // to get the valid WriterConfiguration.  But don't store any non-serializable objects,
    // else Storm will throw a runtime error.
    Function<WriterConfiguration, WriterConfiguration> configurationXform;
    if (bulkMessageWriter instanceof WriterToBulkWriter) {
        configurationXform = WriterToBulkWriter.TRANSFORMATION;
    } else {
        configurationXform = x -> x;
    }
    WriterConfiguration writerconf = configurationXform.apply(new IndexingWriterConfiguration(bulkMessageWriter.getName(), getConfigurations()));
    BatchTimeoutHelper timeoutHelper = new BatchTimeoutHelper(writerconf::getAllConfiguredTimeouts, batchTimeoutDivisor);
    this.requestedTickFreqSecs = timeoutHelper.getRecommendedTickInterval();
    // And while we've got BatchTimeoutHelper handy, capture the defaultBatchTimeout for writerComponent.
    this.defaultBatchTimeout = timeoutHelper.getDefaultBatchTimeout();
    Map<String, Object> conf = super.getComponentConfiguration();
    if (conf == null) {
        conf = new HashMap<String, Object>();
    }
    conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, requestedTickFreqSecs);
    LOG.info("Requesting " + Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS + " set to " + Integer.toString(requestedTickFreqSecs));
    return conf;
}

Example 3 with WriterToBulkWriter

use of org.apache.metron.writer.WriterToBulkWriter in project metron by apache.

the class BulkMessageWriterBolt method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
    if (isTick(tuple)) {
        try {
            if (!(bulkMessageWriter instanceof WriterToBulkWriter)) {
                // WriterToBulkWriter doesn't allow batching, so no need to flush on Tick.
                LOG.debug("Flushing message queues older than their batchTimeouts");
                writerComponent.flushTimeouts(bulkMessageWriter, configurationTransformation.apply(new IndexingWriterConfiguration(bulkMessageWriter.getName(), getConfigurations())), messageGetStrategy);
            }
        } catch (Exception e) {
            throw new RuntimeException("This should have been caught in the writerComponent.  If you see this, file a JIRA", e);
        } finally {
            collector.ack(tuple);
        }
        return;
    }
    try {
        JSONObject message = (JSONObject) messageGetStrategy.get(tuple);
        String sensorType = MessageUtils.getSensorType(message);
        LOG.trace("Writing enrichment message: {}", message);
        WriterConfiguration writerConfiguration = configurationTransformation.apply(new IndexingWriterConfiguration(bulkMessageWriter.getName(), getConfigurations()));
        if (writerConfiguration.isDefault(sensorType)) {
            // want to warn, but not fail the tuple
            collector.reportError(new Exception("WARNING: Default and (likely) unoptimized writer config used for " + bulkMessageWriter.getName() + " writer and sensor " + sensorType));
        }
        writerComponent.write(sensorType, tuple, message, bulkMessageWriter, writerConfiguration, messageGetStrategy);
    } catch (Exception e) {
        throw new RuntimeException("This should have been caught in the writerComponent.  If you see this, file a JIRA", e);
    }
}