Examples with WriterConfiguration org.apache.metron.common.configuration.writer.WriterConfiguration used on opensource projects

Example 11 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testGetHdfsPathFormatConstant.

@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathFormatConstant() {
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, new IndexingConfigurations());
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    JSONObject message = new JSONObject();
    Object result = writer.getHdfsPathExtension(SENSOR_NAME, "FORMAT('/test/folder/')", message);
    writer.close();
    Assert.assertEquals("/test/folder/", result);
}

Example 12 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testGetHdfsPathConstant.

@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathConstant() {
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, new IndexingConfigurations());
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    JSONObject message = new JSONObject();
    Object result = writer.getHdfsPathExtension(SENSOR_NAME, "'new'", message);
    writer.close();
    Assert.assertEquals("new", result);
}

Example 13 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class BulkMessageWriterBolt method getComponentConfiguration.

/**
 * This method is called by TopologyBuilder.createTopology() to obtain topology and
 * bolt specific configuration parameters.  We use it primarily to configure how often
 * a tick tuple will be sent to our bolt.
 * @return conf topology and bolt specific configuration parameters
 */
@Override
public Map<String, Object> getComponentConfiguration() {
    // This is called long before prepare(), so do some of the same stuff as prepare() does,
    // to get the valid WriterConfiguration.  But don't store any non-serializable objects,
    // else Storm will throw a runtime error.
    Function<WriterConfiguration, WriterConfiguration> configurationXform;
    if (bulkMessageWriter instanceof WriterToBulkWriter) {
        configurationXform = WriterToBulkWriter.TRANSFORMATION;
    } else {
        configurationXform = x -> x;
    }
    WriterConfiguration writerconf = configurationXform.apply(new IndexingWriterConfiguration(bulkMessageWriter.getName(), getConfigurations()));
    BatchTimeoutHelper timeoutHelper = new BatchTimeoutHelper(writerconf::getAllConfiguredTimeouts, batchTimeoutDivisor);
    this.requestedTickFreqSecs = timeoutHelper.getRecommendedTickInterval();
    // And while we've got BatchTimeoutHelper handy, capture the defaultBatchTimeout for writerComponent.
    this.defaultBatchTimeout = timeoutHelper.getDefaultBatchTimeout();
    Map<String, Object> conf = super.getComponentConfiguration();
    if (conf == null) {
        conf = new HashMap<String, Object>();
    }
    conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, requestedTickFreqSecs);
    LOG.info("Requesting " + Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS + " set to " + Integer.toString(requestedTickFreqSecs));
    return conf;
}

Example 14 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class BulkMessageWriterBolt method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
    if (isTick(tuple)) {
        try {
            if (!(bulkMessageWriter instanceof WriterToBulkWriter)) {
                // WriterToBulkWriter doesn't allow batching, so no need to flush on Tick.
                LOG.debug("Flushing message queues older than their batchTimeouts");
                writerComponent.flushTimeouts(bulkMessageWriter, configurationTransformation.apply(new IndexingWriterConfiguration(bulkMessageWriter.getName(), getConfigurations())), messageGetStrategy);
            }
        } catch (Exception e) {
            throw new RuntimeException("This should have been caught in the writerComponent.  If you see this, file a JIRA", e);
        } finally {
            collector.ack(tuple);
        }
        return;
    }
    try {
        JSONObject message = (JSONObject) messageGetStrategy.get(tuple);
        String sensorType = MessageUtils.getSensorType(message);
        LOG.trace("Writing enrichment message: {}", message);
        WriterConfiguration writerConfiguration = configurationTransformation.apply(new IndexingWriterConfiguration(bulkMessageWriter.getName(), getConfigurations()));
        if (writerConfiguration.isDefault(sensorType)) {
            // want to warn, but not fail the tuple
            collector.reportError(new Exception("WARNING: Default and (likely) unoptimized writer config used for " + bulkMessageWriter.getName() + " writer and sensor " + sensorType));
        }
        writerComponent.write(sensorType, tuple, message, bulkMessageWriter, writerConfiguration, messageGetStrategy);
    } catch (Exception e) {
        throw new RuntimeException("This should have been caught in the writerComponent.  If you see this, file a JIRA", e);
    }
}

Example 15 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class SimpleHBaseEnrichmentWriterTest method testConfigValidation_enrichment_type_is_empty.

@Test(expected = IllegalArgumentException.class)
public void testConfigValidation_enrichment_type_is_empty() {
    final String sensorType = "dummy";
    SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
    WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>() {

        {
            put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
            put(SimpleHbaseEnrichmentWriter.Configurations.ENRICHMENT_TYPE.getKey(), "  ");
        }
    });
    try {
        writer.configure(sensorType, configuration);
    } catch (IllegalArgumentException ex) {
        Assert.assertEquals(String.format("%s must not be an empty string", SimpleHbaseEnrichmentWriter.Configurations.ENRICHMENT_TYPE.getKey()), ex.getMessage());
        throw ex;
    }
}