Examples with IndexingWriterConfiguration org.apache.metron.common.configuration.writer.IndexingWriterConfiguration used on opensource projects

Example 1 with IndexingWriterConfiguration

use of org.apache.metron.common.configuration.writer.IndexingWriterConfiguration in project metron by apache.

the class IndexingWriterConfigurationTest method testGetAllConfiguredTimeouts.

@Test
public void testGetAllConfiguredTimeouts() throws FileNotFoundException, IOException {
    // default
    IndexingWriterConfiguration config = new IndexingWriterConfiguration("hdfs", new IndexingConfigurations());
    Assert.assertEquals(0, config.getAllConfiguredTimeouts().size());
    // non-default
    IndexingConfigurations iconfigs = new IndexingConfigurations();
    iconfigs.updateSensorIndexingConfig(sensorType, new FileInputStream(sampleSensorIndexingConfigPath));
    config = new IndexingWriterConfiguration("elasticsearch", iconfigs);
    Assert.assertEquals(1, config.getAllConfiguredTimeouts().size());
    Assert.assertEquals(7, (long) config.getAllConfiguredTimeouts().get(0));
}

Example 2 with IndexingWriterConfiguration

use of org.apache.metron.common.configuration.writer.IndexingWriterConfiguration in project metron by apache.

the class IndexingWriterConfigurationTest method testDefaultIndex.

@Test
public void testDefaultIndex() {
    IndexingWriterConfiguration config = new IndexingWriterConfiguration("hdfs", new IndexingConfigurations());
    Assert.assertEquals("foo", config.getIndex("foo"));
}

Example 3 with IndexingWriterConfiguration

use of org.apache.metron.common.configuration.writer.IndexingWriterConfiguration in project metron by apache.

the class IndexingWriterConfigurationTest method testDefaultBatchTimeout.

@Test
public void testDefaultBatchTimeout() {
    IndexingWriterConfiguration config = new IndexingWriterConfiguration("hdfs", new IndexingConfigurations());
    Assert.assertEquals(0, config.getBatchTimeout("foo"));
}

Example 4 with IndexingWriterConfiguration

use of org.apache.metron.common.configuration.writer.IndexingWriterConfiguration in project metron by apache.

the class BulkMessageWriterBolt method prepare.

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.writerComponent = new BulkWriterComponent<>(collector);
    this.collector = collector;
    super.prepare(stormConf, context, collector);
    if (messageGetField != null) {
        messageGetStrategy = MessageGetters.valueOf(messageGetStrategyType).get(messageGetField);
    } else {
        messageGetStrategy = MessageGetters.valueOf(messageGetStrategyType).get();
    }
    if (bulkMessageWriter instanceof WriterToBulkWriter) {
        configurationTransformation = WriterToBulkWriter.TRANSFORMATION;
    } else {
        configurationTransformation = x -> x;
    }
    try {
        WriterConfiguration writerconf = configurationTransformation.apply(new IndexingWriterConfiguration(bulkMessageWriter.getName(), getConfigurations()));
        if (defaultBatchTimeout == 0) {
            // This means getComponentConfiguration was never called to initialize defaultBatchTimeout,
            // probably because we are in a unit test scenario.  So calculate it here.
            BatchTimeoutHelper timeoutHelper = new BatchTimeoutHelper(writerconf::getAllConfiguredTimeouts, batchTimeoutDivisor);
            defaultBatchTimeout = timeoutHelper.getDefaultBatchTimeout();
        }
        writerComponent.setDefaultBatchTimeout(defaultBatchTimeout);
        bulkMessageWriter.init(stormConf, context, writerconf);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 5 with IndexingWriterConfiguration

use of org.apache.metron.common.configuration.writer.IndexingWriterConfiguration in project metron by apache.

the class HdfsWriterTest method testGetHdfsPathStringReturned.

@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathStringReturned() {
    IndexingConfigurations indexingConfig = new IndexingConfigurations();
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    Object result = writer.getHdfsPathExtension(SENSOR_NAME, "TO_UPPER(FORMAT(MAP_GET('key', {'key': 'AbC%s'}), test.key))", message);
    writer.close();
    Assert.assertEquals("ABCTEST.VALUE", result);
}