use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.
the class BulkMessageWriterBolt method prepare.
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this.writerComponent = new BulkWriterComponent<>(collector);
this.collector = collector;
super.prepare(stormConf, context, collector);
if (messageGetField != null) {
messageGetStrategy = MessageGetters.valueOf(messageGetStrategyType).get(messageGetField);
} else {
messageGetStrategy = MessageGetters.valueOf(messageGetStrategyType).get();
}
if (bulkMessageWriter instanceof WriterToBulkWriter) {
configurationTransformation = WriterToBulkWriter.TRANSFORMATION;
} else {
configurationTransformation = x -> x;
}
try {
WriterConfiguration writerconf = configurationTransformation.apply(new IndexingWriterConfiguration(bulkMessageWriter.getName(), getConfigurations()));
if (defaultBatchTimeout == 0) {
// This means getComponentConfiguration was never called to initialize defaultBatchTimeout,
// probably because we are in a unit test scenario. So calculate it here.
BatchTimeoutHelper timeoutHelper = new BatchTimeoutHelper(writerconf::getAllConfiguredTimeouts, batchTimeoutDivisor);
defaultBatchTimeout = timeoutHelper.getDefaultBatchTimeout();
}
writerComponent.setDefaultBatchTimeout(defaultBatchTimeout);
bulkMessageWriter.init(stormConf, context, writerconf);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.
the class SimpleHBaseEnrichmentWriterTest method testConfigValidation_enrichment_type_is_not_a_string.
@Test(expected = IllegalArgumentException.class)
public void testConfigValidation_enrichment_type_is_not_a_string() {
final String sensorType = "dummy";
SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>() {
{
put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
put(SimpleHbaseEnrichmentWriter.Configurations.ENRICHMENT_TYPE.getKey(), 10);
}
});
try {
writer.configure(sensorType, configuration);
} catch (IllegalArgumentException ex) {
Assert.assertEquals(String.format("%s must be a string", SimpleHbaseEnrichmentWriter.Configurations.ENRICHMENT_TYPE.getKey()), ex.getMessage());
throw ex;
}
}
use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.
the class SimpleHBaseEnrichmentWriterTest method testConfigValidation_key_columns_contain_an_empty_value.
@Test(expected = IllegalArgumentException.class)
public void testConfigValidation_key_columns_contain_an_empty_value() {
final String sensorType = "dummy";
SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>() {
{
put(SimpleHbaseEnrichmentWriter.Configurations.ENRICHMENT_TYPE.getKey(), ENRICHMENT_TYPE);
put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), Arrays.asList("ip", " "));
}
});
try {
writer.configure(sensorType, configuration);
} catch (IllegalArgumentException ex) {
Assert.assertEquals("Column name must not be empty", ex.getMessage());
throw ex;
}
}
use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.
the class SimpleHBaseEnrichmentWriterTest method testFilteredKeys.
@Test
public void testFilteredKeys() throws Exception {
final String sensorType = "dummy";
SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {
{
put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), ImmutableList.of("user", "ip"));
}
});
writer.configure(sensorType, configuration);
writer.write(SENSOR_TYPE, configuration, null, new ArrayList<JSONObject>() {
{
add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar")));
}
});
List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues();
Assert.assertEquals(1, values.size());
Assert.assertEquals("localhost", values.get(0).getKey().indicator);
Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user"));
Assert.assertEquals("localhost", values.get(0).getValue().getMetadata().get("ip"));
Assert.assertNull(values.get(0).getValue().getMetadata().get("foo"));
Assert.assertEquals(2, values.get(0).getValue().getMetadata().size());
}
use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.
the class SimpleHBaseEnrichmentWriterTest method testConfigValidation_key_columns_contain_a_null_value.
@Test(expected = IllegalArgumentException.class)
public void testConfigValidation_key_columns_contain_a_null_value() {
final String sensorType = "dummy";
SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>() {
{
put(SimpleHbaseEnrichmentWriter.Configurations.ENRICHMENT_TYPE.getKey(), ENRICHMENT_TYPE);
put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), Arrays.asList("ip", null));
}
});
try {
writer.configure(sensorType, configuration);
} catch (IllegalArgumentException ex) {
Assert.assertEquals("Column name must not be null", ex.getMessage());
throw ex;
}
}
Aggregations