Examples with WriterConfiguration org.apache.metron.common.configuration.writer.WriterConfiguration used on opensource projects

Example 16 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class SimpleHBaseEnrichmentWriterTest method testConfigValidation_missing_key_columns.

@Test(expected = IllegalArgumentException.class)
public void testConfigValidation_missing_key_columns() {
    final String sensorType = "dummy";
    SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
    WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>() {

        {
            put(SimpleHbaseEnrichmentWriter.Configurations.ENRICHMENT_TYPE.getKey(), ENRICHMENT_TYPE);
        }
    });
    try {
        writer.configure(sensorType, configuration);
    } catch (IllegalArgumentException ex) {
        Assert.assertEquals(String.format("%s must be provided", SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey()), ex.getMessage());
        throw ex;
    }
}

Example 17 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class SimpleHBaseEnrichmentWriterTest method testBatchOneNormalPath.

@Test
public void testBatchOneNormalPath() throws Exception {
    final String sensorType = "dummy";
    SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
    WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {

        {
            put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
        }
    });
    writer.configure(sensorType, configuration);
    writer.write(SENSOR_TYPE, configuration, null, new ArrayList<JSONObject>() {

        {
            add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar")));
        }
    });
    List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues();
    Assert.assertEquals(1, values.size());
    Assert.assertEquals("localhost", values.get(0).getKey().indicator);
    Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user"));
    Assert.assertEquals("bar", values.get(0).getValue().getMetadata().get("foo"));
    Assert.assertEquals(2, values.get(0).getValue().getMetadata().size());
}

Example 18 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class SimpleHBaseEnrichmentWriterTest method testFilteredKey.

@Test
public void testFilteredKey() throws Exception {
    final String sensorType = "dummy";
    SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
    WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {

        {
            put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
            put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), "user");
        }
    });
    writer.configure(sensorType, configuration);
    writer.write(SENSOR_TYPE, configuration, null, new ArrayList<JSONObject>() {

        {
            add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar")));
        }
    });
    List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues();
    Assert.assertEquals(1, values.size());
    Assert.assertEquals("localhost", values.get(0).getKey().indicator);
    Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user"));
    Assert.assertNull(values.get(0).getValue().getMetadata().get("foo"));
    Assert.assertEquals(1, values.get(0).getValue().getMetadata().size());
}

Example 19 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class SimpleHBaseEnrichmentWriterTest method testConfigValidation_missing_enrichment_type.

@Test(expected = IllegalArgumentException.class)
public void testConfigValidation_missing_enrichment_type() {
    final String sensorType = "dummy";
    SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
    WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>() {

        {
            put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
        }
    });
    try {
        writer.configure(sensorType, configuration);
    } catch (IllegalArgumentException ex) {
        Assert.assertEquals(String.format("%s must be provided", SimpleHbaseEnrichmentWriter.Configurations.ENRICHMENT_TYPE.getKey()), ex.getMessage());
        throw ex;
    }
}

Example 20 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testGetHdfsPathFormatVariable.

@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathFormatVariable() {
    IndexingConfigurations indexingConfig = new IndexingConfigurations();
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    message.put("test.key.2", "test.value.2");
    message.put("test.key.3", "test.value.3");
    Object result = writer.getHdfsPathExtension(SENSOR_NAME, "FORMAT('%s/%s/%s', test.key, test.key.2, test.key.3)", message);
    writer.close();
    Assert.assertEquals("test.value/test.value.2/test.value.3", result);
}