use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.
the class SimpleHBaseEnrichmentWriterTest method testFilteredKeys.
@Test
public void testFilteredKeys() throws Exception {
final String sensorType = "dummy";
SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {
{
put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), ImmutableList.of("user", "ip"));
}
});
writer.configure(sensorType, configuration);
writer.write(SENSOR_TYPE, configuration, null, new ArrayList<JSONObject>() {
{
add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar")));
}
});
List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues();
Assert.assertEquals(1, values.size());
Assert.assertEquals("localhost", values.get(0).getKey().indicator);
Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user"));
Assert.assertEquals("localhost", values.get(0).getValue().getMetadata().get("ip"));
Assert.assertNull(values.get(0).getValue().getMetadata().get("foo"));
Assert.assertEquals(2, values.get(0).getValue().getMetadata().size());
}
use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.
the class SimpleHBaseEnrichmentWriterTest method getValues.
public static List<LookupKV<EnrichmentKey, EnrichmentValue>> getValues() throws IOException {
MockHTable table = (MockHTable) MockHBaseTableProvider.getFromCache(TABLE_NAME);
Assert.assertNotNull(table);
List<LookupKV<EnrichmentKey, EnrichmentValue>> ret = new ArrayList<>();
EnrichmentConverter converter = new EnrichmentConverter();
for (Result r : table.getScanner(Bytes.toBytes(TABLE_CF))) {
ret.add(converter.fromResult(r, TABLE_CF));
}
return ret;
}
use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.
the class SimpleHBaseEnrichmentWriterTest method testBatchOneNormalPath.
@Test
public void testBatchOneNormalPath() throws Exception {
final String sensorType = "dummy";
SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {
{
put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
}
});
writer.configure(sensorType, configuration);
writer.write(SENSOR_TYPE, configuration, null, new ArrayList<JSONObject>() {
{
add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar")));
}
});
List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues();
Assert.assertEquals(1, values.size());
Assert.assertEquals("localhost", values.get(0).getKey().indicator);
Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user"));
Assert.assertEquals("bar", values.get(0).getValue().getMetadata().get("foo"));
Assert.assertEquals(2, values.get(0).getValue().getMetadata().size());
}
use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.
the class SimpleHBaseEnrichmentWriterTest method testFilteredKey.
@Test
public void testFilteredKey() throws Exception {
final String sensorType = "dummy";
SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {
{
put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), "user");
}
});
writer.configure(sensorType, configuration);
writer.write(SENSOR_TYPE, configuration, null, new ArrayList<JSONObject>() {
{
add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar")));
}
});
List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues();
Assert.assertEquals(1, values.size());
Assert.assertEquals("localhost", values.get(0).getKey().indicator);
Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user"));
Assert.assertNull(values.get(0).getValue().getMetadata().get("foo"));
Assert.assertEquals(1, values.get(0).getValue().getMetadata().size());
}
use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.
the class SimpleHbaseEnrichmentWriterIntegrationTest method test.
@Test
public void test() throws UnableToStartException, IOException {
final String sensorType = "dummy";
final List<byte[]> inputMessages = new ArrayList<byte[]>() {
{
add(Bytes.toBytes("col11,col12,col13"));
add(Bytes.toBytes("col21,col22,col23"));
add(Bytes.toBytes("col31,col32,col33"));
}
};
MockHBaseTableProvider.addToCache(sensorType, "cf");
final Properties topologyProperties = new Properties();
final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties);
final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaComponent.Topic>() {
{
add(new KafkaComponent.Topic(sensorType, 1));
}
});
topologyProperties.setProperty("kafka.broker", kafkaComponent.getBrokerList());
ConfigUploadComponent configUploadComponent = new ConfigUploadComponent().withTopologyProperties(topologyProperties).withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH).withParserSensorConfig(sensorType, JSONUtils.INSTANCE.load(parserConfig, SensorParserConfig.class));
ParserTopologyComponent parserTopologyComponent = new ParserTopologyComponent.Builder().withSensorType(sensorType).withTopologyProperties(topologyProperties).withBrokerUrl(kafkaComponent.getBrokerList()).build();
// UnitTestHelper.verboseLogging();
ComponentRunner runner = new ComponentRunner.Builder().withComponent("zk", zkServerComponent).withComponent("kafka", kafkaComponent).withComponent("config", configUploadComponent).withComponent("org/apache/storm", parserTopologyComponent).withMillisecondsBetweenAttempts(5000).withCustomShutdownOrder(new String[] { "org/apache/storm", "config", "kafka", "zk" }).withNumRetries(10).build();
try {
runner.start();
kafkaComponent.writeMessages(sensorType, inputMessages);
ProcessorResult<List<LookupKV<EnrichmentKey, EnrichmentValue>>> result = runner.process(new Processor<List<LookupKV<EnrichmentKey, EnrichmentValue>>>() {
List<LookupKV<EnrichmentKey, EnrichmentValue>> messages = null;
@Override
public ReadinessState process(ComponentRunner runner) {
MockHTable table = (MockHTable) MockHBaseTableProvider.getFromCache(sensorType);
if (table != null && table.size() == inputMessages.size()) {
EnrichmentConverter converter = new EnrichmentConverter();
messages = new ArrayList<>();
try {
for (Result r : table.getScanner(Bytes.toBytes("cf"))) {
messages.add(converter.fromResult(r, "cf"));
}
} catch (IOException e) {
}
return ReadinessState.READY;
}
return ReadinessState.NOT_READY;
}
@Override
public ProcessorResult<List<LookupKV<EnrichmentKey, EnrichmentValue>>> getResult() {
ProcessorResult.Builder<List<LookupKV<EnrichmentKey, EnrichmentValue>>> builder = new ProcessorResult.Builder();
return builder.withResult(messages).build();
}
});
Set<String> validIndicators = new HashSet<>(ImmutableList.of("col12", "col22", "col32"));
Map<String, Map<String, String>> validMetadata = new HashMap<String, Map<String, String>>() {
{
put("col12", new HashMap<String, String>() {
{
put("col1", "col11");
put("col3", "col13");
}
});
put("col22", new HashMap<String, String>() {
{
put("col1", "col21");
put("col3", "col23");
}
});
put("col32", new HashMap<String, String>() {
{
put("col1", "col31");
put("col3", "col33");
}
});
}
};
for (LookupKV<EnrichmentKey, EnrichmentValue> kv : result.getResult()) {
Assert.assertTrue(validIndicators.contains(kv.getKey().indicator));
Assert.assertEquals(kv.getValue().getMetadata().get("source.type"), "dummy");
Assert.assertNotNull(kv.getValue().getMetadata().get("timestamp"));
Assert.assertNotNull(kv.getValue().getMetadata().get("original_string"));
Map<String, String> metadata = validMetadata.get(kv.getKey().indicator);
for (Map.Entry<String, String> x : metadata.entrySet()) {
Assert.assertEquals(kv.getValue().getMetadata().get(x.getKey()), x.getValue());
}
Assert.assertEquals(metadata.size() + 4, kv.getValue().getMetadata().size());
}
} finally {
if (runner != null) {
runner.stop();
}
}
}
Aggregations