Examples with Enrichment org.apache.metron.enrichment.configuration.Enrichment used on opensource projects

Example 1 with Enrichment

use of org.apache.metron.enrichment.configuration.Enrichment in project metron by apache.

the class EnrichmentSplitterBoltTest method test.

@Test
public void test() throws ParseException, IOException {
    final Enrichment geo = new Enrichment();
    geo.setType("geo");
    final Enrichment host = new Enrichment();
    host.setType("host");
    final Enrichment hbaseEnrichment = new Enrichment();
    hbaseEnrichment.setType("hbaseEnrichment");
    final Enrichment stellarEnrichment = new Enrichment();
    stellarEnrichment.setType("stellar");
    List<Enrichment> enrichments = new ArrayList<Enrichment>() {

        {
            add(geo);
            add(host);
            add(hbaseEnrichment);
            add(stellarEnrichment);
        }
    };
    EnrichmentSplitterBolt enrichmentSplitterBolt = new EnrichmentSplitterBolt("zookeeperUrl").withEnrichments(enrichments);
    enrichmentSplitterBolt.setCuratorFramework(client);
    enrichmentSplitterBolt.setZKCache(cache);
    enrichmentSplitterBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(sampleSensorEnrichmentConfigPath));
    enrichmentSplitterBolt.prepare(new HashMap<>(), topologyContext, outputCollector);
    String key = enrichmentSplitterBolt.getKey(tuple, sampleMessage);
    Assert.assertTrue(key != null && key.length() == 36);
    String someKey = "someKey";
    when(tuple.getStringByField("key")).thenReturn(someKey);
    key = enrichmentSplitterBolt.getKey(tuple, sampleMessage);
    Assert.assertEquals(someKey, key);
    String guid = "sample-guid";
    when(sampleMessage.get("guid")).thenReturn(guid);
    key = enrichmentSplitterBolt.getKey(tuple, sampleMessage);
    Assert.assertEquals(guid, key);
    when(tuple.getBinary(0)).thenReturn(sampleMessageString.getBytes());
    JSONObject generatedMessage = enrichmentSplitterBolt.generateMessage(tuple);
    removeTimingFields(generatedMessage);
    Assert.assertEquals(sampleMessage, generatedMessage);
    String messageFieldName = "messageFieldName";
    enrichmentSplitterBolt.withMessageFieldName(messageFieldName);
    when(tuple.getValueByField(messageFieldName)).thenReturn(sampleMessage);
    generatedMessage = enrichmentSplitterBolt.generateMessage(tuple);
    Assert.assertEquals(sampleMessage, generatedMessage);
    Set<String> actualStreamIds = enrichmentSplitterBolt.getStreamIds();
    Assert.assertEquals(streamIds, actualStreamIds);
    Map<String, List<JSONObject>> actualSplitMessages = enrichmentSplitterBolt.splitMessage(sampleMessage);
    Assert.assertEquals(enrichments.size(), actualSplitMessages.size());
    Assert.assertEquals(ImmutableList.of(geoMessage), actualSplitMessages.get("geo"));
    Assert.assertEquals(ImmutableList.of(hostMessage), actualSplitMessages.get("host"));
    Assert.assertEquals(ImmutableList.of(hbaseEnrichmentMessage), actualSplitMessages.get("hbaseEnrichment"));
}

Example 2 with Enrichment

use of org.apache.metron.enrichment.configuration.Enrichment in project metron by apache.

the class GenericEnrichmentBoltTest method test.

@Test
public void test() throws IOException {
    when(tuple.getSourceComponent()).thenReturn("unit test component");
    when(tuple.getSourceStreamId()).thenReturn("unit test stream");
    String key = "someKey";
    String enrichmentType = "enrichmentType";
    Enrichment<EnrichmentAdapter<CacheKey>> testEnrichment = new Enrichment<>();
    testEnrichment.setType(enrichmentType);
    testEnrichment.setAdapter(enrichmentAdapter);
    GenericEnrichmentBolt genericEnrichmentBolt = new GenericEnrichmentBolt("zookeeperUrl") {

        @Override
        protected void initializeStellar() {
        // do not initialize stellar here.
        }
    };
    genericEnrichmentBolt.setCuratorFramework(client);
    genericEnrichmentBolt.setZKCache(cache);
    genericEnrichmentBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(sampleSensorEnrichmentConfigPath));
    HashMap<String, Object> globalConfig = new HashMap<>();
    String baseDir = UnitTestHelper.findDir("GeoLite");
    File geoHdfsFile = new File(new File(baseDir), "GeoIP2-City-Test.mmdb.gz");
    globalConfig.put(GeoLiteDatabase.GEO_HDFS_FILE, geoHdfsFile.getAbsolutePath());
    genericEnrichmentBolt.getConfigurations().updateGlobalConfig(globalConfig);
    try {
        genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector);
        fail("Should fail if a maxCacheSize property is not set");
    } catch (IllegalStateException e) {
    }
    genericEnrichmentBolt.withMaxCacheSize(100);
    try {
        genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector);
        fail("Should fail if a maxTimeRetain property is not set");
    } catch (IllegalStateException e) {
    }
    genericEnrichmentBolt.withMaxTimeRetain(10000);
    try {
        genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector);
        fail("Should fail if an adapter is not set");
    } catch (IllegalStateException e) {
    }
    genericEnrichmentBolt.withEnrichment(testEnrichment);
    when(enrichmentAdapter.initializeAdapter(globalConfig)).thenReturn(true);
    genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(enrichmentAdapter, times(1)).initializeAdapter(globalConfig);
    when(enrichmentAdapter.initializeAdapter(globalConfig)).thenReturn(false);
    UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.FATAL);
    try {
        genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector);
        fail("An exception should be thrown if enrichment adapter initialization fails");
    } catch (IllegalStateException e) {
    }
    UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.ERROR);
    genericEnrichmentBolt.declareOutputFields(declarer);
    verify(declarer, times(1)).declareStream(eq(enrichmentType), argThat(new FieldsMatcher("key", "message", "subgroup")));
    verify(declarer, times(1)).declareStream(eq("error"), argThat(new FieldsMatcher("message")));
    when(tuple.getStringByField("key")).thenReturn(null);
    UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.FATAL);
    genericEnrichmentBolt.execute(tuple);
    UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.ERROR);
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.ENRICHMENT_ERROR).withThrowable(new Exception("Could not parse binary stream to JSON"));
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
    when(tuple.getStringByField("key")).thenReturn(key);
    when(tuple.getValueByField("message")).thenReturn(originalMessage);
    when(enrichmentAdapter.enrich(any())).thenReturn(new JSONObject());
    genericEnrichmentBolt.execute(tuple);
    verify(outputCollector, times(1)).emit(eq(enrichmentType), argThat(new EnrichedMessageMatcher(key, new JSONObject(ImmutableMap.of("source.type", "test")))));
    reset(enrichmentAdapter);
    SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig.fromBytes(ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH).get(sensorType));
    sensorEnrichmentConfig.getConfiguration().put(GenericEnrichmentBolt.STELLAR_CONTEXT_CONF, genericEnrichmentBolt.getStellarContext());
    CacheKey cacheKey1 = new CacheKey("field1", "value1", sensorEnrichmentConfig);
    CacheKey cacheKey2 = new CacheKey("field2", "value2", sensorEnrichmentConfig);
    genericEnrichmentBolt.cache.invalidateAll();
    when(enrichmentAdapter.getOutputPrefix(cacheKey1)).thenReturn("field1");
    when(enrichmentAdapter.getOutputPrefix(cacheKey2)).thenReturn("field2");
    when(enrichmentAdapter.enrich(cacheKey1)).thenReturn(enrichedField1);
    when(enrichmentAdapter.enrich(cacheKey2)).thenReturn(enrichedField2);
    genericEnrichmentBolt.execute(tuple);
    verify(enrichmentAdapter, times(1)).logAccess(cacheKey1);
    verify(enrichmentAdapter, times(1)).logAccess(cacheKey2);
    verify(outputCollector, times(1)).emit(eq(enrichmentType), argThat(new EnrichedMessageMatcher(key, enrichedMessage)));
    reset(outputCollector);
    genericEnrichmentBolt.cache.invalidateAll();
    when(enrichmentAdapter.enrich(cacheKey1)).thenReturn(null);
    genericEnrichmentBolt.execute(tuple);
    error = new MetronError().withErrorType(Constants.ErrorType.ENRICHMENT_ERROR).withErrorFields(new HashSet<String>() {

        {
            add("field1");
        }
    }).addRawMessage(new JSONObject() {

        {
            put("field1", "value1");
            put("field2", "value2");
            put("source.type", "test");
        }
    }).withThrowable(new Exception("[Metron] Could not enrich string: value1"));
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
}