Examples with MetronError org.apache.metron.common.error.MetronError used on opensource projects

Example 11 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class UnifiedEnrichmentBolt method execute.

/**
 * Fully enrich a message based on the strategy which was used to configure the bolt.
 * Each enrichment is done in parallel and the results are joined together.  Each enrichment
 * will use a cache so computation is avoided if the result has been computed before.
 *
 * Errors in the enrichment result in an error message being sent on the "error" stream.
 * The successful enrichments will be joined with the original message and the message will
 * be sent along the "message" stream.
 *
 * @param input The input tuple to be processed.
 */
@Override
public void execute(Tuple input) {
    JSONObject message = generateMessage(input);
    try {
        String sourceType = MessageUtils.getSensorType(message);
        SensorEnrichmentConfig config = getConfigurations().getSensorEnrichmentConfig(sourceType);
        if (config == null) {
            LOG.debug("Unable to find SensorEnrichmentConfig for sourceType: {}", sourceType);
            config = new SensorEnrichmentConfig();
        }
        // This is an existing kludge for the stellar adapter to pass information along.
        // We should figure out if this can be rearchitected a bit.  This smells.
        config.getConfiguration().putIfAbsent(STELLAR_CONTEXT_CONF, stellarContext);
        String guid = getGUID(input, message);
        // enrich the message
        ParallelEnricher.EnrichmentResult result = enricher.apply(message, strategy, config, perfLog);
        JSONObject enriched = result.getResult();
        enriched = strategy.postProcess(enriched, config, enrichmentContext);
        // we can emit the message now
        collector.emit("message", input, new Values(guid, enriched));
        // and handle each of the errors in turn.  If any adapter errored out, we will have one message per.
        for (Map.Entry<Object, Throwable> t : result.getEnrichmentErrors()) {
            LOG.error("[Metron] Unable to enrich message: {}", message, t);
            MetronError error = new MetronError().withErrorType(strategy.getErrorType()).withMessage(t.getValue().getMessage()).withThrowable(t.getValue()).addRawMessage(t.getKey());
            ErrorUtils.handleError(collector, error);
        }
    } catch (Exception e) {
        // If something terrible and unexpected happens then we want to send an error along, but this
        // really shouldn't be happening.
        LOG.error("[Metron] Unable to enrich message: {}", message, e);
        MetronError error = new MetronError().withErrorType(strategy.getErrorType()).withMessage(e.getMessage()).withThrowable(e).addRawMessage(message);
        ErrorUtils.handleError(collector, error);
    } finally {
        collector.ack(input);
    }
}

Example 12 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class BulkWriterComponentTest method errorAllShouldClearMapsAndHandleErrors.

@Test
public void errorAllShouldClearMapsAndHandleErrors() throws Exception {
    Throwable e = new Exception("test exception");
    MetronError error1 = new MetronError().withSensorType("sensor1").withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Collections.singletonList(message1));
    MetronError error2 = new MetronError().withSensorType("sensor2").withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Collections.singletonList(message2));
    BulkWriterComponent<JSONObject> bulkWriterComponent = new BulkWriterComponent<>(collector);
    bulkWriterComponent.write("sensor1", tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy);
    bulkWriterComponent.write("sensor2", tuple2, message2, bulkMessageWriter, configurations, messageGetStrategy);
    bulkWriterComponent.errorAll(e, messageGetStrategy);
    verifyStatic(times(1));
    ErrorUtils.handleError(collector, error1);
    ErrorUtils.handleError(collector, error2);
    bulkWriterComponent.write("sensor1", tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy);
    verify(bulkMessageWriter, times(0)).write(sensorType, configurations, Collections.singletonList(tuple1), Collections.singletonList(message1));
}

Example 13 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class ParserBoltTest method testEmpty.

@Test
public void testEmpty() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(writer, times(1)).init();
    byte[] sampleBinary = "some binary message".getBytes();
    when(tuple.getBinary(0)).thenReturn(sampleBinary);
    when(parser.parseOptional(sampleBinary)).thenReturn(null);
    parserBolt.execute(tuple);
    verify(parser, times(0)).validate(any());
    verify(writer, times(0)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), any());
    verify(outputCollector, times(1)).ack(tuple);
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(new NullPointerException()).withSensorType(sensorType).addRawMessage(sampleBinary);
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
}

Example 14 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class GenericEnrichmentBoltTest method test.

@Test
public void test() throws IOException {
    when(tuple.getSourceComponent()).thenReturn("unit test component");
    when(tuple.getSourceStreamId()).thenReturn("unit test stream");
    String key = "someKey";
    String enrichmentType = "enrichmentType";
    Enrichment<EnrichmentAdapter<CacheKey>> testEnrichment = new Enrichment<>();
    testEnrichment.setType(enrichmentType);
    testEnrichment.setAdapter(enrichmentAdapter);
    GenericEnrichmentBolt genericEnrichmentBolt = new GenericEnrichmentBolt("zookeeperUrl") {

        @Override
        protected void initializeStellar() {
        // do not initialize stellar here.
        }
    };
    genericEnrichmentBolt.setCuratorFramework(client);
    genericEnrichmentBolt.setZKCache(cache);
    genericEnrichmentBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(sampleSensorEnrichmentConfigPath));
    HashMap<String, Object> globalConfig = new HashMap<>();
    String baseDir = UnitTestHelper.findDir("GeoLite");
    File geoHdfsFile = new File(new File(baseDir), "GeoIP2-City-Test.mmdb.gz");
    globalConfig.put(GeoLiteDatabase.GEO_HDFS_FILE, geoHdfsFile.getAbsolutePath());
    genericEnrichmentBolt.getConfigurations().updateGlobalConfig(globalConfig);
    try {
        genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector);
        fail("Should fail if a maxCacheSize property is not set");
    } catch (IllegalStateException e) {
    }
    genericEnrichmentBolt.withMaxCacheSize(100);
    try {
        genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector);
        fail("Should fail if a maxTimeRetain property is not set");
    } catch (IllegalStateException e) {
    }
    genericEnrichmentBolt.withMaxTimeRetain(10000);
    try {
        genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector);
        fail("Should fail if an adapter is not set");
    } catch (IllegalStateException e) {
    }
    genericEnrichmentBolt.withEnrichment(testEnrichment);
    when(enrichmentAdapter.initializeAdapter(globalConfig)).thenReturn(true);
    genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(enrichmentAdapter, times(1)).initializeAdapter(globalConfig);
    when(enrichmentAdapter.initializeAdapter(globalConfig)).thenReturn(false);
    UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.FATAL);
    try {
        genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector);
        fail("An exception should be thrown if enrichment adapter initialization fails");
    } catch (IllegalStateException e) {
    }
    UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.ERROR);
    genericEnrichmentBolt.declareOutputFields(declarer);
    verify(declarer, times(1)).declareStream(eq(enrichmentType), argThat(new FieldsMatcher("key", "message", "subgroup")));
    verify(declarer, times(1)).declareStream(eq("error"), argThat(new FieldsMatcher("message")));
    when(tuple.getStringByField("key")).thenReturn(null);
    UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.FATAL);
    genericEnrichmentBolt.execute(tuple);
    UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.ERROR);
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.ENRICHMENT_ERROR).withThrowable(new Exception("Could not parse binary stream to JSON"));
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
    when(tuple.getStringByField("key")).thenReturn(key);
    when(tuple.getValueByField("message")).thenReturn(originalMessage);
    when(enrichmentAdapter.enrich(any())).thenReturn(new JSONObject());
    genericEnrichmentBolt.execute(tuple);
    verify(outputCollector, times(1)).emit(eq(enrichmentType), argThat(new EnrichedMessageMatcher(key, new JSONObject(ImmutableMap.of("source.type", "test")))));
    reset(enrichmentAdapter);
    SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig.fromBytes(ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH).get(sensorType));
    sensorEnrichmentConfig.getConfiguration().put(GenericEnrichmentBolt.STELLAR_CONTEXT_CONF, genericEnrichmentBolt.getStellarContext());
    CacheKey cacheKey1 = new CacheKey("field1", "value1", sensorEnrichmentConfig);
    CacheKey cacheKey2 = new CacheKey("field2", "value2", sensorEnrichmentConfig);
    genericEnrichmentBolt.cache.invalidateAll();
    when(enrichmentAdapter.getOutputPrefix(cacheKey1)).thenReturn("field1");
    when(enrichmentAdapter.getOutputPrefix(cacheKey2)).thenReturn("field2");
    when(enrichmentAdapter.enrich(cacheKey1)).thenReturn(enrichedField1);
    when(enrichmentAdapter.enrich(cacheKey2)).thenReturn(enrichedField2);
    genericEnrichmentBolt.execute(tuple);
    verify(enrichmentAdapter, times(1)).logAccess(cacheKey1);
    verify(enrichmentAdapter, times(1)).logAccess(cacheKey2);
    verify(outputCollector, times(1)).emit(eq(enrichmentType), argThat(new EnrichedMessageMatcher(key, enrichedMessage)));
    reset(outputCollector);
    genericEnrichmentBolt.cache.invalidateAll();
    when(enrichmentAdapter.enrich(cacheKey1)).thenReturn(null);
    genericEnrichmentBolt.execute(tuple);
    error = new MetronError().withErrorType(Constants.ErrorType.ENRICHMENT_ERROR).withErrorFields(new HashSet<String>() {

        {
            add("field1");
        }
    }).addRawMessage(new JSONObject() {

        {
            put("field1", "value1");
            put("field2", "value2");
            put("source.type", "test");
        }
    }).withThrowable(new Exception("[Metron] Could not enrich string: value1"));
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
}

Example 15 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class JoinBoltTest method testExecuteShouldReportError.

@SuppressWarnings("unchecked")
@Test
public void testExecuteShouldReportError() throws ExecutionException {
    joinBolt.withMaxCacheSize(100);
    joinBolt.withMaxTimeRetain(10000);
    joinBolt.prepare(new HashMap(), topologyContext, outputCollector);
    when(tuple.getValueByField("key")).thenReturn(key);
    when(tuple.getValueByField("message")).thenReturn(new JSONObject());
    joinBolt.cache = mock(LoadingCache.class);
    when(joinBolt.cache.get(any())).thenThrow(new RuntimeException(new Exception("join exception")));
    joinBolt.execute(tuple);
    RuntimeException expectedExecutionException = new RuntimeException(new Exception("join exception"));
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.ENRICHMENT_ERROR).withMessage("Joining problem: {}").withThrowable(expectedExecutionException).addRawMessage(new JSONObject());
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
    verify(outputCollector, times(1)).reportError(any(ExecutionException.class));
    verify(outputCollector, times(1)).ack(eq(tuple));
    verifyNoMoreInteractions(outputCollector);
}