Examples with MetronError org.apache.metron.common.error.MetronError used on opensource projects

Example 26 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class BulkMessageWriterBolt method handleMissingSensorType.

/**
 * Handles error processing when a message is missing a sensor type.
 *
 * @param tuple The tuple.
 * @param message The message with no sensor type.
 */
private void handleMissingSensorType(Tuple tuple, JSONObject message) {
    // sensor type somehow ended up being null.  We want to error this message directly.
    LOG.debug("Message is missing sensor type");
    String sensorType = "null";
    Exception e = new Exception("Sensor type is not specified for message " + message.toJSONString());
    LOG.error(format("Failing %d tuple(s); sensorType=%s", Iterables.size(ImmutableList.of(tuple)), sensorType), e);
    MetronError error = new MetronError().withSensorType(Collections.singleton(sensorType)).withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).addRawMessage(messageGetStrategy.get(tuple));
    collector.emit(Constants.ERROR_STREAM, new Values(error.getJSONObject()));
    // there is only one error to report for all of the failed tuples
    collector.reportError(e);
    collector.ack(tuple);
}

Example 27 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class ParserRunnerImpl method processMessage.

/**
 * Post-processes parsed messages by:
 * <ul>
 *   <li>Applying field transformations defined in the sensor parser config</li>
 *   <li>Filtering messages using the configured MessageFilter class</li>
 *   <li>Validating messages using the MessageParser validate method</li>
 * </ul>
 * If a message is successfully processed a message is returned in a ProcessResult.  If a message fails
 * validation, a MetronError object is created and returned in a ProcessResult.  If a message is
 * filtered out an empty Optional is returned.
 *
 * @param sensorType Sensor type of the message
 * @param message Message parsed by the MessageParser
 * @param rawMessage Raw message including metadata
 * @param parser MessageParser for the sensor type
 * @param parserConfigurations Parser configurations
 */
@SuppressWarnings("unchecked")
protected Optional<ProcessResult> processMessage(String sensorType, JSONObject message, RawMessage rawMessage, MessageParser<JSONObject> parser, ParserConfigurations parserConfigurations) {
    Optional<ProcessResult> processResult = Optional.empty();
    SensorParserConfig sensorParserConfig = parserConfigurations.getSensorParserConfig(sensorType);
    sensorParserConfig.getRawMessageStrategy().mergeMetadata(message, rawMessage.getMetadata(), sensorParserConfig.getMergeMetadata(), sensorParserConfig.getRawMessageStrategyConfig());
    message.put(Constants.SENSOR_TYPE, sensorType);
    applyFieldTransformations(message, rawMessage, sensorParserConfig);
    if (!message.containsKey(Constants.GUID)) {
        message.put(Constants.GUID, UUID.randomUUID().toString());
    }
    message.putIfAbsent(Fields.ORIGINAL.getName(), new String(rawMessage.getMessage(), parser.getReadCharset()));
    MessageFilter<JSONObject> filter = sensorToParserComponentMap.get(sensorType).getFilter();
    if (filter == null || filter.emit(message, stellarContext)) {
        boolean isInvalid = !parser.validate(message);
        List<FieldValidator> failedValidators = null;
        if (!isInvalid) {
            failedValidators = getFailedValidators(message, parserConfigurations);
            isInvalid = !failedValidators.isEmpty();
        }
        if (isInvalid) {
            MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(Collections.singleton(sensorType)).withMetadata(rawMessage.getMetadata()).addRawMessage(message);
            Set<String> errorFields = failedValidators == null ? null : failedValidators.stream().flatMap(fieldValidator -> fieldValidator.getInput().stream()).collect(Collectors.toSet());
            if (errorFields != null && !errorFields.isEmpty()) {
                error.withErrorFields(errorFields);
            }
            processResult = Optional.of(new ProcessResult(error));
        } else {
            processResult = Optional.of(new ProcessResult(message));
        }
    }
    return processResult;
}

Example 28 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class ParserRunnerImplTest method shouldExecuteWithMasterThrowable.

@Test
public void shouldExecuteWithMasterThrowable() {
    parserRunner = spy(parserRunner);
    RawMessage rawMessage = new RawMessage("raw_message".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    Throwable masterThrowable = mock(Throwable.class);
    MessageParserResult<JSONObject> messageParserResult = new DefaultMessageParserResult<>(masterThrowable);
    when(broParser.parseOptionalResult(rawMessage.getMessage())).thenReturn(Optional.of(messageParserResult));
    parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {

        {
            put("bro", new ParserComponent(broParser, stellarFilter));
        }
    });
    ParserRunnerResults<JSONObject> parserRunnerResults = parserRunner.execute("bro", rawMessage, parserConfigurations);
    verify(parserRunner, times(0)).processMessage(any(), any(), any(), any(), any());
    MetronError expectedError = new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(masterThrowable).withSensorType(Collections.singleton("bro")).addRawMessage(rawMessage.getMessage());
    assertEquals(1, parserRunnerResults.getErrors().size());
    assertTrue(parserRunnerResults.getErrors().contains(expectedError));
}

Example 29 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class ParserRunnerImplTest method shouldReturnMetronErrorOnInvalidMessage.

@Test
public void shouldReturnMetronErrorOnInvalidMessage() {
    Map<String, Object> metadata = new HashMap<>();
    metadata.put("metron.metadata.topic", "bro");
    metadata.put("metron.metadata.partition", 0);
    metadata.put("metron.metadata.offset", 123);
    JSONObject inputMessage = new JSONObject();
    inputMessage.put("guid", "guid");
    RawMessage rawMessage = new RawMessage("raw_message".getBytes(StandardCharsets.UTF_8), metadata);
    JSONObject expectedOutput = new JSONObject();
    expectedOutput.put("guid", "guid");
    expectedOutput.put("source.type", "bro");
    expectedOutput.put(Fields.ORIGINAL.getName(), "raw_message");
    MetronError expectedMetronError = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(Collections.singleton("bro")).withMetadata(metadata).addRawMessage(inputMessage);
    when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true);
    // This is the important switch. Not to be confused with field validators.
    when(broParser.validate(expectedOutput)).thenReturn(false);
    parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {

        {
            put("bro", new ParserComponent(broParser, stellarFilter));
        }
    });
    Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations);
    assertTrue(processResult.isPresent());
    assertTrue(processResult.get().isError());
    assertEquals(expectedMetronError, processResult.get().getError());
}

Example 30 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class GenericEnrichmentBoltTest method test.

@Test
public void test() throws IOException {
    when(tuple.getSourceComponent()).thenReturn("unit test component");
    when(tuple.getSourceStreamId()).thenReturn("unit test stream");
    String key = "someKey";
    String enrichmentType = "enrichmentType";
    Enrichment<EnrichmentAdapter<CacheKey>> testEnrichment = new Enrichment<>();
    testEnrichment.setType(enrichmentType);
    testEnrichment.setAdapter(enrichmentAdapter);
    GenericEnrichmentBolt genericEnrichmentBolt = new GenericEnrichmentBolt("zookeeperUrl") {

        @Override
        protected void initializeStellar() {
        // do not initialize stellar here.
        }
    };
    genericEnrichmentBolt.setCuratorFramework(client);
    genericEnrichmentBolt.setZKCache(cache);
    genericEnrichmentBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(enrichmentConfigPath));
    HashMap<String, Object> globalConfig = new HashMap<>();
    String baseDir = UnitTestHelper.findDir(new File("../metron-enrichment-common"), "GeoLite");
    File geoHdfsFile = new File(new File(baseDir), "GeoLite2-City.mmdb.gz");
    globalConfig.put(GeoLiteCityDatabase.GEO_HDFS_FILE, geoHdfsFile.getAbsolutePath());
    genericEnrichmentBolt.getConfigurations().updateGlobalConfig(globalConfig);
    assertThrows(IllegalStateException.class, () -> genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector), "Should fail if a maxCacheSize property is not set");
    genericEnrichmentBolt.withMaxCacheSize(100);
    assertThrows(IllegalStateException.class, () -> genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector), "Should fail if a maxTimeRetain property is not set");
    genericEnrichmentBolt.withMaxTimeRetain(10000);
    assertThrows(IllegalStateException.class, () -> genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector), "Should fail if an adapter is not set");
    genericEnrichmentBolt.withEnrichment(testEnrichment);
    when(enrichmentAdapter.initializeAdapter(globalConfig)).thenReturn(true);
    genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(enrichmentAdapter, times(1)).initializeAdapter(globalConfig);
    when(enrichmentAdapter.initializeAdapter(globalConfig)).thenReturn(false);
    UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.FATAL);
    assertThrows(IllegalStateException.class, () -> genericEnrichmentBolt.prepare(new HashMap(), topologyContext, outputCollector), "An exception should be thrown if enrichment adapter initialization fails");
    UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.ERROR);
    genericEnrichmentBolt.declareOutputFields(declarer);
    verify(declarer, times(1)).declareStream(eq(enrichmentType), argThat(new FieldsMatcher("key", "message", "subgroup")));
    verify(declarer, times(1)).declareStream(eq("error"), argThat(new FieldsMatcher("message")));
    when(tuple.getStringByField("key")).thenReturn(null);
    UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.FATAL);
    genericEnrichmentBolt.execute(tuple);
    UnitTestHelper.setLog4jLevel(GenericEnrichmentBolt.class, Level.ERROR);
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.ENRICHMENT_ERROR).withThrowable(new Exception("Could not parse binary stream to JSON"));
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
    when(tuple.getStringByField("key")).thenReturn(key);
    when(tuple.getValueByField("message")).thenReturn(originalMessage);
    when(enrichmentAdapter.enrich(any())).thenReturn(new JSONObject());
    genericEnrichmentBolt.execute(tuple);
    verify(outputCollector, times(1)).emit(eq(enrichmentType), argThat(new EnrichedMessageMatcher(key, new JSONObject(ImmutableMap.of("source.type", "test")))));
    reset(enrichmentAdapter);
    SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig.fromBytes(ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(sampleConfigPath).get(sensorType));
    sensorEnrichmentConfig.getConfiguration().put(STELLAR_CONTEXT_CONF, genericEnrichmentBolt.getStellarContext());
    CacheKey cacheKey1 = new CacheKey("field1", "value1", sensorEnrichmentConfig);
    CacheKey cacheKey2 = new CacheKey("field2", "value2", sensorEnrichmentConfig);
    genericEnrichmentBolt.cache.invalidateAll();
    when(enrichmentAdapter.getOutputPrefix(cacheKey1)).thenReturn("field1");
    when(enrichmentAdapter.getOutputPrefix(cacheKey2)).thenReturn("field2");
    when(enrichmentAdapter.enrich(cacheKey1)).thenReturn(enrichedField1);
    when(enrichmentAdapter.enrich(cacheKey2)).thenReturn(enrichedField2);
    genericEnrichmentBolt.execute(tuple);
    verify(enrichmentAdapter, times(1)).logAccess(cacheKey1);
    verify(enrichmentAdapter, times(1)).logAccess(cacheKey2);
    verify(outputCollector, times(1)).emit(eq(enrichmentType), argThat(new EnrichedMessageMatcher(key, enrichedMessage)));
    reset(outputCollector);
    genericEnrichmentBolt.cache.invalidateAll();
    when(enrichmentAdapter.enrich(cacheKey1)).thenReturn(null);
    genericEnrichmentBolt.execute(tuple);
    error = new MetronError().withErrorType(Constants.ErrorType.ENRICHMENT_ERROR).withErrorFields(new HashSet<String>() {

        {
            add("field1");
        }
    }).addRawMessage(new JSONObject() {

        {
            put("field1", "value1");
            put("field2", "value2");
            put("source.type", "test");
        }
    }).withThrowable(new Exception("[Metron] Could not enrich string: value1"));
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
}