Examples with MetronError org.apache.metron.common.error.MetronError used on opensource projects

Example 1 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class ErrorUtilsTest method handleErrorShouldEmitAndReportError.

@Test
public void handleErrorShouldEmitAndReportError() throws Exception {
    Throwable e = new Exception("error");
    MetronError error = new MetronError().withMessage("error message").withThrowable(e);
    OutputCollector collector = mock(OutputCollector.class);
    ErrorUtils.handleError(collector, error);
    verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
    verify(collector, times(1)).reportError(any());
}

Example 2 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class JoinBolt method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
    perfLog.mark("execute");
    String streamId = tuple.getSourceStreamId();
    String key = (String) keyGetStrategy.get(tuple);
    String subgroup = (String) subgroupGetStrategy.get(tuple);
    streamId = Joiner.on(":").join("" + streamId, subgroup == null ? "" : subgroup);
    V message = (V) messageGetStrategy.get(tuple);
    try {
        Map<String, Tuple> streamMessageMap = cache.get(key);
        if (streamMessageMap.containsKey(streamId)) {
            LOG.warn("Received key {} twice for stream {}", key, streamId);
        }
        streamMessageMap.put(streamId, tuple);
        Set<String> streamIds = getStreamIds(message);
        Set<String> streamMessageKeys = streamMessageMap.keySet();
        if (streamMessageKeys.size() == streamIds.size() && Sets.symmetricDifference(streamMessageKeys, streamIds).isEmpty()) {
            perfLog.mark("join-message");
            V joinedMessages = joinMessages(streamMessageMap, this.messageGetStrategy);
            perfLog.log("join-message", "key={}, elapsed time to join messages", key);
            perfLog.mark("emit-message");
            collector.emit("message", tuple, new Values(key, joinedMessages));
            perfLog.log("emit-message", "key={}, elapsed time to emit messages", key);
            cache.invalidate(key);
            Tuple messageTuple = streamMessageMap.get("message:");
            collector.ack(messageTuple);
            LOG.trace("Emitted message for key: {}", key);
        } else {
            cache.put(key, streamMessageMap);
            if (LOG.isDebugEnabled()) {
                LOG.debug("{}: Missed joining portions for {}. Expected {} != {}", getClass().getSimpleName(), key, Joiner.on(",").join(streamIds), Joiner.on(",").join(streamMessageKeys));
            }
        }
    } catch (Exception e) {
        LOG.error("[Metron] Unable to join messages: {}", message, e);
        MetronError error = new MetronError().withErrorType(Constants.ErrorType.ENRICHMENT_ERROR).withMessage("Joining problem: " + message).withThrowable(e).addRawMessage(message);
        ErrorUtils.handleError(collector, error);
        collector.ack(tuple);
    }
    perfLog.log("execute", "key={}, elapsed time to run execute", key);
}

Example 3 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class BulkWriterComponent method error.

public void error(String sensorType, Throwable e, Iterable<Tuple> tuples, MessageGetStrategy messageGetStrategy) {
    tuples.forEach(t -> collector.ack(t));
    MetronError error = new MetronError().withSensorType(sensorType).withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e);
    if (!Iterables.isEmpty(tuples)) {
        LOG.error("Failing {} tuples", Iterables.size(tuples), e);
    }
    tuples.forEach(t -> error.addRawMessage(messageGetStrategy.get(t)));
    ErrorUtils.handleError(collector, error);
}

Example 4 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class ParserBoltTest method testInvalid.

@Test
public void testInvalid() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    buildGlobalConfig(parserBolt);
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    byte[] sampleBinary = "some binary message".getBytes();
    when(tuple.getBinary(0)).thenReturn(sampleBinary);
    JSONObject parsedMessage = new JSONObject();
    parsedMessage.put("field", "invalidValue");
    parsedMessage.put("guid", "this-is-unique-identifier-for-tuple");
    List<JSONObject> messageList = new ArrayList<>();
    messageList.add(parsedMessage);
    when(parser.parseOptional(sampleBinary)).thenReturn(Optional.of(messageList));
    when(parser.validate(parsedMessage)).thenReturn(true);
    parserBolt.execute(tuple);
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(sensorType).withErrorFields(new HashSet<String>() {

        {
            add("field");
        }
    }).addRawMessage(new JSONObject() {

        {
            put("field", "invalidValue");
            put("source.type", "yaf");
            put("guid", "this-is-unique-identifier-for-tuple");
        }
    });
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
}

Example 5 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class ParserRunnerImpl method execute.

/**
 * Parses messages with the appropriate MessageParser based on sensor type.  The resulting list of messages are then
 * post-processed and added to the ParserRunnerResults message list.  Any errors that happen during post-processing are
 * added to the ParserRunnerResults error list.  Any exceptions (including a master exception) thrown by the MessageParser
 * are also added to the ParserRunnerResults error list.
 *
 * @param sensorType Sensor type of the message
 * @param rawMessage Raw message including metadata
 * @param parserConfigurations Parser configurations
 * @return ParserRunnerResults containing a list of messages and a list of errors
 */
@Override
public ParserRunnerResults<JSONObject> execute(String sensorType, RawMessage rawMessage, ParserConfigurations parserConfigurations) {
    DefaultParserRunnerResults parserRunnerResults = new DefaultParserRunnerResults();
    SensorParserConfig sensorParserConfig = parserConfigurations.getSensorParserConfig(sensorType);
    if (sensorParserConfig != null) {
        MessageParser<JSONObject> parser = sensorToParserComponentMap.get(sensorType).getMessageParser();
        Optional<MessageParserResult<JSONObject>> optionalMessageParserResult = parser.parseOptionalResult(rawMessage.getMessage());
        if (optionalMessageParserResult.isPresent()) {
            MessageParserResult<JSONObject> messageParserResult = optionalMessageParserResult.get();
            // Process each message returned from the MessageParser
            messageParserResult.getMessages().forEach(message -> {
                Optional<ProcessResult> processResult = processMessage(sensorType, message, rawMessage, parser, parserConfigurations);
                if (processResult.isPresent()) {
                    if (processResult.get().isError()) {
                        parserRunnerResults.addError(processResult.get().getError());
                    } else {
                        parserRunnerResults.addMessage(processResult.get().getMessage());
                    }
                }
            });
            // If a master exception is thrown by the MessageParser, wrap it with a MetronError and add it to the list of errors
            messageParserResult.getMasterThrowable().ifPresent(throwable -> parserRunnerResults.addError(new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(throwable).withSensorType(Collections.singleton(sensorType)).withMetadata(rawMessage.getMetadata()).addRawMessage(rawMessage.getMessage())));
            // If exceptions are thrown by the MessageParser, wrap them with MetronErrors and add them to the list of errors
            parserRunnerResults.addErrors(messageParserResult.getMessageThrowables().entrySet().stream().map(entry -> new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(entry.getValue()).withSensorType(Collections.singleton(sensorType)).withMetadata(rawMessage.getMetadata()).addRawMessage(entry.getKey())).collect(Collectors.toList()));
        }
    } else {
        throw new IllegalStateException(String.format("Could not execute parser.  Cannot find configuration for sensor %s.", sensorType));
    }
    return parserRunnerResults;
}