Examples with MetronError org.apache.metron.common.error.MetronError used on opensource projects

Example 6 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class ParserBolt method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
    byte[] originalMessage = (byte[]) messageGetStrategy.get(tuple);
    SensorParserConfig sensorParserConfig = getSensorParserConfig();
    try {
        // we want to ack the tuple in the situation where we have are not doing a bulk write
        // otherwise we want to defer to the writerComponent who will ack on bulk commit.
        boolean ackTuple = !writer.handleAck();
        int numWritten = 0;
        if (sensorParserConfig != null) {
            Map<String, Object> metadata = getMetadata(tuple, sensorParserConfig.getReadMetadata());
            List<FieldValidator> fieldValidations = getConfigurations().getFieldValidations();
            Optional<List<JSONObject>> messages = parser.parseOptional(originalMessage);
            for (JSONObject message : messages.orElse(Collections.emptyList())) {
                message.put(Constants.SENSOR_TYPE, getSensorType());
                if (sensorParserConfig.getMergeMetadata()) {
                    message.putAll(metadata);
                }
                for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) {
                    if (handler != null) {
                        if (!sensorParserConfig.getMergeMetadata()) {
                            // if we haven't merged metadata, then we need to pass them along as configuration params.
                            handler.transformAndUpdate(message, stellarContext, sensorParserConfig.getParserConfig(), metadata);
                        } else {
                            handler.transformAndUpdate(message, stellarContext, sensorParserConfig.getParserConfig());
                        }
                    }
                }
                if (!message.containsKey(Constants.GUID)) {
                    message.put(Constants.GUID, UUID.randomUUID().toString());
                }
                if (parser.validate(message) && (filter == null || filter.emitTuple(message, stellarContext))) {
                    numWritten++;
                    List<FieldValidator> failedValidators = getFailedValidators(message, fieldValidations);
                    if (failedValidators.size() > 0) {
                        MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(getSensorType()).addRawMessage(message);
                        Set<String> errorFields = failedValidators.stream().flatMap(fieldValidator -> fieldValidator.getInput().stream()).collect(Collectors.toSet());
                        if (!errorFields.isEmpty()) {
                            error.withErrorFields(errorFields);
                        }
                        ErrorUtils.handleError(collector, error);
                    } else {
                        writer.write(getSensorType(), tuple, message, getConfigurations(), messageGetStrategy);
                    }
                }
            }
        }
        // then we want to handle the ack ourselves.
        if (ackTuple || numWritten == 0) {
            collector.ack(tuple);
        }
    } catch (Throwable ex) {
        handleError(originalMessage, tuple, ex, collector);
    }
}

Example 7 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class WriterBolt method execute.

@Override
public void execute(Tuple tuple) {
    JSONObject message = null;
    try {
        message = (JSONObject) messageGetStrategy.get(tuple);
        handler.write(sensorType, tuple, message, configuration, messageGetStrategy);
        if (!handler.handleAck()) {
            collector.ack(tuple);
        }
    } catch (Throwable e) {
        MetronError error = new MetronError().withErrorType(errorType).withThrowable(e).withSensorType(sensorType).addRawMessage(message);
        ErrorUtils.handleError(collector, error);
        collector.ack(tuple);
    }
}

Example 8 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class ParserBoltTest method testInvalid.

@Test
public void testInvalid() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    buildGlobalConfig(parserBolt);
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    byte[] sampleBinary = "some binary message".getBytes();
    when(tuple.getBinary(0)).thenReturn(sampleBinary);
    JSONObject parsedMessage = new JSONObject();
    parsedMessage.put("field", "invalidValue");
    parsedMessage.put("guid", "this-is-unique-identifier-for-tuple");
    List<JSONObject> messageList = new ArrayList<>();
    messageList.add(parsedMessage);
    when(parser.parseOptional(sampleBinary)).thenReturn(Optional.of(messageList));
    when(parser.validate(parsedMessage)).thenReturn(true);
    parserBolt.execute(tuple);
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(sensorType).withErrorFields(new HashSet<String>() {

        {
            add("field");
        }
    }).addRawMessage(new JSONObject() {

        {
            put("field", "invalidValue");
            put("source.type", "yaf");
            put("guid", "this-is-unique-identifier-for-tuple");
        }
    });
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
}

Example 9 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class WriterBoltTest method testNonBatchErrorPathErrorInWrite.

@Test
public void testNonBatchErrorPathErrorInWrite() throws Exception {
    ParserConfigurations configurations = getConfigurations(1);
    String sensorType = "test";
    Tuple t = mock(Tuple.class);
    when(t.toString()).thenReturn("tuple");
    when(t.getValueByField(eq("message"))).thenReturn(new JSONObject());
    WriterBolt bolt = new WriterBolt(new WriterHandler(writer), configurations, sensorType);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    doThrow(new Exception("write error")).when(writer).write(any(), any(), any(), any());
    verify(writer, times(1)).init();
    bolt.execute(t);
    verify(outputCollector, times(1)).ack(t);
    verify(writer, times(1)).write(eq(sensorType), any(), any(), any());
    verify(outputCollector, times(1)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.DEFAULT_ERROR).withThrowable(new IllegalStateException("Unhandled bulk errors in response: {java.lang.Exception: write error=[tuple]}")).withSensorType(sensorType).addRawMessage(new JSONObject());
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
}

Example 10 with MetronError

use of org.apache.metron.common.error.MetronError in project metron by apache.

the class BulkWriterComponentTest method writeShouldProperlyHandleWriterException.

@Test
public void writeShouldProperlyHandleWriterException() throws Exception {
    Throwable e = new Exception("test exception");
    MetronError error = new MetronError().withSensorType(sensorType).withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Arrays.asList(message1, message2));
    BulkWriterResponse response = new BulkWriterResponse();
    response.addAllErrors(e, tupleList);
    when(bulkMessageWriter.write(sensorType, configurations, Arrays.asList(tuple1, tuple2), Arrays.asList(message1, message2))).thenThrow(e);
    BulkWriterComponent<JSONObject> bulkWriterComponent = new BulkWriterComponent<>(collector);
    bulkWriterComponent.write(sensorType, tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy);
    bulkWriterComponent.write(sensorType, tuple2, message2, bulkMessageWriter, configurations, messageGetStrategy);
    verifyStatic(times(1));
    ErrorUtils.handleError(collector, error);
}