Examples with FieldValidator org.apache.metron.common.configuration.FieldValidator used on opensource projects

Example 1 with FieldValidator

use of org.apache.metron.common.configuration.FieldValidator in project metron by apache.

the class BaseValidationTest method execute.

public boolean execute(String config, Map<String, Object> input) throws IOException {
    Configurations configurations = getConfiguration(config);
    FieldValidator validator = getValidator(configurations);
    return validator.isValid(new JSONObject(input), configurations.getGlobalConfig(), Context.EMPTY_CONTEXT());
}

Example 2 with FieldValidator

use of org.apache.metron.common.configuration.FieldValidator in project metron by apache.

the class ParserBolt method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
    byte[] originalMessage = (byte[]) messageGetStrategy.get(tuple);
    SensorParserConfig sensorParserConfig = getSensorParserConfig();
    try {
        // we want to ack the tuple in the situation where we have are not doing a bulk write
        // otherwise we want to defer to the writerComponent who will ack on bulk commit.
        boolean ackTuple = !writer.handleAck();
        int numWritten = 0;
        if (sensorParserConfig != null) {
            Map<String, Object> metadata = getMetadata(tuple, sensorParserConfig.getReadMetadata());
            List<FieldValidator> fieldValidations = getConfigurations().getFieldValidations();
            Optional<List<JSONObject>> messages = parser.parseOptional(originalMessage);
            for (JSONObject message : messages.orElse(Collections.emptyList())) {
                message.put(Constants.SENSOR_TYPE, getSensorType());
                if (sensorParserConfig.getMergeMetadata()) {
                    message.putAll(metadata);
                }
                for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) {
                    if (handler != null) {
                        if (!sensorParserConfig.getMergeMetadata()) {
                            // if we haven't merged metadata, then we need to pass them along as configuration params.
                            handler.transformAndUpdate(message, stellarContext, sensorParserConfig.getParserConfig(), metadata);
                        } else {
                            handler.transformAndUpdate(message, stellarContext, sensorParserConfig.getParserConfig());
                        }
                    }
                }
                if (!message.containsKey(Constants.GUID)) {
                    message.put(Constants.GUID, UUID.randomUUID().toString());
                }
                if (parser.validate(message) && (filter == null || filter.emitTuple(message, stellarContext))) {
                    numWritten++;
                    List<FieldValidator> failedValidators = getFailedValidators(message, fieldValidations);
                    if (failedValidators.size() > 0) {
                        MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(getSensorType()).addRawMessage(message);
                        Set<String> errorFields = failedValidators.stream().flatMap(fieldValidator -> fieldValidator.getInput().stream()).collect(Collectors.toSet());
                        if (!errorFields.isEmpty()) {
                            error.withErrorFields(errorFields);
                        }
                        ErrorUtils.handleError(collector, error);
                    } else {
                        writer.write(getSensorType(), tuple, message, getConfigurations(), messageGetStrategy);
                    }
                }
            }
        }
        // then we want to handle the ack ourselves.
        if (ackTuple || numWritten == 0) {
            collector.ack(tuple);
        }
    } catch (Throwable ex) {
        handleError(originalMessage, tuple, ex, collector);
    }
}