Examples with SensorParserConfig org.apache.metron.common.configuration.SensorParserConfig used on opensource projects

Example 91 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class SensorParserConfigServiceImpl method getAll.

@Override
public Map<String, SensorParserConfig> getAll() throws RestException {
    Map<String, SensorParserConfig> sensorParserConfigs = new HashMap<>();
    List<String> sensorNames = getAllTypes();
    for (String name : sensorNames) {
        SensorParserConfig config = findOne(name);
        if (config != null) {
            sensorParserConfigs.put(name, config);
        }
    }
    return sensorParserConfigs;
}

Example 92 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserRunnerImpl method initializeParsers.

/**
 * Initializes MessageParsers and MessageFilters for sensor types configured in this ParserRunner.  Objects are created
 * using reflection and the MessageParser configure and init methods are called.
 * @param parserConfigSupplier Parser configurations
 */
private void initializeParsers(Supplier<ParserConfigurations> parserConfigSupplier) {
    LOG.info("Initializing parsers...");
    sensorToParserComponentMap = new HashMap<>();
    for (String sensorType : sensorTypes) {
        if (parserConfigSupplier.get().getSensorParserConfig(sensorType) == null) {
            throw new IllegalStateException(String.format("Could not initialize parsers.  Cannot find configuration for sensor %s.", sensorType));
        }
        SensorParserConfig parserConfig = parserConfigSupplier.get().getSensorParserConfig(sensorType);
        LOG.info("Creating parser for sensor {} with parser class = {} and filter class = {} ", sensorType, parserConfig.getParserClassName(), parserConfig.getFilterClassName());
        // create message parser
        MessageParser<JSONObject> parser = createParserInstance(parserConfig);
        // create message filter
        MessageFilter<JSONObject> filter = null;
        parserConfig.getParserConfig().putIfAbsent("stellarContext", stellarContext);
        filter = getMessageFilter(parserConfig, filter);
        parser.configure(parserConfig.getParserConfig());
        parser.init();
        sensorToParserComponentMap.put(sensorType, new ParserComponent(parser, filter));
    }
}

Example 93 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserRunnerImpl method processMessage.

/**
 * Post-processes parsed messages by:
 * <ul>
 *   <li>Applying field transformations defined in the sensor parser config</li>
 *   <li>Filtering messages using the configured MessageFilter class</li>
 *   <li>Validating messages using the MessageParser validate method</li>
 * </ul>
 * If a message is successfully processed a message is returned in a ProcessResult.  If a message fails
 * validation, a MetronError object is created and returned in a ProcessResult.  If a message is
 * filtered out an empty Optional is returned.
 *
 * @param sensorType Sensor type of the message
 * @param message Message parsed by the MessageParser
 * @param rawMessage Raw message including metadata
 * @param parser MessageParser for the sensor type
 * @param parserConfigurations Parser configurations
 */
@SuppressWarnings("unchecked")
protected Optional<ProcessResult> processMessage(String sensorType, JSONObject message, RawMessage rawMessage, MessageParser<JSONObject> parser, ParserConfigurations parserConfigurations) {
    Optional<ProcessResult> processResult = Optional.empty();
    SensorParserConfig sensorParserConfig = parserConfigurations.getSensorParserConfig(sensorType);
    sensorParserConfig.getRawMessageStrategy().mergeMetadata(message, rawMessage.getMetadata(), sensorParserConfig.getMergeMetadata(), sensorParserConfig.getRawMessageStrategyConfig());
    message.put(Constants.SENSOR_TYPE, sensorType);
    applyFieldTransformations(message, rawMessage, sensorParserConfig);
    if (!message.containsKey(Constants.GUID)) {
        message.put(Constants.GUID, UUID.randomUUID().toString());
    }
    message.putIfAbsent(Fields.ORIGINAL.getName(), new String(rawMessage.getMessage(), parser.getReadCharset()));
    MessageFilter<JSONObject> filter = sensorToParserComponentMap.get(sensorType).getFilter();
    if (filter == null || filter.emit(message, stellarContext)) {
        boolean isInvalid = !parser.validate(message);
        List<FieldValidator> failedValidators = null;
        if (!isInvalid) {
            failedValidators = getFailedValidators(message, parserConfigurations);
            isInvalid = !failedValidators.isEmpty();
        }
        if (isInvalid) {
            MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(Collections.singleton(sensorType)).withMetadata(rawMessage.getMetadata()).addRawMessage(message);
            Set<String> errorFields = failedValidators == null ? null : failedValidators.stream().flatMap(fieldValidator -> fieldValidator.getInput().stream()).collect(Collectors.toSet());
            if (errorFields != null && !errorFields.isEmpty()) {
                error.withErrorFields(errorFields);
            }
            processResult = Optional.of(new ProcessResult(error));
        } else {
            processResult = Optional.of(new ProcessResult(message));
        }
    }
    return processResult;
}

Example 94 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserRunnerImplTest method setup.

@BeforeEach
@SuppressWarnings("unchecked")
public void setup() throws IOException {
    parserConfigurations = new ParserConfigurations();
    SensorParserConfig broConfig = SensorParserConfig.fromBytes(broConfigString.getBytes(StandardCharsets.UTF_8));
    SensorParserConfig snortConfig = SensorParserConfig.fromBytes(snortConfigString.getBytes(StandardCharsets.UTF_8));
    parserConfigurations.updateSensorParserConfig("bro", broConfig);
    parserConfigurations.updateSensorParserConfig("snort", snortConfig);
    parserConfigurations.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfigString, JSONUtils.MAP_SUPPLIER));
    parserRunner = new ParserRunnerImpl(new HashSet<>(Arrays.asList("bro", "snort")));
    broParser = mock(MessageParser.class);
    snortParser = mock(MessageParser.class);
    stellarFilter = mock(StellarFilter.class);
    // mockStatic(ReflectionUtils.class);
    // mockStatic(Filters.class);
    when(broParser.getReadCharset()).thenReturn(StandardCharsets.UTF_8);
// when(ReflectionUtils.createInstance("org.apache.metron.parsers.bro.BasicBroParser")).thenReturn(broParser);
// when(ReflectionUtils.createInstance("org.apache.metron.parsers.snort.BasicSnortParser")).thenReturn(snortParser);
// when(Filters.get("org.apache.metron.parsers.filters.StellarFilter", broConfig.getParserConfig()))
// .thenReturn(stellarFilter);
}

Example 95 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class CSVParserTest method test.

@Test
public void test() throws IOException {
    CSVParser parser = new CSVParser();
    SensorParserConfig config = JSONUtils.INSTANCE.load(parserConfig, SensorParserConfig.class);
    parser.init();
    parser.configure(config.getParserConfig());
    {
        String line = "#foo,bar,grok";
        assertEquals(0, parser.parse(Bytes.toBytes(line)).size());
    }
    {
        String line = "";
        assertEquals(0, parser.parse(Bytes.toBytes(line)).size());
    }
    {
        String line = "foo,bar,grok";
        List<JSONObject> results = parser.parse(Bytes.toBytes(line));
        assertEquals(1, results.size());
        JSONObject o = results.get(0);
        assertTrue(parser.validate(o));
        assertEquals(5, o.size());
        assertEquals("foo", o.get("col1"));
        assertEquals("bar", o.get("col2"));
        assertEquals("grok", o.get("col3"));
    }
    {
        String line = "\"foo\", \"bar\",\"grok\"";
        List<JSONObject> results = parser.parse(Bytes.toBytes(line));
        assertEquals(1, results.size());
        JSONObject o = results.get(0);
        assertTrue(parser.validate(o));
        assertEquals(5, o.size());
        assertEquals("foo", o.get("col1"));
        assertEquals("bar", o.get("col2"));
        assertEquals("grok", o.get("col3"));
    }
    {
        String line = "foo, bar, grok";
        List<JSONObject> results = parser.parse(Bytes.toBytes(line));
        assertEquals(1, results.size());
        JSONObject o = results.get(0);
        assertTrue(parser.validate(o));
        assertEquals(5, o.size());
        assertEquals("foo", o.get("col1"));
        assertEquals("bar", o.get("col2"));
        assertEquals("grok", o.get("col3"));
    }
    {
        String line = " foo , bar , grok ";
        List<JSONObject> results = parser.parse(Bytes.toBytes(line));
        assertEquals(1, results.size());
        JSONObject o = results.get(0);
        assertTrue(parser.validate(o));
        assertEquals(5, o.size());
        assertEquals("foo", o.get("col1"));
        assertEquals("bar", o.get("col2"));
        assertEquals("grok", o.get("col3"));
        assertNull(o.get(" col2"));
        assertNull(o.get("col3 "));
    }
    {
        UnitTestHelper.setLog4jLevel(CSVParser.class, Level.FATAL);
        String line = "foo";
        assertThrows(IllegalStateException.class, () -> parser.parse(Bytes.toBytes(line)));
        UnitTestHelper.setLog4jLevel(CSVParser.class, Level.ERROR);
    }
}