Examples with SensorParserConfig org.apache.metron.common.configuration.SensorParserConfig used on opensource projects

Example 11 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class SensorParserConfigServiceImplTest method getAllShouldProperlyReturnSensorParserConfigs.

@Test
public void getAllShouldProperlyReturnSensorParserConfigs() throws Exception {
    final SensorParserConfig broSensorParserConfig = getTestBroSensorParserConfig();
    final SensorParserConfig squidSensorParserConfig = getTestSquidSensorParserConfig();
    ParserConfigurations configs = new ParserConfigurations() {

        @Override
        public Map<String, Object> getConfigurations() {
            return ImmutableMap.of(ParserConfigurations.getKey("bro"), broSensorParserConfig, ParserConfigurations.getKey("squid"), squidSensorParserConfig);
        }
    };
    when(cache.get(eq(ParserConfigurations.class))).thenReturn(configs);
    assertEquals(new HashMap() {

        {
            put("bro", getTestBroSensorParserConfig());
            put("squid", getTestSquidSensorParserConfig());
        }
    }, sensorParserConfigService.getAll());
}

Example 12 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class SensorParserConfigServiceImplTest method getTestBroSensorParserConfig.

private SensorParserConfig getTestBroSensorParserConfig() {
    SensorParserConfig sensorParserConfig = new SensorParserConfig();
    sensorParserConfig.setSensorTopic("bro");
    sensorParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser");
    return sensorParserConfig;
}

Example 13 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserConfigFunctionsTest method transform.

public Map<String, Object> transform(String parserConfig, Map<String, Object> variables) {
    JSONObject ret = new JSONObject(variables);
    SensorParserConfig sensorParserConfig = (SensorParserConfig) PARSER.deserialize(parserConfig);
    sensorParserConfig.init();
    for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) {
        if (handler != null) {
            handler.transformAndUpdate(ret, context, sensorParserConfig.getParserConfig());
        }
    }
    return ret;
}

Example 14 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserBolt method prepare.

@SuppressWarnings("unchecked")
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    super.prepare(stormConf, context, collector);
    messageGetStrategy = MessageGetters.DEFAULT_BYTES_FROM_POSITION.get();
    this.collector = collector;
    initializeStellar();
    if (getSensorParserConfig() != null && filter == null) {
        getSensorParserConfig().getParserConfig().putIfAbsent("stellarContext", stellarContext);
        if (!StringUtils.isEmpty(getSensorParserConfig().getFilterClassName())) {
            filter = Filters.get(getSensorParserConfig().getFilterClassName(), getSensorParserConfig().getParserConfig());
        }
    }
    parser.init();
    writer.init(stormConf, context, collector, getConfigurations());
    SensorParserConfig config = getSensorParserConfig();
    if (config != null) {
        config.init();
    } else {
        throw new IllegalStateException("Unable to retrieve a parser config for " + getSensorType());
    }
    parser.configure(config.getParserConfig());
}

Example 15 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserBolt method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
    byte[] originalMessage = (byte[]) messageGetStrategy.get(tuple);
    SensorParserConfig sensorParserConfig = getSensorParserConfig();
    try {
        // we want to ack the tuple in the situation where we have are not doing a bulk write
        // otherwise we want to defer to the writerComponent who will ack on bulk commit.
        boolean ackTuple = !writer.handleAck();
        int numWritten = 0;
        if (sensorParserConfig != null) {
            Map<String, Object> metadata = getMetadata(tuple, sensorParserConfig.getReadMetadata());
            List<FieldValidator> fieldValidations = getConfigurations().getFieldValidations();
            Optional<List<JSONObject>> messages = parser.parseOptional(originalMessage);
            for (JSONObject message : messages.orElse(Collections.emptyList())) {
                message.put(Constants.SENSOR_TYPE, getSensorType());
                if (sensorParserConfig.getMergeMetadata()) {
                    message.putAll(metadata);
                }
                for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) {
                    if (handler != null) {
                        if (!sensorParserConfig.getMergeMetadata()) {
                            // if we haven't merged metadata, then we need to pass them along as configuration params.
                            handler.transformAndUpdate(message, stellarContext, sensorParserConfig.getParserConfig(), metadata);
                        } else {
                            handler.transformAndUpdate(message, stellarContext, sensorParserConfig.getParserConfig());
                        }
                    }
                }
                if (!message.containsKey(Constants.GUID)) {
                    message.put(Constants.GUID, UUID.randomUUID().toString());
                }
                if (parser.validate(message) && (filter == null || filter.emitTuple(message, stellarContext))) {
                    numWritten++;
                    List<FieldValidator> failedValidators = getFailedValidators(message, fieldValidations);
                    if (failedValidators.size() > 0) {
                        MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(getSensorType()).addRawMessage(message);
                        Set<String> errorFields = failedValidators.stream().flatMap(fieldValidator -> fieldValidator.getInput().stream()).collect(Collectors.toSet());
                        if (!errorFields.isEmpty()) {
                            error.withErrorFields(errorFields);
                        }
                        ErrorUtils.handleError(collector, error);
                    } else {
                        writer.write(getSensorType(), tuple, message, getConfigurations(), messageGetStrategy);
                    }
                }
            }
        }
        // then we want to handle the ack ourselves.
        if (ackTuple || numWritten == 0) {
            collector.ack(tuple);
        }
    } catch (Throwable ex) {
        handleError(originalMessage, tuple, ex, collector);
    }
}