Examples with SensorParserConfig org.apache.metron.common.configuration.SensorParserConfig used on opensource projects

Example 86 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class SensorParserConfigServiceImplTest method parseMessageShouldProperlyReturnParsedResults.

@Test
public void parseMessageShouldProperlyReturnParsedResults() throws Exception {
    final SensorParserConfig sensorParserConfig = getTestSquidSensorParserConfig();
    String grokStatement = "SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?";
    String sampleData = "1461576382.642    161 127.0.0.1 TCP_MISS/200 103701 GET http://www.cnn.com/ - DIRECT/199.27.79.73 text/html";
    ParseMessageRequest parseMessageRequest = new ParseMessageRequest();
    parseMessageRequest.setSensorParserConfig(sensorParserConfig);
    parseMessageRequest.setGrokStatement(grokStatement);
    parseMessageRequest.setSampleData(sampleData);
    File grokRoot = new File("./target", user);
    grokRoot.mkdir();
    File patternFile = new File(grokRoot, "squid");
    Writer writer = new OutputStreamWriter(new FileOutputStream(patternFile), StandardCharsets.UTF_8);
    writer.write(grokStatement);
    writer.close();
    assertEquals(new HashMap<String, Object>() {

        {
            put("elapsed", 161);
            put("code", 200);
            put("ip_dst_addr", "199.27.79.73");
            put("ip_src_addr", "127.0.0.1");
            put("action", "TCP_MISS");
            put("bytes", 103701);
            put("method", "GET");
            put("url", "http://www.cnn.com/");
            put("timestamp", 1461576382642L);
            put("original_string", "1461576382.642    161 127.0.0.1 TCP_MISS/200 103701 GET http://www.cnn.com/ - DIRECT/199.27.79.73 text/html");
        }
    }, sensorParserConfigService.parseMessage(parseMessageRequest));
}

Example 87 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class StellarServiceImplTest method applyTransformationsShouldProperlyTransformData.

@Test
public void applyTransformationsShouldProperlyTransformData() {
    SensorParserConfig sensorParserConfig = new SensorParserConfig();
    FieldTransformer fieldTransformater = new FieldTransformer();
    fieldTransformater.setOutput("url_host");
    fieldTransformater.setTransformation("STELLAR");
    fieldTransformater.setConfig(new LinkedHashMap<String, Object>() {

        {
            put("url_host", "TO_LOWER(URL_TO_HOST(url))");
        }
    });
    sensorParserConfig.setFieldTransformations(ImmutableList.of(fieldTransformater));
    SensorParserContext sensorParserContext = new SensorParserContext();
    sensorParserContext.setSensorParserConfig(sensorParserConfig);
    sensorParserContext.setSampleData(new HashMap<String, Object>() {

        {
            put("url", "https://caseystella.com/blog");
        }
    });
    Map<String, Object> results = stellarService.applyTransformations(sensorParserContext);
    assertEquals(2, results.size());
    assertEquals("https://caseystella.com/blog", results.get("url"));
    assertEquals("caseystella.com", results.get("url_host"));
}

Example 88 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class StormControllerIntegrationTest method test.

@Test
public void test() throws Exception {
    this.mockMvc.perform(get(stormUrl).with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$", hasSize(0)));
    this.mockMvc.perform(get(stormUrl + "/broTest").with(httpBasic(user, password))).andExpect(status().isNotFound());
    Map<String, Object> globalConfig = globalConfigService.get();
    if (globalConfig == null) {
        globalConfig = new HashMap<>();
    }
    globalConfigService.delete();
    sensorParserConfigService.delete("broTest");
    this.mockMvc.perform(get(stormUrl + "/parser/stop/broTest?stopNow=true").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString()));
    this.mockMvc.perform(get(stormUrl + "/parser/activate/broTest").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
    this.mockMvc.perform(get(stormUrl + "/parser/deactivate/broTest").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
    this.mockMvc.perform(get(stormUrl + "/parser/start/broTest").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.GLOBAL_CONFIG_MISSING.name()));
    globalConfigService.save(globalConfig);
    {
        final Map<String, Object> expectedGlobalConfig = globalConfig;
        // we must wait for the config to find its way into the config.
        TestUtils.assertEventually(() -> assertEquals(expectedGlobalConfig, globalConfigService.get()));
    }
    this.mockMvc.perform(get(stormUrl + "/parser/start/broTest").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.SENSOR_PARSER_CONFIG_MISSING.name()));
    SensorParserConfig sensorParserConfig = new SensorParserConfig();
    sensorParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser");
    sensorParserConfig.setSensorTopic("broTest");
    sensorParserConfigService.save("broTest", sensorParserConfig);
    {
        final SensorParserConfig expectedSensorParserConfig = sensorParserConfig;
        // we must wait for the config to find its way into the config.
        TestUtils.assertEventually(() -> assertEquals(expectedSensorParserConfig, sensorParserConfigService.findOne("broTest")));
    }
    this.mockMvc.perform(get(stormUrl + "/parser/start/broTest").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.name()));
    this.mockMvc.perform(get(stormUrl + "/supervisors").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.supervisors[0]").exists()).andExpect(jsonPath("$.supervisors[0].id").exists()).andExpect(jsonPath("$.supervisors[0].host").exists()).andExpect(jsonPath("$.supervisors[0].uptime").exists()).andExpect(jsonPath("$.supervisors[0].slotsTotal").exists()).andExpect(jsonPath("$.supervisors[0].slotsUsed").exists());
    this.mockMvc.perform(get(stormUrl + "/broTest").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.name").value("broTest")).andExpect(jsonPath("$.id", containsString("broTest"))).andExpect(jsonPath("$.status").value("ACTIVE")).andExpect(jsonPath("$.latency").exists()).andExpect(jsonPath("$.throughput").exists()).andExpect(jsonPath("$.emitted").exists()).andExpect(jsonPath("$.acked").exists());
    this.mockMvc.perform(get(stormUrl).with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$[?(@.name == 'broTest' && @.status == 'ACTIVE')]").exists());
    this.mockMvc.perform(get(stormUrl + "/parser/stop/broTest?stopNow=true").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name()));
    this.mockMvc.perform(get(stormUrl + "/enrichment").with(httpBasic(user, password))).andExpect(status().isNotFound());
    this.mockMvc.perform(get(stormUrl + "/enrichment/activate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
    this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
    this.mockMvc.perform(get(stormUrl + "/enrichment/stop?stopNow=true").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString()));
    this.mockMvc.perform(get(stormUrl + "/enrichment/start").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.toString()));
    this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name()));
    this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name()));
    this.mockMvc.perform(get(stormUrl + "/enrichment/activate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.ACTIVE.name()));
    this.mockMvc.perform(get(stormUrl + "/enrichment").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.name").value("enrichment")).andExpect(jsonPath("$.id", containsString("enrichment"))).andExpect(jsonPath("$.status").value("ACTIVE")).andExpect(jsonPath("$.latency").exists()).andExpect(jsonPath("$.throughput").exists()).andExpect(jsonPath("$.emitted").exists()).andExpect(jsonPath("$.acked").exists());
    this.mockMvc.perform(get(stormUrl).with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$[?(@.name == 'enrichment' && @.status == 'ACTIVE')]").exists());
    this.mockMvc.perform(get(stormUrl + "/enrichment/stop").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name()));
    for (String type : ImmutableList.of("randomaccess", "batch")) {
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type).with(httpBasic(user, password))).andExpect(status().isNotFound());
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/activate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/deactivate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/stop?stopNow=true").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString()));
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/start").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.toString()));
        ResultActions actions = this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/deactivate").with(httpBasic(user, password)));
        actions.andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name()));
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/activate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.ACTIVE.name()));
        String topologyName = type.equals("randomaccess") ? MetronRestConstants.RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME : MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME;
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type).with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.name").value(topologyName)).andExpect(jsonPath("$.id", containsString("indexing"))).andExpect(jsonPath("$.status").value("ACTIVE")).andExpect(jsonPath("$.latency").exists()).andExpect(jsonPath("$.throughput").exists()).andExpect(jsonPath("$.emitted").exists()).andExpect(jsonPath("$.acked").exists());
        this.mockMvc.perform(get(stormUrl).with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$[?(@.name == '" + topologyName + "' && @.status == 'ACTIVE')]").exists());
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/stop").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name()));
    }
    this.mockMvc.perform(get(stormUrl + "/client/status").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.stormClientVersionInstalled").value("1.0.1")).andExpect(jsonPath("$.parserScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_parser_topology.sh")).andExpect(jsonPath("$.enrichmentScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_enrichment_topology.sh")).andExpect(jsonPath("$.randomAccessIndexingScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_elasticsearch_topology.sh")).andExpect(jsonPath("$.batchIndexingScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_hdfs_topology.sh"));
    globalConfigService.delete();
    sensorParserConfigService.delete("broTest");
}

Example 89 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ConfiguredParserBoltTest method test.

@Test
public void test() throws Exception {
    ParserConfigurations sampleConfigurations = new ParserConfigurations();
    UnitTestHelper.setLog4jLevel(ConfiguredBolt.class, Level.FATAL);
    StandAloneConfiguredParserBolt configuredBoltNullZk = new ConfiguredParserBoltTest.StandAloneConfiguredParserBolt(null);
    assertThrows(RuntimeException.class, () -> configuredBoltNullZk.prepare(new HashMap(), topologyContext, outputCollector), "A valid zookeeper url must be supplied");
    UnitTestHelper.setLog4jLevel(ConfiguredBolt.class, Level.ERROR);
    configsUpdated = new HashSet<>();
    sampleConfigurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile("../" + TestConstants.SAMPLE_CONFIG_PATH));
    Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile("../" + TestConstants.PARSER_CONFIGS_PATH);
    for (String sensorType : sensorParserConfigs.keySet()) {
        sampleConfigurations.updateSensorParserConfig(sensorType, sensorParserConfigs.get(sensorType));
    }
    StandAloneConfiguredParserBolt configuredBolt = new StandAloneConfiguredParserBolt(zookeeperUrl);
    configuredBolt.prepare(new HashMap(), topologyContext, outputCollector);
    waitForConfigUpdate(parserConfigurationTypes);
    assertEquals(sampleConfigurations, configuredBolt.getConfigurations());
    configsUpdated = new HashSet<>();
    Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig();
    sampleGlobalConfig.put("newGlobalField", "newGlobalValue");
    ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl);
    waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName());
    assertEquals(sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig(), "Add global config field");
    configsUpdated = new HashSet<>();
    sampleGlobalConfig.remove("newGlobalField");
    ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl);
    waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName());
    assertEquals(sampleConfigurations, configuredBolt.getConfigurations(), "Remove global config field");
    configsUpdated = new HashSet<>();
    String sensorType = "testSensorConfig";
    SensorParserConfig testSensorConfig = new SensorParserConfig();
    testSensorConfig.setParserClassName("className");
    testSensorConfig.setSensorTopic("sensorTopic");
    testSensorConfig.setParserConfig(new HashMap<String, Object>() {

        {
            put("configName", "configObject");
        }
    });
    sampleConfigurations.updateSensorParserConfig(sensorType, testSensorConfig);
    ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorType, testSensorConfig, zookeeperUrl);
    waitForConfigUpdate(sensorType);
    ParserConfigurations configuredBoltConfigs = configuredBolt.getConfigurations();
    if (!sampleConfigurations.equals(configuredBoltConfigs)) {
        // before we fail, let's try to dump out some info.
        if (sampleConfigurations.getFieldValidations().size() != configuredBoltConfigs.getFieldValidations().size()) {
            System.out.println("Field validations don't line up");
        }
        for (int i = 0; i < sampleConfigurations.getFieldValidations().size(); ++i) {
            FieldValidator l = sampleConfigurations.getFieldValidations().get(i);
            FieldValidator r = configuredBoltConfigs.getFieldValidations().get(i);
            if (!l.equals(r)) {
                System.out.println(l + " != " + r);
            }
        }
        if (sampleConfigurations.getConfigurations().size() != configuredBoltConfigs.getConfigurations().size()) {
            System.out.println("Configs don't line up");
        }
        for (Map.Entry<String, Object> kv : sampleConfigurations.getConfigurations().entrySet()) {
            Object l = kv.getValue();
            Object r = configuredBoltConfigs.getConfigurations().get(kv.getKey());
            if (!l.equals(r)) {
                System.out.println(kv.getKey() + " config does not line up: ");
                System.out.println(l);
                System.out.println(r);
            }
        }
        assertEquals(sampleConfigurations, configuredBoltConfigs, "Add new sensor config");
    }
    assertEquals(sampleConfigurations, configuredBoltConfigs, "Add new sensor config");
    configuredBolt.cleanup();
}

Example 90 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class KafkaWriterTest method createConfiguration.

public WriterConfiguration createConfiguration(final Map<String, Object> parserConfig) {
    ParserConfigurations configurations = new ParserConfigurations();
    configurations.updateSensorParserConfig(SENSOR_TYPE, new SensorParserConfig() {

        {
            setParserConfig(parserConfig);
        }
    });
    return new ParserWriterConfiguration(configurations);
}