Examples with SensorParserConfig org.apache.metron.common.configuration.SensorParserConfig used on opensource projects

Example 21 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyCLITest method testErrorParallelism.

@Test
public void testErrorParallelism() throws Exception {
    testConfigOption(ParserTopologyCLI.ParserOptions.ERROR_WRITER_PARALLELISM, "10", input -> input.getErrorParallelism().equals(10), () -> {
        SensorParserConfig config = getBaseConfig();
        config.setErrorWriterParallelism(20);
        return config;
    }, input -> input.getErrorParallelism().equals(20));
}

Example 22 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyCLITest method testSpoutParallelism.

@Test
public void testSpoutParallelism() throws Exception {
    testConfigOption(ParserTopologyCLI.ParserOptions.SPOUT_PARALLELISM, "10", input -> input.getSpoutParallelism().equals(10), () -> {
        SensorParserConfig config = getBaseConfig();
        config.setSpoutParallelism(20);
        return config;
    }, input -> input.getSpoutParallelism().equals(20));
}

Example 23 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class SensorParserConfigServiceImpl method getAll.

@Override
public Map<String, SensorParserConfig> getAll() throws RestException {
    Map<String, SensorParserConfig> sensorParserConfigs = new HashMap<>();
    List<String> sensorNames = getAllTypes();
    for (String name : sensorNames) {
        SensorParserConfig config = findOne(name);
        if (config != null) {
            sensorParserConfigs.put(name, config);
        }
    }
    return sensorParserConfigs;
}

Example 24 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class StormControllerIntegrationTest method test.

@Test
public void test() throws Exception {
    this.mockMvc.perform(get(stormUrl).with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$", hasSize(0)));
    this.mockMvc.perform(get(stormUrl + "/broTest").with(httpBasic(user, password))).andExpect(status().isNotFound());
    Map<String, Object> globalConfig = globalConfigService.get();
    if (globalConfig == null) {
        globalConfig = new HashMap<>();
    }
    globalConfigService.delete();
    sensorParserConfigService.delete("broTest");
    this.mockMvc.perform(get(stormUrl + "/parser/stop/broTest?stopNow=true").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString()));
    this.mockMvc.perform(get(stormUrl + "/parser/activate/broTest").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
    this.mockMvc.perform(get(stormUrl + "/parser/deactivate/broTest").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
    this.mockMvc.perform(get(stormUrl + "/parser/start/broTest").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.GLOBAL_CONFIG_MISSING.name()));
    globalConfigService.save(globalConfig);
    {
        final Map<String, Object> expectedGlobalConfig = globalConfig;
        // we must wait for the config to find its way into the config.
        TestUtils.assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, globalConfigService.get()));
    }
    this.mockMvc.perform(get(stormUrl + "/parser/start/broTest").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.SENSOR_PARSER_CONFIG_MISSING.name()));
    SensorParserConfig sensorParserConfig = new SensorParserConfig();
    sensorParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser");
    sensorParserConfig.setSensorTopic("broTest");
    sensorParserConfigService.save("broTest", sensorParserConfig);
    {
        final SensorParserConfig expectedSensorParserConfig = sensorParserConfig;
        // we must wait for the config to find its way into the config.
        TestUtils.assertEventually(() -> Assert.assertEquals(expectedSensorParserConfig, sensorParserConfigService.findOne("broTest")));
    }
    this.mockMvc.perform(get(stormUrl + "/parser/start/broTest").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.name()));
    this.mockMvc.perform(get(stormUrl + "/supervisors").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.supervisors[0]").exists()).andExpect(jsonPath("$.supervisors[0].id").exists()).andExpect(jsonPath("$.supervisors[0].host").exists()).andExpect(jsonPath("$.supervisors[0].uptime").exists()).andExpect(jsonPath("$.supervisors[0].slotsTotal").exists()).andExpect(jsonPath("$.supervisors[0].slotsUsed").exists());
    this.mockMvc.perform(get(stormUrl + "/broTest").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.name").value("broTest")).andExpect(jsonPath("$.id", containsString("broTest"))).andExpect(jsonPath("$.status").value("ACTIVE")).andExpect(jsonPath("$.latency").exists()).andExpect(jsonPath("$.throughput").exists()).andExpect(jsonPath("$.emitted").exists()).andExpect(jsonPath("$.acked").exists());
    this.mockMvc.perform(get(stormUrl).with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$[?(@.name == 'broTest' && @.status == 'ACTIVE')]").exists());
    this.mockMvc.perform(get(stormUrl + "/parser/stop/broTest?stopNow=true").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name()));
    this.mockMvc.perform(get(stormUrl + "/enrichment").with(httpBasic(user, password))).andExpect(status().isNotFound());
    this.mockMvc.perform(get(stormUrl + "/enrichment/activate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
    this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
    this.mockMvc.perform(get(stormUrl + "/enrichment/stop?stopNow=true").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString()));
    this.mockMvc.perform(get(stormUrl + "/enrichment/start").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.toString()));
    this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name()));
    this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name()));
    this.mockMvc.perform(get(stormUrl + "/enrichment/activate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.ACTIVE.name()));
    this.mockMvc.perform(get(stormUrl + "/enrichment").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.name").value("enrichment")).andExpect(jsonPath("$.id", containsString("enrichment"))).andExpect(jsonPath("$.status").value("ACTIVE")).andExpect(jsonPath("$.latency").exists()).andExpect(jsonPath("$.throughput").exists()).andExpect(jsonPath("$.emitted").exists()).andExpect(jsonPath("$.acked").exists());
    this.mockMvc.perform(get(stormUrl).with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$[?(@.name == 'enrichment' && @.status == 'ACTIVE')]").exists());
    this.mockMvc.perform(get(stormUrl + "/enrichment/stop").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name()));
    for (String type : ImmutableList.of("randomaccess", "batch")) {
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type).with(httpBasic(user, password))).andExpect(status().isNotFound());
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/activate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/deactivate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/stop?stopNow=true").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("ERROR")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString()));
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/start").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.toString()));
        ResultActions actions = this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/deactivate").with(httpBasic(user, password)));
        actions.andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name()));
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/activate").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.ACTIVE.name()));
        String topologyName = type.equals("randomaccess") ? MetronRestConstants.RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME : MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME;
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type).with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.name").value(topologyName)).andExpect(jsonPath("$.id", containsString("indexing"))).andExpect(jsonPath("$.status").value("ACTIVE")).andExpect(jsonPath("$.latency").exists()).andExpect(jsonPath("$.throughput").exists()).andExpect(jsonPath("$.emitted").exists()).andExpect(jsonPath("$.acked").exists());
        this.mockMvc.perform(get(stormUrl).with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$[?(@.name == '" + topologyName + "' && @.status == 'ACTIVE')]").exists());
        this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/stop").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.status").value("SUCCESS")).andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name()));
    }
    this.mockMvc.perform(get(stormUrl + "/client/status").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(jsonPath("$.stormClientVersionInstalled").value("1.0.1")).andExpect(jsonPath("$.parserScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_parser_topology.sh")).andExpect(jsonPath("$.enrichmentScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_enrichment_topology.sh")).andExpect(jsonPath("$.randomAccessIndexingScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_elasticsearch_topology.sh")).andExpect(jsonPath("$.batchIndexingScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_hdfs_topology.sh"));
    globalConfigService.delete();
    sensorParserConfigService.delete("broTest");
}

Example 25 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class SensorParserConfigServiceImplTest method saveShouldWrapExceptionInRestException.

@Test
public void saveShouldWrapExceptionInRestException() throws Exception {
    exception.expect(RestException.class);
    SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
    when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes())).thenThrow(Exception.class);
    when(curatorFramework.setData()).thenReturn(setDataBuilder);
    final SensorParserConfig sensorParserConfig = new SensorParserConfig();
    sensorParserConfig.setSensorTopic("bro");
    sensorParserConfigService.save("bro", sensorParserConfig);
}