Examples with SensorParserConfig org.apache.metron.common.configuration.SensorParserConfig used on opensource projects

Example 41 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class CSVParserTest method test.

@Test
public void test() throws IOException {
    CSVParser parser = new CSVParser();
    SensorParserConfig config = JSONUtils.INSTANCE.load(parserConfig, SensorParserConfig.class);
    parser.init();
    parser.configure(config.getParserConfig());
    {
        String line = "#foo,bar,grok";
        Assert.assertEquals(0, parser.parse(Bytes.toBytes(line)).size());
    }
    {
        String line = "";
        Assert.assertEquals(0, parser.parse(Bytes.toBytes(line)).size());
    }
    {
        String line = "foo,bar,grok";
        List<JSONObject> results = parser.parse(Bytes.toBytes(line));
        Assert.assertEquals(1, results.size());
        JSONObject o = results.get(0);
        Assert.assertTrue(parser.validate(o));
        Assert.assertEquals(5, o.size());
        Assert.assertEquals("foo", o.get("col1"));
        Assert.assertEquals("bar", o.get("col2"));
        Assert.assertEquals("grok", o.get("col3"));
    }
    {
        String line = "\"foo\", \"bar\",\"grok\"";
        List<JSONObject> results = parser.parse(Bytes.toBytes(line));
        Assert.assertEquals(1, results.size());
        JSONObject o = results.get(0);
        Assert.assertTrue(parser.validate(o));
        Assert.assertEquals(5, o.size());
        Assert.assertEquals("foo", o.get("col1"));
        Assert.assertEquals("bar", o.get("col2"));
        Assert.assertEquals("grok", o.get("col3"));
    }
    {
        String line = "foo, bar, grok";
        List<JSONObject> results = parser.parse(Bytes.toBytes(line));
        Assert.assertEquals(1, results.size());
        JSONObject o = results.get(0);
        Assert.assertTrue(parser.validate(o));
        Assert.assertEquals(5, o.size());
        Assert.assertEquals("foo", o.get("col1"));
        Assert.assertEquals("bar", o.get("col2"));
        Assert.assertEquals("grok", o.get("col3"));
    }
    {
        String line = " foo , bar , grok ";
        List<JSONObject> results = parser.parse(Bytes.toBytes(line));
        Assert.assertEquals(1, results.size());
        JSONObject o = results.get(0);
        Assert.assertTrue(parser.validate(o));
        Assert.assertEquals(5, o.size());
        Assert.assertEquals("foo", o.get("col1"));
        Assert.assertEquals("bar", o.get("col2"));
        Assert.assertEquals("grok", o.get("col3"));
        Assert.assertEquals(null, o.get(" col2"));
        Assert.assertEquals(null, o.get("col3 "));
    }
    {
        UnitTestHelper.setLog4jLevel(CSVParser.class, Level.FATAL);
        String line = "foo";
        try {
            List<JSONObject> results = parser.parse(Bytes.toBytes(line));
            Assert.fail("Expected exception");
        } catch (IllegalStateException iae) {
        }
        UnitTestHelper.setLog4jLevel(CSVParser.class, Level.ERROR);
    }
}

Example 42 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyCLITest method testSpoutConfig.

@Test
public void testSpoutConfig() throws Exception {
    File extraConfig = File.createTempFile("spoutConfig", "json");
    extraConfig.deleteOnExit();
    writeMap(extraConfig, new HashMap<String, Object>() {

        {
            put("extra_config", "from_file");
        }
    });
    EnumMap<ParserTopologyCLI.ParserOptions, String> cliOptions = new EnumMap<ParserTopologyCLI.ParserOptions, String>(ParserTopologyCLI.ParserOptions.class) {

        {
            put(ParserTopologyCLI.ParserOptions.SPOUT_CONFIG, extraConfig.getAbsolutePath());
        }
    };
    Predicate<ParserInput> cliOverrideExpected = input -> {
        return input.getSpoutConfig().get("extra_config").equals("from_file");
    };
    Predicate<ParserInput> configOverrideExpected = input -> {
        return input.getSpoutConfig().get("extra_config").equals("from_zk");
    };
    Supplier<SensorParserConfig> configSupplier = () -> {
        SensorParserConfig config = getBaseConfig();
        config.setSpoutConfig(new HashMap<String, Object>() {

            {
                put("extra_config", "from_zk");
            }
        });
        return config;
    };
    testConfigOption(cliOptions, cliOverrideExpected, configSupplier, configOverrideExpected);
}

Example 43 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyCLITest method testParserParallelism.

@Test
public void testParserParallelism() throws Exception {
    testConfigOption(ParserTopologyCLI.ParserOptions.PARSER_PARALLELISM, "10", input -> input.getParserParallelism().equals(10), () -> {
        SensorParserConfig config = getBaseConfig();
        config.setParserParallelism(20);
        return config;
    }, input -> input.getParserParallelism().equals(20));
}

Example 44 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyCLITest method testTopologyConfig.

@Test
public void testTopologyConfig() throws Exception {
    File extraConfig = File.createTempFile("topologyConfig", "json");
    extraConfig.deleteOnExit();
    writeMap(extraConfig, new HashMap<String, Object>() {

        {
            put(Config.TOPOLOGY_DEBUG, true);
        }
    });
    testConfigOption(new EnumMap<ParserTopologyCLI.ParserOptions, String>(ParserTopologyCLI.ParserOptions.class) {

        {
            put(ParserTopologyCLI.ParserOptions.NUM_WORKERS, "10");
            put(ParserTopologyCLI.ParserOptions.NUM_ACKERS, "20");
            put(ParserTopologyCLI.ParserOptions.EXTRA_OPTIONS, extraConfig.getAbsolutePath());
        }
    }, input -> {
        Config c = input.getStormConf();
        return (int) c.get(Config.TOPOLOGY_WORKERS) == 10 && (int) c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 20 && (boolean) c.get(Config.TOPOLOGY_DEBUG);
    }, () -> {
        SensorParserConfig config = getBaseConfig();
        config.setStormConfig(new HashMap<String, Object>() {

            {
                put(Config.TOPOLOGY_WORKERS, 100);
                put(Config.TOPOLOGY_ACKER_EXECUTORS, 200);
            }
        });
        return config;
    }, input -> {
        Config c = input.getStormConf();
        return (int) c.get(Config.TOPOLOGY_WORKERS) == 100 && (int) c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 200 && !c.containsKey(Config.TOPOLOGY_DEBUG);
    });
}

Example 45 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyCLITest method testTopologyConfig_fromConfigExplicitly.

@Test
public void testTopologyConfig_fromConfigExplicitly() throws Exception {
    testConfigOption(new EnumMap<ParserTopologyCLI.ParserOptions, String>(ParserTopologyCLI.ParserOptions.class) {

        {
            put(ParserTopologyCLI.ParserOptions.NUM_WORKERS, "10");
            put(ParserTopologyCLI.ParserOptions.NUM_ACKERS, "20");
        }
    }, input -> {
        Config c = input.getStormConf();
        return (int) c.get(Config.TOPOLOGY_WORKERS) == 10 && (int) c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 20;
    }, () -> {
        SensorParserConfig config = getBaseConfig();
        config.setNumWorkers(100);
        config.setNumAckers(200);
        return config;
    }, input -> {
        Config c = input.getStormConf();
        return (int) c.get(Config.TOPOLOGY_WORKERS) == 100 && (int) c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 200;
    });
}