Examples with SensorParserConfig org.apache.metron.common.configuration.SensorParserConfig used on opensource projects

Example 6 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class StellarTransformationTest method testStellar_multi.

/**
 * A more complicated test where we are transforming multiple fields:
 * 1. Convert a timestamp field in yyyy-MM-dd HH:mm:ss format to unix epoch while
 *    looking up the timezone based on a second field, dc, in a map being kept in the parser config.
 *    If the data center isn't in the map, then the default is UTC
 * 2. Extract the host from a URL field and convert to lowercase
 * 3. Extract the protocol of the URL field
 */
@Test
public void testStellar_multi() throws Exception {
    SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(stellarConfig_multi));
    FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
    {
        // We need a timestamp field, a URL field and a data center field
        JSONObject input = new JSONObject(new HashMap<String, Object>() {

            {
                put("timestamp", "2016-01-05 17:02:30");
                put("url", "https://caseystella.com/blog");
                // looking up the data center in portland, which doesn't exist in the map, so we default to UTC
                put("dc", "portland");
            }
        });
        handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
        long expected = 1452013350000L;
        Assert.assertEquals(expected, input.get("utc_timestamp"));
        Assert.assertEquals("caseystella.com", input.get("url_host"));
        Assert.assertEquals("https", input.get("url_protocol"));
        Assert.assertTrue(input.containsKey("timestamp"));
        Assert.assertTrue(input.containsKey("url"));
    }
    {
        // now we see what happens when we change the data center to london, which is in the map
        JSONObject input = new JSONObject(new HashMap<String, Object>() {

            {
                put("timestamp", "2016-01-05 17:02:30");
                put("url", "https://caseystella.com/blog");
                put("dc", "london");
            }
        });
        handler.transformAndUpdate(input, Context.EMPTY_CONTEXT(), c.getParserConfig());
        long expected = 1452013350000L;
        Assert.assertEquals(expected, input.get("utc_timestamp"));
        Assert.assertEquals("caseystella.com", input.get("url_host"));
        Assert.assertEquals("https", input.get("url_protocol"));
        Assert.assertTrue(input.containsKey("timestamp"));
        Assert.assertTrue(input.containsKey("url"));
    }
    // now we ensure that because we don't have a data center field at all, it's defaulted to UTC.
    {
        JSONObject input = new JSONObject(new HashMap<String, Object>() {

            {
                put("timestamp", "2016-01-05 17:02:30");
                put("url", "https://caseystella.com/blog");
            }
        });
        handler.transformAndUpdate(input, Context.EMPTY_CONTEXT(), c.getParserConfig());
        long expected = 1452013350000L;
        Assert.assertEquals(expected, input.get("utc_timestamp"));
        Assert.assertEquals("caseystella.com", input.get("url_host"));
        Assert.assertEquals("https", input.get("url_protocol"));
        Assert.assertTrue(input.containsKey("timestamp"));
        Assert.assertTrue(input.containsKey("url"));
    }
}

Example 7 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class StellarTransformationTest method testStellar.

/**
 * Test the happy path.  This ensures that a simple transformation, converting a timestamp in a yyyy-MM-dd HH:mm:ss
 * format can be converted to the expected UTC MS since Epoch.
 */
@Test
public void testStellar() throws Exception {
    SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(stellarConfig));
    FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
    JSONObject input = new JSONObject(new HashMap<String, Object>() {

        {
            put("timestamp", "2016-01-05 17:02:30");
        }
    });
    handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
    long expected = 1452013350000L;
    Assert.assertEquals(expected, input.get("utc_timestamp"));
    Assert.assertTrue(input.containsKey("timestamp"));
}

Example 8 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class SensorParserConfigServiceImplTest method findOneShouldProperlyReturnSensorParserConfig.

@Test
public void findOneShouldProperlyReturnSensorParserConfig() throws Exception {
    final SensorParserConfig sensorParserConfig = getTestBroSensorParserConfig();
    ParserConfigurations configs = new ParserConfigurations() {

        @Override
        public Map<String, Object> getConfigurations() {
            return ImmutableMap.of(ParserConfigurations.getKey("bro"), sensorParserConfig);
        }
    };
    when(cache.get(eq(ParserConfigurations.class))).thenReturn(configs);
    // We only have bro, so we should expect it to be returned
    assertEquals(getTestBroSensorParserConfig(), sensorParserConfigService.findOne("bro"));
    // and blah should be a miss.
    assertNull(sensorParserConfigService.findOne("blah"));
}

Example 9 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class SensorParserConfigServiceImplTest method saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave.

@Test
public void saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave() throws Exception {
    final SensorParserConfig sensorParserConfig = getTestBroSensorParserConfig();
    when(objectMapper.writeValueAsString(sensorParserConfig)).thenReturn(broJson);
    SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
    when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes())).thenReturn(new Stat());
    when(curatorFramework.setData()).thenReturn(setDataBuilder);
    assertEquals(getTestBroSensorParserConfig(), sensorParserConfigService.save("bro", sensorParserConfig));
    verify(setDataBuilder).forPath(eq(ConfigurationType.PARSER.getZookeeperRoot() + "/bro"), eq(broJson.getBytes()));
}

Example 10 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class SensorParserConfigServiceImplTest method invalidParserClassShouldThrowRestException.

@Test
public void invalidParserClassShouldThrowRestException() throws Exception {
    exception.expect(RestException.class);
    final SensorParserConfig sensorParserConfig = new SensorParserConfig();
    sensorParserConfig.setSensorTopic("squid");
    sensorParserConfig.setParserClassName("bad.class.package.BadClassName");
    ParseMessageRequest parseMessageRequest = new ParseMessageRequest();
    parseMessageRequest.setSensorParserConfig(sensorParserConfig);
    sensorParserConfigService.parseMessage(parseMessageRequest);
}