Examples with FieldTransformer org.apache.metron.common.configuration.FieldTransformer used on opensource projects

Example 1 with FieldTransformer

use of org.apache.metron.common.configuration.FieldTransformer in project metron by apache.

the class ParserConfigFunctionsTest method transform.

public Map<String, Object> transform(String parserConfig, Map<String, Object> variables) {
    JSONObject ret = new JSONObject(variables);
    SensorParserConfig sensorParserConfig = (SensorParserConfig) PARSER.deserialize(parserConfig);
    sensorParserConfig.init();
    for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) {
        if (handler != null) {
            handler.transformAndUpdate(ret, context, sensorParserConfig.getParserConfig());
        }
    }
    return ret;
}

Example 2 with FieldTransformer

use of org.apache.metron.common.configuration.FieldTransformer in project metron by apache.

the class ParserConfigFunctions method getStellarTransformer.

private static FieldTransformer getStellarTransformer(SensorParserConfig config) {
    List<FieldTransformer> fieldTransformations = config.getFieldTransformations();
    FieldTransformer stellarTransformer = null;
    for (FieldTransformer transformer : fieldTransformations) {
        if (transformer.getFieldTransformation().getClass().getName().equals(FieldTransformations.STELLAR.getMappingClass().getName())) {
            stellarTransformer = transformer;
        }
    }
    if (stellarTransformer == null) {
        stellarTransformer = new FieldTransformer();
        stellarTransformer.setConfig(new LinkedHashMap<>());
        stellarTransformer.setTransformation(FieldTransformations.STELLAR.toString());
        fieldTransformations.add(stellarTransformer);
    }
    return stellarTransformer;
}

Example 3 with FieldTransformer

use of org.apache.metron.common.configuration.FieldTransformer in project metron by apache.

the class FieldTransformationTest method testComplexMapping.

@Test
public void testComplexMapping() throws IOException {
    SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(complexConfig));
    FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
    assertNotNull(handler);
    assertEquals(ImmutableMap.of("output", "field1=value1,field2=value2"), handler.transform(new JSONObject(ImmutableMap.of("field1", "value1", "field2", "value2")), Context.EMPTY_CONTEXT(), c.getParserConfig()));
}

Example 4 with FieldTransformer

use of org.apache.metron.common.configuration.FieldTransformer in project metron by apache.

the class StellarTransformationTest method testIntermediateValues.

@Test
public void testIntermediateValues() throws Exception {
    SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(intermediateValuesConfig));
    FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
    JSONObject input = new JSONObject(new HashMap<String, Object>() {

        {
        }
    });
    handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
    int expected = 3;
    assertEquals(expected, input.get("final_value"));
    assertFalse(input.containsKey("value1"));
    assertFalse(input.containsKey("value2"));
}

Example 5 with FieldTransformer

use of org.apache.metron.common.configuration.FieldTransformer in project metron by apache.

the class StellarTransformationTest method testStellar_multi.

/**
 * A more complicated test where we are transforming multiple fields:
 * 1. Convert a timestamp field in yyyy-MM-dd HH:mm:ss format to unix epoch while
 *    looking up the timezone based on a second field, dc, in a map being kept in the parser config.
 *    If the data center isn't in the map, then the default is UTC
 * 2. Extract the host from a URL field and convert to lowercase
 * 3. Extract the protocol of the URL field
 */
@Test
public void testStellar_multi() throws Exception {
    SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(stellarConfig_multi));
    FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
    {
        // We need a timestamp field, a URL field and a data center field
        JSONObject input = new JSONObject(new HashMap<String, Object>() {

            {
                put("timestamp", "2016-01-05 17:02:30");
                put("url", "https://caseystella.com/blog");
                // looking up the data center in portland, which doesn't exist in the map, so we default to UTC
                put("dc", "portland");
            }
        });
        handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
        long expected = 1452013350000L;
        assertEquals(expected, input.get("utc_timestamp"));
        assertEquals("caseystella.com", input.get("url_host"));
        assertEquals("https", input.get("url_protocol"));
        assertTrue(input.containsKey("timestamp"));
        assertTrue(input.containsKey("url"));
    }
    {
        // now we see what happens when we change the data center to london, which is in the map
        JSONObject input = new JSONObject(new HashMap<String, Object>() {

            {
                put("timestamp", "2016-01-05 17:02:30");
                put("url", "https://caseystella.com/blog");
                put("dc", "london");
            }
        });
        handler.transformAndUpdate(input, Context.EMPTY_CONTEXT(), c.getParserConfig());
        long expected = 1452013350000L;
        assertEquals(expected, input.get("utc_timestamp"));
        assertEquals("caseystella.com", input.get("url_host"));
        assertEquals("https", input.get("url_protocol"));
        assertTrue(input.containsKey("timestamp"));
        assertTrue(input.containsKey("url"));
    }
    // now we ensure that because we don't have a data center field at all, it's defaulted to UTC.
    {
        JSONObject input = new JSONObject(new HashMap<String, Object>() {

            {
                put("timestamp", "2016-01-05 17:02:30");
                put("url", "https://caseystella.com/blog");
            }
        });
        handler.transformAndUpdate(input, Context.EMPTY_CONTEXT(), c.getParserConfig());
        long expected = 1452013350000L;
        assertEquals(expected, input.get("utc_timestamp"));
        assertEquals("caseystella.com", input.get("url_host"));
        assertEquals("https", input.get("url_protocol"));
        assertTrue(input.containsKey("timestamp"));
        assertTrue(input.containsKey("url"));
    }
}