Examples with EnrichmentValue org.apache.metron.enrichment.converter.EnrichmentValue used on opensource projects

Example 6 with EnrichmentValue

use of org.apache.metron.enrichment.converter.EnrichmentValue in project metron by apache.

the class ExtractorTest method testExtractionLoading.

@Test
public void testExtractionLoading() throws Exception {
    /**
     *         config:
     *         {
     *            "config" : {}
     *            ,"extractor" : "org.apache.metron.dataloads.extractor.ExtractorTest$DummyExtractor"
     *         }
     */
    String config = "{\n" + "            \"config\" : {}\n" + "            ,\"extractor\" : \"org.apache.metron.dataloads.extractor.ExtractorTest$DummyExtractor\"\n" + "         }";
    ExtractorHandler handler = ExtractorHandler.load(config);
    LookupKV results = Iterables.getFirst(handler.getExtractor().extract(null), null);
    EnrichmentKey key = (EnrichmentKey) results.getKey();
    EnrichmentValue value = (EnrichmentValue) results.getValue();
    Assert.assertEquals("dummy", key.indicator);
    Assert.assertEquals("type", key.type);
    Assert.assertEquals("dummy", value.getMetadata().get("indicator"));
}

Example 7 with EnrichmentValue

use of org.apache.metron.enrichment.converter.EnrichmentValue in project metron by apache.

the class TransformFilterExtractorDecoratorTest method transforms_values_and_indicators.

@Test
public void transforms_values_and_indicators() throws IOException {
    final String indicatorVal = "val2";
    EnrichmentKey lookupKey = new EnrichmentKey("testenrichment", indicatorVal);
    EnrichmentValue lookupValue = new EnrichmentValue(new HashMap<String, Object>() {

        {
            put("foo", "val1");
            put("bar", indicatorVal);
            put("baz", "val3");
        }
    });
    LookupKV lkv = new LookupKV<>(lookupKey, lookupValue);
    List<LookupKV> extractedLkvs = new ArrayList<>();
    extractedLkvs.add(lkv);
    Mockito.when(extractor.extract("val1,val2,val3")).thenReturn(extractedLkvs);
    Iterable<LookupKV> extracted = decorator.extract("val1,val2,val3");
    EnrichmentKey expectedLookupKey = new EnrichmentKey("testenrichment", "VAL2");
    EnrichmentValue expectedLookupValue = new EnrichmentValue(new HashMap<String, Object>() {

        {
            put("foo", "VAL1");
            put("bar", "val2");
            put("baz", "val3");
            put("newvar", "VAL1");
            put("lowernewvar", "val1");
        }
    });
    LookupKV expectedLkv = new LookupKV<>(expectedLookupKey, expectedLookupValue);
    List<LookupKV> expectedLkvs = new ArrayList<>();
    expectedLkvs.add(expectedLkv);
    Assert.assertThat(extracted, CoreMatchers.equalTo(expectedLkvs));
}

Example 8 with EnrichmentValue

use of org.apache.metron.enrichment.converter.EnrichmentValue in project metron by apache.

the class TransformFilterExtractorDecoratorTest method filters_values.

@Test
public void filters_values() throws Exception {
    final String indicatorVal = "val2";
    EnrichmentKey lookupKey = new EnrichmentKey("testenrichment", indicatorVal);
    EnrichmentValue lookupValue = new EnrichmentValue(new HashMap<String, Object>() {

        {
            put("foo", "val1");
            put("bar", indicatorVal);
            put("baz", "");
        }
    });
    LookupKV lkv = new LookupKV<>(lookupKey, lookupValue);
    List<LookupKV> extractedLkvs = new ArrayList<>();
    extractedLkvs.add(lkv);
    Mockito.when(extractor.extract("val1,val2,")).thenReturn(extractedLkvs);
    Iterable<LookupKV> extracted = decorator.extract("val1,val2,");
    Assert.assertThat(extracted, CoreMatchers.equalTo(new ArrayList<>()));
}

Example 9 with EnrichmentValue

use of org.apache.metron.enrichment.converter.EnrichmentValue in project metron by apache.

the class SimpleHBaseAdapter method enrich.

@Override
public JSONObject enrich(CacheKey value) {
    JSONObject enriched = new JSONObject();
    if (!isInitialized()) {
        initializeAdapter(null);
    }
    List<String> enrichmentTypes = value.getConfig().getEnrichment().getFieldToTypeMap().get(EnrichmentUtils.toTopLevelField(value.getField()));
    if (isInitialized() && enrichmentTypes != null && value.getValue() != null) {
        try {
            for (LookupKV<EnrichmentKey, EnrichmentValue> kv : lookup.get(Iterables.transform(enrichmentTypes, new EnrichmentUtils.TypeToKey(value.coerceValue(String.class), lookup.getTable(), value.getConfig().getEnrichment())), false)) {
                if (kv != null && kv.getValue() != null && kv.getValue().getMetadata() != null) {
                    for (Map.Entry<String, Object> values : kv.getValue().getMetadata().entrySet()) {
                        enriched.put(kv.getKey().type + "." + values.getKey(), values.getValue());
                    }
                    LOG.trace("Enriched type {} => {}", kv.getKey().type, enriched);
                }
            }
        } catch (IOException e) {
            LOG.error("Unable to retrieve value: {}", e.getMessage(), e);
            initializeAdapter(null);
            throw new RuntimeException("Unable to retrieve value: " + e.getMessage(), e);
        }
    }
    LOG.trace("SimpleHBaseAdapter succeeded: {}", enriched);
    return enriched;
}

Example 10 with EnrichmentValue

use of org.apache.metron.enrichment.converter.EnrichmentValue in project metron by apache.

the class SimpleHbaseEnrichmentWriter method write.

@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
    Map<String, Object> sensorConfig = configurations.getSensorConfig(sensorType);
    HTableInterface table = getTable(sensorConfig);
    KeyTransformer transformer = getTransformer(sensorConfig);
    Object enrichmentTypeObj = Configurations.ENRICHMENT_TYPE.get(sensorConfig);
    String enrichmentType = enrichmentTypeObj == null ? null : enrichmentTypeObj.toString();
    Set<String> valueColumns = new HashSet<>(getColumns(Configurations.VALUE_COLUMNS.get(sensorConfig), true));
    List<Put> puts = new ArrayList<>();
    for (JSONObject message : messages) {
        EnrichmentKey key = getKey(message, transformer, enrichmentType);
        EnrichmentValue value = getValue(message, transformer.keySet, valueColumns);
        if (key == null || value == null) {
            continue;
        }
        Put put = converter.toPut(this.cf, key, value);
        if (put != null) {
            LOG.debug("Put: {Column Family: '{}', Key: '{}', Value: '{}'}", this.cf, key, value);
            puts.add(put);
        }
    }
    BulkWriterResponse response = new BulkWriterResponse();
    try {
        table.put(puts);
    } catch (Exception e) {
        response.addAllErrors(e, tuples);
        return response;
    }
    // Can return no errors, because put will throw Exception on error.
    response.addAllSuccesses(tuples);
    return response;
}