Examples with EnrichmentValue org.apache.metron.enrichment.converter.EnrichmentValue used on opensource projects

Example 1 with EnrichmentValue

use of org.apache.metron.enrichment.converter.EnrichmentValue in project metron by apache.

the class SimpleHbaseEnrichmentWriter method write.

@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
    Map<String, Object> sensorConfig = configurations.getSensorConfig(sensorType);
    HTableInterface table = getTable(sensorConfig);
    KeyTransformer transformer = getTransformer(sensorConfig);
    Object enrichmentTypeObj = Configurations.ENRICHMENT_TYPE.get(sensorConfig);
    String enrichmentType = enrichmentTypeObj == null ? null : enrichmentTypeObj.toString();
    Set<String> valueColumns = new HashSet<>(getColumns(Configurations.VALUE_COLUMNS.get(sensorConfig), true));
    List<Put> puts = new ArrayList<>();
    for (JSONObject message : messages) {
        EnrichmentKey key = getKey(message, transformer, enrichmentType);
        EnrichmentValue value = getValue(message, transformer.keySet, valueColumns);
        if (key == null || value == null) {
            continue;
        }
        Put put = converter.toPut(this.cf, key, value);
        if (put != null) {
            LOG.debug("Put: {Column Family: '{}', Key: '{}', Value: '{}'}", this.cf, key, value);
            puts.add(put);
        }
    }
    BulkWriterResponse response = new BulkWriterResponse();
    try {
        table.put(puts);
    } catch (Exception e) {
        response.addAllErrors(e, tuples);
        return response;
    }
    // Can return no errors, because put will throw Exception on error.
    response.addAllSuccesses(tuples);
    return response;
}

Example 2 with EnrichmentValue

use of org.apache.metron.enrichment.converter.EnrichmentValue in project metron by apache.

the class TaxiiHandler method run.

/**
 * The action to be performed by this timer task.
 */
@Override
public void run() {
    if (inProgress) {
        return;
    }
    Date ts = new Date();
    LOG.info("Polling...{}", new SimpleDateFormat().format(ts));
    try {
        inProgress = true;
        // Prepare the message to send.
        String sessionID = MessageHelper.generateMessageId();
        PollRequest request = messageFactory.get().createPollRequest().withMessageId(sessionID).withCollectionName(collection);
        if (subscriptionId != null) {
            request = request.withSubscriptionID(subscriptionId);
        } else {
            request = request.withPollParameters(messageFactory.get().createPollParametersType());
        }
        if (beginTime != null) {
            Calendar gc = GregorianCalendar.getInstance();
            gc.setTime(beginTime);
            XMLGregorianCalendar gTime = null;
            try {
                gTime = DatatypeFactory.newInstance().newXMLGregorianCalendar((GregorianCalendar) gc).normalize();
            } catch (DatatypeConfigurationException e) {
                RuntimeErrors.ILLEGAL_STATE.throwRuntime("Unable to set the begin time due to", e);
            }
            gTime.setFractionalSecond(null);
            LOG.info("Begin Time: {}", gTime);
            request.setExclusiveBeginTimestamp(gTime);
        }
        try {
            PollResponse response = call(request, PollResponse.class);
            LOG.info("Got Poll Response with {} blocks", response.getContentBlocks().size());
            int numProcessed = 0;
            long avgTimeMS = 0;
            long timeStartedBlock = System.currentTimeMillis();
            for (ContentBlock block : response.getContentBlocks()) {
                AnyMixedContentType content = block.getContent();
                for (Object o : content.getContent()) {
                    numProcessed++;
                    long timeS = System.currentTimeMillis();
                    String xml = null;
                    if (o instanceof Element) {
                        Element element = (Element) o;
                        xml = getStringFromDocument(element.getOwnerDocument());
                        if (LOG.isDebugEnabled() && Math.random() < 0.01) {
                            LOG.debug("Random Stix doc: {}", xml);
                        }
                        for (LookupKV<EnrichmentKey, EnrichmentValue> kv : extractor.extract(xml)) {
                            if (allowedIndicatorTypes.isEmpty() || allowedIndicatorTypes.contains(kv.getKey().type)) {
                                kv.getValue().getMetadata().put("source_type", "taxii");
                                kv.getValue().getMetadata().put("taxii_url", endpoint.toString());
                                kv.getValue().getMetadata().put("taxii_collection", collection);
                                Put p = converter.toPut(columnFamily, kv.getKey(), kv.getValue());
                                Table table = getTable(hbaseTable);
                                table.put(p);
                                LOG.info("Found Threat Intel: {} => ", kv.getKey(), kv.getValue());
                            }
                        }
                    }
                    avgTimeMS += System.currentTimeMillis() - timeS;
                }
                if ((numProcessed + 1) % 100 == 0) {
                    LOG.info("Processed {}  in {} ms, avg time: {}", numProcessed, System.currentTimeMillis() - timeStartedBlock, avgTimeMS / content.getContent().size());
                    timeStartedBlock = System.currentTimeMillis();
                    avgTimeMS = 0;
                    numProcessed = 0;
                }
            }
        } catch (Exception e) {
            LOG.error(e.getMessage(), e);
            throw new RuntimeException("Unable to make request", e);
        }
    } finally {
        inProgress = false;
        beginTime = ts;
    }
}

Example 3 with EnrichmentValue

use of org.apache.metron.enrichment.converter.EnrichmentValue in project metron by apache.

the class ExtractorTest method testExtractionLoading.

@Test
public void testExtractionLoading() throws Exception {
    /**
     *         config:
     *         {
     *            "config" : {}
     *            ,"extractor" : "org.apache.metron.dataloads.extractor.ExtractorTest$DummyExtractor"
     *         }
     */
    String config = "{\n" + "            \"config\" : {}\n" + "            ,\"extractor\" : \"org.apache.metron.dataloads.extractor.ExtractorTest$DummyExtractor\"\n" + "         }";
    ExtractorHandler handler = ExtractorHandler.load(config);
    LookupKV results = Iterables.getFirst(handler.getExtractor().extract(null), null);
    EnrichmentKey key = (EnrichmentKey) results.getKey();
    EnrichmentValue value = (EnrichmentValue) results.getValue();
    assertEquals("dummy", key.indicator);
    assertEquals("type", key.type);
    assertEquals("dummy", value.getMetadata().get("indicator"));
}

Example 4 with EnrichmentValue

use of org.apache.metron.enrichment.converter.EnrichmentValue in project metron by apache.

the class ExtractorTest method testDummyExtractor.

@Test
public void testDummyExtractor() throws IllegalAccessException, InstantiationException, ClassNotFoundException, IOException, NoSuchMethodException, InvocationTargetException {
    Extractor extractor = Extractors.create(DummyExtractor.class.getName());
    LookupKV results = Iterables.getFirst(extractor.extract(null), null);
    EnrichmentKey key = (EnrichmentKey) results.getKey();
    EnrichmentValue value = (EnrichmentValue) results.getValue();
    assertEquals("dummy", key.indicator);
    assertEquals("type", key.type);
    assertEquals("dummy", value.getMetadata().get("indicator"));
}

Example 5 with EnrichmentValue

use of org.apache.metron.enrichment.converter.EnrichmentValue in project metron by apache.

the class TransformFilterExtractorDecoratorTest method transforms_values_and_indicators.

@Test
public void transforms_values_and_indicators() throws IOException {
    final String indicatorVal = "val2";
    EnrichmentKey lookupKey = new EnrichmentKey("testenrichment", indicatorVal);
    EnrichmentValue lookupValue = new EnrichmentValue(new HashMap<String, Object>() {

        {
            put("foo", "val1");
            put("bar", indicatorVal);
            put("baz", "val3");
        }
    });
    LookupKV lkv = new LookupKV<>(lookupKey, lookupValue);
    List<LookupKV> extractedLkvs = new ArrayList<>();
    extractedLkvs.add(lkv);
    Mockito.when(extractor.extract("val1,val2,val3")).thenReturn(extractedLkvs);
    Iterable<LookupKV> extracted = decorator.extract("val1,val2,val3");
    EnrichmentKey expectedLookupKey = new EnrichmentKey("testenrichment", "VAL2");
    EnrichmentValue expectedLookupValue = new EnrichmentValue(new HashMap<String, Object>() {

        {
            put("foo", "VAL1");
            put("bar", "val2");
            put("baz", "val3");
            put("newvar", "VAL1");
            put("lowernewvar", "val1");
        }
    });
    LookupKV expectedLkv = new LookupKV<>(expectedLookupKey, expectedLookupValue);
    List<LookupKV> expectedLkvs = new ArrayList<>();
    expectedLkvs.add(expectedLkv);
    assertThat(extracted, CoreMatchers.equalTo(expectedLkvs));
}