Examples with EnrichmentKey org.apache.metron.enrichment.converter.EnrichmentKey used on opensource projects

Example 1 with EnrichmentKey

use of org.apache.metron.enrichment.converter.EnrichmentKey in project metron by apache.

the class AddressHandler method extract.

@Override
public Iterable<LookupKV> extract(final Address type, Map<String, Object> config) throws IOException {
    List<LookupKV> ret = new ArrayList<>();
    final CategoryTypeEnum category = type.getCategory();
    if (!SUPPORTED_CATEGORIES.contains(category)) {
        return ret;
    }
    String typeStr = getType();
    if (config != null) {
        if (config.containsKey(SPECIFIC_CATEGORY_CONFIG)) {
            List<CategoryTypeEnum> categories = new ArrayList<>();
            for (String c : Splitter.on(",").split(config.get(SPECIFIC_CATEGORY_CONFIG).toString())) {
                categories.add(CategoryTypeEnum.valueOf(c));
            }
            EnumSet<CategoryTypeEnum> specificCategories = EnumSet.copyOf(categories);
            if (!specificCategories.contains(category)) {
                return ret;
            }
        }
        if (config.containsKey(TYPE_CONFIG)) {
            typeStr = config.get(TYPE_CONFIG).toString();
        }
    }
    StringObjectPropertyType value = type.getAddressValue();
    for (String token : StixExtractor.split(value)) {
        final String indicatorType = typeStr + ":" + category;
        LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, token), new EnrichmentValue(new HashMap<String, Object>() {

            {
                put("source-type", "STIX");
                put("indicator-type", indicatorType);
                put("source", type.toXMLString());
            }
        }));
        ret.add(results);
    }
    return ret;
}

Example 2 with EnrichmentKey

use of org.apache.metron.enrichment.converter.EnrichmentKey in project metron by apache.

the class DomainHandler method extract.

@Override
public Iterable<LookupKV> extract(final DomainName type, Map<String, Object> config) throws IOException {
    List<LookupKV> ret = new ArrayList<>();
    String typeStr = getType();
    if (config != null) {
        Object o = config.get(TYPE_CONFIG);
        if (o != null) {
            typeStr = o.toString();
        }
    }
    final DomainNameTypeEnum domainType = type.getType();
    if (domainType == null || SUPPORTED_TYPES.contains(domainType)) {
        StringObjectPropertyType value = type.getValue();
        for (String token : StixExtractor.split(value)) {
            final String indicatorType = typeStr + ":" + DomainNameTypeEnum.FQDN;
            LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, token), new EnrichmentValue(new HashMap<String, Object>() {

                {
                    put("source-type", "STIX");
                    put("indicator-type", indicatorType);
                    put("source", type.toXMLString());
                }
            }));
            ret.add(results);
        }
    }
    return ret;
}

Example 3 with EnrichmentKey

use of org.apache.metron.enrichment.converter.EnrichmentKey in project metron by apache.

the class URIHandler method extract.

@Override
public Iterable<LookupKV> extract(URIObjectType type, Map<String, Object> config) throws IOException {
    List<LookupKV> ret = new ArrayList<>();
    if (type != null) {
        AnyURIObjectPropertyType val = type.getValue();
        if (val != null) {
            Object v = val.getValue();
            if (v != null) {
                final String indicatorType = getType();
                LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, v.toString()), new EnrichmentValue(new HashMap<String, Object>() {

                    {
                        put("source-type", "STIX");
                        put("uri", v.toString());
                        put("indicator-type", indicatorType);
                        put("source", type.toXMLString());
                    }
                }));
                ret.add(results);
            }
        }
    }
    return ret;
}

Example 4 with EnrichmentKey

use of org.apache.metron.enrichment.converter.EnrichmentKey in project metron by apache.

the class TaxiiHandler method run.

/**
 * The action to be performed by this timer task.
 */
@Override
public void run() {
    if (inProgress) {
        return;
    }
    Date ts = new Date();
    LOG.info("Polling...{}", new SimpleDateFormat().format(ts));
    try {
        inProgress = true;
        // Prepare the message to send.
        String sessionID = MessageHelper.generateMessageId();
        PollRequest request = messageFactory.get().createPollRequest().withMessageId(sessionID).withCollectionName(collection);
        if (subscriptionId != null) {
            request = request.withSubscriptionID(subscriptionId);
        } else {
            request = request.withPollParameters(messageFactory.get().createPollParametersType());
        }
        if (beginTime != null) {
            Calendar gc = GregorianCalendar.getInstance();
            gc.setTime(beginTime);
            XMLGregorianCalendar gTime = null;
            try {
                gTime = DatatypeFactory.newInstance().newXMLGregorianCalendar((GregorianCalendar) gc).normalize();
            } catch (DatatypeConfigurationException e) {
                ErrorUtils.RuntimeErrors.ILLEGAL_STATE.throwRuntime("Unable to set the begin time due to", e);
            }
            gTime.setFractionalSecond(null);
            LOG.info("Begin Time: {}", gTime);
            request.setExclusiveBeginTimestamp(gTime);
        }
        try {
            PollResponse response = call(request, PollResponse.class);
            LOG.info("Got Poll Response with {} blocks", response.getContentBlocks().size());
            int numProcessed = 0;
            long avgTimeMS = 0;
            long timeStartedBlock = System.currentTimeMillis();
            for (ContentBlock block : response.getContentBlocks()) {
                AnyMixedContentType content = block.getContent();
                for (Object o : content.getContent()) {
                    numProcessed++;
                    long timeS = System.currentTimeMillis();
                    String xml = null;
                    if (o instanceof Element) {
                        Element element = (Element) o;
                        xml = getStringFromDocument(element.getOwnerDocument());
                        if (LOG.isDebugEnabled() && Math.random() < 0.01) {
                            LOG.debug("Random Stix doc: {}", xml);
                        }
                        for (LookupKV<EnrichmentKey, EnrichmentValue> kv : extractor.extract(xml)) {
                            if (allowedIndicatorTypes.isEmpty() || allowedIndicatorTypes.contains(kv.getKey().type)) {
                                kv.getValue().getMetadata().put("source_type", "taxii");
                                kv.getValue().getMetadata().put("taxii_url", endpoint.toString());
                                kv.getValue().getMetadata().put("taxii_collection", collection);
                                Put p = converter.toPut(columnFamily, kv.getKey(), kv.getValue());
                                HTableInterface table = getTable(hbaseTable);
                                table.put(p);
                                LOG.info("Found Threat Intel: {} => ", kv.getKey(), kv.getValue());
                            }
                        }
                    }
                    avgTimeMS += System.currentTimeMillis() - timeS;
                }
                if ((numProcessed + 1) % 100 == 0) {
                    LOG.info("Processed {}  in {} ms, avg time: {}", numProcessed, System.currentTimeMillis() - timeStartedBlock, avgTimeMS / content.getContent().size());
                    timeStartedBlock = System.currentTimeMillis();
                    avgTimeMS = 0;
                    numProcessed = 0;
                }
            }
        } catch (Exception e) {
            LOG.error(e.getMessage(), e);
            throw new RuntimeException("Unable to make request", e);
        }
    } finally {
        inProgress = false;
        beginTime = ts;
    }
}

Example 5 with EnrichmentKey

use of org.apache.metron.enrichment.converter.EnrichmentKey in project metron by apache.

the class ExtractorTest method testDummyExtractor.

@Test
public void testDummyExtractor() throws IllegalAccessException, InstantiationException, ClassNotFoundException, IOException, NoSuchMethodException, InvocationTargetException {
    Extractor extractor = Extractors.create(DummyExtractor.class.getName());
    LookupKV results = Iterables.getFirst(extractor.extract(null), null);
    EnrichmentKey key = (EnrichmentKey) results.getKey();
    EnrichmentValue value = (EnrichmentValue) results.getValue();
    Assert.assertEquals("dummy", key.indicator);
    Assert.assertEquals("type", key.type);
    Assert.assertEquals("dummy", value.getMetadata().get("indicator"));
}