Search in sources :

Example 1 with ContentBlock

use of org.mitre.taxii.messages.xml11.ContentBlock in project metron by apache.

the class TaxiiHandler method run.

/**
 * The action to be performed by this timer task.
 */
@Override
public void run() {
    if (inProgress) {
        return;
    }
    Date ts = new Date();
    LOG.info("Polling...{}", new SimpleDateFormat().format(ts));
    try {
        inProgress = true;
        // Prepare the message to send.
        String sessionID = MessageHelper.generateMessageId();
        PollRequest request = messageFactory.get().createPollRequest().withMessageId(sessionID).withCollectionName(collection);
        if (subscriptionId != null) {
            request = request.withSubscriptionID(subscriptionId);
        } else {
            request = request.withPollParameters(messageFactory.get().createPollParametersType());
        }
        if (beginTime != null) {
            Calendar gc = GregorianCalendar.getInstance();
            gc.setTime(beginTime);
            XMLGregorianCalendar gTime = null;
            try {
                gTime = DatatypeFactory.newInstance().newXMLGregorianCalendar((GregorianCalendar) gc).normalize();
            } catch (DatatypeConfigurationException e) {
                RuntimeErrors.ILLEGAL_STATE.throwRuntime("Unable to set the begin time due to", e);
            }
            gTime.setFractionalSecond(null);
            LOG.info("Begin Time: {}", gTime);
            request.setExclusiveBeginTimestamp(gTime);
        }
        try {
            PollResponse response = call(request, PollResponse.class);
            LOG.info("Got Poll Response with {} blocks", response.getContentBlocks().size());
            int numProcessed = 0;
            long avgTimeMS = 0;
            long timeStartedBlock = System.currentTimeMillis();
            for (ContentBlock block : response.getContentBlocks()) {
                AnyMixedContentType content = block.getContent();
                for (Object o : content.getContent()) {
                    numProcessed++;
                    long timeS = System.currentTimeMillis();
                    String xml = null;
                    if (o instanceof Element) {
                        Element element = (Element) o;
                        xml = getStringFromDocument(element.getOwnerDocument());
                        if (LOG.isDebugEnabled() && Math.random() < 0.01) {
                            LOG.debug("Random Stix doc: {}", xml);
                        }
                        for (LookupKV<EnrichmentKey, EnrichmentValue> kv : extractor.extract(xml)) {
                            if (allowedIndicatorTypes.isEmpty() || allowedIndicatorTypes.contains(kv.getKey().type)) {
                                kv.getValue().getMetadata().put("source_type", "taxii");
                                kv.getValue().getMetadata().put("taxii_url", endpoint.toString());
                                kv.getValue().getMetadata().put("taxii_collection", collection);
                                Put p = converter.toPut(columnFamily, kv.getKey(), kv.getValue());
                                Table table = getTable(hbaseTable);
                                table.put(p);
                                LOG.info("Found Threat Intel: {} => ", kv.getKey(), kv.getValue());
                            }
                        }
                    }
                    avgTimeMS += System.currentTimeMillis() - timeS;
                }
                if ((numProcessed + 1) % 100 == 0) {
                    LOG.info("Processed {}  in {} ms, avg time: {}", numProcessed, System.currentTimeMillis() - timeStartedBlock, avgTimeMS / content.getContent().size());
                    timeStartedBlock = System.currentTimeMillis();
                    avgTimeMS = 0;
                    numProcessed = 0;
                }
            }
        } catch (Exception e) {
            LOG.error(e.getMessage(), e);
            throw new RuntimeException("Unable to make request", e);
        }
    } finally {
        inProgress = false;
        beginTime = ts;
    }
}
Also used : Table(org.apache.hadoop.hbase.client.Table) PollRequest(org.mitre.taxii.messages.xml11.PollRequest) GregorianCalendar(java.util.GregorianCalendar) Calendar(java.util.Calendar) XMLGregorianCalendar(javax.xml.datatype.XMLGregorianCalendar) Element(org.w3c.dom.Element) Date(java.util.Date) EnrichmentKey(org.apache.metron.enrichment.converter.EnrichmentKey) Put(org.apache.hadoop.hbase.client.Put) DatatypeConfigurationException(javax.xml.datatype.DatatypeConfigurationException) JAXBException(javax.xml.bind.JAXBException) TransformerException(javax.xml.transform.TransformerException) IOException(java.io.IOException) XMLGregorianCalendar(javax.xml.datatype.XMLGregorianCalendar) DatatypeConfigurationException(javax.xml.datatype.DatatypeConfigurationException) ContentBlock(org.mitre.taxii.messages.xml11.ContentBlock) PollResponse(org.mitre.taxii.messages.xml11.PollResponse) AnyMixedContentType(org.mitre.taxii.messages.xml11.AnyMixedContentType) SimpleDateFormat(java.text.SimpleDateFormat) EnrichmentValue(org.apache.metron.enrichment.converter.EnrichmentValue)

Aggregations

IOException (java.io.IOException)1 SimpleDateFormat (java.text.SimpleDateFormat)1 Calendar (java.util.Calendar)1 Date (java.util.Date)1 GregorianCalendar (java.util.GregorianCalendar)1 JAXBException (javax.xml.bind.JAXBException)1 DatatypeConfigurationException (javax.xml.datatype.DatatypeConfigurationException)1 XMLGregorianCalendar (javax.xml.datatype.XMLGregorianCalendar)1 TransformerException (javax.xml.transform.TransformerException)1 Put (org.apache.hadoop.hbase.client.Put)1 Table (org.apache.hadoop.hbase.client.Table)1 EnrichmentKey (org.apache.metron.enrichment.converter.EnrichmentKey)1 EnrichmentValue (org.apache.metron.enrichment.converter.EnrichmentValue)1 AnyMixedContentType (org.mitre.taxii.messages.xml11.AnyMixedContentType)1 ContentBlock (org.mitre.taxii.messages.xml11.ContentBlock)1 PollRequest (org.mitre.taxii.messages.xml11.PollRequest)1 PollResponse (org.mitre.taxii.messages.xml11.PollResponse)1 Element (org.w3c.dom.Element)1