Examples with Document org.apache.metron.indexing.dao.update.Document used on opensource projects

Example 26 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class ElasticsearchMetaAlertDaoTest method testCalculateMetaScoresList.

@Test
public void testCalculateMetaScoresList() {
    final double delta = 0.001;
    List<Map<String, Object>> alertList = new ArrayList<>();
    // add an alert with a threat score
    alertList.add(Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0f));
    // add a second alert with a threat score
    alertList.add(Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 20.0f));
    // add a third alert with NO threat score
    alertList.add(Collections.singletonMap("alert3", "has no threat score"));
    // create the metaalert
    Map<String, Object> docMap = new HashMap<>();
    docMap.put(MetaAlertDao.ALERT_FIELD, alertList);
    Document metaalert = new Document(docMap, "guid", MetaAlertDao.METAALERT_TYPE, 0L);
    // calculate the threat score for the metaalert
    ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao();
    metaAlertDao.calculateMetaScores(metaalert);
    Object threatScore = metaalert.getDocument().get(ElasticsearchMetaAlertDao.THREAT_FIELD_DEFAULT);
    // the metaalert must contain a summary of all child threat scores
    assertEquals(20D, (Double) metaalert.getDocument().get("max"), delta);
    assertEquals(10D, (Double) metaalert.getDocument().get("min"), delta);
    assertEquals(15D, (Double) metaalert.getDocument().get("average"), delta);
    assertEquals(2L, metaalert.getDocument().get("count"));
    assertEquals(30D, (Double) metaalert.getDocument().get("sum"), delta);
    assertEquals(15D, (Double) metaalert.getDocument().get("median"), delta);
    // it must contain an overall threat score; a float to match the type of the threat score of the other sensor indices
    assertTrue(threatScore instanceof Float);
    // by default, the overall threat score is the sum of all child threat scores
    assertEquals(30.0F, threatScore);
}

Example 27 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class ElasticsearchUpdateIntegrationTest method test.

@Test
public void test() throws Exception {
    List<Map<String, Object>> inputData = new ArrayList<>();
    for (int i = 0; i < 10; ++i) {
        final String name = "message" + i;
        inputData.add(new HashMap<String, Object>() {

            {
                put("source:type", SENSOR_NAME);
                put("name", name);
                put("timestamp", System.currentTimeMillis());
                put(Constants.GUID, name);
            }
        });
    }
    es.add(index, SENSOR_NAME, Iterables.transform(inputData, m -> {
        try {
            return JSONUtils.INSTANCE.toJSON(m, true);
        } catch (JsonProcessingException e) {
            throw new IllegalStateException(e.getMessage(), e);
        }
    }));
    List<Map<String, Object>> docs = null;
    for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
        docs = es.getAllIndexedDocs(index, SENSOR_NAME + "_doc");
        if (docs.size() >= 10) {
            break;
        }
    }
    Assert.assertEquals(10, docs.size());
    // modify the first message and add a new field
    {
        Map<String, Object> message0 = new HashMap<String, Object>(inputData.get(0)) {

            {
                put("new-field", "metron");
            }
        };
        String guid = "" + message0.get(Constants.GUID);
        dao.replace(new ReplaceRequest() {

            {
                setReplacement(message0);
                setGuid(guid);
                setSensorType(SENSOR_NAME);
            }
        }, Optional.empty());
        Assert.assertEquals(1, table.size());
        Document doc = dao.getLatest(guid, SENSOR_NAME);
        Assert.assertEquals(message0, doc.getDocument());
        {
            // ensure hbase is up to date
            Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME)));
            Result r = table.get(g);
            NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes());
            Assert.assertEquals(1, columns.size());
            Assert.assertEquals(message0, JSONUtils.INSTANCE.load(new String(columns.lastEntry().getValue()), JSONUtils.MAP_SUPPLIER));
        }
        {
            // ensure ES is up-to-date
            long cnt = 0;
            for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) {
                docs = es.getAllIndexedDocs(index, SENSOR_NAME + "_doc");
                cnt = docs.stream().filter(d -> message0.get("new-field").equals(d.get("new-field"))).count();
            }
            Assert.assertNotEquals("Elasticsearch is not updated!", cnt, 0);
        }
    }
    // modify the same message and modify the new field
    {
        Map<String, Object> message0 = new HashMap<String, Object>(inputData.get(0)) {

            {
                put("new-field", "metron2");
            }
        };
        String guid = "" + message0.get(Constants.GUID);
        dao.replace(new ReplaceRequest() {

            {
                setReplacement(message0);
                setGuid(guid);
                setSensorType(SENSOR_NAME);
            }
        }, Optional.empty());
        Assert.assertEquals(1, table.size());
        Document doc = dao.getLatest(guid, SENSOR_NAME);
        Assert.assertEquals(message0, doc.getDocument());
        {
            // ensure hbase is up to date
            Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME)));
            Result r = table.get(g);
            NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes());
            Assert.assertEquals(2, columns.size());
            Assert.assertEquals(message0, JSONUtils.INSTANCE.load(new String(columns.lastEntry().getValue()), JSONUtils.MAP_SUPPLIER));
            Assert.assertNotEquals(message0, JSONUtils.INSTANCE.load(new String(columns.firstEntry().getValue()), JSONUtils.MAP_SUPPLIER));
        }
        {
            // ensure ES is up-to-date
            long cnt = 0;
            for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) {
                docs = es.getAllIndexedDocs(index, SENSOR_NAME + "_doc");
                cnt = docs.stream().filter(d -> message0.get("new-field").equals(d.get("new-field"))).count();
            }
            Assert.assertNotEquals("Elasticsearch is not updated!", cnt, 0);
        }
    }
}

Example 28 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class IndexDao method replace.

/**
 * Replace a document in an index.
 * @param request The replacement request.
 * @param timestamp The timestamp (optional) of the update.  If not specified, then current time will be used.
 * @throws IOException
 */
default void replace(ReplaceRequest request, Optional<Long> timestamp) throws IOException {
    Document d = new Document(request.getReplacement(), request.getGuid(), request.getSensorType(), timestamp.orElse(System.currentTimeMillis()));
    update(d, Optional.ofNullable(request.getIndex()));
}

Example 29 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class IndexDao method getPatchedDocument.

default Document getPatchedDocument(PatchRequest request, Optional<Long> timestamp) throws OriginalNotFoundException, IOException {
    Map<String, Object> latest = request.getSource();
    if (latest == null) {
        Document latestDoc = getLatest(request.getGuid(), request.getSensorType());
        if (latestDoc != null && latestDoc.getDocument() != null) {
            latest = latestDoc.getDocument();
        } else {
            throw new OriginalNotFoundException("Unable to patch an document that doesn't exist and isn't specified.");
        }
    }
    Map<String, Object> updated = JSONUtils.INSTANCE.applyPatch(request.getPatch(), latest);
    return new Document(updated, request.getGuid(), request.getSensorType(), timestamp.orElse(System.currentTimeMillis()));
}

Example 30 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class InMemoryDao method getAllLatest.

@Override
public Iterable<Document> getAllLatest(List<GetRequest> getRequests) throws IOException {
    List<Document> documents = new ArrayList<>();
    for (Map.Entry<String, List<String>> kv : BACKING_STORE.entrySet()) {
        for (String doc : kv.getValue()) {
            Map<String, Object> docParsed = parse(doc);
            String guid = (String) docParsed.getOrDefault(Constants.GUID, "");
            for (GetRequest getRequest : getRequests) {
                if (getRequest.getGuid().equals(guid)) {
                    documents.add(new Document(doc, guid, getRequest.getSensorType(), 0L));
                }
            }
        }
    }
    return documents;
}