Examples with Document org.apache.metron.indexing.dao.update.Document used on opensource projects

Example 16 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class HBaseDaoIntegrationTest method shouldGetLatest.

@Test
public void shouldGetLatest() throws Exception {
    // Load alerts
    List<Document> alerts = buildAlerts(3);
    Map<Document, Optional<String>> updates = alerts.stream().collect(Collectors.toMap(document -> document, document -> Optional.empty()));
    hbaseDao.batchUpdate(updates);
    Document actualDocument = hbaseDao.getLatest("message_1", SENSOR_TYPE);
    Document expectedDocument = alerts.get(1);
    Assert.assertEquals(expectedDocument, actualDocument);
}

Example 17 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class HBaseDaoIntegrationTest method shouldGetAllLatest.

@Test
public void shouldGetAllLatest() throws Exception {
    // Load alerts
    List<Document> alerts = buildAlerts(15);
    alerts.stream().collect(Collectors.toMap(Document::getGuid, document -> Optional.empty()));
    Map<Document, Optional<String>> updates = alerts.stream().collect(Collectors.toMap(document -> document, document -> Optional.empty()));
    hbaseDao.batchUpdate(updates);
    int expectedCount = 12;
    List<GetRequest> getRequests = new ArrayList<>();
    for (int i = 1; i < expectedCount + 1; i++) {
        getRequests.add(new GetRequest("message_" + i, SENSOR_TYPE));
    }
    Iterator<Document> results = hbaseDao.getAllLatest(getRequests).iterator();
    for (int i = 0; i < expectedCount; i++) {
        Document expectedDocument = alerts.get(i + 1);
        Document actualDocument = results.next();
        Assert.assertEquals(expectedDocument, actualDocument);
    }
    Assert.assertFalse("Result size should be 12 but was greater", results.hasNext());
}

Example 18 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class HBaseDaoIntegrationTest method buildAlerts.

protected List<Document> buildAlerts(int count) throws IOException {
    List<Document> alerts = new ArrayList<>();
    for (int i = 0; i < count; ++i) {
        String guid = "message_" + i;
        String json = "{\"guid\":\"message_" + i + "\", \"source:type\":\"test\"}";
        Document alert = new Document(json, guid, SENSOR_TYPE, System.currentTimeMillis());
        alerts.add(alert);
    }
    return alerts;
}

Example 19 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class ElasticsearchDao method batchUpdate.

@Override
public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
    String indexPostfix = ElasticsearchUtils.getIndexFormat(accessConfig.getGlobalConfigSupplier().get()).format(new Date());
    BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    // Get the indices we'll actually be using for each Document.
    for (Map.Entry<Document, Optional<String>> updateEntry : updates.entrySet()) {
        Document update = updateEntry.getKey();
        String sensorType = update.getSensorType();
        String indexName = getIndexName(update, updateEntry.getValue(), indexPostfix);
        IndexRequest indexRequest = buildIndexRequest(update, sensorType, indexName);
        bulkRequestBuilder.add(indexRequest);
    }
    BulkResponse bulkResponse = bulkRequestBuilder.get();
    if (bulkResponse.hasFailures()) {
        LOG.error("Bulk Request has failures: {}", bulkResponse.buildFailureMessage());
        throw new IOException("ElasticsearchDao upsert failed: " + bulkResponse.buildFailureMessage());
    }
}

Example 20 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class ElasticsearchDao method toDocument.

private Optional<Document> toDocument(final String guid, SearchHit hit) {
    Long ts = 0L;
    String doc = hit.getSourceAsString();
    String sourceType = toSourceType(hit.getType());
    try {
        return Optional.of(new Document(doc, guid, sourceType, ts));
    } catch (IOException e) {
        throw new IllegalStateException("Unable to retrieve latest: " + e.getMessage(), e);
    }
}