Examples with Document org.apache.metron.indexing.dao.update.Document used on opensource projects

Example 21 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class ElasticsearchDao method getAllLatest.

@Override
public Iterable<Document> getAllLatest(final List<GetRequest> getRequests) throws IOException {
    Collection<String> guids = new HashSet<>();
    Collection<String> sensorTypes = new HashSet<>();
    for (GetRequest getRequest : getRequests) {
        guids.add(getRequest.getGuid());
        sensorTypes.add(getRequest.getSensorType());
    }
    List<Document> documents = searchByGuids(guids, sensorTypes, hit -> {
        Long ts = 0L;
        String doc = hit.getSourceAsString();
        String sourceType = Iterables.getFirst(Splitter.on("_doc").split(hit.getType()), null);
        try {
            return Optional.of(new Document(doc, hit.getId(), sourceType, ts));
        } catch (IOException e) {
            throw new IllegalStateException("Unable to retrieve latest: " + e.getMessage(), e);
        }
    });
    return documents;
}

Example 22 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class ElasticsearchMetaAlertDao method buildCreateDocument.

/**
 * Build the Document representing a meta alert to be created.
 * @param alerts The Elasticsearch results for the meta alerts child documents
 * @param groups The groups used to create this meta alert
 * @return A Document representing the new meta alert
 */
protected Document buildCreateDocument(Iterable<Document> alerts, List<String> groups) {
    // Need to create a Document from the multiget. Scores will be calculated later
    Map<String, Object> metaSource = new HashMap<>();
    List<Map<String, Object>> alertList = new ArrayList<>();
    for (Document alert : alerts) {
        alertList.add(alert.getDocument());
    }
    metaSource.put(ALERT_FIELD, alertList);
    // Add any meta fields
    String guid = UUID.randomUUID().toString();
    metaSource.put(GUID, guid);
    metaSource.put(Constants.Fields.TIMESTAMP.getName(), System.currentTimeMillis());
    metaSource.put(GROUPS_FIELD, groups);
    metaSource.put(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
    return new Document(metaSource, guid, METAALERT_TYPE, System.currentTimeMillis());
}

Example 23 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class ElasticsearchMetaAlertDao method patch.

/**
 * Does not allow patches on the "alerts" or "status" fields.  These fields must be updated with their
 * dedicated methods.
 *
 * @param request The patch request
 * @param timestamp Optionally a timestamp to set. If not specified then current time is used.
 * @throws OriginalNotFoundException
 * @throws IOException
 */
@Override
public void patch(PatchRequest request, Optional<Long> timestamp) throws OriginalNotFoundException, IOException {
    if (isPatchAllowed(request)) {
        Document d = getPatchedDocument(request, timestamp);
        indexDao.update(d, Optional.ofNullable(request.getIndex()));
    } else {
        throw new IllegalArgumentException("Meta alert patches are not allowed for /alert or /status paths.  " + "Please use the add/remove alert or update status functions instead.");
    }
}

Example 24 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class ElasticsearchMetaAlertDao method createMetaAlert.

@Override
@SuppressWarnings("unchecked")
public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) throws InvalidCreateException, IOException {
    List<GetRequest> alertRequests = request.getAlerts();
    if (request.getAlerts().isEmpty()) {
        throw new InvalidCreateException("MetaAlertCreateRequest must contain alerts");
    }
    if (request.getGroups().isEmpty()) {
        throw new InvalidCreateException("MetaAlertCreateRequest must contain UI groups");
    }
    // Retrieve the documents going into the meta alert and build it
    Iterable<Document> alerts = indexDao.getAllLatest(alertRequests);
    Document metaAlert = buildCreateDocument(alerts, request.getGroups());
    calculateMetaScores(metaAlert);
    // Add source type to be consistent with other sources and allow filtering
    metaAlert.getDocument().put(SOURCE_TYPE, MetaAlertDao.METAALERT_TYPE);
    // Start a list of updates / inserts we need to run
    Map<Document, Optional<String>> updates = new HashMap<>();
    updates.put(metaAlert, Optional.of(MetaAlertDao.METAALERTS_INDEX));
    try {
        // We need to update the associated alerts with the new meta alerts, making sure existing
        // links are maintained.
        Map<String, Optional<String>> guidToIndices = alertRequests.stream().collect(Collectors.toMap(GetRequest::getGuid, GetRequest::getIndex));
        Map<String, String> guidToSensorTypes = alertRequests.stream().collect(Collectors.toMap(GetRequest::getGuid, GetRequest::getSensorType));
        for (Document alert : alerts) {
            if (addMetaAlertToAlert(metaAlert.getGuid(), alert)) {
                // Use the index in the request if it exists
                Optional<String> index = guidToIndices.get(alert.getGuid());
                if (!index.isPresent()) {
                    // Look up the index from Elasticsearch if one is not supplied in the request
                    index = elasticsearchDao.getIndexName(alert.getGuid(), guidToSensorTypes.get(alert.getGuid()));
                    if (!index.isPresent()) {
                        throw new IllegalArgumentException("Could not find index for " + alert.getGuid());
                    }
                }
                updates.put(alert, index);
            }
        }
        // Kick off any updates.
        indexDaoUpdate(updates);
        MetaAlertCreateResponse createResponse = new MetaAlertCreateResponse();
        createResponse.setCreated(true);
        createResponse.setGuid(metaAlert.getGuid());
        return createResponse;
    } catch (IOException ioe) {
        throw new InvalidCreateException("Unable to create meta alert", ioe);
    }
}

Example 25 with Document

use of org.apache.metron.indexing.dao.update.Document in project metron by apache.

the class ElasticsearchMetaAlertDaoTest method testInvalidInit.

@Test(expected = IllegalArgumentException.class)
public void testInvalidInit() {
    IndexDao dao = new IndexDao() {

        @Override
        public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException {
            return null;
        }

        @Override
        public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException {
            return null;
        }

        @Override
        public void init(AccessConfig config) {
        }

        @Override
        public Document getLatest(String guid, String sensorType) throws IOException {
            return null;
        }

        @Override
        public Iterable<Document> getAllLatest(List<GetRequest> getRequests) throws IOException {
            return null;
        }

        @Override
        public void update(Document update, Optional<String> index) throws IOException {
        }

        @Override
        public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
        }

        @Override
        public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException {
            return null;
        }
    };
    ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao();
    metaAlertDao.init(dao);
}