Examples with SearchResponse org.apache.metron.indexing.dao.search.SearchResponse used on opensource projects

Example 1 with SearchResponse

use of org.apache.metron.indexing.dao.search.SearchResponse in project metron by apache.

the class ElasticsearchDao method buildSearchResponse.

/**
 * Builds a search response.
 *
 * This effectively transforms an Elasticsearch search response into a Metron search response.
 *
 * @param searchRequest The Metron search request.
 * @param esResponse The Elasticsearch search response.
 * @return A Metron search response.
 * @throws InvalidSearchException
 */
private SearchResponse buildSearchResponse(SearchRequest searchRequest, org.elasticsearch.action.search.SearchResponse esResponse) throws InvalidSearchException {
    SearchResponse searchResponse = new SearchResponse();
    searchResponse.setTotal(esResponse.getHits().getTotalHits());
    // search hits --> search results
    List<SearchResult> results = new ArrayList<>();
    for (SearchHit hit : esResponse.getHits().getHits()) {
        results.add(getSearchResult(hit, searchRequest.getFields()));
    }
    searchResponse.setResults(results);
    // handle facet fields
    if (searchRequest.getFacetFields() != null) {
        List<String> facetFields = searchRequest.getFacetFields();
        Map<String, FieldType> commonColumnMetadata;
        try {
            commonColumnMetadata = getColumnMetadata(searchRequest.getIndices());
        } catch (IOException e) {
            throw new InvalidSearchException(String.format("Could not get common column metadata for indices %s", Arrays.toString(searchRequest.getIndices().toArray())));
        }
        searchResponse.setFacetCounts(getFacetCounts(facetFields, esResponse.getAggregations(), commonColumnMetadata));
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Built search response; response={}", ElasticsearchUtils.toJSON(searchResponse).orElse("???"));
    }
    return searchResponse;
}

Example 2 with SearchResponse

use of org.apache.metron.indexing.dao.search.SearchResponse in project metron by apache.

the class ElasticsearchMetaAlertIntegrationTest method shouldSearchByNestedAlert.

@Test
public void shouldSearchByNestedAlert() throws Exception {
    // Load alerts
    List<Map<String, Object>> alerts = buildAlerts(4);
    alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
    alerts.get(0).put("ip_src_addr", "192.168.1.1");
    alerts.get(0).put("ip_src_port", 8010);
    alerts.get(1).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
    alerts.get(1).put("ip_src_addr", "192.168.1.2");
    alerts.get(1).put("ip_src_port", 8009);
    alerts.get(2).put("ip_src_addr", "192.168.1.3");
    alerts.get(2).put("ip_src_port", 8008);
    alerts.get(3).put("ip_src_addr", "192.168.1.4");
    alerts.get(3).put("ip_src_port", 8007);
    elasticsearchAdd(alerts, INDEX, SENSOR_NAME);
    // Put the nested type into the test index, so that it'll match appropriately
    ((ElasticsearchDao) esDao).getClient().admin().indices().preparePutMapping(INDEX).setType("test_doc").setSource(nestedAlertMapping).get();
    // Load metaAlerts
    Map<String, Object> activeMetaAlert = buildMetaAlert("meta_active", MetaAlertStatus.ACTIVE, Optional.of(Arrays.asList(alerts.get(0), alerts.get(1))));
    Map<String, Object> inactiveMetaAlert = buildMetaAlert("meta_inactive", MetaAlertStatus.INACTIVE, Optional.of(Arrays.asList(alerts.get(2), alerts.get(3))));
    // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
    elasticsearchAdd(Arrays.asList(activeMetaAlert, inactiveMetaAlert), METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE);
    // Verify load was successful
    findCreatedDocs(Arrays.asList(new GetRequest("message_0", SENSOR_NAME), new GetRequest("message_1", SENSOR_NAME), new GetRequest("message_2", SENSOR_NAME), new GetRequest("message_3", SENSOR_NAME), new GetRequest("meta_active", METAALERT_TYPE), new GetRequest("meta_inactive", METAALERT_TYPE)));
    SearchResponse searchResponse = metaDao.search(new SearchRequest() {

        {
            setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8009) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8009)");
            setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE));
            setFrom(0);
            setSize(5);
            setSort(Collections.singletonList(new SortField() {

                {
                    setField(Constants.GUID);
                }
            }));
        }
    });
    // Should not have results because nested alerts shouldn't be flattened
    Assert.assertEquals(0, searchResponse.getTotal());
    // Query against all indices. Only the single active meta alert should be returned.
    // The child alerts should be hidden.
    searchResponse = metaDao.search(new SearchRequest() {

        {
            setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8010)" + " OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8010)");
            setIndices(Collections.singletonList("*"));
            setFrom(0);
            setSize(5);
            setSort(Collections.singletonList(new SortField() {

                {
                    setField(Constants.GUID);
                }
            }));
        }
    });
    // Nested query should match a nested alert
    Assert.assertEquals(1, searchResponse.getTotal());
    Assert.assertEquals("meta_active", searchResponse.getResults().get(0).getSource().get("guid"));
    // Query against all indices. The child alert has no actual attached meta alerts, and should
    // be returned on its own.
    searchResponse = metaDao.search(new SearchRequest() {

        {
            setQuery("(ip_src_addr:192.168.1.3 AND ip_src_port:8008)" + " OR (alert.ip_src_addr:192.168.1.3 AND alert.ip_src_port:8008)");
            setIndices(Collections.singletonList("*"));
            setFrom(0);
            setSize(1);
            setSort(Collections.singletonList(new SortField() {

                {
                    setField(Constants.GUID);
                }
            }));
        }
    });
    // Nested query should match a plain alert
    Assert.assertEquals(1, searchResponse.getTotal());
    Assert.assertEquals("message_2", searchResponse.getResults().get(0).getSource().get("guid"));
}

Example 3 with SearchResponse

use of org.apache.metron.indexing.dao.search.SearchResponse in project metron by apache.

the class ElasticsearchMetaAlertIntegrationTest method shouldGetAllMetaAlertsForAlert.

@Test
public void shouldGetAllMetaAlertsForAlert() throws Exception {
    // Load alerts
    List<Map<String, Object>> alerts = buildAlerts(3);
    elasticsearchAdd(alerts, INDEX, SENSOR_NAME);
    // Load metaAlerts
    List<Map<String, Object>> metaAlerts = buildMetaAlerts(12, MetaAlertStatus.ACTIVE, Optional.of(Collections.singletonList(alerts.get(0))));
    metaAlerts.add(buildMetaAlert("meta_active_12", MetaAlertStatus.ACTIVE, Optional.of(Arrays.asList(alerts.get(0), alerts.get(2)))));
    metaAlerts.add(buildMetaAlert("meta_inactive", MetaAlertStatus.INACTIVE, Optional.of(Arrays.asList(alerts.get(0), alerts.get(2)))));
    // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
    elasticsearchAdd(metaAlerts, METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE);
    // Verify load was successful
    List<GetRequest> createdDocs = metaAlerts.stream().map(metaAlert -> new GetRequest((String) metaAlert.get(Constants.GUID), METAALERT_TYPE)).collect(Collectors.toList());
    createdDocs.addAll(alerts.stream().map(alert -> new GetRequest((String) alert.get(Constants.GUID), SENSOR_NAME)).collect(Collectors.toList()));
    findCreatedDocs(createdDocs);
    int previousPageSize = ((ElasticsearchMetaAlertDao) metaDao).getPageSize();
    ((ElasticsearchMetaAlertDao) metaDao).setPageSize(5);
    {
        // Verify searches successfully return more than 10 results
        SearchResponse searchResponse0 = metaDao.getAllMetaAlertsForAlert("message_0");
        List<SearchResult> searchResults0 = searchResponse0.getResults();
        Assert.assertEquals(13, searchResults0.size());
        Set<Map<String, Object>> resultSet = new HashSet<>();
        Iterables.addAll(resultSet, Iterables.transform(searchResults0, r -> r.getSource()));
        StringBuffer reason = new StringBuffer("Unable to find " + metaAlerts.get(0) + "\n");
        reason.append(Joiner.on("\n").join(resultSet));
        Assert.assertTrue(reason.toString(), resultSet.contains(metaAlerts.get(0)));
        // Verify no meta alerts are returned because message_1 was not added to any
        SearchResponse searchResponse1 = metaDao.getAllMetaAlertsForAlert("message_1");
        List<SearchResult> searchResults1 = searchResponse1.getResults();
        Assert.assertEquals(0, searchResults1.size());
        // Verify only the meta alert message_2 was added to is returned
        SearchResponse searchResponse2 = metaDao.getAllMetaAlertsForAlert("message_2");
        List<SearchResult> searchResults2 = searchResponse2.getResults();
        Assert.assertEquals(1, searchResults2.size());
        Assert.assertEquals(metaAlerts.get(12), searchResults2.get(0).getSource());
    }
    ((ElasticsearchMetaAlertDao) metaDao).setPageSize(previousPageSize);
}

Example 4 with SearchResponse

use of org.apache.metron.indexing.dao.search.SearchResponse in project metron by apache.

the class SearchIntegrationTest method sort_descending_with_missing_fields.

@Test
public void sort_descending_with_missing_fields() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    Assert.assertEquals(10, results.size());
    // validate sorted order - there are only 2 with a 'threat:triage:score'
    Assert.assertEquals("20", results.get(0).getSource().get("threat:triage:score"));
    Assert.assertEquals("10", results.get(1).getSource().get("threat:triage:score"));
    // the remaining are missing the 'threat:triage:score' and should be sorted last
    for (int i = 2; i < 10; i++) {
        Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score"));
    }
}

Example 5 with SearchResponse

use of org.apache.metron.indexing.dao.search.SearchResponse in project metron by apache.

the class SearchIntegrationTest method no_results_returned_when_query_does_not_match.

@Test
public void no_results_returned_when_query_does_not_match() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(0, response.getTotal());
}