Examples with SortField org.apache.metron.indexing.dao.search.SortField used on opensource projects

Example 1 with SortField

use of org.apache.metron.indexing.dao.search.SortField in project metron by apache.

the class ElasticsearchDao method buildSearchRequest.

/**
 * Builds an Elasticsearch search request.
 * @param searchRequest The Metron search request.
 * @param queryBuilder
 * @return An Elasticsearch search request.
 */
private org.elasticsearch.action.search.SearchRequest buildSearchRequest(SearchRequest searchRequest, QueryBuilder queryBuilder) throws InvalidSearchException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Got search request; request={}", ElasticsearchUtils.toJSON(searchRequest).orElse("???"));
    }
    SearchSourceBuilder searchBuilder = new SearchSourceBuilder().size(searchRequest.getSize()).from(searchRequest.getFrom()).query(queryBuilder).trackScores(true);
    List<String> fields = searchRequest.getFields();
    // column metadata needed to understand the type of each sort field
    Map<String, FieldType> meta;
    try {
        meta = getColumnMetadata(searchRequest.getIndices());
    } catch (IOException e) {
        throw new InvalidSearchException("Unable to get column metadata", e);
    }
    // handle sort fields
    for (SortField sortField : searchRequest.getSort()) {
        // what type is the sort field?
        FieldType sortFieldType = meta.getOrDefault(sortField.getField(), FieldType.OTHER);
        // sort order - if ascending missing values sorted last. otherwise, missing values sorted first
        org.elasticsearch.search.sort.SortOrder sortOrder = getElasticsearchSortOrder(sortField.getSortOrder());
        String missingSortOrder;
        if (sortOrder == org.elasticsearch.search.sort.SortOrder.DESC) {
            missingSortOrder = SORT_MISSING_LAST;
        } else {
            missingSortOrder = SORT_MISSING_FIRST;
        }
        // sort by the field - missing fields always last
        FieldSortBuilder sortBy = new FieldSortBuilder(sortField.getField()).order(sortOrder).missing(missingSortOrder).unmappedType(sortFieldType.getFieldType());
        searchBuilder.sort(sortBy);
    }
    // handle search fields
    if (fields != null) {
        searchBuilder.fetchSource("*", null);
    } else {
        searchBuilder.fetchSource(true);
    }
    List<String> facetFields = searchRequest.getFacetFields();
    // handle facet fields
    if (facetFields != null) {
        // https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/_bucket_aggregations.html
        for (String field : facetFields) {
            String name = getFacetAggregationName(field);
            TermsAggregationBuilder terms = AggregationBuilders.terms(name).field(field);
            // new TermsBuilder(name).field(field);
            searchBuilder.aggregation(terms);
        }
    }
    // return the search request
    String[] indices = wildcardIndices(searchRequest.getIndices());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Built Elasticsearch request; indices={}, request={}", indices, searchBuilder.toString());
    }
    return new org.elasticsearch.action.search.SearchRequest().indices(indices).source(searchBuilder);
}

Example 2 with SortField

use of org.apache.metron.indexing.dao.search.SortField in project metron by apache.

the class ElasticsearchDaoTest method sortBy.

private SortField sortBy(String field, SortOrder order) {
    SortField sortField = new SortField();
    sortField.setField(field);
    sortField.setSortOrder(order.toString());
    return sortField;
}

Example 3 with SortField

use of org.apache.metron.indexing.dao.search.SortField in project metron by apache.

the class ElasticsearchMetaAlertIntegrationTest method shouldSearchByNestedAlert.

@Test
public void shouldSearchByNestedAlert() throws Exception {
    // Load alerts
    List<Map<String, Object>> alerts = buildAlerts(4);
    alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
    alerts.get(0).put("ip_src_addr", "192.168.1.1");
    alerts.get(0).put("ip_src_port", 8010);
    alerts.get(1).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
    alerts.get(1).put("ip_src_addr", "192.168.1.2");
    alerts.get(1).put("ip_src_port", 8009);
    alerts.get(2).put("ip_src_addr", "192.168.1.3");
    alerts.get(2).put("ip_src_port", 8008);
    alerts.get(3).put("ip_src_addr", "192.168.1.4");
    alerts.get(3).put("ip_src_port", 8007);
    elasticsearchAdd(alerts, INDEX, SENSOR_NAME);
    // Put the nested type into the test index, so that it'll match appropriately
    ((ElasticsearchDao) esDao).getClient().admin().indices().preparePutMapping(INDEX).setType("test_doc").setSource(nestedAlertMapping).get();
    // Load metaAlerts
    Map<String, Object> activeMetaAlert = buildMetaAlert("meta_active", MetaAlertStatus.ACTIVE, Optional.of(Arrays.asList(alerts.get(0), alerts.get(1))));
    Map<String, Object> inactiveMetaAlert = buildMetaAlert("meta_inactive", MetaAlertStatus.INACTIVE, Optional.of(Arrays.asList(alerts.get(2), alerts.get(3))));
    // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
    elasticsearchAdd(Arrays.asList(activeMetaAlert, inactiveMetaAlert), METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE);
    // Verify load was successful
    findCreatedDocs(Arrays.asList(new GetRequest("message_0", SENSOR_NAME), new GetRequest("message_1", SENSOR_NAME), new GetRequest("message_2", SENSOR_NAME), new GetRequest("message_3", SENSOR_NAME), new GetRequest("meta_active", METAALERT_TYPE), new GetRequest("meta_inactive", METAALERT_TYPE)));
    SearchResponse searchResponse = metaDao.search(new SearchRequest() {

        {
            setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8009) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8009)");
            setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE));
            setFrom(0);
            setSize(5);
            setSort(Collections.singletonList(new SortField() {

                {
                    setField(Constants.GUID);
                }
            }));
        }
    });
    // Should not have results because nested alerts shouldn't be flattened
    Assert.assertEquals(0, searchResponse.getTotal());
    // Query against all indices. Only the single active meta alert should be returned.
    // The child alerts should be hidden.
    searchResponse = metaDao.search(new SearchRequest() {

        {
            setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8010)" + " OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8010)");
            setIndices(Collections.singletonList("*"));
            setFrom(0);
            setSize(5);
            setSort(Collections.singletonList(new SortField() {

                {
                    setField(Constants.GUID);
                }
            }));
        }
    });
    // Nested query should match a nested alert
    Assert.assertEquals(1, searchResponse.getTotal());
    Assert.assertEquals("meta_active", searchResponse.getResults().get(0).getSource().get("guid"));
    // Query against all indices. The child alert has no actual attached meta alerts, and should
    // be returned on its own.
    searchResponse = metaDao.search(new SearchRequest() {

        {
            setQuery("(ip_src_addr:192.168.1.3 AND ip_src_port:8008)" + " OR (alert.ip_src_addr:192.168.1.3 AND alert.ip_src_port:8008)");
            setIndices(Collections.singletonList("*"));
            setFrom(0);
            setSize(1);
            setSort(Collections.singletonList(new SortField() {

                {
                    setField(Constants.GUID);
                }
            }));
        }
    });
    // Nested query should match a plain alert
    Assert.assertEquals(1, searchResponse.getTotal());
    Assert.assertEquals("message_2", searchResponse.getResults().get(0).getSource().get("guid"));
}

Example 4 with SortField

use of org.apache.metron.indexing.dao.search.SortField in project metron by apache.

the class MetaAlertIntegrationTest method shouldSearchByStatus.

@Test
public void shouldSearchByStatus() throws Exception {
    // Load alert
    List<Map<String, Object>> alerts = buildAlerts(1);
    alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
    alerts.get(0).put("ip_src_addr", "192.168.1.1");
    alerts.get(0).put("ip_src_port", 8010);
    // Load metaAlerts
    Map<String, Object> activeMetaAlert = buildMetaAlert("meta_active", MetaAlertStatus.ACTIVE, Optional.of(Collections.singletonList(alerts.get(0))));
    Map<String, Object> inactiveMetaAlert = buildMetaAlert("meta_inactive", MetaAlertStatus.INACTIVE, Optional.empty());
    // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
    addRecords(Arrays.asList(activeMetaAlert, inactiveMetaAlert), getMetaAlertIndex(), METAALERT_TYPE);
    // Verify load was successful
    findCreatedDocs(Arrays.asList(new GetRequest("meta_active", METAALERT_TYPE), new GetRequest("meta_inactive", METAALERT_TYPE)));
    SearchResponse searchResponse = metaDao.search(new SearchRequest() {

        {
            setQuery("*:*");
            setIndices(Collections.singletonList(METAALERT_TYPE));
            setFrom(0);
            setSize(5);
            setSort(Collections.singletonList(new SortField() {

                {
                    setField(Constants.GUID);
                }
            }));
        }
    });
    // Verify only active meta alerts are returned
    assertEquals(1, searchResponse.getTotal());
    assertEquals(MetaAlertStatus.ACTIVE.getStatusString(), searchResponse.getResults().get(0).getSource().get(STATUS_FIELD));
}

Example 5 with SortField

use of org.apache.metron.indexing.dao.search.SortField in project metron by apache.

the class MetaAlertIntegrationTest method shouldSortMetaAlertsByAlertStatus.

@Test
public void shouldSortMetaAlertsByAlertStatus() throws Exception {
    final String guid = "meta_alert";
    setupTypings();
    // should be able to sort meta-alert search results by 'alert_status'
    SortField sortField = new SortField();
    sortField.setField("alert_status");
    sortField.setSortOrder("asc");
    // when no meta-alerts exist, it should work
    assertEquals(0, searchForSortedMetaAlerts(sortField).getTotal());
    // when meta-alert just created, it should work
    createMetaAlert(guid);
    assertEquals(1, searchForSortedMetaAlerts(sortField).getTotal());
    // when meta-alert 'esclated', it should work
    escalateMetaAlert(guid);
    assertEquals(1, searchForSortedMetaAlerts(sortField).getTotal());
}