use of org.apache.metron.indexing.dao.search.SearchRequest in project metron by apache.
the class SearchServiceImplTest method saveShouldWrapExceptionInRestException.
@Test
public void saveShouldWrapExceptionInRestException() throws Exception {
exception.expect(RestException.class);
when(dao.search(any(SearchRequest.class))).thenThrow(InvalidSearchException.class);
SearchRequest searchRequest = new SearchRequest();
searchRequest.setIndices(Arrays.asList("bro"));
searchRequest.setFacetFields(Arrays.asList("ip_src_addr"));
searchService.search(searchRequest);
}
use of org.apache.metron.indexing.dao.search.SearchRequest in project metron by apache.
the class SearchServiceImplTest method searchShouldProperlySearchDefaultIndices.
@Test
public void searchShouldProperlySearchDefaultIndices() throws Exception {
when(environment.getProperty(INDEX_WRITER_NAME)).thenReturn("elasticsearch");
when(sensorIndexingConfigService.getAllIndices("elasticsearch")).thenReturn(Arrays.asList("bro", "snort", "error"));
SearchRequest searchRequest = new SearchRequest();
searchService.search(searchRequest);
SearchRequest expectedSearchRequest = new SearchRequest();
expectedSearchRequest.setIndices(Arrays.asList("bro", "snort", "metaalert"));
verify(dao).search(eq(expectedSearchRequest));
verifyNoMoreInteractions(dao);
}
use of org.apache.metron.indexing.dao.search.SearchRequest in project metron by apache.
the class ElasticsearchDao method buildSearchRequest.
/**
* Builds an Elasticsearch search request.
* @param searchRequest The Metron search request.
* @param queryBuilder
* @return An Elasticsearch search request.
*/
private org.elasticsearch.action.search.SearchRequest buildSearchRequest(SearchRequest searchRequest, QueryBuilder queryBuilder) throws InvalidSearchException {
if (LOG.isDebugEnabled()) {
LOG.debug("Got search request; request={}", ElasticsearchUtils.toJSON(searchRequest).orElse("???"));
}
SearchSourceBuilder searchBuilder = new SearchSourceBuilder().size(searchRequest.getSize()).from(searchRequest.getFrom()).query(queryBuilder).trackScores(true);
List<String> fields = searchRequest.getFields();
// column metadata needed to understand the type of each sort field
Map<String, FieldType> meta;
try {
meta = getColumnMetadata(searchRequest.getIndices());
} catch (IOException e) {
throw new InvalidSearchException("Unable to get column metadata", e);
}
// handle sort fields
for (SortField sortField : searchRequest.getSort()) {
// what type is the sort field?
FieldType sortFieldType = meta.getOrDefault(sortField.getField(), FieldType.OTHER);
// sort order - if ascending missing values sorted last. otherwise, missing values sorted first
org.elasticsearch.search.sort.SortOrder sortOrder = getElasticsearchSortOrder(sortField.getSortOrder());
String missingSortOrder;
if (sortOrder == org.elasticsearch.search.sort.SortOrder.DESC) {
missingSortOrder = SORT_MISSING_LAST;
} else {
missingSortOrder = SORT_MISSING_FIRST;
}
// sort by the field - missing fields always last
FieldSortBuilder sortBy = new FieldSortBuilder(sortField.getField()).order(sortOrder).missing(missingSortOrder).unmappedType(sortFieldType.getFieldType());
searchBuilder.sort(sortBy);
}
// handle search fields
if (fields != null) {
searchBuilder.fetchSource("*", null);
} else {
searchBuilder.fetchSource(true);
}
List<String> facetFields = searchRequest.getFacetFields();
// handle facet fields
if (facetFields != null) {
// https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/_bucket_aggregations.html
for (String field : facetFields) {
String name = getFacetAggregationName(field);
TermsAggregationBuilder terms = AggregationBuilders.terms(name).field(field);
// new TermsBuilder(name).field(field);
searchBuilder.aggregation(terms);
}
}
// return the search request
String[] indices = wildcardIndices(searchRequest.getIndices());
if (LOG.isDebugEnabled()) {
LOG.debug("Built Elasticsearch request; indices={}, request={}", indices, searchBuilder.toString());
}
return new org.elasticsearch.action.search.SearchRequest().indices(indices).source(searchBuilder);
}
use of org.apache.metron.indexing.dao.search.SearchRequest in project metron by apache.
the class ElasticsearchDaoTest method searchShouldWildcardIndices.
@Test
public void searchShouldWildcardIndices() throws Exception {
// setup the dao
setup(RestStatus.OK, 25);
// "sort by" fields for the search request
SortField[] expectedSortFields = { sortBy("sortByStringDesc", SortOrder.DESC), sortBy("sortByIntAsc", SortOrder.ASC), sortBy("sortByUndefinedDesc", SortOrder.DESC) };
// create a metron search request
final List<String> indices = Arrays.asList("bro", "snort");
SearchRequest searchRequest = new SearchRequest();
searchRequest.setSize(2);
searchRequest.setIndices(indices);
searchRequest.setFrom(5);
searchRequest.setSort(Arrays.asList(expectedSortFields));
searchRequest.setQuery("some query");
// submit the metron search request
SearchResponse searchResponse = dao.search(searchRequest);
assertNotNull(searchResponse);
// capture the elasticsearch search request that was created
ArgumentCaptor<org.elasticsearch.action.search.SearchRequest> argument = ArgumentCaptor.forClass(org.elasticsearch.action.search.SearchRequest.class);
verify(requestSubmitter).submitSearch(argument.capture());
org.elasticsearch.action.search.SearchRequest request = argument.getValue();
// transform the request to JSON for validation
JSONParser parser = new JSONParser();
JSONObject json = (JSONObject) parser.parse(ElasticsearchUtils.toJSON(request).orElse("???"));
// ensure that the index names are 'wildcard-ed'
String[] expected = { "bro_index*", "snort_index*" };
assertArrayEquals(expected, request.indices());
}
use of org.apache.metron.indexing.dao.search.SearchRequest in project metron by apache.
the class ElasticsearchMetaAlertIntegrationTest method shouldSearchByNestedAlert.
@Test
public void shouldSearchByNestedAlert() throws Exception {
// Load alerts
List<Map<String, Object>> alerts = buildAlerts(4);
alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
alerts.get(0).put("ip_src_addr", "192.168.1.1");
alerts.get(0).put("ip_src_port", 8010);
alerts.get(1).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
alerts.get(1).put("ip_src_addr", "192.168.1.2");
alerts.get(1).put("ip_src_port", 8009);
alerts.get(2).put("ip_src_addr", "192.168.1.3");
alerts.get(2).put("ip_src_port", 8008);
alerts.get(3).put("ip_src_addr", "192.168.1.4");
alerts.get(3).put("ip_src_port", 8007);
elasticsearchAdd(alerts, INDEX, SENSOR_NAME);
// Put the nested type into the test index, so that it'll match appropriately
((ElasticsearchDao) esDao).getClient().admin().indices().preparePutMapping(INDEX).setType("test_doc").setSource(nestedAlertMapping).get();
// Load metaAlerts
Map<String, Object> activeMetaAlert = buildMetaAlert("meta_active", MetaAlertStatus.ACTIVE, Optional.of(Arrays.asList(alerts.get(0), alerts.get(1))));
Map<String, Object> inactiveMetaAlert = buildMetaAlert("meta_inactive", MetaAlertStatus.INACTIVE, Optional.of(Arrays.asList(alerts.get(2), alerts.get(3))));
// We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
elasticsearchAdd(Arrays.asList(activeMetaAlert, inactiveMetaAlert), METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE);
// Verify load was successful
findCreatedDocs(Arrays.asList(new GetRequest("message_0", SENSOR_NAME), new GetRequest("message_1", SENSOR_NAME), new GetRequest("message_2", SENSOR_NAME), new GetRequest("message_3", SENSOR_NAME), new GetRequest("meta_active", METAALERT_TYPE), new GetRequest("meta_inactive", METAALERT_TYPE)));
SearchResponse searchResponse = metaDao.search(new SearchRequest() {
{
setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8009) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8009)");
setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE));
setFrom(0);
setSize(5);
setSort(Collections.singletonList(new SortField() {
{
setField(Constants.GUID);
}
}));
}
});
// Should not have results because nested alerts shouldn't be flattened
Assert.assertEquals(0, searchResponse.getTotal());
// Query against all indices. Only the single active meta alert should be returned.
// The child alerts should be hidden.
searchResponse = metaDao.search(new SearchRequest() {
{
setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8010)" + " OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8010)");
setIndices(Collections.singletonList("*"));
setFrom(0);
setSize(5);
setSort(Collections.singletonList(new SortField() {
{
setField(Constants.GUID);
}
}));
}
});
// Nested query should match a nested alert
Assert.assertEquals(1, searchResponse.getTotal());
Assert.assertEquals("meta_active", searchResponse.getResults().get(0).getSource().get("guid"));
// Query against all indices. The child alert has no actual attached meta alerts, and should
// be returned on its own.
searchResponse = metaDao.search(new SearchRequest() {
{
setQuery("(ip_src_addr:192.168.1.3 AND ip_src_port:8008)" + " OR (alert.ip_src_addr:192.168.1.3 AND alert.ip_src_port:8008)");
setIndices(Collections.singletonList("*"));
setFrom(0);
setSize(1);
setSort(Collections.singletonList(new SortField() {
{
setField(Constants.GUID);
}
}));
}
});
// Nested query should match a plain alert
Assert.assertEquals(1, searchResponse.getTotal());
Assert.assertEquals("message_2", searchResponse.getResults().get(0).getSource().get("guid"));
}
Aggregations