use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.AggregationBuilder in project uavstack by uavorg.
the class ThreadAnalysisQueryHandler method queryDistinct.
/**
* 官网上的 【Top Hits Aggregation】 JAVA API 运行报错,
*
* @see https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/_metrics_aggregations.html
*
* <pre>
*
* {@code
* AggregationBuilder aggregation =
* AggregationBuilders
* .terms("agg").field("gender")
* .subAggregation(
* AggregationBuilders.topHits("top")
* .explain(true)
* .size(1)
* .from(10)
* );
* }
* </pre>
*
* Caused by: java.lang.IllegalArgumentException: An SPI class of type org.apache.lucene.codecs.PostingsFormat
* with name 'Lucene50' does not exist. You need to add the corresponding JAR file supporting this SPI to your
* classpath. The current classpath supports the following names: [completion, completion090]
* <p>
* 以下满足张真要求的ES的查询可以工作,但找不到对应的JAVA API(原因在上)??!!!
*
* <pre>
*
* {@code
* {
* "aggs": {
* "time": {
* "terms": {
* "field": "time",
* "order":{"_term":"desc"},
* "size": 1000
* },
* "aggs": {
* "example": {
* "top_hits": {
* "sort": [
* {
* "percpu": {
* "order": "desc"
* }
* }
* ],
* "size": 1
* }
* }
* }
* }
* },
* "from": 0,
* "size": 0
*
* }
* }
* </pre>
*
* @param data
*/
private void queryDistinct(UAVHttpMessage data) {
try {
String ipport = data.getRequest("ipport");
// TODO ES aggregation 默认最多查10条, 这里暂时改到1000,待refine
AggregationBuilder agg = AggregationBuilders.terms("unique_time").field("time").size(1000).order(Terms.Order.term(false)).subAggregation(AggregationBuilders.terms("unique_user").field("user").size(1000));
String date = data.getRequest("indexdate");
String currentIndex;
if (date != null) {
// 指定index
currentIndex = this.indexMgr.getIndexByDate(date);
} else {
// current index
currentIndex = this.indexMgr.getCurrentIndex();
}
SearchResponse sResponse = client.getClient().prepareSearch(currentIndex).setTypes(ThreadAnalysisIndexMgr.JTA_TABLE).setQuery(QueryBuilders.boolQuery().must(QueryBuilders.termQuery("ipport", ipport))).setSize(0).addAggregation(agg).execute().actionGet();
// sr is here your SearchResponse object
Terms aggs = sResponse.getAggregations().get("unique_time");
List<Map<String, Object>> records = new ArrayList<Map<String, Object>>();
// For each entry
for (Terms.Bucket entry : aggs.getBuckets()) {
// bucket key
String key = entry.getKey().toString();
// Doc count
long docCount = entry.getDocCount();
Map<String, Object> record = new HashMap<String, Object>();
record.put("time", key);
record.put("threadcount", docCount);
Terms userAggs = entry.getAggregations().get("unique_user");
List<Terms.Bucket> users = userAggs.getBuckets();
if (!users.isEmpty()) {
record.put("user", users.get(0).getKey().toString());
}
records.add(record);
}
data.putResponse("rs", JSONHelper.toString(records));
// 返回总的条数
data.putResponse("count", aggs.getBuckets().size() + "");
} catch (Exception e) {
if (e.getMessage().indexOf("no such index") >= 0) {
data.putResponse("rs", "NO_INDEX");
} else {
data.putResponse("rs", "ERR");
log.err(this, "query distinct FAILED. " + JSONHelper.toString(data), e);
}
}
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.AggregationBuilder in project incubator-sdap-mudrod by apache.
the class CrawlerDetection method checkByRate.
private int checkByRate(ESDriver es, String user) {
int rate = Integer.parseInt(props.getProperty(MudrodConstants.REQUEST_RATE));
Pattern pattern = Pattern.compile("get (.*?) http/*");
Matcher matcher;
BoolQueryBuilder filterSearch = new BoolQueryBuilder();
filterSearch.must(QueryBuilders.termQuery("IP", user));
AggregationBuilder aggregation = AggregationBuilders.dateHistogram("by_minute").field("Time").dateHistogramInterval(DateHistogramInterval.MINUTE).order(Order.COUNT_DESC);
SearchResponse checkRobot = es.getClient().prepareSearch(logIndex).setTypes(httpType, ftpType).setQuery(filterSearch).setSize(0).addAggregation(aggregation).execute().actionGet();
Histogram agg = checkRobot.getAggregations().get("by_minute");
List<? extends Histogram.Bucket> botList = agg.getBuckets();
long maxCount = botList.get(0).getDocCount();
if (maxCount >= rate) {
return 0;
} else {
DateTime dt1 = null;
int toLast = 0;
SearchResponse scrollResp = es.getClient().prepareSearch(logIndex).setTypes(httpType, ftpType).setScroll(new TimeValue(60000)).setQuery(filterSearch).setSize(100).execute().actionGet();
while (true) {
for (SearchHit hit : scrollResp.getHits().getHits()) {
Map<String, Object> result = hit.getSource();
String logtype = (String) result.get("LogType");
if (logtype.equals(MudrodConstants.HTTP_LOG)) {
String request = (String) result.get("Request");
matcher = pattern.matcher(request.trim().toLowerCase());
boolean find = false;
while (matcher.find()) {
request = matcher.group(1);
result.put("RequestUrl", props.getProperty(MudrodConstants.BASE_URL) + request);
find = true;
}
if (!find) {
result.put("RequestUrl", request);
}
} else {
result.put("RequestUrl", result.get("Request"));
}
DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
DateTime dt2 = fmt.parseDateTime((String) result.get("Time"));
if (dt1 == null) {
toLast = 0;
} else {
toLast = Math.abs(Seconds.secondsBetween(dt1, dt2).getSeconds());
}
result.put("ToLast", toLast);
IndexRequest ir = new IndexRequest(logIndex, cleanupType).source(result);
es.getBulkProcessor().add(ir);
dt1 = dt2;
}
scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
if (scrollResp.getHits().getHits().length == 0) {
break;
}
}
}
return 1;
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.AggregationBuilder in project vertexium by visallo.
the class ElasticsearchSearchQueryBase method getElasticsearchGeohashAggregations.
protected List<AggregationBuilder> getElasticsearchGeohashAggregations(GeohashAggregation agg) {
List<AggregationBuilder> aggs = new ArrayList<>();
PropertyDefinition propertyDefinition = getPropertyDefinition(agg.getFieldName());
if (propertyDefinition == null) {
throw new VertexiumException("Unknown property " + agg.getFieldName() + " for geohash aggregation.");
}
if (propertyDefinition.getDataType() != GeoPoint.class) {
throw new VertexiumNotSupportedException("Only GeoPoint properties are valid for Geohash aggregation. Invalid property " + agg.getFieldName());
}
for (String propertyName : getPropertyNames(agg.getFieldName())) {
String visibilityHash = getSearchIndex().getPropertyVisibilityHashFromPropertyName(propertyName);
String aggName = createAggregationName(agg.getAggregationName(), visibilityHash);
GeoGridAggregationBuilder geoHashAgg = AggregationBuilders.geohashGrid(aggName);
geoHashAgg.field(propertyName + Elasticsearch5SearchIndex.GEO_POINT_PROPERTY_NAME_SUFFIX);
geoHashAgg.precision(agg.getPrecision());
aggs.add(geoHashAgg);
}
return aggs;
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.AggregationBuilder in project vertexium by visallo.
the class ElasticsearchSearchQueryBase method getElasticsearchHistogramAggregations.
protected List<AggregationBuilder> getElasticsearchHistogramAggregations(HistogramAggregation agg) {
List<AggregationBuilder> aggs = new ArrayList<>();
PropertyDefinition propertyDefinition = getPropertyDefinition(agg.getFieldName());
if (propertyDefinition == null) {
throw new VertexiumException("Could not find mapping for property: " + agg.getFieldName());
}
Class propertyDataType = propertyDefinition.getDataType();
for (String propertyName : getPropertyNames(agg.getFieldName())) {
String visibilityHash = getSearchIndex().getPropertyVisibilityHashFromPropertyName(propertyName);
String aggName = createAggregationName(agg.getAggregationName(), visibilityHash);
if (propertyDataType == Date.class) {
DateHistogramAggregationBuilder dateAgg = AggregationBuilders.dateHistogram(aggName);
dateAgg.field(propertyName);
String interval = agg.getInterval();
if (Pattern.matches("^[0-9\\.]+$", interval)) {
interval += "ms";
}
dateAgg.dateHistogramInterval(new DateHistogramInterval(interval));
dateAgg.minDocCount(1L);
if (agg.getMinDocumentCount() != null) {
dateAgg.minDocCount(agg.getMinDocumentCount());
}
if (agg.getExtendedBounds() != null) {
HistogramAggregation.ExtendedBounds<?> bounds = agg.getExtendedBounds();
if (bounds.getMinMaxType().isAssignableFrom(Long.class)) {
dateAgg.extendedBounds(new ExtendedBounds((Long) bounds.getMin(), (Long) bounds.getMax()));
} else if (bounds.getMinMaxType().isAssignableFrom(Date.class)) {
dateAgg.extendedBounds(new ExtendedBounds(new DateTime(bounds.getMin()).toString(), new DateTime(bounds.getMax()).toString()));
} else if (bounds.getMinMaxType().isAssignableFrom(String.class)) {
dateAgg.extendedBounds(new ExtendedBounds((String) bounds.getMin(), (String) bounds.getMax()));
} else {
throw new VertexiumException("Unhandled extended bounds type. Expected Long, String, or Date. Found: " + bounds.getMinMaxType().getName());
}
}
for (AggregationBuilder subAgg : getElasticsearchAggregations(agg.getNestedAggregations())) {
dateAgg.subAggregation(subAgg);
}
aggs.add(dateAgg);
} else {
HistogramAggregationBuilder histogramAgg = AggregationBuilders.histogram(aggName);
histogramAgg.field(propertyName);
histogramAgg.interval(Long.parseLong(agg.getInterval()));
histogramAgg.minDocCount(1L);
if (agg.getMinDocumentCount() != null) {
histogramAgg.minDocCount(agg.getMinDocumentCount());
}
if (agg.getExtendedBounds() != null) {
HistogramAggregation.ExtendedBounds<?> bounds = agg.getExtendedBounds();
if (bounds.getMinMaxType().isAssignableFrom(Long.class)) {
histogramAgg.extendedBounds((Long) bounds.getMin(), (Long) bounds.getMax());
} else {
throw new VertexiumException("Unhandled extended bounds type. Expected Long. Found: " + bounds.getMinMaxType().getName());
}
}
for (AggregationBuilder subAgg : getElasticsearchAggregations(agg.getNestedAggregations())) {
histogramAgg.subAggregation(subAgg);
}
aggs.add(histogramAgg);
}
}
return aggs;
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.AggregationBuilder in project vertexium by visallo.
the class ElasticsearchSearchQueryBase method getElasticsearchRangeAggregations.
protected List<AggregationBuilder> getElasticsearchRangeAggregations(RangeAggregation agg) {
List<AggregationBuilder> aggs = new ArrayList<>();
PropertyDefinition propertyDefinition = getPropertyDefinition(agg.getFieldName());
if (propertyDefinition == null) {
throw new VertexiumException("Could not find mapping for property: " + agg.getFieldName());
}
Class propertyDataType = propertyDefinition.getDataType();
for (String propertyName : getPropertyNames(agg.getFieldName())) {
String visibilityHash = getSearchIndex().getPropertyVisibilityHashFromPropertyName(propertyName);
String aggName = createAggregationName(agg.getAggregationName(), visibilityHash);
if (propertyDataType == Date.class) {
DateRangeAggregationBuilder dateRangeBuilder = AggregationBuilders.dateRange(aggName);
dateRangeBuilder.field(propertyName);
if (!Strings.isNullOrEmpty(agg.getFormat())) {
dateRangeBuilder.format(agg.getFormat());
}
for (RangeAggregation.Range range : agg.getRanges()) {
applyRange(dateRangeBuilder, range);
}
for (AggregationBuilder subAgg : getElasticsearchAggregations(agg.getNestedAggregations())) {
dateRangeBuilder.subAggregation(subAgg);
}
aggs.add(dateRangeBuilder);
} else {
RangeAggregationBuilder rangeBuilder = AggregationBuilders.range(aggName);
rangeBuilder.field(propertyName);
if (!Strings.isNullOrEmpty(agg.getFormat())) {
throw new VertexiumException("Invalid use of format for property: " + agg.getFieldName() + ". Format is only valid for date properties");
}
for (RangeAggregation.Range range : agg.getRanges()) {
Object from = range.getFrom();
Object to = range.getTo();
if ((from != null && !(from instanceof Number)) || (to != null && !(to instanceof Number))) {
throw new VertexiumException("Invalid range for property: " + agg.getFieldName() + ". Both to and from must be Numeric.");
}
rangeBuilder.addRange(range.getKey(), from == null ? Double.MIN_VALUE : ((Number) from).doubleValue(), to == null ? Double.MAX_VALUE : ((Number) to).doubleValue());
}
for (AggregationBuilder subAgg : getElasticsearchAggregations(agg.getNestedAggregations())) {
rangeBuilder.subAggregation(subAgg);
}
aggs.add(rangeBuilder);
}
}
return aggs;
}
Aggregations