Examples with AggregationBuilder org.elasticsearch.search.aggregations.AggregationBuilder used on opensource projects

Example 6 with AggregationBuilder

use of org.elasticsearch.search.aggregations.AggregationBuilder in project pancm_project by xuwujing.

the class EsAggregationSearchTest method groupbySearch.

/**
 * @Author pancm
 * @Description 多个聚合条件测试
 * SQL: select age, name, count(*) as count1 from student group by age, name;
 * @Date  2019/7/3
 * @Param []
 * @return void
 */
private static void groupbySearch() throws IOException {
    String buk = "group";
    AggregationBuilder aggregation = AggregationBuilders.terms("age").field("age");
    AggregationBuilder aggregation2 = AggregationBuilders.terms("name").field("name");
    // 根据创建时间按天分组
    // AggregationBuilder aggregation3 = AggregationBuilders.dateHistogram("createtm")
    // .field("createtm")
    // .format("yyyy-MM-dd")
    // .dateHistogramInterval(DateHistogramInterval.DAY);
    // 
    // aggregation2.subAggregation(aggregation3);
    aggregation.subAggregation(aggregation2);
    agg(aggregation, buk);
}

Example 7 with AggregationBuilder

use of org.elasticsearch.search.aggregations.AggregationBuilder in project pancm_project by xuwujing.

the class EsAggregationSearchTest2 method test.

private static void test() throws IOException {
    // TermsAggregationBuilder termsBuilder = AggregationBuilders.terms("ecid").field("ecid").size(99999);
    // List<FieldSortBuilder> fieldSorts=new ArrayList<>();
    // fieldSorts.add(new FieldSortBuilder("sum_field").order(SortOrder.DESC));
    // termsBuilder.subAggregation(new BucketSortPipelineAggregationBuilder("bucket_field", fieldSorts).from(6000).size(10));
    // 
    AggregationBuilder aggregation = AggregationBuilders.terms("ecid").field("ecid");
    List<FieldSortBuilder> fieldSorts = new ArrayList<>();
    fieldSorts.add(new FieldSortBuilder("id_").order(SortOrder.DESC));
    aggregation.subAggregation(new BucketSortPipelineAggregationBuilder("id", fieldSorts).from(0).size(10));
    SearchResponse searchResponse = search(aggregation);
    // 获取聚合结果
    Aggregations aggregations = searchResponse.getAggregations();
    Map<String, Object> map = new HashMap<>();
    List<Map<String, Object>> list = new ArrayList<>();
    agg(map, list, aggregations);
    logger.info("聚合查询结果:" + list);
    logger.info("------------------------------------");
}

Example 8 with AggregationBuilder

use of org.elasticsearch.search.aggregations.AggregationBuilder in project pancm_project by xuwujing.

the class EsAggregationSearchTest2 method test2.

private static void test2() throws IOException {
    AggregationBuilder aggregation = AggregationBuilders.filters("ecid", QueryBuilders.termQuery("id", 1));
    SearchResponse searchResponse = search(aggregation);
    // 获取聚合结果
    Aggregations aggregations = searchResponse.getAggregations();
    Map<String, Object> map = new HashMap<>();
    List<Map<String, Object>> list = new ArrayList<>();
    agg(map, list, aggregations);
    logger.info("test2聚合查询结果:" + list);
    logger.info("------------------------------------");
}

Example 9 with AggregationBuilder

use of org.elasticsearch.search.aggregations.AggregationBuilder in project uavstack by uavorg.

the class ThreadAnalysisQueryHandler method queryDistinct.

/**
 * 官网上的 【Top Hits Aggregation】 JAVA API 运行报错，
 *
 * @see https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/_metrics_aggregations.html
 *
 *      <pre>
 *
 * {@code
 * AggregationBuilder aggregation =
 *        AggregationBuilders
 *            .terms("agg").field("gender")
 *            .subAggregation(
 *                    AggregationBuilders.topHits("top")
 *                        .explain(true)
 *                        .size(1)
 *                        .from(10)
 *            );
 * }
 *      </pre>
 *
 *      Caused by: java.lang.IllegalArgumentException: An SPI class of type org.apache.lucene.codecs.PostingsFormat
 *      with name 'Lucene50' does not exist. You need to add the corresponding JAR file supporting this SPI to your
 *      classpath. The current classpath supports the following names: [completion, completion090]
 *      <p>
 *      以下满足张真要求的ES的查询可以工作，但找不到对应的JAVA API（原因在上）？？！！！
 *
 *      <pre>
 *
 * {@code
 * {
 *    "aggs": {
 *        "time": {
 *            "terms": {
 *                "field": "time",
 *                "order":{"_term":"desc"},
 *                "size": 1000
 *            },
 *            "aggs": {
 *                "example": {
 *                    "top_hits": {
 *                        "sort": [
 *                            {
 *                                "percpu": {
 *                                    "order": "desc"
 *                                }
 *                            }
 *                        ],
 *                        "size": 1
 *                    }
 *                }
 *            }
 *        }
 *    },
 *    "from": 0,
 *    "size": 0
 *
 *    }
 * }
 *      </pre>
 *
 * @param data
 */
private void queryDistinct(UAVHttpMessage data) {
    try {
        String ipport = data.getRequest("ipport");
        // TODO ES aggregation 默认最多查10条, 这里暂时改到1000，待refine
        AggregationBuilder agg = AggregationBuilders.terms("unique_time").field("time").size(1000).order(Terms.Order.term(false)).subAggregation(AggregationBuilders.terms("unique_user").field("user").size(1000));
        String date = data.getRequest("indexdate");
        String currentIndex;
        if (date != null) {
            // 指定index
            currentIndex = this.indexMgr.getIndexByDate(date);
        } else {
            // current index
            currentIndex = this.indexMgr.getCurrentIndex();
        }
        SearchResponse sResponse = client.getClient().prepareSearch(currentIndex).setTypes(ThreadAnalysisIndexMgr.JTA_TABLE).setQuery(QueryBuilders.boolQuery().must(QueryBuilders.termQuery("ipport", ipport))).setSize(0).addAggregation(agg).execute().actionGet();
        // sr is here your SearchResponse object
        Terms aggs = sResponse.getAggregations().get("unique_time");
        List<Map<String, Object>> records = new ArrayList<Map<String, Object>>();
        // For each entry
        for (Terms.Bucket entry : aggs.getBuckets()) {
            // bucket key
            String key = entry.getKey().toString();
            // Doc count
            long docCount = entry.getDocCount();
            Map<String, Object> record = new HashMap<String, Object>();
            record.put("time", key);
            record.put("threadcount", docCount);
            Terms userAggs = entry.getAggregations().get("unique_user");
            List<Terms.Bucket> users = userAggs.getBuckets();
            if (!users.isEmpty()) {
                record.put("user", users.get(0).getKey().toString());
            }
            records.add(record);
        }
        data.putResponse("rs", JSONHelper.toString(records));
        // 返回总的条数
        data.putResponse("count", aggs.getBuckets().size() + "");
    } catch (Exception e) {
        if (e.getMessage().indexOf("no such index") >= 0) {
            data.putResponse("rs", "NO_INDEX");
        } else {
            data.putResponse("rs", "ERR");
            log.err(this, "query distinct FAILED. " + JSONHelper.toString(data), e);
        }
    }
}

Example 10 with AggregationBuilder

use of org.elasticsearch.search.aggregations.AggregationBuilder in project incubator-sdap-mudrod by apache.

the class CrawlerDetection method checkByRate.

private int checkByRate(ESDriver es, String user) {
    int rate = Integer.parseInt(props.getProperty(MudrodConstants.REQUEST_RATE));
    Pattern pattern = Pattern.compile("get (.*?) http/*");
    Matcher matcher;
    BoolQueryBuilder filterSearch = new BoolQueryBuilder();
    filterSearch.must(QueryBuilders.termQuery("IP", user));
    AggregationBuilder aggregation = AggregationBuilders.dateHistogram("by_minute").field("Time").dateHistogramInterval(DateHistogramInterval.MINUTE).order(Order.COUNT_DESC);
    SearchResponse checkRobot = es.getClient().prepareSearch(logIndex).setTypes(httpType, ftpType).setQuery(filterSearch).setSize(0).addAggregation(aggregation).execute().actionGet();
    Histogram agg = checkRobot.getAggregations().get("by_minute");
    List<? extends Histogram.Bucket> botList = agg.getBuckets();
    long maxCount = botList.get(0).getDocCount();
    if (maxCount >= rate) {
        return 0;
    } else {
        DateTime dt1 = null;
        int toLast = 0;
        SearchResponse scrollResp = es.getClient().prepareSearch(logIndex).setTypes(httpType, ftpType).setScroll(new TimeValue(60000)).setQuery(filterSearch).setSize(100).execute().actionGet();
        while (true) {
            for (SearchHit hit : scrollResp.getHits().getHits()) {
                Map<String, Object> result = hit.getSource();
                String logtype = (String) result.get("LogType");
                if (logtype.equals(MudrodConstants.HTTP_LOG)) {
                    String request = (String) result.get("Request");
                    matcher = pattern.matcher(request.trim().toLowerCase());
                    boolean find = false;
                    while (matcher.find()) {
                        request = matcher.group(1);
                        result.put("RequestUrl", props.getProperty(MudrodConstants.BASE_URL) + request);
                        find = true;
                    }
                    if (!find) {
                        result.put("RequestUrl", request);
                    }
                } else {
                    result.put("RequestUrl", result.get("Request"));
                }
                DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
                DateTime dt2 = fmt.parseDateTime((String) result.get("Time"));
                if (dt1 == null) {
                    toLast = 0;
                } else {
                    toLast = Math.abs(Seconds.secondsBetween(dt1, dt2).getSeconds());
                }
                result.put("ToLast", toLast);
                IndexRequest ir = new IndexRequest(logIndex, cleanupType).source(result);
                es.getBulkProcessor().add(ir);
                dt1 = dt2;
            }
            scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
            if (scrollResp.getHits().getHits().length == 0) {
                break;
            }
        }
    }
    return 1;
}