Examples with AggregationBuilder org.elasticsearch.search.aggregations.AggregationBuilder used on opensource projects

Example 1 with AggregationBuilder

use of org.elasticsearch.search.aggregations.AggregationBuilder in project incubator-sdap-mudrod by apache.

the class CrawlerDetection method checkByRate.

private int checkByRate(ESDriver es, String user) {
    int rate = Integer.parseInt(props.getProperty(MudrodConstants.REQUEST_RATE));
    Pattern pattern = Pattern.compile("get (.*?) http/*");
    Matcher matcher;
    BoolQueryBuilder filterSearch = new BoolQueryBuilder();
    filterSearch.must(QueryBuilders.termQuery("IP", user));
    AggregationBuilder aggregation = AggregationBuilders.dateHistogram("by_minute").field("Time").dateHistogramInterval(DateHistogramInterval.MINUTE).order(Order.COUNT_DESC);
    SearchResponse checkRobot = es.getClient().prepareSearch(logIndex).setTypes(httpType, ftpType).setQuery(filterSearch).setSize(0).addAggregation(aggregation).execute().actionGet();
    Histogram agg = checkRobot.getAggregations().get("by_minute");
    List<? extends Histogram.Bucket> botList = agg.getBuckets();
    long maxCount = botList.get(0).getDocCount();
    if (maxCount >= rate) {
        return 0;
    } else {
        DateTime dt1 = null;
        int toLast = 0;
        SearchResponse scrollResp = es.getClient().prepareSearch(logIndex).setTypes(httpType, ftpType).setScroll(new TimeValue(60000)).setQuery(filterSearch).setSize(100).execute().actionGet();
        while (true) {
            for (SearchHit hit : scrollResp.getHits().getHits()) {
                Map<String, Object> result = hit.getSource();
                String logtype = (String) result.get("LogType");
                if (logtype.equals(MudrodConstants.HTTP_LOG)) {
                    String request = (String) result.get("Request");
                    matcher = pattern.matcher(request.trim().toLowerCase());
                    boolean find = false;
                    while (matcher.find()) {
                        request = matcher.group(1);
                        result.put("RequestUrl", props.getProperty(MudrodConstants.BASE_URL) + request);
                        find = true;
                    }
                    if (!find) {
                        result.put("RequestUrl", request);
                    }
                } else {
                    result.put("RequestUrl", result.get("Request"));
                }
                DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
                DateTime dt2 = fmt.parseDateTime((String) result.get("Time"));
                if (dt1 == null) {
                    toLast = 0;
                } else {
                    toLast = Math.abs(Seconds.secondsBetween(dt1, dt2).getSeconds());
                }
                result.put("ToLast", toLast);
                IndexRequest ir = new IndexRequest(logIndex, cleanupType).source(result);
                es.getBulkProcessor().add(ir);
                dt1 = dt2;
            }
            scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
            if (scrollResp.getHits().getHits().length == 0) {
                break;
            }
        }
    }
    return 1;
}

Example 2 with AggregationBuilder

use of org.elasticsearch.search.aggregations.AggregationBuilder in project vertexium by visallo.

the class ElasticsearchSearchQueryBase method getElasticsearchGeohashAggregations.

protected List<AggregationBuilder> getElasticsearchGeohashAggregations(GeohashAggregation agg) {
    List<AggregationBuilder> aggs = new ArrayList<>();
    PropertyDefinition propertyDefinition = getPropertyDefinition(agg.getFieldName());
    if (propertyDefinition == null) {
        throw new VertexiumException("Unknown property " + agg.getFieldName() + " for geohash aggregation.");
    }
    if (propertyDefinition.getDataType() != GeoPoint.class) {
        throw new VertexiumNotSupportedException("Only GeoPoint properties are valid for Geohash aggregation. Invalid property " + agg.getFieldName());
    }
    for (String propertyName : getPropertyNames(agg.getFieldName())) {
        String visibilityHash = getSearchIndex().getPropertyVisibilityHashFromPropertyName(propertyName);
        String aggName = createAggregationName(agg.getAggregationName(), visibilityHash);
        GeoGridAggregationBuilder geoHashAgg = AggregationBuilders.geohashGrid(aggName);
        geoHashAgg.field(propertyName + Elasticsearch5SearchIndex.GEO_POINT_PROPERTY_NAME_SUFFIX);
        geoHashAgg.precision(agg.getPrecision());
        aggs.add(geoHashAgg);
    }
    return aggs;
}

Example 3 with AggregationBuilder

use of org.elasticsearch.search.aggregations.AggregationBuilder in project vertexium by visallo.

the class ElasticsearchSearchQueryBase method getElasticsearchHistogramAggregations.

protected List<AggregationBuilder> getElasticsearchHistogramAggregations(HistogramAggregation agg) {
    List<AggregationBuilder> aggs = new ArrayList<>();
    PropertyDefinition propertyDefinition = getPropertyDefinition(agg.getFieldName());
    if (propertyDefinition == null) {
        throw new VertexiumException("Could not find mapping for property: " + agg.getFieldName());
    }
    Class propertyDataType = propertyDefinition.getDataType();
    for (String propertyName : getPropertyNames(agg.getFieldName())) {
        String visibilityHash = getSearchIndex().getPropertyVisibilityHashFromPropertyName(propertyName);
        String aggName = createAggregationName(agg.getAggregationName(), visibilityHash);
        if (propertyDataType == Date.class) {
            DateHistogramAggregationBuilder dateAgg = AggregationBuilders.dateHistogram(aggName);
            dateAgg.field(propertyName);
            String interval = agg.getInterval();
            if (Pattern.matches("^[0-9\\.]+$", interval)) {
                interval += "ms";
            }
            dateAgg.dateHistogramInterval(new DateHistogramInterval(interval));
            dateAgg.minDocCount(1L);
            if (agg.getMinDocumentCount() != null) {
                dateAgg.minDocCount(agg.getMinDocumentCount());
            }
            if (agg.getExtendedBounds() != null) {
                HistogramAggregation.ExtendedBounds<?> bounds = agg.getExtendedBounds();
                if (bounds.getMinMaxType().isAssignableFrom(Long.class)) {
                    dateAgg.extendedBounds(new ExtendedBounds((Long) bounds.getMin(), (Long) bounds.getMax()));
                } else if (bounds.getMinMaxType().isAssignableFrom(Date.class)) {
                    dateAgg.extendedBounds(new ExtendedBounds(new DateTime(bounds.getMin()).toString(), new DateTime(bounds.getMax()).toString()));
                } else if (bounds.getMinMaxType().isAssignableFrom(String.class)) {
                    dateAgg.extendedBounds(new ExtendedBounds((String) bounds.getMin(), (String) bounds.getMax()));
                } else {
                    throw new VertexiumException("Unhandled extended bounds type. Expected Long, String, or Date. Found: " + bounds.getMinMaxType().getName());
                }
            }
            for (AggregationBuilder subAgg : getElasticsearchAggregations(agg.getNestedAggregations())) {
                dateAgg.subAggregation(subAgg);
            }
            aggs.add(dateAgg);
        } else {
            HistogramAggregationBuilder histogramAgg = AggregationBuilders.histogram(aggName);
            histogramAgg.field(propertyName);
            histogramAgg.interval(Long.parseLong(agg.getInterval()));
            histogramAgg.minDocCount(1L);
            if (agg.getMinDocumentCount() != null) {
                histogramAgg.minDocCount(agg.getMinDocumentCount());
            }
            if (agg.getExtendedBounds() != null) {
                HistogramAggregation.ExtendedBounds<?> bounds = agg.getExtendedBounds();
                if (bounds.getMinMaxType().isAssignableFrom(Long.class)) {
                    histogramAgg.extendedBounds((Long) bounds.getMin(), (Long) bounds.getMax());
                } else {
                    throw new VertexiumException("Unhandled extended bounds type. Expected Long. Found: " + bounds.getMinMaxType().getName());
                }
            }
            for (AggregationBuilder subAgg : getElasticsearchAggregations(agg.getNestedAggregations())) {
                histogramAgg.subAggregation(subAgg);
            }
            aggs.add(histogramAgg);
        }
    }
    return aggs;
}

Example 4 with AggregationBuilder

use of org.elasticsearch.search.aggregations.AggregationBuilder in project vertexium by visallo.

the class ElasticsearchSearchQueryBase method getElasticsearchRangeAggregations.

protected List<AggregationBuilder> getElasticsearchRangeAggregations(RangeAggregation agg) {
    List<AggregationBuilder> aggs = new ArrayList<>();
    PropertyDefinition propertyDefinition = getPropertyDefinition(agg.getFieldName());
    if (propertyDefinition == null) {
        throw new VertexiumException("Could not find mapping for property: " + agg.getFieldName());
    }
    Class propertyDataType = propertyDefinition.getDataType();
    for (String propertyName : getPropertyNames(agg.getFieldName())) {
        String visibilityHash = getSearchIndex().getPropertyVisibilityHashFromPropertyName(propertyName);
        String aggName = createAggregationName(agg.getAggregationName(), visibilityHash);
        if (propertyDataType == Date.class) {
            DateRangeAggregationBuilder dateRangeBuilder = AggregationBuilders.dateRange(aggName);
            dateRangeBuilder.field(propertyName);
            if (!Strings.isNullOrEmpty(agg.getFormat())) {
                dateRangeBuilder.format(agg.getFormat());
            }
            for (RangeAggregation.Range range : agg.getRanges()) {
                applyRange(dateRangeBuilder, range);
            }
            for (AggregationBuilder subAgg : getElasticsearchAggregations(agg.getNestedAggregations())) {
                dateRangeBuilder.subAggregation(subAgg);
            }
            aggs.add(dateRangeBuilder);
        } else {
            RangeAggregationBuilder rangeBuilder = AggregationBuilders.range(aggName);
            rangeBuilder.field(propertyName);
            if (!Strings.isNullOrEmpty(agg.getFormat())) {
                throw new VertexiumException("Invalid use of format for property: " + agg.getFieldName() + ". Format is only valid for date properties");
            }
            for (RangeAggregation.Range range : agg.getRanges()) {
                Object from = range.getFrom();
                Object to = range.getTo();
                if ((from != null && !(from instanceof Number)) || (to != null && !(to instanceof Number))) {
                    throw new VertexiumException("Invalid range for property: " + agg.getFieldName() + ". Both to and from must be Numeric.");
                }
                rangeBuilder.addRange(range.getKey(), from == null ? Double.MIN_VALUE : ((Number) from).doubleValue(), to == null ? Double.MAX_VALUE : ((Number) to).doubleValue());
            }
            for (AggregationBuilder subAgg : getElasticsearchAggregations(agg.getNestedAggregations())) {
                rangeBuilder.subAggregation(subAgg);
            }
            aggs.add(rangeBuilder);
        }
    }
    return aggs;
}

Example 5 with AggregationBuilder

use of org.elasticsearch.search.aggregations.AggregationBuilder in project alien4cloud by alien4cloud.

the class AbstractToscaIndexSearchService method search.

public FacetedSearchResult search(Class<? extends T> clazz, String query, Integer size, Map<String, String[]> filters) {
    TopHitsBuilder topHitAggregation = AggregationBuilders.topHits("highest_version").setSize(1).addSort(new FieldSortBuilder("nestedVersion.majorVersion").order(SortOrder.DESC)).addSort(new FieldSortBuilder("nestedVersion.minorVersion").order(SortOrder.DESC)).addSort(new FieldSortBuilder("nestedVersion.incrementalVersion").order(SortOrder.DESC)).addSort(new FieldSortBuilder("nestedVersion.qualifier").order(SortOrder.DESC).missing("_first"));
    AggregationBuilder aggregation = AggregationBuilders.terms("query_aggregation").field(getAggregationField()).size(size).subAggregation(topHitAggregation);
    FacetedSearchResult<? extends T> searchResult = alienDAO.buildSearchQuery(clazz, query).setFilters(FilterUtil.singleKeyFilter(filters, "workspace", AlienConstants.GLOBAL_WORKSPACE_ID)).prepareSearch().setFetchContext(FetchContext.SUMMARY, topHitAggregation).facetedSearch(new IAggregationQueryManager() {

        @Override
        public AggregationBuilder getQueryAggregation() {
            return aggregation;
        }

        @Override
        @SneakyThrows({ IOException.class })
        public void setData(ObjectMapper objectMapper, Function getClassFromType, FacetedSearchResult result, Aggregation aggregation) {
            List<Object> resultData = Lists.newArrayList();
            List<String> resultTypes = Lists.newArrayList();
            if (aggregation == null) {
                result.setData(getArray(0));
                result.setTypes(new String[0]);
            }
            for (Terms.Bucket bucket : safe(((Terms) aggregation).getBuckets())) {
                TopHits topHits = bucket.getAggregations().get("highest_version");
                for (SearchHit hit : topHits.getHits()) {
                    resultTypes.add(hit.getType());
                    resultData.add(objectMapper.readValue(hit.getSourceAsString(), ((Function<String, Class>) getClassFromType).apply(hit.getType())));
                }
            }
            result.setData(resultData.toArray(getArray(resultData.size())));
            result.setTypes(resultTypes.toArray(new String[resultTypes.size()]));
            result.setFrom(0);
            result.setTo(resultData.size());
            if (size == Integer.MAX_VALUE || resultData.size() < size) {
                result.setTotalResults(resultData.size());
            } else {
                // just to show that there is more results to fetch but iteration is not possible through aggregations.
                result.setTotalResults(resultData.size() + ((Terms) aggregation).getSumOfOtherDocCounts());
            }
        }
    });
    return searchResult;
}