Examples with QueryBuilder org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.QueryBuilder used on opensource projects

Example 61 with QueryBuilder

use of org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.QueryBuilder in project incubator-sdap-mudrod by apache.

the class SessionGenerator method combineShortSessions.

public void combineShortSessions(ESDriver es, String user, int timeThres) throws ElasticsearchException, IOException {
    BoolQueryBuilder filterSearch = new BoolQueryBuilder();
    filterSearch.must(QueryBuilders.termQuery("IP", user));
    String[] indexArr = new String[] { logIndex };
    String[] typeArr = new String[] { cleanupType };
    int docCount = es.getDocCount(indexArr, typeArr, filterSearch);
    if (docCount < 3) {
        deleteInvalid(es, user);
        return;
    }
    BoolQueryBuilder filterCheck = new BoolQueryBuilder();
    filterCheck.must(QueryBuilders.termQuery("IP", user)).must(QueryBuilders.termQuery("Referer", "-"));
    SearchResponse checkReferer = es.getClient().prepareSearch(logIndex).setTypes(this.cleanupType).setScroll(new TimeValue(60000)).setQuery(filterCheck).setSize(0).execute().actionGet();
    long numInvalid = checkReferer.getHits().getTotalHits();
    double invalidRate = numInvalid / docCount;
    if (invalidRate >= 0.8) {
        deleteInvalid(es, user);
        return;
    }
    StatsAggregationBuilder statsAgg = AggregationBuilders.stats("Stats").field("Time");
    SearchResponse srSession = es.getClient().prepareSearch(logIndex).setTypes(this.cleanupType).setScroll(new TimeValue(60000)).setQuery(filterSearch).addAggregation(AggregationBuilders.terms("Sessions").field("SessionID").size(docCount).subAggregation(statsAgg)).execute().actionGet();
    Terms sessions = srSession.getAggregations().get("Sessions");
    List<Session> sessionList = new ArrayList<>();
    for (Terms.Bucket session : sessions.getBuckets()) {
        Stats agg = session.getAggregations().get("Stats");
        Session sess = new Session(props, es, agg.getMinAsString(), agg.getMaxAsString(), session.getKey().toString());
        sessionList.add(sess);
    }
    Collections.sort(sessionList);
    DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
    String last = null;
    String lastnewID = null;
    String lastoldID = null;
    String current = null;
    for (Session s : sessionList) {
        current = s.getEndTime();
        if (last != null) {
            if (Seconds.secondsBetween(fmt.parseDateTime(last), fmt.parseDateTime(current)).getSeconds() < timeThres) {
                if (lastnewID == null) {
                    s.setNewID(lastoldID);
                } else {
                    s.setNewID(lastnewID);
                }
                QueryBuilder fs = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("SessionID", s.getID()));
                SearchResponse scrollResp = es.getClient().prepareSearch(logIndex).setTypes(this.cleanupType).setScroll(new TimeValue(60000)).setQuery(fs).setSize(100).execute().actionGet();
                while (true) {
                    for (SearchHit hit : scrollResp.getHits().getHits()) {
                        if (lastnewID == null) {
                            update(es, logIndex, this.cleanupType, hit.getId(), "SessionID", lastoldID);
                        } else {
                            update(es, logIndex, this.cleanupType, hit.getId(), "SessionID", lastnewID);
                        }
                    }
                    scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
                    if (scrollResp.getHits().getHits().length == 0) {
                        break;
                    }
                }
            }
        }
        lastoldID = s.getID();
        lastnewID = s.getNewID();
        last = current;
    }
}

Example 62 with QueryBuilder

use of org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.QueryBuilder in project incubator-sdap-mudrod by apache.

the class ESDriver method searchByQuery.

@SuppressWarnings("unchecked")
public String searchByQuery(String index, String type, String query, Boolean bDetail) throws IOException, InterruptedException, ExecutionException {
    boolean exists = getClient().admin().indices().prepareExists(index).execute().actionGet().isExists();
    if (!exists) {
        return null;
    }
    QueryBuilder qb = QueryBuilders.queryStringQuery(query);
    SearchResponse response = getClient().prepareSearch(index).setTypes(type).setQuery(qb).setSize(500).execute().actionGet();
    // Map of K,V pairs where key is the field name from search result and value is the that should be returned for that field. Not always the same.
    Map<String, String> fieldsToReturn = new HashMap<>();
    fieldsToReturn.put("Dataset-ShortName", "Short Name");
    fieldsToReturn.put("Dataset-LongName", "Long Name");
    fieldsToReturn.put("DatasetParameter-Topic", "Topic");
    fieldsToReturn.put("Dataset-Description", "Dataset-Description");
    fieldsToReturn.put("DatasetCitation-ReleaseDateLong", "Release Date");
    if (bDetail) {
        fieldsToReturn.put("DatasetPolicy-DataFormat", "DataFormat");
        fieldsToReturn.put("Dataset-Doi", "Dataset-Doi");
        fieldsToReturn.put("Dataset-ProcessingLevel", "Processing Level");
        fieldsToReturn.put("DatasetCitation-Version", "Version");
        fieldsToReturn.put("DatasetSource-Sensor-ShortName", "DatasetSource-Sensor-ShortName");
        fieldsToReturn.put("DatasetProject-Project-ShortName", "DatasetProject-Project-ShortName");
        fieldsToReturn.put("DatasetParameter-Category", "DatasetParameter-Category");
        fieldsToReturn.put("DatasetLocationPolicy-BasePath", "DatasetLocationPolicy-BasePath");
        fieldsToReturn.put("DatasetParameter-Variable-Full", "DatasetParameter-Variable-Full");
        fieldsToReturn.put("DatasetParameter-Term-Full", "DatasetParameter-Term-Full");
        fieldsToReturn.put("DatasetParameter-VariableDetail", "DatasetParameter-VariableDetail");
        fieldsToReturn.put("DatasetRegion-Region", "Region");
        fieldsToReturn.put("DatasetCoverage-NorthLat", "NorthLat");
        fieldsToReturn.put("DatasetCoverage-SouthLat", "SouthLat");
        fieldsToReturn.put("DatasetCoverage-WestLon", "WestLon");
        fieldsToReturn.put("DatasetCoverage-EastLon", "EastLon");
        fieldsToReturn.put("DatasetCoverage-StartTimeLong-Long", "DatasetCoverage-StartTimeLong-Long");
        fieldsToReturn.put("Dataset-DatasetCoverage-StopTimeLong", "Dataset-DatasetCoverage-StopTimeLong");
        fieldsToReturn.put("Dataset-TemporalResolution", "Dataset-TemporalResolution");
        fieldsToReturn.put("Dataset-TemporalRepeat", "Dataset-TemporalRepeat");
        fieldsToReturn.put("Dataset-LatitudeResolution", "Dataset-LatitudeResolution");
        fieldsToReturn.put("Dataset-LongitudeResolution", "Dataset-LongitudeResolution");
        fieldsToReturn.put("Dataset-AcrossTrackResolution", "Dataset-AcrossTrackResolution");
        fieldsToReturn.put("Dataset-AlongTrackResolution", "Dataset-AlongTrackResolution");
    }
    List<Map<String, Object>> searchResults = new ArrayList<>();
    for (SearchHit hit : response.getHits().getHits()) {
        Map<String, Object> source = hit.getSource();
        Map<String, Object> searchResult = source.entrySet().stream().filter(entry -> fieldsToReturn.keySet().contains(entry.getKey())).collect(Collectors.toMap(entry -> fieldsToReturn.get(entry.getKey()), Entry::getValue));
        // searchResult is now a map where the key = value from fieldsToReturn and the value = value from search result
        // Some results require special handling/formatting:
        // Release Date formatting
        LocalDate releaseDate = Instant.ofEpochMilli(Long.parseLong(((ArrayList<String>) searchResult.get("Release Date")).get(0))).atZone(ZoneId.of("Z")).toLocalDate();
        searchResult.put("Release Date", releaseDate.format(DateTimeFormatter.ISO_DATE));
        if (bDetail) {
            // DataFormat value, translate RAW to BINARY
            if ("RAW".equals(searchResult.get("DataFormat"))) {
                searchResult.put("DataFormat", "BINARY");
            }
            // DatasetLocationPolicy-BasePath Should only contain ftp, http, or https URLs
            List<String> urls = ((List<String>) searchResult.get("DatasetLocationPolicy-BasePath")).stream().filter(url -> url.startsWith("ftp") || url.startsWith("http")).collect(Collectors.toList());
            searchResult.put("DatasetLocationPolicy-BasePath", urls);
            // Time Span Formatting
            LocalDate startDate = Instant.ofEpochMilli((Long) searchResult.get("DatasetCoverage-StartTimeLong-Long")).atZone(ZoneId.of("Z")).toLocalDate();
            LocalDate endDate = "".equals(searchResult.get("Dataset-DatasetCoverage-StopTimeLong")) ? null : Instant.ofEpochMilli(Long.parseLong(searchResult.get("Dataset-DatasetCoverage-StopTimeLong").toString())).atZone(ZoneId.of("Z")).toLocalDate();
            searchResult.put("Time Span", startDate.format(DateTimeFormatter.ISO_DATE) + " to " + (endDate == null ? "Present" : endDate.format(DateTimeFormatter.ISO_DATE)));
            // Temporal resolution can come from one of two fields
            searchResult.put("TemporalResolution", "".equals(searchResult.get("Dataset-TemporalResolution")) ? searchResult.get("Dataset-TemporalRepeat") : searchResult.get("Dataset-TemporalResolution"));
            // Special formatting for spatial resolution
            String latResolution = (String) searchResult.get("Dataset-LatitudeResolution");
            String lonResolution = (String) searchResult.get("Dataset-LongitudeResolution");
            if (!latResolution.isEmpty() && !lonResolution.isEmpty()) {
                searchResult.put("SpatialResolution", latResolution + " degrees (latitude) x " + lonResolution + " degrees (longitude)");
            } else {
                String acrossResolution = (String) searchResult.get("Dataset-AcrossTrackResolution");
                String alonResolution = (String) searchResult.get("Dataset-AlongTrackResolution");
                double dAcrossResolution = Double.parseDouble(acrossResolution) / 1000;
                double dAlonResolution = Double.parseDouble(alonResolution) / 1000;
                searchResult.put("SpatialResolution", dAlonResolution + " km (Along) x " + dAcrossResolution + " km (Across)");
            }
            // Measurement is a list of hierarchies that goes Topic -> Term -> Variable -> Variable Detail. Need to construct these hierarchies.
            List<List<String>> measurements = buildMeasurementHierarchies((List<String>) searchResult.get("Topic"), (List<String>) searchResult.get("DatasetParameter-Term-Full"), (List<String>) searchResult.get("DatasetParameter-Variable-Full"), (List<String>) searchResult.get("DatasetParameter-VariableDetail"));
            searchResult.put("Measurements", measurements);
        }
        searchResults.add(searchResult);
    }
    Map<String, List<?>> pdResults = new HashMap<>();
    pdResults.put("PDResults", searchResults);
    return new GsonBuilder().create().toJson(pdResults);
}

Example 63 with QueryBuilder

use of org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.QueryBuilder in project ff4j by ff4j.

the class ElasticQueryBuilder method queryGetEventQueryDefinition.

public Search queryGetEventQueryDefinition(EventQueryDefinition query, String action) {
    BoolQueryBuilder booleanQuery = new BoolQueryBuilder();
    // Optional constant for action filter
    if (action != null) {
        query.getActionFilters().add(action);
    }
    QueryBuilder typeQuery = QueryBuilders.termQuery("type", EventConstants.TARGET_FEATURE);
    // Timestamp filter
    RangeQueryBuilder timestampFilter = // 
    QueryBuilders.rangeQuery("timestamp").gt(// 
    query.getFrom().longValue()).lt(// 
    query.getTo().longValue()).includeLower(// 
    false).includeUpper(false);
    booleanQuery.must(typeQuery);
    booleanQuery.must(timestampFilter);
    // Optional filters
    addOptionalFilters(booleanQuery, query.getActionFilters(), "action");
    addOptionalFilters(booleanQuery, query.getHostFilters(), "hostName");
    addOptionalFilters(booleanQuery, query.getNamesFilter(), "name");
    addOptionalFilters(booleanQuery, query.getSourceFilters(), "source");
    // Warning : default size is set to 10 results, that's why it's
    // overridden
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(100);
    Search searchQuery = // 
    new Search.Builder(searchSourceBuilder.query(booleanQuery.toString()).toString()).addIndex(// 
    connection.getIndexName()).addType(// 
    ElasticConstants.TYPE_EVENT).build();
    return searchQuery;
}

Example 64 with QueryBuilder

use of org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.QueryBuilder in project tutorials by eugenp.

the class GeoQueriesTest method givenGeoPointData_whenExecutedGeoPolygonQuery_thenResultNonEmpty.

@Test
public void givenGeoPointData_whenExecutedGeoPolygonQuery_thenResultNonEmpty() {
    String jsonObject = "{\"name\":\"The Great Rann of Kutch\",\"location\":[69.859741,23.733732]}";
    IndexResponse response = client.prepareIndex(WONDERS_OF_WORLD, WONDERS).setSource(jsonObject).get();
    String greatRannOfKutchid = response.getId();
    client.admin().indices().prepareRefresh(WONDERS_OF_WORLD).get();
    QueryBuilder qb = QueryBuilders.geoPolygonQuery("location").addPoint(22.733, 68.859).addPoint(24.733, 68.859).addPoint(23, 70.859);
    SearchResponse searchResponse = client.prepareSearch(WONDERS_OF_WORLD).setTypes(WONDERS).setQuery(qb).execute().actionGet();
    List<String> ids = Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList());
    assertTrue(ids.contains(greatRannOfKutchid));
}

Example 65 with QueryBuilder

use of org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.QueryBuilder in project tutorials by eugenp.

the class ElasticSearchQueryIntegrationTest method givenBoolQuery_whenQueryByAuthorsName_thenFoundArticlesByThatAuthorAndFilteredTag.

@Test
public void givenBoolQuery_whenQueryByAuthorsName_thenFoundArticlesByThatAuthorAndFilteredTag() {
    final QueryBuilder builder = boolQuery().must(nestedQuery("authors", boolQuery().must(termQuery("authors.name", "doe")))).filter(termQuery("tags", "elasticsearch"));
    final SearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(builder).build();
    final List<Article> articles = elasticsearchTemplate.queryForList(searchQuery, Article.class);
    assertEquals(2, articles.size());
}