Search in sources :

Example 1 with ArrayListTotal

use of io.kestra.core.repositories.ArrayListTotal in project kestra by kestra-io.

the class AbstractElasticSearchRepository method findDistinctNamespace.

protected List<String> findDistinctNamespace(String index) {
    BoolQueryBuilder query = this.defaultFilter();
    // We want to keep only "distinct" values of field "namespace"
    // @TODO: use includeExclude(new IncludeExclude(0, 10)) to partition results
    TermsAggregationBuilder termsAggregationBuilder = AggregationBuilders.terms("distinct_namespace").field("namespace").size(10000).order(BucketOrder.key(true));
    SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(query).aggregation(termsAggregationBuilder);
    SearchRequest searchRequest = searchRequest(index, sourceBuilder, false);
    try {
        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        Terms namespaces = searchResponse.getAggregations().get("distinct_namespace");
        return new ArrayListTotal<>(namespaces.getBuckets().stream().map(o -> {
            return o.getKey().toString();
        }).collect(Collectors.toList()), namespaces.getBuckets().size());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
Also used : SearchRequest(org.opensearch.action.search.SearchRequest) TermsAggregationBuilder(org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) Terms(org.opensearch.search.aggregations.bucket.terms.Terms) IOException(java.io.IOException) ArrayListTotal(io.kestra.core.repositories.ArrayListTotal) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) SearchResponse(org.opensearch.action.search.SearchResponse)

Example 2 with ArrayListTotal

use of io.kestra.core.repositories.ArrayListTotal in project kestra by kestra-io.

the class ElasticSearchExecutionRepository method findTaskRun.

@Override
public ArrayListTotal<TaskRun> findTaskRun(String query, Pageable pageable, @Nullable List<State.Type> state) {
    BoolQueryBuilder filterAggQuery = QueryBuilders.boolQuery().filter(QueryBuilders.queryStringQuery(query));
    if (state != null) {
        filterAggQuery = filterAggQuery.must(QueryBuilders.termsQuery("taskRunList.state.current", stateConvert(state)));
    }
    NestedAggregationBuilder nestedAgg = AggregationBuilders.nested("NESTED", "taskRunList").subAggregation(AggregationBuilders.filter("FILTER", filterAggQuery).subAggregation(AggregationBuilders.topHits("TOPHITS").size(pageable.getSize()).sorts(defaultSorts(pageable, true)).from(Math.toIntExact(pageable.getOffset() - pageable.getSize()))));
    BoolQueryBuilder mainQuery = this.defaultFilter().filter(QueryBuilders.nestedQuery("taskRunList", QueryBuilders.queryStringQuery(query).field("*.fulltext"), ScoreMode.Total));
    SearchSourceBuilder sourceBuilder = this.searchSource(mainQuery, Optional.of(List.of(nestedAgg)), null).fetchSource(false);
    SearchRequest searchRequest = searchRequest(INDEX_NAME, sourceBuilder, false);
    try {
        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        ParsedNested pn = searchResponse.getAggregations().get("NESTED");
        Filter fa = pn.getAggregations().get("FILTER");
        long docCount = fa.getDocCount();
        TopHits th = fa.getAggregations().get("TOPHITS");
        List<TaskRun> collect = Arrays.stream(th.getHits().getHits()).map(documentFields -> {
            try {
                return MAPPER.readValue(documentFields.getSourceAsString(), TaskRun.class);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }).collect(Collectors.toList());
        return new ArrayListTotal<>(collect, docCount);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
Also used : TaskRun(io.kestra.core.models.executions.TaskRun) RequestOptions(org.opensearch.client.RequestOptions) java.util(java.util) ExecutionRepositoryInterface(io.kestra.core.repositories.ExecutionRepositoryInterface) Flow(io.kestra.core.models.executions.statistics.Flow) ParsedStringTerms(org.opensearch.search.aggregations.bucket.terms.ParsedStringTerms) ZonedDateTime(java.time.ZonedDateTime) DateHistogramAggregationBuilder(org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder) DateHistogramInterval(org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval) ExecutionCount(io.kestra.core.models.executions.statistics.ExecutionCount) ParsedDateHistogram(org.opensearch.search.aggregations.bucket.histogram.ParsedDateHistogram) State(io.kestra.core.models.flows.State) LongBounds(org.opensearch.search.aggregations.bucket.histogram.LongBounds) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) FiltersAggregator(org.opensearch.search.aggregations.bucket.filter.FiltersAggregator) Nullable(io.micronaut.core.annotation.Nullable) Duration(java.time.Duration) Pageable(io.micronaut.data.model.Pageable) SearchRequest(org.opensearch.action.search.SearchRequest) SearchResponse(org.opensearch.action.search.SearchResponse) RestHighLevelClient(org.opensearch.client.RestHighLevelClient) ParsedFilters(org.opensearch.search.aggregations.bucket.filter.ParsedFilters) QueryBuilders(org.opensearch.index.query.QueryBuilders) Filter(org.opensearch.search.aggregations.bucket.filter.Filter) TopHits(org.opensearch.search.aggregations.metrics.TopHits) Singleton(jakarta.inject.Singleton) Settings(org.opensearch.common.settings.Settings) IOException(java.io.IOException) ScoreMode(org.apache.lucene.search.join.ScoreMode) Terms(org.opensearch.search.aggregations.bucket.terms.Terms) Execution(io.kestra.core.models.executions.Execution) Collectors(java.util.stream.Collectors) DailyExecutionStatistics(io.kestra.core.models.executions.statistics.DailyExecutionStatistics) ZoneId(java.time.ZoneId) AggregationBuilders(org.opensearch.search.aggregations.AggregationBuilders) ParsedStats(org.opensearch.search.aggregations.metrics.ParsedStats) ArrayListTotal(io.kestra.core.repositories.ArrayListTotal) ExecutorsUtils(io.kestra.core.utils.ExecutorsUtils) TermsAggregationBuilder(org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) LocalDate(java.time.LocalDate) DateTimeFormatter(java.time.format.DateTimeFormatter) IndexSettings(org.opensearch.index.IndexSettings) ModelValidator(io.kestra.core.models.validations.ModelValidator) NestedAggregationBuilder(org.opensearch.search.aggregations.bucket.nested.NestedAggregationBuilder) Inject(jakarta.inject.Inject) ParsedNested(org.opensearch.search.aggregations.bucket.nested.ParsedNested) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) SearchRequest(org.opensearch.action.search.SearchRequest) ParsedNested(org.opensearch.search.aggregations.bucket.nested.ParsedNested) IOException(java.io.IOException) ArrayListTotal(io.kestra.core.repositories.ArrayListTotal) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) SearchResponse(org.opensearch.action.search.SearchResponse) TaskRun(io.kestra.core.models.executions.TaskRun) TopHits(org.opensearch.search.aggregations.metrics.TopHits) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) Filter(org.opensearch.search.aggregations.bucket.filter.Filter) NestedAggregationBuilder(org.opensearch.search.aggregations.bucket.nested.NestedAggregationBuilder)

Example 3 with ArrayListTotal

use of io.kestra.core.repositories.ArrayListTotal in project kestra by kestra-io.

the class ElasticSearchFlowRepository method findSourceCode.

@Override
public ArrayListTotal<SearchResult<Flow>> findSourceCode(String query, Pageable pageable) {
    BoolQueryBuilder bool = this.defaultFilter().must(QueryBuilders.queryStringQuery(query).field("sourceCode"));
    SearchSourceBuilder sourceBuilder = this.searchSource(bool, Optional.empty(), pageable);
    sourceBuilder.fetchSource("*", "sourceCode");
    sourceBuilder.highlighter(new HighlightBuilder().preTags("[mark]").postTags("[/mark]").field("sourceCode"));
    SearchRequest searchRequest = searchRequest(INDEX_NAME, sourceBuilder, false);
    try {
        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        return new ArrayListTotal<>(Arrays.stream(searchResponse.getHits().getHits()).map(documentFields -> {
            try {
                return new SearchResult<>(MAPPER.readValue(documentFields.getSourceAsString(), this.cls), documentFields.getHighlightFields().get("sourceCode") != null ? Arrays.stream(documentFields.getHighlightFields().get("sourceCode").getFragments()).map(Text::string).collect(Collectors.toList()) : Collections.emptyList());
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }).collect(Collectors.toList()), searchResponse.getHits().getTotalHits().value);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
Also used : SearchRequest(org.opensearch.action.search.SearchRequest) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) SearchResult(io.kestra.core.models.SearchResult) IOException(java.io.IOException) ArrayListTotal(io.kestra.core.repositories.ArrayListTotal) HighlightBuilder(org.opensearch.search.fetch.subphase.highlight.HighlightBuilder) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) SearchResponse(org.opensearch.action.search.SearchResponse)

Aggregations

ArrayListTotal (io.kestra.core.repositories.ArrayListTotal)3 IOException (java.io.IOException)3 SearchRequest (org.opensearch.action.search.SearchRequest)3 SearchResponse (org.opensearch.action.search.SearchResponse)3 BoolQueryBuilder (org.opensearch.index.query.BoolQueryBuilder)3 SearchSourceBuilder (org.opensearch.search.builder.SearchSourceBuilder)3 Terms (org.opensearch.search.aggregations.bucket.terms.Terms)2 TermsAggregationBuilder (org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder)2 SearchResult (io.kestra.core.models.SearchResult)1 Execution (io.kestra.core.models.executions.Execution)1 TaskRun (io.kestra.core.models.executions.TaskRun)1 DailyExecutionStatistics (io.kestra.core.models.executions.statistics.DailyExecutionStatistics)1 ExecutionCount (io.kestra.core.models.executions.statistics.ExecutionCount)1 Flow (io.kestra.core.models.executions.statistics.Flow)1 State (io.kestra.core.models.flows.State)1 ModelValidator (io.kestra.core.models.validations.ModelValidator)1 ExecutionRepositoryInterface (io.kestra.core.repositories.ExecutionRepositoryInterface)1 ExecutorsUtils (io.kestra.core.utils.ExecutorsUtils)1 Nullable (io.micronaut.core.annotation.Nullable)1 Pageable (io.micronaut.data.model.Pageable)1