Search in sources :

Example 1 with CheckedBiConsumer

use of org.opensearch.common.CheckedBiConsumer in project OpenSearch by opensearch-project.

the class RequestConvertersTests method testMultiSearch.

public void testMultiSearch() throws IOException {
    int numberOfSearchRequests = randomIntBetween(0, 32);
    MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
    for (int i = 0; i < numberOfSearchRequests; i++) {
        SearchRequest searchRequest = randomSearchRequest(() -> {
            // No need to return a very complex SearchSourceBuilder here, that is tested
            // elsewhere
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.from(randomInt(10));
            searchSourceBuilder.size(randomIntBetween(20, 100));
            return searchSourceBuilder;
        });
        // scroll is not supported in the current msearch api, so unset it:
        searchRequest.scroll((Scroll) null);
        // only expand_wildcards, ignore_unavailable and allow_no_indices can be
        // specified from msearch api, so unset other options:
        IndicesOptions randomlyGenerated = searchRequest.indicesOptions();
        IndicesOptions msearchDefault = new MultiSearchRequest().indicesOptions();
        searchRequest.indicesOptions(IndicesOptions.fromOptions(randomlyGenerated.ignoreUnavailable(), randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(), randomlyGenerated.expandWildcardsClosed(), msearchDefault.expandWildcardsHidden(), msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases(), msearchDefault.ignoreThrottled()));
        multiSearchRequest.add(searchRequest);
    }
    Map<String, String> expectedParams = new HashMap<>();
    expectedParams.put(RestSearchAction.TYPED_KEYS_PARAM, "true");
    if (randomBoolean()) {
        multiSearchRequest.maxConcurrentSearchRequests(randomIntBetween(1, 8));
        expectedParams.put("max_concurrent_searches", Integer.toString(multiSearchRequest.maxConcurrentSearchRequests()));
    }
    Request request = RequestConverters.multiSearch(multiSearchRequest);
    assertEquals("/_msearch", request.getEndpoint());
    assertEquals(HttpPost.METHOD_NAME, request.getMethod());
    assertEquals(expectedParams, request.getParameters());
    List<SearchRequest> requests = new ArrayList<>();
    CheckedBiConsumer<SearchRequest, XContentParser, IOException> consumer = (searchRequest, p) -> {
        SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(p, false);
        if (searchSourceBuilder.equals(new SearchSourceBuilder()) == false) {
            searchRequest.source(searchSourceBuilder);
        }
        requests.add(searchRequest);
    };
    MultiSearchRequest.readMultiLineFormat(new BytesArray(EntityUtils.toByteArray(request.getEntity())), REQUEST_BODY_CONTENT_TYPE.xContent(), consumer, null, multiSearchRequest.indicesOptions(), null, null, null, xContentRegistry(), true, deprecationLogger);
    assertEquals(requests, multiSearchRequest.requests());
}
Also used : HttpPost(org.apache.http.client.methods.HttpPost) Arrays(java.util.Arrays) NByteArrayEntity(org.apache.http.nio.entity.NByteArrayEntity) Strings(org.opensearch.common.Strings) ScriptType(org.opensearch.script.ScriptType) RandomSearchRequestGenerator.randomSearchRequest(org.opensearch.search.RandomSearchRequestGenerator.randomSearchRequest) MasterNodeRequest(org.opensearch.action.support.master.MasterNodeRequest) WriteRequest(org.opensearch.action.support.WriteRequest) AbstractBulkByScrollRequest(org.opensearch.index.reindex.AbstractBulkByScrollRequest) RatedRequest(org.opensearch.index.rankeval.RatedRequest) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) DeleteRequest(org.opensearch.action.delete.DeleteRequest) ValueType(org.opensearch.search.aggregations.support.ValueType) TermVectorsRequest(org.opensearch.client.core.TermVectorsRequest) TimeValue(org.opensearch.common.unit.TimeValue) PrecisionAtK(org.opensearch.index.rankeval.PrecisionAtK) AcknowledgedRequest(org.opensearch.action.support.master.AcknowledgedRequest) FieldCapabilitiesRequest(org.opensearch.action.fieldcaps.FieldCapabilitiesRequest) VersionType(org.opensearch.index.VersionType) SuggestBuilder(org.opensearch.search.suggest.SuggestBuilder) BytesArray(org.opensearch.common.bytes.BytesArray) CollapseBuilder(org.opensearch.search.collapse.CollapseBuilder) UpdateRequest(org.opensearch.action.update.UpdateRequest) XContentType(org.opensearch.common.xcontent.XContentType) OpenSearchAssertions.assertToXContentEquivalent(org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent) GetSourceRequest(org.opensearch.client.core.GetSourceRequest) HttpHead(org.apache.http.client.methods.HttpHead) MultiSearchRequest(org.opensearch.action.search.MultiSearchRequest) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) DocWriteRequest(org.opensearch.action.DocWriteRequest) RemoteInfo(org.opensearch.index.reindex.RemoteInfo) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) RestSearchAction(org.opensearch.rest.action.search.RestSearchAction) SearchScrollRequest(org.opensearch.action.search.SearchScrollRequest) Streams(org.opensearch.common.io.Streams) ExplainRequest(org.opensearch.action.explain.ExplainRequest) SearchRequest(org.opensearch.action.search.SearchRequest) QueryBuilders(org.opensearch.index.query.QueryBuilders) Versions(org.opensearch.common.lucene.uid.Versions) QueryRescorerBuilder(org.opensearch.search.rescore.QueryRescorerBuilder) SearchType(org.opensearch.action.search.SearchType) TaskId(org.opensearch.tasks.TaskId) IOException(java.io.IOException) PutStoredScriptRequest(org.opensearch.action.admin.cluster.storedscripts.PutStoredScriptRequest) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) XContentHelper(org.opensearch.common.xcontent.XContentHelper) RankEvalSpec(org.opensearch.index.rankeval.RankEvalSpec) EndpointBuilder(org.opensearch.client.RequestConverters.EndpointBuilder) RestRankEvalAction(org.opensearch.index.rankeval.RestRankEvalAction) DeleteByQueryRequest(org.opensearch.index.reindex.DeleteByQueryRequest) RequestConverters.enforceSameContentType(org.opensearch.client.RequestConverters.enforceSameContentType) CompletionSuggestionBuilder(org.opensearch.search.suggest.completion.CompletionSuggestionBuilder) MultiGetRequest(org.opensearch.action.get.MultiGetRequest) ToXContent(org.opensearch.common.xcontent.ToXContent) BiFunction(java.util.function.BiFunction) BulkRequest(org.opensearch.action.bulk.BulkRequest) Matchers.hasKey(org.hamcrest.Matchers.hasKey) EntityUtils(org.apache.http.util.EntityUtils) XContentParser(org.opensearch.common.xcontent.XContentParser) MapperService(org.opensearch.index.mapper.MapperService) QueryBuilders.matchAllQuery(org.opensearch.index.query.QueryBuilders.matchAllQuery) Locale(java.util.Locale) GetStoredScriptRequest(org.opensearch.action.admin.cluster.storedscripts.GetStoredScriptRequest) ReplicationRequest(org.opensearch.action.support.replication.ReplicationRequest) AnalyzeRequest(org.opensearch.client.indices.AnalyzeRequest) Scroll(org.opensearch.search.Scroll) RandomObjects(org.opensearch.test.RandomObjects) CountRequest(org.opensearch.client.core.CountRequest) BulkShardRequest(org.opensearch.action.bulk.BulkShardRequest) Script(org.opensearch.script.Script) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) HttpEntity(org.apache.http.HttpEntity) Tuple(org.opensearch.common.collect.Tuple) List(java.util.List) HttpGet(org.apache.http.client.methods.HttpGet) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) SearchTemplateRequest(org.opensearch.script.mustache.SearchTemplateRequest) RankEvalRequest(org.opensearch.index.rankeval.RankEvalRequest) BytesReference(org.opensearch.common.bytes.BytesReference) CheckedBiConsumer(org.opensearch.common.CheckedBiConsumer) HighlightBuilder(org.opensearch.search.fetch.subphase.highlight.HighlightBuilder) HashMap(java.util.HashMap) IndicesOptions(org.opensearch.action.support.IndicesOptions) Function(java.util.function.Function) REQUEST_BODY_CONTENT_TYPE(org.opensearch.client.RequestConverters.REQUEST_BODY_CONTENT_TYPE) DeprecationLogger(org.opensearch.common.logging.DeprecationLogger) HashSet(java.util.HashSet) HttpDelete(org.apache.http.client.methods.HttpDelete) Collections.singletonMap(java.util.Collections.singletonMap) Matchers.hasEntry(org.hamcrest.Matchers.hasEntry) DeleteStoredScriptRequest(org.opensearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest) Collections.emptyMap(java.util.Collections.emptyMap) GetRequest(org.opensearch.action.get.GetRequest) MultiSearchTemplateRequest(org.opensearch.script.mustache.MultiSearchTemplateRequest) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) Matchers(org.hamcrest.Matchers) ActiveShardCount(org.opensearch.action.support.ActiveShardCount) Consumer(java.util.function.Consumer) UpdateByQueryRequest(org.opensearch.index.reindex.UpdateByQueryRequest) HttpPut(org.apache.http.client.methods.HttpPut) MultiTermVectorsRequest(org.opensearch.client.core.MultiTermVectorsRequest) TermsAggregationBuilder(org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder) ClearScrollRequest(org.opensearch.action.search.ClearScrollRequest) StringJoiner(java.util.StringJoiner) JsonXContent(org.opensearch.common.xcontent.json.JsonXContent) IndexRequest(org.opensearch.action.index.IndexRequest) MatchAllQueryBuilder(org.opensearch.index.query.MatchAllQueryBuilder) ReindexRequest(org.opensearch.index.reindex.ReindexRequest) Collections(java.util.Collections) FetchSourceContext(org.opensearch.search.fetch.subphase.FetchSourceContext) InputStream(java.io.InputStream) RandomSearchRequestGenerator.randomSearchRequest(org.opensearch.search.RandomSearchRequestGenerator.randomSearchRequest) MultiSearchRequest(org.opensearch.action.search.MultiSearchRequest) SearchRequest(org.opensearch.action.search.SearchRequest) BytesArray(org.opensearch.common.bytes.BytesArray) HashMap(java.util.HashMap) RandomSearchRequestGenerator.randomSearchRequest(org.opensearch.search.RandomSearchRequestGenerator.randomSearchRequest) MasterNodeRequest(org.opensearch.action.support.master.MasterNodeRequest) WriteRequest(org.opensearch.action.support.WriteRequest) AbstractBulkByScrollRequest(org.opensearch.index.reindex.AbstractBulkByScrollRequest) RatedRequest(org.opensearch.index.rankeval.RatedRequest) DeleteRequest(org.opensearch.action.delete.DeleteRequest) TermVectorsRequest(org.opensearch.client.core.TermVectorsRequest) AcknowledgedRequest(org.opensearch.action.support.master.AcknowledgedRequest) FieldCapabilitiesRequest(org.opensearch.action.fieldcaps.FieldCapabilitiesRequest) UpdateRequest(org.opensearch.action.update.UpdateRequest) GetSourceRequest(org.opensearch.client.core.GetSourceRequest) MultiSearchRequest(org.opensearch.action.search.MultiSearchRequest) DocWriteRequest(org.opensearch.action.DocWriteRequest) SearchScrollRequest(org.opensearch.action.search.SearchScrollRequest) ExplainRequest(org.opensearch.action.explain.ExplainRequest) SearchRequest(org.opensearch.action.search.SearchRequest) PutStoredScriptRequest(org.opensearch.action.admin.cluster.storedscripts.PutStoredScriptRequest) DeleteByQueryRequest(org.opensearch.index.reindex.DeleteByQueryRequest) MultiGetRequest(org.opensearch.action.get.MultiGetRequest) BulkRequest(org.opensearch.action.bulk.BulkRequest) GetStoredScriptRequest(org.opensearch.action.admin.cluster.storedscripts.GetStoredScriptRequest) ReplicationRequest(org.opensearch.action.support.replication.ReplicationRequest) AnalyzeRequest(org.opensearch.client.indices.AnalyzeRequest) CountRequest(org.opensearch.client.core.CountRequest) BulkShardRequest(org.opensearch.action.bulk.BulkShardRequest) SearchTemplateRequest(org.opensearch.script.mustache.SearchTemplateRequest) RankEvalRequest(org.opensearch.index.rankeval.RankEvalRequest) DeleteStoredScriptRequest(org.opensearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest) GetRequest(org.opensearch.action.get.GetRequest) MultiSearchTemplateRequest(org.opensearch.script.mustache.MultiSearchTemplateRequest) UpdateByQueryRequest(org.opensearch.index.reindex.UpdateByQueryRequest) MultiTermVectorsRequest(org.opensearch.client.core.MultiTermVectorsRequest) ClearScrollRequest(org.opensearch.action.search.ClearScrollRequest) IndexRequest(org.opensearch.action.index.IndexRequest) ReindexRequest(org.opensearch.index.reindex.ReindexRequest) ArrayList(java.util.ArrayList) IOException(java.io.IOException) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) MultiSearchRequest(org.opensearch.action.search.MultiSearchRequest) IndicesOptions(org.opensearch.action.support.IndicesOptions) XContentParser(org.opensearch.common.xcontent.XContentParser)

Example 2 with CheckedBiConsumer

use of org.opensearch.common.CheckedBiConsumer in project OpenSearch by opensearch-project.

the class AutoDateHistogramAggregatorTests method testAsSubAggInManyBuckets.

public void testAsSubAggInManyBuckets() throws IOException {
    CheckedBiConsumer<RandomIndexWriter, DateFieldMapper.DateFieldType, IOException> buildIndex = (iw, dft) -> {
        long start = dft.parse("2020-01-01T00:00:00Z");
        long end = dft.parse("2021-01-01T00:00:00Z");
        long anHour = dft.resolution().convert(Instant.ofEpochSecond(TimeUnit.HOURS.toSeconds(1)));
        List<List<IndexableField>> docs = new ArrayList<>();
        int n = 0;
        for (long d = start; d < end; d += anHour) {
            docs.add(org.opensearch.common.collect.List.of(new SortedNumericDocValuesField(AGGREGABLE_DATE, d), new SortedNumericDocValuesField("n", n % 100)));
            n++;
        }
        /*
             * Intentionally add all documents at once to put them on the
             * same shard to make the reduce behavior consistent.
             */
        iw.addDocuments(docs);
    };
    AggregationBuilder builder = new HistogramAggregationBuilder("n").field("n").interval(1).subAggregation(new AutoDateHistogramAggregationBuilder("dh").field(AGGREGABLE_DATE).setNumBuckets(4).subAggregation(new MaxAggregationBuilder("max").field("n")));
    asSubAggTestCase(builder, buildIndex, (InternalHistogram histo) -> {
        assertThat(histo.getBuckets(), hasSize(100));
        for (int n = 0; n < 100; n++) {
            InternalHistogram.Bucket b = histo.getBuckets().get(n);
            InternalAutoDateHistogram dh = b.getAggregations().get("dh");
            assertThat(bucketCountsAsMap(dh), hasEntry(equalTo("2020-01-01T00:00:00.000Z"), either(equalTo(21)).or(equalTo(22))));
            assertThat(bucketCountsAsMap(dh), hasEntry(equalTo("2020-04-01T00:00:00.000Z"), either(equalTo(21)).or(equalTo(22))));
            assertThat(bucketCountsAsMap(dh), hasEntry(equalTo("2020-07-01T00:00:00.000Z"), either(equalTo(22)).or(equalTo(23))));
            assertThat(bucketCountsAsMap(dh), hasEntry(equalTo("2020-10-01T00:00:00.000Z"), either(equalTo(22)).or(equalTo(23))));
            Map<String, Double> expectedMax = new TreeMap<>();
            expectedMax.put("2020-01-01T00:00:00.000Z", (double) n);
            expectedMax.put("2020-04-01T00:00:00.000Z", (double) n);
            expectedMax.put("2020-07-01T00:00:00.000Z", (double) n);
            expectedMax.put("2020-10-01T00:00:00.000Z", (double) n);
            assertThat(maxAsMap(dh), equalTo(expectedMax));
        }
    });
}
Also used : Query(org.apache.lucene.search.Query) Arrays(java.util.Arrays) Aggregation(org.opensearch.search.aggregations.Aggregation) Matchers.either(org.hamcrest.Matchers.either) MatchNoDocsQuery(org.apache.lucene.search.MatchNoDocsQuery) IndexableField(org.apache.lucene.index.IndexableField) ZonedDateTime(java.time.ZonedDateTime) SortedNumericDocValuesField(org.apache.lucene.document.SortedNumericDocValuesField) Version(org.opensearch.Version) Document(org.apache.lucene.document.Document) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) InternalMax(org.opensearch.search.aggregations.metrics.InternalMax) Locale(java.util.Locale) Directory(org.apache.lucene.store.Directory) Map(java.util.Map) ZoneOffset(java.time.ZoneOffset) AggregationInspectionHelper(org.opensearch.search.aggregations.support.AggregationInspectionHelper) SortedSetDocValuesField(org.apache.lucene.document.SortedSetDocValuesField) BytesRef(org.apache.lucene.util.BytesRef) MappedFieldType(org.opensearch.index.mapper.MappedFieldType) DirectoryReader(org.apache.lucene.index.DirectoryReader) DerivativePipelineAggregationBuilder(org.opensearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder) Settings(org.opensearch.common.settings.Settings) MatchAllDocsQuery(org.apache.lucene.search.MatchAllDocsQuery) Instant(java.time.Instant) List(java.util.List) LocalDate(java.time.LocalDate) Matchers.equalTo(org.hamcrest.Matchers.equalTo) IndexSettings(org.opensearch.index.IndexSettings) InternalStats(org.opensearch.search.aggregations.metrics.InternalStats) MultiBucketConsumerService(org.opensearch.search.aggregations.MultiBucketConsumerService) IndexReader(org.apache.lucene.index.IndexReader) IndexSearcher(org.apache.lucene.search.IndexSearcher) InternalSimpleValue(org.opensearch.search.aggregations.pipeline.InternalSimpleValue) LongPoint(org.apache.lucene.document.LongPoint) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) StringTerms(org.opensearch.search.aggregations.bucket.terms.StringTerms) CheckedBiConsumer(org.opensearch.common.CheckedBiConsumer) HashMap(java.util.HashMap) NumberFieldMapper(org.opensearch.index.mapper.NumberFieldMapper) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) Matchers.hasSize(org.hamcrest.Matchers.hasSize) Matchers.hasEntry(org.hamcrest.Matchers.hasEntry) Matchers(org.hamcrest.Matchers) IOException(java.io.IOException) MaxAggregationBuilder(org.opensearch.search.aggregations.metrics.MaxAggregationBuilder) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) AggregationBuilders(org.opensearch.search.aggregations.AggregationBuilders) TreeMap(java.util.TreeMap) DateFieldMapper(org.opensearch.index.mapper.DateFieldMapper) TermsAggregationBuilder(org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder) YearMonth(java.time.YearMonth) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter) Collections(java.util.Collections) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) DerivativePipelineAggregationBuilder(org.opensearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder) MaxAggregationBuilder(org.opensearch.search.aggregations.metrics.MaxAggregationBuilder) TermsAggregationBuilder(org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder) IOException(java.io.IOException) TreeMap(java.util.TreeMap) LongPoint(org.apache.lucene.document.LongPoint) IndexableField(org.apache.lucene.index.IndexableField) MaxAggregationBuilder(org.opensearch.search.aggregations.metrics.MaxAggregationBuilder) SortedNumericDocValuesField(org.apache.lucene.document.SortedNumericDocValuesField) List(java.util.List) ArrayList(java.util.ArrayList) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter)

Example 3 with CheckedBiConsumer

use of org.opensearch.common.CheckedBiConsumer in project OpenSearch by opensearch-project.

the class AutoDateHistogramAggregatorTests method testAsSubAggWithIncreasedRounding.

public void testAsSubAggWithIncreasedRounding() throws IOException {
    CheckedBiConsumer<RandomIndexWriter, DateFieldMapper.DateFieldType, IOException> buildIndex = (iw, dft) -> {
        long start = dft.parse("2020-01-01T00:00:00Z");
        long end = dft.parse("2021-01-01T00:00:00Z");
        long useC = dft.parse("2020-07-01T00:00Z");
        long anHour = dft.resolution().convert(Instant.ofEpochSecond(TimeUnit.HOURS.toSeconds(1)));
        List<List<IndexableField>> docs = new ArrayList<>();
        BytesRef aBytes = new BytesRef("a");
        BytesRef bBytes = new BytesRef("b");
        BytesRef cBytes = new BytesRef("c");
        int n = 0;
        for (long d = start; d < end; d += anHour) {
            docs.add(org.opensearch.common.collect.List.of(new SortedNumericDocValuesField(AGGREGABLE_DATE, d), new SortedSetDocValuesField("k1", aBytes), new SortedSetDocValuesField("k1", d < useC ? bBytes : cBytes), new SortedNumericDocValuesField("n", n++)));
        }
        /*
             * Intentionally add all documents at once to put them on the
             * same shard to make the reduce behavior consistent.
             */
        iw.addDocuments(docs);
    };
    AggregationBuilder builder = new TermsAggregationBuilder("k1").field("k1").subAggregation(new AutoDateHistogramAggregationBuilder("dh").field(AGGREGABLE_DATE).setNumBuckets(4).subAggregation(new MaxAggregationBuilder("max").field("n")));
    asSubAggTestCase(builder, buildIndex, (StringTerms terms) -> {
        StringTerms.Bucket a = terms.getBucketByKey("a");
        InternalAutoDateHistogram adh = a.getAggregations().get("dh");
        Map<String, Integer> expectedDocCount = new TreeMap<>();
        expectedDocCount.put("2020-01-01T00:00:00.000Z", 2184);
        expectedDocCount.put("2020-04-01T00:00:00.000Z", 2184);
        expectedDocCount.put("2020-07-01T00:00:00.000Z", 2208);
        expectedDocCount.put("2020-10-01T00:00:00.000Z", 2208);
        assertThat(bucketCountsAsMap(adh), equalTo(expectedDocCount));
        Map<String, Double> expectedMax = new TreeMap<>();
        expectedMax.put("2020-01-01T00:00:00.000Z", 2183.0);
        expectedMax.put("2020-04-01T00:00:00.000Z", 4367.0);
        expectedMax.put("2020-07-01T00:00:00.000Z", 6575.0);
        expectedMax.put("2020-10-01T00:00:00.000Z", 8783.0);
        assertThat(maxAsMap(adh), equalTo(expectedMax));
        StringTerms.Bucket b = terms.getBucketByKey("b");
        InternalAutoDateHistogram bdh = b.getAggregations().get("dh");
        expectedDocCount.clear();
        expectedDocCount.put("2020-01-01T00:00:00.000Z", 2184);
        expectedDocCount.put("2020-04-01T00:00:00.000Z", 2184);
        assertThat(bucketCountsAsMap(bdh), equalTo(expectedDocCount));
        expectedMax.clear();
        expectedMax.put("2020-01-01T00:00:00.000Z", 2183.0);
        expectedMax.put("2020-04-01T00:00:00.000Z", 4367.0);
        assertThat(maxAsMap(bdh), equalTo(expectedMax));
        StringTerms.Bucket c = terms.getBucketByKey("c");
        InternalAutoDateHistogram cdh = c.getAggregations().get("dh");
        expectedDocCount.clear();
        expectedDocCount.put("2020-07-01T00:00:00.000Z", 2208);
        expectedDocCount.put("2020-10-01T00:00:00.000Z", 2208);
        assertThat(bucketCountsAsMap(cdh), equalTo(expectedDocCount));
        expectedMax.clear();
        expectedMax.put("2020-07-01T00:00:00.000Z", 6575.0);
        expectedMax.put("2020-10-01T00:00:00.000Z", 8783.0);
        assertThat(maxAsMap(cdh), equalTo(expectedMax));
    });
}
Also used : Query(org.apache.lucene.search.Query) Arrays(java.util.Arrays) Aggregation(org.opensearch.search.aggregations.Aggregation) Matchers.either(org.hamcrest.Matchers.either) MatchNoDocsQuery(org.apache.lucene.search.MatchNoDocsQuery) IndexableField(org.apache.lucene.index.IndexableField) ZonedDateTime(java.time.ZonedDateTime) SortedNumericDocValuesField(org.apache.lucene.document.SortedNumericDocValuesField) Version(org.opensearch.Version) Document(org.apache.lucene.document.Document) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) InternalMax(org.opensearch.search.aggregations.metrics.InternalMax) Locale(java.util.Locale) Directory(org.apache.lucene.store.Directory) Map(java.util.Map) ZoneOffset(java.time.ZoneOffset) AggregationInspectionHelper(org.opensearch.search.aggregations.support.AggregationInspectionHelper) SortedSetDocValuesField(org.apache.lucene.document.SortedSetDocValuesField) BytesRef(org.apache.lucene.util.BytesRef) MappedFieldType(org.opensearch.index.mapper.MappedFieldType) DirectoryReader(org.apache.lucene.index.DirectoryReader) DerivativePipelineAggregationBuilder(org.opensearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder) Settings(org.opensearch.common.settings.Settings) MatchAllDocsQuery(org.apache.lucene.search.MatchAllDocsQuery) Instant(java.time.Instant) List(java.util.List) LocalDate(java.time.LocalDate) Matchers.equalTo(org.hamcrest.Matchers.equalTo) IndexSettings(org.opensearch.index.IndexSettings) InternalStats(org.opensearch.search.aggregations.metrics.InternalStats) MultiBucketConsumerService(org.opensearch.search.aggregations.MultiBucketConsumerService) IndexReader(org.apache.lucene.index.IndexReader) IndexSearcher(org.apache.lucene.search.IndexSearcher) InternalSimpleValue(org.opensearch.search.aggregations.pipeline.InternalSimpleValue) LongPoint(org.apache.lucene.document.LongPoint) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) StringTerms(org.opensearch.search.aggregations.bucket.terms.StringTerms) CheckedBiConsumer(org.opensearch.common.CheckedBiConsumer) HashMap(java.util.HashMap) NumberFieldMapper(org.opensearch.index.mapper.NumberFieldMapper) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) Matchers.hasSize(org.hamcrest.Matchers.hasSize) Matchers.hasEntry(org.hamcrest.Matchers.hasEntry) Matchers(org.hamcrest.Matchers) IOException(java.io.IOException) MaxAggregationBuilder(org.opensearch.search.aggregations.metrics.MaxAggregationBuilder) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) AggregationBuilders(org.opensearch.search.aggregations.AggregationBuilders) TreeMap(java.util.TreeMap) DateFieldMapper(org.opensearch.index.mapper.DateFieldMapper) TermsAggregationBuilder(org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder) YearMonth(java.time.YearMonth) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter) Collections(java.util.Collections) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) DerivativePipelineAggregationBuilder(org.opensearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder) MaxAggregationBuilder(org.opensearch.search.aggregations.metrics.MaxAggregationBuilder) TermsAggregationBuilder(org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder) IOException(java.io.IOException) TreeMap(java.util.TreeMap) IndexableField(org.apache.lucene.index.IndexableField) MaxAggregationBuilder(org.opensearch.search.aggregations.metrics.MaxAggregationBuilder) SortedNumericDocValuesField(org.apache.lucene.document.SortedNumericDocValuesField) TermsAggregationBuilder(org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder) StringTerms(org.opensearch.search.aggregations.bucket.terms.StringTerms) List(java.util.List) ArrayList(java.util.ArrayList) SortedSetDocValuesField(org.apache.lucene.document.SortedSetDocValuesField) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter) BytesRef(org.apache.lucene.util.BytesRef)

Example 4 with CheckedBiConsumer

use of org.opensearch.common.CheckedBiConsumer in project OpenSearch by opensearch-project.

the class DateHistogramAggregatorTestCase method asSubAggTestCase.

protected final <R extends InternalAggregation> void asSubAggTestCase(AggregationBuilder builder, Consumer<R> verify) throws IOException {
    CheckedBiConsumer<RandomIndexWriter, DateFieldMapper.DateFieldType, IOException> buildIndex = (iw, dft) -> {
        iw.addDocument(org.opensearch.common.collect.List.of(new SortedNumericDocValuesField(AGGREGABLE_DATE, dft.parse("2020-02-01T00:00:00Z")), new SortedSetDocValuesField("k1", new BytesRef("a")), new SortedSetDocValuesField("k2", new BytesRef("a")), new SortedNumericDocValuesField("n", 1)));
        iw.addDocument(org.opensearch.common.collect.List.of(new SortedNumericDocValuesField(AGGREGABLE_DATE, dft.parse("2020-03-01T00:00:00Z")), new SortedSetDocValuesField("k1", new BytesRef("a")), new SortedSetDocValuesField("k2", new BytesRef("a")), new SortedNumericDocValuesField("n", 2)));
        iw.addDocument(org.opensearch.common.collect.List.of(new SortedNumericDocValuesField(AGGREGABLE_DATE, dft.parse("2021-02-01T00:00:00Z")), new SortedSetDocValuesField("k1", new BytesRef("a")), new SortedSetDocValuesField("k2", new BytesRef("a")), new SortedNumericDocValuesField("n", 3)));
        iw.addDocument(org.opensearch.common.collect.List.of(new SortedNumericDocValuesField(AGGREGABLE_DATE, dft.parse("2021-03-01T00:00:00Z")), new SortedSetDocValuesField("k1", new BytesRef("a")), new SortedSetDocValuesField("k2", new BytesRef("b")), new SortedNumericDocValuesField("n", 4)));
        iw.addDocument(org.opensearch.common.collect.List.of(new SortedNumericDocValuesField(AGGREGABLE_DATE, dft.parse("2020-02-01T00:00:00Z")), new SortedSetDocValuesField("k1", new BytesRef("b")), new SortedSetDocValuesField("k2", new BytesRef("b")), new SortedNumericDocValuesField("n", 5)));
    };
    asSubAggTestCase(builder, buildIndex, verify);
}
Also used : SortedSetDocValuesField(org.apache.lucene.document.SortedSetDocValuesField) AggregatorTestCase(org.opensearch.search.aggregations.AggregatorTestCase) BytesRef(org.apache.lucene.util.BytesRef) CheckedBiConsumer(org.opensearch.common.CheckedBiConsumer) SortedNumericDocValuesField(org.apache.lucene.document.SortedNumericDocValuesField) IOException(java.io.IOException) MatchAllDocsQuery(org.apache.lucene.search.MatchAllDocsQuery) NumberFieldMapper(org.opensearch.index.mapper.NumberFieldMapper) Consumer(java.util.function.Consumer) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) DateFieldMapper(org.opensearch.index.mapper.DateFieldMapper) NumberType(org.opensearch.index.mapper.NumberFieldMapper.NumberType) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter) KeywordFieldMapper(org.opensearch.index.mapper.KeywordFieldMapper) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) Collections(java.util.Collections) SortedNumericDocValuesField(org.apache.lucene.document.SortedNumericDocValuesField) IOException(java.io.IOException) SortedSetDocValuesField(org.apache.lucene.document.SortedSetDocValuesField) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter) BytesRef(org.apache.lucene.util.BytesRef)

Example 5 with CheckedBiConsumer

use of org.opensearch.common.CheckedBiConsumer in project OpenSearch by opensearch-project.

the class MultiSearchRequestTests method testMultiLineSerialization.

public void testMultiLineSerialization() throws IOException {
    int iters = 16;
    for (int i = 0; i < iters; i++) {
        // The only formats that support stream separator
        XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
        MultiSearchRequest originalRequest = createMultiSearchRequest();
        byte[] originalBytes = MultiSearchRequest.writeMultiLineFormat(originalRequest, xContentType.xContent());
        MultiSearchRequest parsedRequest = new MultiSearchRequest();
        CheckedBiConsumer<SearchRequest, XContentParser, IOException> consumer = (r, p) -> {
            SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(p, false);
            if (searchSourceBuilder.equals(new SearchSourceBuilder()) == false) {
                r.source(searchSourceBuilder);
            }
            parsedRequest.add(r);
        };
        MultiSearchRequest.readMultiLineFormat(new BytesArray(originalBytes), xContentType.xContent(), consumer, null, null, null, null, null, xContentRegistry(), true, deprecationLogger);
        assertEquals(originalRequest, parsedRequest);
    }
}
Also used : BytesReference(org.opensearch.common.bytes.BytesReference) CheckedBiConsumer(org.opensearch.common.CheckedBiConsumer) Version(org.opensearch.Version) IndicesOptions(org.opensearch.action.support.IndicesOptions) ParseField(org.opensearch.common.ParseField) Strings(org.opensearch.common.Strings) NamedWriteableRegistry(org.opensearch.common.io.stream.NamedWriteableRegistry) ArrayList(java.util.ArrayList) Collections.singletonList(java.util.Collections.singletonList) DeprecationLogger(org.opensearch.common.logging.DeprecationLogger) XContentParser(org.opensearch.common.xcontent.XContentParser) FakeRestRequest(org.opensearch.test.rest.FakeRestRequest) RandomSearchRequestGenerator.randomSearchRequest(org.opensearch.search.RandomSearchRequestGenerator.randomSearchRequest) StreamsUtils(org.opensearch.test.StreamsUtils) VersionUtils(org.opensearch.test.VersionUtils) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) Scroll(org.opensearch.search.Scroll) EqualsHashCodeTestUtils.checkEqualsAndHashCode(org.opensearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode) TimeValue(org.opensearch.common.unit.TimeValue) RestRequest(org.opensearch.rest.RestRequest) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) IOException(java.io.IOException) StandardCharsets(java.nio.charset.StandardCharsets) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) XContentHelper(org.opensearch.common.xcontent.XContentHelper) TimeUnit(java.util.concurrent.TimeUnit) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) List(java.util.List) QueryBuilder(org.opensearch.index.query.QueryBuilder) BytesArray(org.opensearch.common.bytes.BytesArray) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) RestMultiSearchAction(org.opensearch.rest.action.search.RestMultiSearchAction) Matchers.equalTo(org.hamcrest.Matchers.equalTo) CheckedRunnable(org.opensearch.common.CheckedRunnable) JsonXContent(org.opensearch.common.xcontent.json.JsonXContent) XContentType(org.opensearch.common.xcontent.XContentType) Matchers.is(org.hamcrest.Matchers.is) MatchAllQueryBuilder(org.opensearch.index.query.MatchAllQueryBuilder) Collections(java.util.Collections) RandomSearchRequestGenerator.randomSearchRequest(org.opensearch.search.RandomSearchRequestGenerator.randomSearchRequest) BytesArray(org.opensearch.common.bytes.BytesArray) XContentType(org.opensearch.common.xcontent.XContentType) IOException(java.io.IOException) XContentParser(org.opensearch.common.xcontent.XContentParser) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder)

Aggregations

IOException (java.io.IOException)5 Collections (java.util.Collections)5 CheckedBiConsumer (org.opensearch.common.CheckedBiConsumer)5 ArrayList (java.util.ArrayList)4 List (java.util.List)4 Map (java.util.Map)4 Consumer (java.util.function.Consumer)4 Arrays (java.util.Arrays)3 HashMap (java.util.HashMap)3 Locale (java.util.Locale)3 TimeUnit (java.util.concurrent.TimeUnit)3 SortedNumericDocValuesField (org.apache.lucene.document.SortedNumericDocValuesField)3 SortedSetDocValuesField (org.apache.lucene.document.SortedSetDocValuesField)3 RandomIndexWriter (org.apache.lucene.index.RandomIndexWriter)3 MatchAllDocsQuery (org.apache.lucene.search.MatchAllDocsQuery)3 BytesRef (org.apache.lucene.util.BytesRef)3 Matchers (org.hamcrest.Matchers)3 Matchers.equalTo (org.hamcrest.Matchers.equalTo)3 Matchers.hasEntry (org.hamcrest.Matchers.hasEntry)3 DateFieldMapper (org.opensearch.index.mapper.DateFieldMapper)3