Examples with InternalMax org.opensearch.search.aggregations.metrics.InternalMax used on opensource projects

Example 1 with InternalMax

use of org.opensearch.search.aggregations.metrics.InternalMax in project OpenSearch by opensearch-project.

the class SearchPhaseControllerTests method testConsumerConcurrently.

public void testConsumerConcurrently() throws Exception {
    int expectedNumResults = randomIntBetween(1, 100);
    int bufferSize = randomIntBetween(2, 200);
    SearchRequest request = randomSearchRequest();
    request.source(new SearchSourceBuilder().aggregation(AggregationBuilders.avg("foo")));
    request.setBatchedReduceSize(bufferSize);
    ArraySearchPhaseResults<SearchPhaseResult> consumer = searchPhaseController.newSearchPhaseResults(fixedExecutor, new NoopCircuitBreaker(CircuitBreaker.REQUEST), SearchProgressListener.NOOP, request, expectedNumResults, exc -> {
    });
    AtomicInteger max = new AtomicInteger();
    Thread[] threads = new Thread[expectedNumResults];
    CountDownLatch latch = new CountDownLatch(expectedNumResults);
    for (int i = 0; i < expectedNumResults; i++) {
        int id = i;
        threads[i] = new Thread(() -> {
            int number = randomIntBetween(1, 1000);
            max.updateAndGet(prev -> Math.max(prev, number));
            QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId("", id), new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE), null);
            result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] { new ScoreDoc(0, number) }), number), new DocValueFormat[0]);
            InternalAggregations aggs = InternalAggregations.from(Collections.singletonList(new InternalMax("test", (double) number, DocValueFormat.RAW, Collections.emptyMap())));
            result.aggregations(aggs);
            result.setShardIndex(id);
            result.size(1);
            consumer.consumeResult(result, latch::countDown);
        });
        threads[i].start();
    }
    for (int i = 0; i < expectedNumResults; i++) {
        threads[i].join();
    }
    latch.await();
    SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
    assertAggReduction(request);
    InternalMax internalMax = (InternalMax) reduce.aggregations.asList().get(0);
    assertEquals(max.get(), internalMax.getValue(), 0.0D);
    assertEquals(1, reduce.sortedTopDocs.scoreDocs.length);
    assertEquals(max.get(), reduce.maxScore, 0.0f);
    assertEquals(expectedNumResults, reduce.totalHits.value);
    assertEquals(max.get(), reduce.sortedTopDocs.scoreDocs[0].score, 0.0f);
    assertFalse(reduce.sortedTopDocs.isSortedByField);
    assertNull(reduce.sortedTopDocs.sortFields);
    assertNull(reduce.sortedTopDocs.collapseField);
    assertNull(reduce.sortedTopDocs.collapseValues);
}

Example 2 with InternalMax

use of org.opensearch.search.aggregations.metrics.InternalMax in project OpenSearch by opensearch-project.

the class SearchPhaseControllerTests method testConsumerOnlyAggs.

public void testConsumerOnlyAggs() throws Exception {
    int expectedNumResults = randomIntBetween(1, 100);
    int bufferSize = randomIntBetween(2, 200);
    SearchRequest request = randomSearchRequest();
    request.source(new SearchSourceBuilder().aggregation(AggregationBuilders.avg("foo")).size(0));
    request.setBatchedReduceSize(bufferSize);
    QueryPhaseResultConsumer consumer = searchPhaseController.newSearchPhaseResults(fixedExecutor, new NoopCircuitBreaker(CircuitBreaker.REQUEST), SearchProgressListener.NOOP, request, expectedNumResults, exc -> {
    });
    AtomicInteger max = new AtomicInteger();
    CountDownLatch latch = new CountDownLatch(expectedNumResults);
    for (int i = 0; i < expectedNumResults; i++) {
        int number = randomIntBetween(1, 1000);
        max.updateAndGet(prev -> Math.max(prev, number));
        QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
        result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), number), new DocValueFormat[0]);
        InternalAggregations aggs = InternalAggregations.from(Collections.singletonList(new InternalMax("test", (double) number, DocValueFormat.RAW, Collections.emptyMap())));
        result.aggregations(aggs);
        result.setShardIndex(i);
        result.size(1);
        consumer.consumeResult(result, latch::countDown);
    }
    latch.await();
    SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
    assertAggReduction(request);
    InternalMax internalMax = (InternalMax) reduce.aggregations.asList().get(0);
    assertEquals(max.get(), internalMax.getValue(), 0.0D);
    assertEquals(0, reduce.sortedTopDocs.scoreDocs.length);
    assertEquals(max.get(), reduce.maxScore, 0.0f);
    assertEquals(expectedNumResults, reduce.totalHits.value);
    assertFalse(reduce.sortedTopDocs.isSortedByField);
    assertNull(reduce.sortedTopDocs.sortFields);
    assertNull(reduce.sortedTopDocs.collapseField);
    assertNull(reduce.sortedTopDocs.collapseValues);
}

Example 3 with InternalMax

use of org.opensearch.search.aggregations.metrics.InternalMax in project OpenSearch by opensearch-project.

the class SearchPhaseControllerTests method testProgressListener.

public void testProgressListener() throws Exception {
    int expectedNumResults = randomIntBetween(10, 100);
    for (int bufferSize : new int[] { expectedNumResults, expectedNumResults / 2, expectedNumResults / 4, 2 }) {
        SearchRequest request = randomSearchRequest();
        request.source(new SearchSourceBuilder().aggregation(AggregationBuilders.avg("foo")));
        request.setBatchedReduceSize(bufferSize);
        AtomicInteger numQueryResultListener = new AtomicInteger();
        AtomicInteger numQueryFailureListener = new AtomicInteger();
        AtomicInteger numReduceListener = new AtomicInteger();
        AtomicReference<InternalAggregations> finalAggsListener = new AtomicReference<>();
        AtomicReference<TotalHits> totalHitsListener = new AtomicReference<>();
        SearchProgressListener progressListener = new SearchProgressListener() {

            @Override
            public void onQueryResult(int shardIndex) {
                assertThat(shardIndex, lessThan(expectedNumResults));
                numQueryResultListener.incrementAndGet();
            }

            @Override
            public void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
                assertThat(shardIndex, lessThan(expectedNumResults));
                numQueryFailureListener.incrementAndGet();
            }

            @Override
            public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
                assertEquals(numReduceListener.incrementAndGet(), reducePhase);
            }

            @Override
            public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
                totalHitsListener.set(totalHits);
                finalAggsListener.set(aggs);
                assertEquals(numReduceListener.incrementAndGet(), reducePhase);
            }
        };
        QueryPhaseResultConsumer consumer = searchPhaseController.newSearchPhaseResults(fixedExecutor, new NoopCircuitBreaker(CircuitBreaker.REQUEST), progressListener, request, expectedNumResults, exc -> {
        });
        AtomicInteger max = new AtomicInteger();
        Thread[] threads = new Thread[expectedNumResults];
        CountDownLatch latch = new CountDownLatch(expectedNumResults);
        for (int i = 0; i < expectedNumResults; i++) {
            int id = i;
            threads[i] = new Thread(() -> {
                int number = randomIntBetween(1, 1000);
                max.updateAndGet(prev -> Math.max(prev, number));
                QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId("", id), new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE), null);
                result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] { new ScoreDoc(0, number) }), number), new DocValueFormat[0]);
                InternalAggregations aggs = InternalAggregations.from(Collections.singletonList(new InternalMax("test", (double) number, DocValueFormat.RAW, Collections.emptyMap())));
                result.aggregations(aggs);
                result.setShardIndex(id);
                result.size(1);
                consumer.consumeResult(result, latch::countDown);
            });
            threads[i].start();
        }
        for (int i = 0; i < expectedNumResults; i++) {
            threads[i].join();
        }
        latch.await();
        SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
        assertAggReduction(request);
        InternalMax internalMax = (InternalMax) reduce.aggregations.asList().get(0);
        assertEquals(max.get(), internalMax.getValue(), 0.0D);
        assertEquals(1, reduce.sortedTopDocs.scoreDocs.length);
        assertEquals(max.get(), reduce.maxScore, 0.0f);
        assertEquals(expectedNumResults, reduce.totalHits.value);
        assertEquals(max.get(), reduce.sortedTopDocs.scoreDocs[0].score, 0.0f);
        assertFalse(reduce.sortedTopDocs.isSortedByField);
        assertNull(reduce.sortedTopDocs.sortFields);
        assertNull(reduce.sortedTopDocs.collapseField);
        assertNull(reduce.sortedTopDocs.collapseValues);
        assertEquals(reduce.aggregations, finalAggsListener.get());
        assertEquals(reduce.totalHits, totalHitsListener.get());
        assertEquals(expectedNumResults, numQueryResultListener.get());
        assertEquals(0, numQueryFailureListener.get());
        assertEquals(numReduceListener.get(), reduce.numReducePhases);
    }
}

Example 4 with InternalMax

use of org.opensearch.search.aggregations.metrics.InternalMax in project OpenSearch by opensearch-project.

the class CompositeAggregatorTests method testWithTermsSubAggExecutionMode.

public void testWithTermsSubAggExecutionMode() throws Exception {
    // test with no bucket
    for (Aggregator.SubAggCollectionMode mode : Aggregator.SubAggCollectionMode.values()) {
        testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("keyword")), Collections.singletonList(createDocument()), () -> {
            TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder("keyword").field("keyword");
            return new CompositeAggregationBuilder("name", Collections.singletonList(terms)).subAggregation(new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING).field("terms").collectMode(mode).subAggregation(new MaxAggregationBuilder("max").field("long")));
        }, (result) -> {
            assertEquals(0, result.getBuckets().size());
        });
    }
    final List<Map<String, List<Object>>> dataset = new ArrayList<>();
    dataset.addAll(Arrays.asList(createDocument("keyword", "a", "terms", "a", "long", 50L), createDocument("keyword", "c", "terms", "d", "long", 78L), createDocument("keyword", "a", "terms", "w", "long", 78L), createDocument("keyword", "d", "terms", "y", "long", 76L), createDocument("keyword", "c", "terms", "y", "long", 70L)));
    for (Aggregator.SubAggCollectionMode mode : Aggregator.SubAggCollectionMode.values()) {
        testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("keyword")), dataset, () -> {
            TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder("keyword").field("keyword");
            return new CompositeAggregationBuilder("name", Collections.singletonList(terms)).subAggregation(new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING).field("terms").collectMode(mode).subAggregation(new MaxAggregationBuilder("max").field("long")));
        }, (result) -> {
            assertEquals(3, result.getBuckets().size());
            assertEquals("{keyword=a}", result.getBuckets().get(0).getKeyAsString());
            assertEquals(2L, result.getBuckets().get(0).getDocCount());
            StringTerms subTerms = result.getBuckets().get(0).getAggregations().get("terms");
            assertEquals(2, subTerms.getBuckets().size());
            assertEquals("a", subTerms.getBuckets().get(0).getKeyAsString());
            assertEquals("w", subTerms.getBuckets().get(1).getKeyAsString());
            InternalMax max = subTerms.getBuckets().get(0).getAggregations().get("max");
            assertEquals(50L, (long) max.getValue());
            max = subTerms.getBuckets().get(1).getAggregations().get("max");
            assertEquals(78L, (long) max.getValue());
            assertEquals("{keyword=c}", result.getBuckets().get(1).getKeyAsString());
            assertEquals(2L, result.getBuckets().get(1).getDocCount());
            subTerms = result.getBuckets().get(1).getAggregations().get("terms");
            assertEquals(2, subTerms.getBuckets().size());
            assertEquals("d", subTerms.getBuckets().get(0).getKeyAsString());
            assertEquals("y", subTerms.getBuckets().get(1).getKeyAsString());
            max = subTerms.getBuckets().get(0).getAggregations().get("max");
            assertEquals(78L, (long) max.getValue());
            max = subTerms.getBuckets().get(1).getAggregations().get("max");
            assertEquals(70L, (long) max.getValue());
            assertEquals("{keyword=d}", result.getBuckets().get(2).getKeyAsString());
            assertEquals(1L, result.getBuckets().get(2).getDocCount());
            subTerms = result.getBuckets().get(2).getAggregations().get("terms");
            assertEquals(1, subTerms.getBuckets().size());
            assertEquals("y", subTerms.getBuckets().get(0).getKeyAsString());
            max = subTerms.getBuckets().get(0).getAggregations().get("max");
            assertEquals(76L, (long) max.getValue());
        });
    }
}

Example 5 with InternalMax

use of org.opensearch.search.aggregations.metrics.InternalMax in project OpenSearch by opensearch-project.

the class NestedAggregatorTests method testNestedWithPipeline.

/**
 * This tests to make sure pipeline aggs embedded under a SingleBucket agg (like nested)
 * are properly reduced
 */
public void testNestedWithPipeline() throws IOException {
    int numRootDocs = randomIntBetween(1, 20);
    int expectedNestedDocs = 0;
    double expectedMaxValue = Double.NEGATIVE_INFINITY;
    try (Directory directory = newDirectory()) {
        try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
            for (int i = 0; i < numRootDocs; i++) {
                List<Document> documents = new ArrayList<>();
                expectedMaxValue = Math.max(expectedMaxValue, generateMaxDocs(documents, 1, i, NESTED_OBJECT, VALUE_FIELD_NAME));
                expectedNestedDocs += 1;
                Document document = new Document();
                document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.FIELD_TYPE));
                document.add(sequenceIDFields.primaryTerm);
                documents.add(document);
                iw.addDocuments(documents);
            }
            iw.commit();
        }
        try (IndexReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) {
            NestedAggregationBuilder nestedBuilder = new NestedAggregationBuilder(NESTED_AGG, NESTED_OBJECT).subAggregation(new TermsAggregationBuilder("terms").field(VALUE_FIELD_NAME).userValueTypeHint(ValueType.NUMERIC).subAggregation(new MaxAggregationBuilder(MAX_AGG_NAME).field(VALUE_FIELD_NAME)).subAggregation(new BucketScriptPipelineAggregationBuilder("bucketscript", Collections.singletonMap("_value", MAX_AGG_NAME), new Script(ScriptType.INLINE, MockScriptEngine.NAME, INVERSE_SCRIPT, Collections.emptyMap()))));
            MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG);
            InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType);
            assertEquals(expectedNestedDocs, nested.getDocCount());
            assertEquals(NESTED_AGG, nested.getName());
            assertEquals(expectedNestedDocs, nested.getDocCount());
            InternalTerms<?, LongTerms.Bucket> terms = (InternalTerms) nested.getProperty("terms");
            assertNotNull(terms);
            for (LongTerms.Bucket bucket : terms.getBuckets()) {
                InternalMax max = (InternalMax) bucket.getAggregations().asMap().get(MAX_AGG_NAME);
                InternalSimpleValue bucketScript = (InternalSimpleValue) bucket.getAggregations().asMap().get("bucketscript");
                assertNotNull(max);
                assertNotNull(bucketScript);
                assertEquals(max.getValue(), -bucketScript.getValue(), Double.MIN_VALUE);
            }
            assertTrue(AggregationInspectionHelper.hasValue(nested));
        }
    }
}