Search in sources :

Example 21 with SearchRequest

use of com.yelp.nrtsearch.server.grpc.SearchRequest in project nrtsearch by Yelp.

the class DocCollectorTest method testUsesDefaultTerminateAfter.

@Test
public void testUsesDefaultTerminateAfter() {
    IndexState indexState = Mockito.mock(IndexState.class);
    when(indexState.getDefaultTerminateAfter()).thenReturn(100);
    SearchRequest request = SearchRequest.newBuilder().setTopHits(10).build();
    TestDocCollector docCollector = new TestDocCollector(request, indexState);
    assertTrue(docCollector.getManager() instanceof TestDocCollector.TestCollectorManager);
    assertTrue(docCollector.getWrappedManager() instanceof TerminateAfterWrapper);
    assertEquals(100, ((TerminateAfterWrapper<?>) docCollector.getWrappedManager()).getTerminateAfter());
}
Also used : SearchRequest(com.yelp.nrtsearch.server.grpc.SearchRequest) TerminateAfterWrapper(com.yelp.nrtsearch.server.luceneserver.search.TerminateAfterWrapper) IndexState(com.yelp.nrtsearch.server.luceneserver.IndexState) Test(org.junit.Test)

Example 22 with SearchRequest

use of com.yelp.nrtsearch.server.grpc.SearchRequest in project nrtsearch by Yelp.

the class SearchRequestProcessor method buildDocCollector.

/**
 * Build {@link DocCollector} to provide the {@link org.apache.lucene.search.CollectorManager} for
 * collecting hits for this query.
 *
 * @return collector
 */
private static DocCollector buildDocCollector(CollectorCreatorContext collectorCreatorContext) {
    SearchRequest searchRequest = collectorCreatorContext.getRequest();
    List<AdditionalCollectorManager<? extends Collector, ? extends CollectorResult>> additionalCollectors = searchRequest.getCollectorsMap().entrySet().stream().map(e -> CollectorCreator.getInstance().createCollectorManager(collectorCreatorContext, e.getKey(), e.getValue())).collect(Collectors.toList());
    if (searchRequest.getQuerySort().getFields().getSortedFieldsList().isEmpty()) {
        if (hasLargeNumHits(searchRequest)) {
            return new LargeNumHitsCollector(collectorCreatorContext, additionalCollectors);
        } else {
            return new RelevanceCollector(collectorCreatorContext, additionalCollectors);
        }
    } else {
        return new SortFieldCollector(collectorCreatorContext, additionalCollectors);
    }
}
Also used : SearchResponse(com.yelp.nrtsearch.server.grpc.SearchResponse) ScriptParamsUtils(com.yelp.nrtsearch.server.utils.ScriptParamsUtils) SearcherTaxonomyManager(org.apache.lucene.facet.taxonomy.SearcherTaxonomyManager) Query(org.apache.lucene.search.Query) QueryRescorer(com.yelp.nrtsearch.server.grpc.QueryRescorer) PluginRescorer(com.yelp.nrtsearch.server.grpc.PluginRescorer) HashMap(java.util.HashMap) AdditionalCollectorManager(com.yelp.nrtsearch.server.luceneserver.search.collectors.AdditionalCollectorManager) VirtualField(com.yelp.nrtsearch.server.grpc.VirtualField) CollectorCreator(com.yelp.nrtsearch.server.luceneserver.search.collectors.CollectorCreator) ArrayList(java.util.ArrayList) QueryBuilder(org.apache.lucene.util.QueryBuilder) MultiFieldQueryParser(org.apache.lucene.queryparser.classic.MultiFieldQueryParser) VirtualFieldDef(com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef) IndexableFieldDef(com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef) Map(java.util.Map) DrillDownQuery(org.apache.lucene.facet.DrillDownQuery) ShardState(com.yelp.nrtsearch.server.luceneserver.ShardState) ScoreScript(com.yelp.nrtsearch.server.luceneserver.script.ScoreScript) DefaultSharedDocContext(com.yelp.nrtsearch.server.luceneserver.doc.DefaultSharedDocContext) SimpleQueryParser(org.apache.lucene.queryparser.simple.SimpleQueryParser) QueryParserBase(org.apache.lucene.queryparser.classic.QueryParserBase) CollectorResult(com.yelp.nrtsearch.server.grpc.CollectorResult) FieldDef(com.yelp.nrtsearch.server.luceneserver.field.FieldDef) CollectorCreatorContext(com.yelp.nrtsearch.server.luceneserver.search.collectors.CollectorCreatorContext) RescoreTask(com.yelp.nrtsearch.server.luceneserver.rescore.RescoreTask) QueryNodeMapper(com.yelp.nrtsearch.server.luceneserver.QueryNodeMapper) QueryRescore(com.yelp.nrtsearch.server.luceneserver.rescore.QueryRescore) RescoreOperation(com.yelp.nrtsearch.server.luceneserver.rescore.RescoreOperation) DocCollector(com.yelp.nrtsearch.server.luceneserver.search.collectors.DocCollector) IOException(java.io.IOException) IndexState(com.yelp.nrtsearch.server.luceneserver.IndexState) Collector(org.apache.lucene.search.Collector) ProfileResult(com.yelp.nrtsearch.server.grpc.ProfileResult) Collectors(java.util.stream.Collectors) LargeNumHitsCollector(com.yelp.nrtsearch.server.luceneserver.search.collectors.LargeNumHitsCollector) ScriptService(com.yelp.nrtsearch.server.luceneserver.script.ScriptService) SearchRequest(com.yelp.nrtsearch.server.grpc.SearchRequest) List(java.util.List) RescorerCreator(com.yelp.nrtsearch.server.luceneserver.rescore.RescorerCreator) SortFieldCollector(com.yelp.nrtsearch.server.luceneserver.search.collectors.SortFieldCollector) RelevanceCollector(com.yelp.nrtsearch.server.luceneserver.search.collectors.RelevanceCollector) Collections(java.util.Collections) IndexSearcher(org.apache.lucene.search.IndexSearcher) SearchRequest(com.yelp.nrtsearch.server.grpc.SearchRequest) CollectorResult(com.yelp.nrtsearch.server.grpc.CollectorResult) AdditionalCollectorManager(com.yelp.nrtsearch.server.luceneserver.search.collectors.AdditionalCollectorManager) LargeNumHitsCollector(com.yelp.nrtsearch.server.luceneserver.search.collectors.LargeNumHitsCollector) DocCollector(com.yelp.nrtsearch.server.luceneserver.search.collectors.DocCollector) Collector(org.apache.lucene.search.Collector) LargeNumHitsCollector(com.yelp.nrtsearch.server.luceneserver.search.collectors.LargeNumHitsCollector) SortFieldCollector(com.yelp.nrtsearch.server.luceneserver.search.collectors.SortFieldCollector) RelevanceCollector(com.yelp.nrtsearch.server.luceneserver.search.collectors.RelevanceCollector) RelevanceCollector(com.yelp.nrtsearch.server.luceneserver.search.collectors.RelevanceCollector) SortFieldCollector(com.yelp.nrtsearch.server.luceneserver.search.collectors.SortFieldCollector)

Example 23 with SearchRequest

use of com.yelp.nrtsearch.server.grpc.SearchRequest in project nrtsearch by Yelp.

the class CollectorTest method testCustomCollector.

/**
 * Collect the top 3 reviews for each business *
 */
@Test
public void testCustomCollector() throws InvalidProtocolBufferException {
    // topHits = 0 results in java.lang.IllegalArgumentException:
    // numHits must be > 0; please use TotalHitCountCollector if you just need the total hit count
    SearchRequest request = SearchRequest.newBuilder().setTopHits(1).setIndexName(DEFAULT_TEST_INDEX).addRetrieveFields("review_id").setQuery(Query.newBuilder().setFunctionScoreQuery(FunctionScoreQuery.newBuilder().setScript(Script.newBuilder().setLang("js").setSource("review_count").build()).build()).build()).putCollectors(COLLECTOR_NAME, Collector.newBuilder().setPluginCollector(PluginCollector.newBuilder().setName(PLUGIN_COLLECTOR).setParams(Struct.newBuilder().putFields("size", Value.newBuilder().setNumberValue(3).build())).build()).build()).build();
    Map<String, CollectorResult> collectorsResult = getGrpcServer().getBlockingStub().search(request).getCollectorResultsMap();
    assertEquals(1, collectorsResult.size());
    // review_id -> score
    Map<String, Double> expectedHits = Map.of("0", 100.0, "1", 99.0, "2", 98.0, "3", 97.0, "4", 96.0, "5", 95.0, "6", 94.0, "7", 93.0, "8", 92.0);
    Map<String, Double> collectedHits = collectorsResult.get(COLLECTOR_NAME).getAnyResult().unpack(SearchResponse.class).getHitsList().stream().collect(Collectors.toMap(hit -> hit.getFieldsMap().get("review_id").getFieldValue(0).getTextValue(), SearchResponse.Hit::getScore));
    assertEquals(expectedHits, collectedHits);
}
Also used : SearchResponse(com.yelp.nrtsearch.server.grpc.SearchResponse) AddDocumentRequest(com.yelp.nrtsearch.server.grpc.AddDocumentRequest) NoMergePolicy(org.apache.lucene.index.NoMergePolicy) LeafCollector(org.apache.lucene.search.LeafCollector) Query(com.yelp.nrtsearch.server.grpc.Query) PriorityQueue(java.util.PriorityQueue) Collector(com.yelp.nrtsearch.server.grpc.Collector) HashMap(java.util.HashMap) Scorable(org.apache.lucene.search.Scorable) Struct(com.google.protobuf.Struct) ArrayList(java.util.ArrayList) IndexableFieldDef(com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef) Map(java.util.Map) Assert.fail(org.junit.Assert.fail) Value(com.google.protobuf.Value) LeafReaderContext(org.apache.lucene.index.LeafReaderContext) Script(com.yelp.nrtsearch.server.grpc.Script) ClassRule(org.junit.ClassRule) CollectorResult(com.yelp.nrtsearch.server.grpc.CollectorResult) FieldDefRequest(com.yelp.nrtsearch.server.grpc.FieldDefRequest) SearchHandler(com.yelp.nrtsearch.server.luceneserver.SearchHandler) LoadedDocValues(com.yelp.nrtsearch.server.luceneserver.doc.LoadedDocValues) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) Plugin(com.yelp.nrtsearch.server.plugins.Plugin) Collection(java.util.Collection) ServerTestCase(com.yelp.nrtsearch.server.luceneserver.ServerTestCase) Assert.assertTrue(org.junit.Assert.assertTrue) PluginCollector(com.yelp.nrtsearch.server.grpc.PluginCollector) IOException(java.io.IOException) Test(org.junit.Test) Collectors(java.util.stream.Collectors) FunctionScoreQuery(com.yelp.nrtsearch.server.grpc.FunctionScoreQuery) StatusRuntimeException(io.grpc.StatusRuntimeException) ScoreMode(org.apache.lucene.search.ScoreMode) CollectorPlugin(com.yelp.nrtsearch.server.plugins.CollectorPlugin) SearchRequest(com.yelp.nrtsearch.server.grpc.SearchRequest) AbstractMap(java.util.AbstractMap) List(java.util.List) IndexWriter(org.apache.lucene.index.IndexWriter) GrpcCleanupRule(io.grpc.testing.GrpcCleanupRule) Any(com.google.protobuf.Any) SearchContext(com.yelp.nrtsearch.server.luceneserver.search.SearchContext) Comparator(java.util.Comparator) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) SearchRequest(com.yelp.nrtsearch.server.grpc.SearchRequest) CollectorResult(com.yelp.nrtsearch.server.grpc.CollectorResult) SearchResponse(com.yelp.nrtsearch.server.grpc.SearchResponse) Test(org.junit.Test)

Example 24 with SearchRequest

use of com.yelp.nrtsearch.server.grpc.SearchRequest in project nrtsearch by Yelp.

the class CollectorTest method testUnknownCollector.

@Test
public void testUnknownCollector() {
    SearchRequest request = SearchRequest.newBuilder().setTopHits(1).setIndexName(DEFAULT_TEST_INDEX).addRetrieveFields("review_id").setQuery(Query.newBuilder().setFunctionScoreQuery(FunctionScoreQuery.newBuilder().setScript(Script.newBuilder().setLang("js").setSource("review_count").build()).build()).build()).putCollectors(COLLECTOR_NAME, com.yelp.nrtsearch.server.grpc.Collector.newBuilder().setPluginCollector(PluginCollector.newBuilder().setName("invalid").build()).build()).build();
    try {
        getGrpcServer().getBlockingStub().search(request);
        fail();
    } catch (StatusRuntimeException e) {
        String expectedMessage = "Invalid collector name: invalid, must be one of: [plugin_collector]";
        assertTrue(e.getMessage().contains(expectedMessage));
    }
}
Also used : SearchRequest(com.yelp.nrtsearch.server.grpc.SearchRequest) StatusRuntimeException(io.grpc.StatusRuntimeException) Test(org.junit.Test)

Example 25 with SearchRequest

use of com.yelp.nrtsearch.server.grpc.SearchRequest in project nrtsearch by Yelp.

the class DocCollectorTest method testUsesDefaultTimeout.

@Test
public void testUsesDefaultTimeout() {
    IndexState indexState = Mockito.mock(IndexState.class);
    when(indexState.getDefaultSearchTimeoutSec()).thenReturn(3.0);
    when(indexState.getDefaultSearchTimeoutCheckEvery()).thenReturn(0);
    SearchRequest request = SearchRequest.newBuilder().setTopHits(10).build();
    TestDocCollector docCollector = new TestDocCollector(request, indexState);
    assertTrue(docCollector.getManager() instanceof TestDocCollector.TestCollectorManager);
    assertTrue(docCollector.getWrappedManager() instanceof SearchCutoffWrapper);
    SearchCutoffWrapper<?> cutoffWrapper = (SearchCutoffWrapper<?>) docCollector.getWrappedManager();
    assertEquals(3.0, cutoffWrapper.getTimeoutSec(), 0.0);
    assertEquals(0, cutoffWrapper.getCheckEvery());
}
Also used : SearchRequest(com.yelp.nrtsearch.server.grpc.SearchRequest) SearchCutoffWrapper(com.yelp.nrtsearch.server.luceneserver.search.SearchCutoffWrapper) IndexState(com.yelp.nrtsearch.server.luceneserver.IndexState) Test(org.junit.Test)

Aggregations

SearchRequest (com.yelp.nrtsearch.server.grpc.SearchRequest)28 Test (org.junit.Test)21 IndexState (com.yelp.nrtsearch.server.luceneserver.IndexState)10 SearchCutoffWrapper (com.yelp.nrtsearch.server.luceneserver.search.SearchCutoffWrapper)7 SearchResponse (com.yelp.nrtsearch.server.grpc.SearchResponse)6 Path (java.nio.file.Path)5 SearchHandler (com.yelp.nrtsearch.server.luceneserver.SearchHandler)4 TerminateAfterWrapper (com.yelp.nrtsearch.server.luceneserver.search.TerminateAfterWrapper)4 BufferedWriter (java.io.BufferedWriter)4 ArrayList (java.util.ArrayList)4 SearchStatsWrapper (com.yelp.nrtsearch.server.luceneserver.search.SearchStatsWrapper)3 StatusRuntimeException (io.grpc.StatusRuntimeException)3 IOException (java.io.IOException)3 CollectorResult (com.yelp.nrtsearch.server.grpc.CollectorResult)2 IndexableFieldDef (com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef)2 SearchCollectorManager (com.yelp.nrtsearch.server.luceneserver.search.SearchCollectorManager)2 Collections (java.util.Collections)2 HashMap (java.util.HashMap)2 List (java.util.List)2 Map (java.util.Map)2