Examples with IndexState com.yelp.nrtsearch.server.luceneserver.IndexState used on opensource projects

Example 21 with IndexState

use of com.yelp.nrtsearch.server.luceneserver.IndexState in project nrtsearch by Yelp.

the class BucketedTieredMergePolicyTest method setLiveSettings.

private void setLiveSettings(int virtualShards, int maxDocs, int maxSegments) throws IOException {
    IndexState indexState = getGlobalState().getIndex(DEFAULT_TEST_INDEX);
    indexState.setVirtualShards(virtualShards);
    indexState.setSliceMaxDocs(maxDocs);
    indexState.setSliceMaxSegments(maxSegments);
}

Example 22 with IndexState

use of com.yelp.nrtsearch.server.luceneserver.IndexState in project nrtsearch by Yelp.

the class DrillSidewaysImpl method getFieldFacetResult.

private static com.yelp.nrtsearch.server.grpc.FacetResult getFieldFacetResult(FacetsCollector drillDowns, Map<String, FacetsCollector> dsDimMap, ShardState shardState, Facet facet, Map<String, FieldDef> dynamicFields, SearcherTaxonomyManager.SearcherAndTaxonomy searcherAndTaxonomyManager, Map<String, Facets> indexFieldNameToFacets) throws IOException {
    IndexState indexState = shardState.indexState;
    String fieldName = facet.getDim();
    FieldDef fieldDef = dynamicFields.get(fieldName);
    if (fieldDef == null) {
        throw new IllegalArgumentException(String.format("field %s was not registered and was not specified as a dynamic field ", fieldName));
    }
    FacetResult facetResult;
    if (!(fieldDef instanceof IndexableFieldDef) && !(fieldDef instanceof VirtualFieldDef)) {
        throw new IllegalArgumentException(String.format("field %s is neither a virtual field nor registered as an indexable field. Facets are supported only for these types", fieldName));
    }
    if (!facet.getNumericRangeList().isEmpty()) {
        if (fieldDef.getFacetValueType() != IndexableFieldDef.FacetValueType.NUMERIC_RANGE) {
            throw new IllegalArgumentException(String.format("field %s was not registered with facet=numericRange", fieldDef.getName()));
        }
        if (fieldDef instanceof IntFieldDef || fieldDef instanceof LongFieldDef) {
            List<NumericRangeType> rangeList = facet.getNumericRangeList();
            LongRange[] ranges = new LongRange[rangeList.size()];
            for (int i = 0; i < ranges.length; i++) {
                NumericRangeType numericRangeType = rangeList.get(i);
                ranges[i] = new LongRange(numericRangeType.getLabel(), numericRangeType.getMin(), numericRangeType.getMinInclusive(), numericRangeType.getMax(), numericRangeType.getMaxInclusive());
            }
            FacetsCollector c = dsDimMap.get(fieldDef.getName());
            if (c == null) {
                c = drillDowns;
            }
            LongRangeFacetCounts longRangeFacetCounts = new LongRangeFacetCounts(fieldDef.getName(), c, ranges);
            facetResult = longRangeFacetCounts.getTopChildren(0, fieldDef.getName(), facet.getPathsList().toArray(new String[facet.getPathsCount()]));
        } else if (fieldDef instanceof FloatFieldDef) {
            throw new IllegalArgumentException(String.format("field %s is of type float with FloatFieldDocValues which do not support numeric_range faceting", fieldDef.getName()));
        } else if (fieldDef instanceof DoubleFieldDef || fieldDef instanceof VirtualFieldDef) {
            List<NumericRangeType> rangeList = facet.getNumericRangeList();
            DoubleRange[] ranges = new DoubleRange[rangeList.size()];
            for (int i = 0; i < ranges.length; i++) {
                NumericRangeType numericRangeType = rangeList.get(i);
                ranges[i] = new DoubleRange(numericRangeType.getLabel(), numericRangeType.getMin(), numericRangeType.getMinInclusive(), numericRangeType.getMax(), numericRangeType.getMaxInclusive());
            }
            FacetsCollector c = dsDimMap.get(fieldDef.getName());
            if (c == null) {
                c = drillDowns;
            }
            DoubleRangeFacetCounts doubleRangeFacetCounts;
            if (fieldDef instanceof VirtualFieldDef) {
                VirtualFieldDef virtualFieldDef = (VirtualFieldDef) fieldDef;
                doubleRangeFacetCounts = new DoubleRangeFacetCounts(virtualFieldDef.getName(), virtualFieldDef.getValuesSource(), c, ranges);
            } else {
                doubleRangeFacetCounts = new DoubleRangeFacetCounts(fieldDef.getName(), c, ranges);
            }
            facetResult = doubleRangeFacetCounts.getTopChildren(0, fieldDef.getName(), facet.getPathsList().toArray(new String[facet.getPathsCount()]));
        } else {
            throw new IllegalArgumentException(String.format("numericRanges must be provided only on field type numeric e.g. int, double, flat"));
        }
    } else if (fieldDef.getFacetValueType() == IndexableFieldDef.FacetValueType.SORTED_SET_DOC_VALUES) {
        FacetsCollector c = dsDimMap.get(fieldDef.getName());
        if (c == null) {
            c = drillDowns;
        }
        if (facet.getLabelsCount() > 0) {
            // filter facet if a label list is provided
            FilteredSSDVFacetCounts filteredSSDVFacetCounts = new FilteredSSDVFacetCounts(facet.getLabelsList(), fieldDef.getName(), shardState.getSSDVState(searcherAndTaxonomyManager, fieldDef), c);
            facetResult = filteredSSDVFacetCounts.getTopChildren(facet.getTopN(), fieldDef.getName());
        } else {
            SortedSetDocValuesFacetCounts sortedSetDocValuesFacetCounts = new SortedSetDocValuesFacetCounts(shardState.getSSDVState(searcherAndTaxonomyManager, fieldDef), c);
            facetResult = sortedSetDocValuesFacetCounts.getTopChildren(facet.getTopN(), fieldDef.getName());
        }
    } else if (fieldDef.getFacetValueType() != IndexableFieldDef.FacetValueType.NO_FACETS) {
        // Taxonomy  facets
        if (fieldDef.getFacetValueType() == IndexableFieldDef.FacetValueType.NUMERIC_RANGE) {
            throw new IllegalArgumentException(String.format("%s was registered with facet = numericRange; must pass numericRanges in the request", fieldDef.getName()));
        }
        String[] path;
        if (!facet.getPathsList().isEmpty()) {
            ProtocolStringList pathList = facet.getPathsList();
            path = new String[facet.getPathsList().size()];
            for (int idx = 0; idx < path.length; idx++) {
                path[idx] = pathList.get(idx);
            }
        } else {
            path = new String[0];
        }
        FacetsCollector c = dsDimMap.get(fieldDef.getName());
        boolean useCachedOrds = facet.getUseOrdsCache();
        Facets luceneFacets;
        if (c != null) {
            // This dimension was used in
            // drill-down; compute its facet counts from the
            // drill-sideways collector:
            String indexFieldName = indexState.facetsConfig.getDimConfig(fieldDef.getName()).indexFieldName;
            if (useCachedOrds) {
                luceneFacets = new TaxonomyFacetCounts(shardState.getOrdsCache(indexFieldName), searcherAndTaxonomyManager.taxonomyReader, indexState.facetsConfig, c);
            } else {
                luceneFacets = new FastTaxonomyFacetCounts(indexFieldName, searcherAndTaxonomyManager.taxonomyReader, indexState.facetsConfig, c);
            }
        } else {
            // nocommit test both normal & ssdv facets in same index
            // See if we already computed facet
            // counts for this indexFieldName:
            String indexFieldName = indexState.facetsConfig.getDimConfig(fieldDef.getName()).indexFieldName;
            Map<String, Facets> facetsMap = indexFieldNameToFacets;
            luceneFacets = facetsMap.get(indexFieldName);
            if (luceneFacets == null) {
                if (useCachedOrds) {
                    luceneFacets = new TaxonomyFacetCounts(shardState.getOrdsCache(indexFieldName), searcherAndTaxonomyManager.taxonomyReader, indexState.facetsConfig, drillDowns);
                } else {
                    luceneFacets = new FastTaxonomyFacetCounts(indexFieldName, searcherAndTaxonomyManager.taxonomyReader, indexState.facetsConfig, drillDowns);
                }
                facetsMap.put(indexFieldName, luceneFacets);
            }
        }
        if (facet.getTopN() != 0) {
            facetResult = luceneFacets.getTopChildren(facet.getTopN(), fieldDef.getName(), path);
        } else if (!facet.getLabelsList().isEmpty()) {
            List<LabelAndValue> results = new ArrayList<LabelAndValue>();
            for (String label : facet.getLabelsList()) {
                results.add(new LabelAndValue(label, luceneFacets.getSpecificValue(fieldDef.getName(), label)));
            }
            facetResult = new FacetResult(fieldDef.getName(), path, -1, results.toArray(new LabelAndValue[results.size()]), -1);
        } else {
            throw new IllegalArgumentException(String.format("each facet request must have either topN or labels"));
        }
    } else {
        // if no facet type is enabled on the field, try using the field doc values
        if (!(fieldDef instanceof IndexableFieldDef)) {
            throw new IllegalArgumentException("Doc values facet requires an indexable field : " + fieldName);
        }
        IndexableFieldDef indexableFieldDef = (IndexableFieldDef) fieldDef;
        if (!indexableFieldDef.hasDocValues()) {
            throw new IllegalArgumentException("Doc values facet requires doc values enabled : " + fieldName);
        }
        return getDocValuesFacetResult(facet, drillDowns, indexableFieldDef);
    }
    return buildFacetResultGrpc(facetResult, facet.getName());
}

Example 23 with IndexState

use of com.yelp.nrtsearch.server.luceneserver.IndexState in project nrtsearch by Yelp.

the class DrillSidewaysImpl method fillFacetResults.

static void fillFacetResults(FacetsCollector drillDowns, FacetsCollector[] drillSideways, String[] drillSidewaysDims, ShardState shardState, List<Facet> grpcFacets, Map<String, FieldDef> dynamicFields, SearcherTaxonomyManager.SearcherAndTaxonomy searcherAndTaxonomyManager, List<com.yelp.nrtsearch.server.grpc.FacetResult> grpcFacetResults, Diagnostics.Builder diagnostics) throws IOException {
    IndexState indexState = shardState.indexState;
    Map<String, FacetsCollector> dsDimMap = new HashMap<String, FacetsCollector>();
    if (drillSidewaysDims != null) {
        for (int i = 0; i < drillSidewaysDims.length; i++) {
            dsDimMap.put(drillSidewaysDims[i], drillSideways[i]);
        }
    }
    // Holds already computed Facets, since more
    // than one dimension can share a single
    // index field name.  We need one map for "normal" and
    // another for SSDV facets because an app can index both
    // into the same Lucene field (this is the default):
    Map<String, Facets> indexFieldNameToFacets = new HashMap<String, Facets>();
    Map<String, Facets> indexFieldNameToSSDVFacets = new HashMap<String, Facets>();
    for (Facet facet : grpcFacets) {
        // these facets will be created from the top docs
        if (facet.getSampleTopDocs() != 0) {
            continue;
        }
        long startNS = System.nanoTime();
        com.yelp.nrtsearch.server.grpc.FacetResult facetResult;
        if (facet.hasScript()) {
            // this facet is a FacetScript, run script against all matching documents
            facetResult = getScriptFacetResult(facet, drillDowns, indexState);
        } else {
            facetResult = getFieldFacetResult(drillDowns, dsDimMap, shardState, facet, dynamicFields, searcherAndTaxonomyManager, indexFieldNameToFacets);
        }
        if (facetResult != null) {
            grpcFacetResults.add(facetResult);
        }
        long endNS = System.nanoTime();
        diagnostics.putFacetTimeMs(facet.getName(), (endNS - startNS) / 1000000.0);
    }
}