Examples with VirtualFieldDef com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef used on opensource projects

Example 6 with VirtualFieldDef

use of com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef in project nrtsearch by Yelp.

the class JsScriptBindingsTest method testExtendedIntIndex.

@Test(expected = IllegalArgumentException.class)
public void testExtendedIntIndex() {
    Map<String, Object> params = new HashMap<>();
    params.put("param1", 100);
    params.put("param2", 1.11);
    DoubleValuesSource field1Source = new DummyValuesSource();
    DoubleValuesSource field2Source = new DummyValuesSource();
    Map<String, FieldDef> fieldDefMap = new HashMap<>();
    fieldDefMap.put("field1", new VirtualFieldDef("field1", field1Source));
    fieldDefMap.put("field2", new VirtualFieldDef("field2", field2Source));
    JsScriptBindings bindings = new JsScriptBindings(new FieldDefBindings(fieldDefMap), params);
    bindings.getDoubleValuesSource("doc[1].value");
}

Example 7 with VirtualFieldDef

use of com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef in project nrtsearch by Yelp.

the class JsScriptBindingsTest method testExtendedInvalidPrefix.

@Test(expected = IllegalArgumentException.class)
public void testExtendedInvalidPrefix() {
    Map<String, Object> params = new HashMap<>();
    params.put("param1", 100);
    params.put("param2", 1.11);
    DoubleValuesSource field1Source = new DummyValuesSource();
    DoubleValuesSource field2Source = new DummyValuesSource();
    Map<String, FieldDef> fieldDefMap = new HashMap<>();
    fieldDefMap.put("field1", new VirtualFieldDef("field1", field1Source));
    fieldDefMap.put("field2", new VirtualFieldDef("field2", field2Source));
    JsScriptBindings bindings = new JsScriptBindings(new FieldDefBindings(fieldDefMap), params);
    bindings.getDoubleValuesSource("not_doc['field1'].value");
}

Example 8 with VirtualFieldDef

use of com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef in project nrtsearch by Yelp.

the class DrillSidewaysImpl method getFieldFacetResult.

private static com.yelp.nrtsearch.server.grpc.FacetResult getFieldFacetResult(FacetsCollector drillDowns, Map<String, FacetsCollector> dsDimMap, ShardState shardState, Facet facet, Map<String, FieldDef> dynamicFields, SearcherTaxonomyManager.SearcherAndTaxonomy searcherAndTaxonomyManager, Map<String, Facets> indexFieldNameToFacets) throws IOException {
    IndexState indexState = shardState.indexState;
    String fieldName = facet.getDim();
    FieldDef fieldDef = dynamicFields.get(fieldName);
    if (fieldDef == null) {
        throw new IllegalArgumentException(String.format("field %s was not registered and was not specified as a dynamic field ", fieldName));
    }
    FacetResult facetResult;
    if (!(fieldDef instanceof IndexableFieldDef) && !(fieldDef instanceof VirtualFieldDef)) {
        throw new IllegalArgumentException(String.format("field %s is neither a virtual field nor registered as an indexable field. Facets are supported only for these types", fieldName));
    }
    if (!facet.getNumericRangeList().isEmpty()) {
        if (fieldDef.getFacetValueType() != IndexableFieldDef.FacetValueType.NUMERIC_RANGE) {
            throw new IllegalArgumentException(String.format("field %s was not registered with facet=numericRange", fieldDef.getName()));
        }
        if (fieldDef instanceof IntFieldDef || fieldDef instanceof LongFieldDef) {
            List<NumericRangeType> rangeList = facet.getNumericRangeList();
            LongRange[] ranges = new LongRange[rangeList.size()];
            for (int i = 0; i < ranges.length; i++) {
                NumericRangeType numericRangeType = rangeList.get(i);
                ranges[i] = new LongRange(numericRangeType.getLabel(), numericRangeType.getMin(), numericRangeType.getMinInclusive(), numericRangeType.getMax(), numericRangeType.getMaxInclusive());
            }
            FacetsCollector c = dsDimMap.get(fieldDef.getName());
            if (c == null) {
                c = drillDowns;
            }
            LongRangeFacetCounts longRangeFacetCounts = new LongRangeFacetCounts(fieldDef.getName(), c, ranges);
            facetResult = longRangeFacetCounts.getTopChildren(0, fieldDef.getName(), facet.getPathsList().toArray(new String[facet.getPathsCount()]));
        } else if (fieldDef instanceof FloatFieldDef) {
            throw new IllegalArgumentException(String.format("field %s is of type float with FloatFieldDocValues which do not support numeric_range faceting", fieldDef.getName()));
        } else if (fieldDef instanceof DoubleFieldDef || fieldDef instanceof VirtualFieldDef) {
            List<NumericRangeType> rangeList = facet.getNumericRangeList();
            DoubleRange[] ranges = new DoubleRange[rangeList.size()];
            for (int i = 0; i < ranges.length; i++) {
                NumericRangeType numericRangeType = rangeList.get(i);
                ranges[i] = new DoubleRange(numericRangeType.getLabel(), numericRangeType.getMin(), numericRangeType.getMinInclusive(), numericRangeType.getMax(), numericRangeType.getMaxInclusive());
            }
            FacetsCollector c = dsDimMap.get(fieldDef.getName());
            if (c == null) {
                c = drillDowns;
            }
            DoubleRangeFacetCounts doubleRangeFacetCounts;
            if (fieldDef instanceof VirtualFieldDef) {
                VirtualFieldDef virtualFieldDef = (VirtualFieldDef) fieldDef;
                doubleRangeFacetCounts = new DoubleRangeFacetCounts(virtualFieldDef.getName(), virtualFieldDef.getValuesSource(), c, ranges);
            } else {
                doubleRangeFacetCounts = new DoubleRangeFacetCounts(fieldDef.getName(), c, ranges);
            }
            facetResult = doubleRangeFacetCounts.getTopChildren(0, fieldDef.getName(), facet.getPathsList().toArray(new String[facet.getPathsCount()]));
        } else {
            throw new IllegalArgumentException(String.format("numericRanges must be provided only on field type numeric e.g. int, double, flat"));
        }
    } else if (fieldDef.getFacetValueType() == IndexableFieldDef.FacetValueType.SORTED_SET_DOC_VALUES) {
        FacetsCollector c = dsDimMap.get(fieldDef.getName());
        if (c == null) {
            c = drillDowns;
        }
        if (facet.getLabelsCount() > 0) {
            // filter facet if a label list is provided
            FilteredSSDVFacetCounts filteredSSDVFacetCounts = new FilteredSSDVFacetCounts(facet.getLabelsList(), fieldDef.getName(), shardState.getSSDVState(searcherAndTaxonomyManager, fieldDef), c);
            facetResult = filteredSSDVFacetCounts.getTopChildren(facet.getTopN(), fieldDef.getName());
        } else {
            SortedSetDocValuesFacetCounts sortedSetDocValuesFacetCounts = new SortedSetDocValuesFacetCounts(shardState.getSSDVState(searcherAndTaxonomyManager, fieldDef), c);
            facetResult = sortedSetDocValuesFacetCounts.getTopChildren(facet.getTopN(), fieldDef.getName());
        }
    } else if (fieldDef.getFacetValueType() != IndexableFieldDef.FacetValueType.NO_FACETS) {
        // Taxonomy  facets
        if (fieldDef.getFacetValueType() == IndexableFieldDef.FacetValueType.NUMERIC_RANGE) {
            throw new IllegalArgumentException(String.format("%s was registered with facet = numericRange; must pass numericRanges in the request", fieldDef.getName()));
        }
        String[] path;
        if (!facet.getPathsList().isEmpty()) {
            ProtocolStringList pathList = facet.getPathsList();
            path = new String[facet.getPathsList().size()];
            for (int idx = 0; idx < path.length; idx++) {
                path[idx] = pathList.get(idx);
            }
        } else {
            path = new String[0];
        }
        FacetsCollector c = dsDimMap.get(fieldDef.getName());
        boolean useCachedOrds = facet.getUseOrdsCache();
        Facets luceneFacets;
        if (c != null) {
            // This dimension was used in
            // drill-down; compute its facet counts from the
            // drill-sideways collector:
            String indexFieldName = indexState.facetsConfig.getDimConfig(fieldDef.getName()).indexFieldName;
            if (useCachedOrds) {
                luceneFacets = new TaxonomyFacetCounts(shardState.getOrdsCache(indexFieldName), searcherAndTaxonomyManager.taxonomyReader, indexState.facetsConfig, c);
            } else {
                luceneFacets = new FastTaxonomyFacetCounts(indexFieldName, searcherAndTaxonomyManager.taxonomyReader, indexState.facetsConfig, c);
            }
        } else {
            // nocommit test both normal & ssdv facets in same index
            // See if we already computed facet
            // counts for this indexFieldName:
            String indexFieldName = indexState.facetsConfig.getDimConfig(fieldDef.getName()).indexFieldName;
            Map<String, Facets> facetsMap = indexFieldNameToFacets;
            luceneFacets = facetsMap.get(indexFieldName);
            if (luceneFacets == null) {
                if (useCachedOrds) {
                    luceneFacets = new TaxonomyFacetCounts(shardState.getOrdsCache(indexFieldName), searcherAndTaxonomyManager.taxonomyReader, indexState.facetsConfig, drillDowns);
                } else {
                    luceneFacets = new FastTaxonomyFacetCounts(indexFieldName, searcherAndTaxonomyManager.taxonomyReader, indexState.facetsConfig, drillDowns);
                }
                facetsMap.put(indexFieldName, luceneFacets);
            }
        }
        if (facet.getTopN() != 0) {
            facetResult = luceneFacets.getTopChildren(facet.getTopN(), fieldDef.getName(), path);
        } else if (!facet.getLabelsList().isEmpty()) {
            List<LabelAndValue> results = new ArrayList<LabelAndValue>();
            for (String label : facet.getLabelsList()) {
                results.add(new LabelAndValue(label, luceneFacets.getSpecificValue(fieldDef.getName(), label)));
            }
            facetResult = new FacetResult(fieldDef.getName(), path, -1, results.toArray(new LabelAndValue[results.size()]), -1);
        } else {
            throw new IllegalArgumentException(String.format("each facet request must have either topN or labels"));
        }
    } else {
        // if no facet type is enabled on the field, try using the field doc values
        if (!(fieldDef instanceof IndexableFieldDef)) {
            throw new IllegalArgumentException("Doc values facet requires an indexable field : " + fieldName);
        }
        IndexableFieldDef indexableFieldDef = (IndexableFieldDef) fieldDef;
        if (!indexableFieldDef.hasDocValues()) {
            throw new IllegalArgumentException("Doc values facet requires doc values enabled : " + fieldName);
        }
        return getDocValuesFacetResult(facet, drillDowns, indexableFieldDef);
    }
    return buildFacetResultGrpc(facetResult, facet.getName());
}

Example 9 with VirtualFieldDef

use of com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef in project nrtsearch by Yelp.

the class RegisterFieldsHandler method parseOneVirtualFieldType.

private FieldDef parseOneVirtualFieldType(IndexState indexState, Map<String, FieldDef> pendingFieldDefs, String fieldName, Field currentField) throws RegisterFieldsException {
    ScoreScript.Factory factory = ScriptService.getInstance().compile(currentField.getScript(), ScoreScript.CONTEXT);
    Map<String, Object> params = ScriptParamsUtils.decodeParams(currentField.getScript().getParamsMap());
    // doing this (or maybe updating index state in general).
    if (currentField.getScript().getLang().equals(JsScriptEngine.LANG)) {
        params = new HashMap<>(params);
        Map<String, FieldDef> allFields = new HashMap<>(indexState.getAllFields());
        allFields.putAll(pendingFieldDefs);
        params.put("bindings", new FieldDefBindings(allFields));
    }
    DoubleValuesSource values = factory.newFactory(params, indexState.docLookup);
    return new VirtualFieldDef(fieldName, values);
}