Examples with RankProfile com.yahoo.searchdefinition.RankProfile used on opensource projects

Example 1 with RankProfile

use of com.yahoo.searchdefinition.RankProfile in project vespa by vespa-engine.

the class TensorFlowFeatureConverter method reduceBatchDimensions.

/**
 * Check if batch dimensions of inputs can be reduced out. If the input
 * macro specifies that a single exemplar should be evaluated, we can
 * reduce the batch dimension out.
 */
private void reduceBatchDimensions(RankingExpression expression, TensorFlowModel model, RankProfile profile, QueryProfileRegistry queryProfiles) {
    TypeContext<Reference> typeContext = profile.typeContext(queryProfiles);
    TensorType typeBeforeReducing = expression.getRoot().type(typeContext);
    // Check generated macros for inputs to reduce
    Set<String> macroNames = new HashSet<>();
    addMacroNamesIn(expression.getRoot(), macroNames, model);
    for (String macroName : macroNames) {
        if (!model.macros().containsKey(macroName)) {
            continue;
        }
        RankProfile.Macro macro = profile.getMacros().get(macroName);
        if (macro == null) {
            throw new IllegalArgumentException("Model refers to generated macro '" + macroName + "but this macro is not present in " + profile);
        }
        RankingExpression macroExpression = macro.getRankingExpression();
        macroExpression.setRoot(reduceBatchDimensionsAtInput(macroExpression.getRoot(), model, typeContext));
    }
    // Check expression for inputs to reduce
    ExpressionNode root = expression.getRoot();
    root = reduceBatchDimensionsAtInput(root, model, typeContext);
    TensorType typeAfterReducing = root.type(typeContext);
    root = expandBatchDimensionsAtOutput(root, typeBeforeReducing, typeAfterReducing);
    expression.setRoot(root);
}

Example 2 with RankProfile

use of com.yahoo.searchdefinition.RankProfile in project vespa by vespa-engine.

the class TensorFlowFeatureConverter method transformLargeConstant.

private void transformLargeConstant(ModelStore store, RankProfile profile, QueryProfileRegistry queryProfiles, Set<String> constantsReplacedByMacros, String constantName, Tensor constantValue) {
    RankProfile.Macro macroOverridingConstant = profile.getMacros().get(constantName);
    if (macroOverridingConstant != null) {
        TensorType macroType = macroOverridingConstant.getRankingExpression().type(profile.typeContext(queryProfiles));
        if (!macroType.equals(constantValue.type()))
            throw new IllegalArgumentException("Macro '" + constantName + "' replaces the constant with this name. " + "The required type of this is " + constantValue.type() + ", but the macro returns " + macroType);
        // will replace constant(constantName) by constantName later
        constantsReplacedByMacros.add(constantName);
    } else {
        Path constantPath = store.writeLargeConstant(constantName, constantValue);
        if (!profile.getSearch().getRankingConstants().containsKey(constantName)) {
            profile.getSearch().addRankingConstant(new RankingConstant(constantName, constantValue.type(), constantPath.toString()));
        }
    }
}

Example 3 with RankProfile

use of com.yahoo.searchdefinition.RankProfile in project vespa by vespa-engine.

the class TensorFlowFeatureConverter method verifyRequiredMacros.

/**
 * Verify that the macros referred in the given expression exists in the given rank profile,
 * and return tensors of the types specified in requiredMacros.
 */
private void verifyRequiredMacros(RankingExpression expression, TensorFlowModel model, RankProfile profile, QueryProfileRegistry queryProfiles) {
    Set<String> macroNames = new HashSet<>();
    addMacroNamesIn(expression.getRoot(), macroNames, model);
    for (String macroName : macroNames) {
        TensorType requiredType = model.requiredMacros().get(macroName);
        // Not a required macro
        if (requiredType == null)
            continue;
        RankProfile.Macro macro = profile.getMacros().get(macroName);
        if (macro == null)
            throw new IllegalArgumentException("Model refers Placeholder '" + macroName + "' of type " + requiredType + " but this macro is not present in " + profile);
        // TODO: We should verify this in the (function reference(s) this is invoked (starting from first/second
        // phase and summary features), as it may only resolve correctly given those bindings
        // Or, probably better, annotate the macros with type constraints here and verify during general
        // type verification
        TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext(queryProfiles));
        if (actualType == null)
            throw new IllegalArgumentException("Model refers Placeholder '" + macroName + "' of type " + requiredType + " which must be produced by a macro in the rank profile, but " + "this macro references a feature which is not declared");
        if (!actualType.isAssignableTo(requiredType))
            throw new IllegalArgumentException("Model refers Placeholder '" + macroName + "' of type " + requiredType + " which must be produced by a macro in the rank profile, but " + "this macro produces type " + actualType);
    }
}

Example 4 with RankProfile

use of com.yahoo.searchdefinition.RankProfile in project vespa by vespa-engine.

the class LiteralBoostTestCase method testLiteralBoost.

/**
 * Tests adding of literal boost constructs
 */
@Test
public void testLiteralBoost() {
    Search search = new Search("literalboost", null);
    RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search);
    SDDocumentType document = new SDDocumentType("literalboost");
    search.addDocument(document);
    SDField field1 = document.addField("a", DataType.STRING);
    field1.parseIndexingScript("{ index }");
    field1.setLiteralBoost(20);
    RankProfile other = new RankProfile("other", search, rankProfileRegistry);
    rankProfileRegistry.addRankProfile(other);
    other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333));
    Processing.process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true);
    DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry());
    // Check attribute fields
    // TODO: assert content
    derived.getAttributeFields();
    // Check il script addition
    assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"SHORTEST\" | index a; }", "clear_state | guard { input a | tokenize | index a_literal; }"), search);
    // Check index info addition
    IndexInfo indexInfo = derived.getIndexInfo();
    assertTrue(indexInfo.hasCommand("a", "literal-boost"));
}

Example 5 with RankProfile

use of com.yahoo.searchdefinition.RankProfile in project vespa by vespa-engine.

the class RankingExpressionTypeValidatorTestCase method testTensorMacroInvocationTypes_Nested.

@Test
public void testTensorMacroInvocationTypes_Nested() throws Exception {
    SearchBuilder builder = new SearchBuilder();
    builder.importString(joinLines("search test {", "  document test { ", "    field a type tensor(x[],y[]) {", "      indexing: attribute", "    }", "    field b type tensor(z[10]) {", "      indexing: attribute", "    }", "  }", "  rank-profile my_rank_profile {", "    macro return_a() {", "      expression: return_first(attribute(a), attribute(b))", "    }", "    macro return_b() {", "      expression: return_second(attribute(a), attribute(b))", "    }", "    macro return_first(e1, e2) {", "      expression: e1", "    }", "    macro return_second(e1, e2) {", "      expression: return_first(e2, e1)", "    }", "    summary-features {", "      return_a", "      return_b", "    }", "  }", "}"));
    builder.build();
    RankProfile profile = builder.getRankProfileRegistry().getRankProfile(builder.getSearch(), "my_rank_profile");
    assertEquals(TensorType.fromSpec("tensor(x[],y[])"), summaryFeatures(profile).get("return_a").type(profile.typeContext(builder.getQueryProfileRegistry())));
    assertEquals(TensorType.fromSpec("tensor(z[10])"), summaryFeatures(profile).get("return_b").type(profile.typeContext(builder.getQueryProfileRegistry())));
}