Examples with Index com.yahoo.searchdefinition.Index used on opensource projects

Example 1 with Index

use of com.yahoo.searchdefinition.Index in project vespa by vespa-engine.

the class Derived method derive.

/**
 * Derives the content of this configuration. This
 * default calls derive(Document) for each document
 * and derive(SDField) for each search definition level field
 * AND sets the name of this to the name of the input search definition
 */
protected void derive(Search search) {
    setName(search.getName());
    derive(search.getDocument(), search);
    for (Index index : search.getExplicitIndices()) {
        derive(index, search);
    }
    for (SDField field : search.allExtraFields()) {
        derive(field, search);
    }
    search.allImportedFields().forEach(importedField -> derive(importedField, search));
}

Example 2 with Index

use of com.yahoo.searchdefinition.Index in project vespa by vespa-engine.

the class IndexInfo method derive.

@Override
protected void derive(Search search) {
    // Derive per field
    super.derive(search);
    this.search = search;
    // Populate fieldsets with actual field objects, bit late to do that here but
    for (FieldSet fs : fieldSets.values()) {
        for (String fieldName : fs.getFieldNames()) {
            fs.fields().add(search.getField(fieldName));
        }
    }
    // Must follow, because index settings overrides field settings
    for (Index index : search.getExplicitIndices()) {
        derive(index, search);
    }
    // TODO: Move to fieldinfo and implement differently. This is not right
    for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values()) {
        if (summaryField.getTransform().isTeaser()) {
            addIndexCommand(summaryField.getName(), CMD_DYNTEASER);
        }
        if (summaryField.getTransform().isBolded()) {
            addIndexCommand(summaryField.getName(), CMD_HIGHLIGHT);
        }
    }
}

Example 3 with Index

use of com.yahoo.searchdefinition.Index in project vespa by vespa-engine.

the class IndexInfo method derive.

@Override
protected void derive(ImmutableSDField field, Search search) {
    if (field.getDataType().equals(DataType.PREDICATE)) {
        Index index = field.getIndex(field.getName());
        if (index != null) {
            BooleanIndexDefinition options = index.getBooleanIndexDefiniton();
            if (options.hasLowerBound() || options.hasUpperBound()) {
                addIndexCommand(field.getName(), CMD_PREDICATE_BOUNDS + " [" + (options.hasLowerBound() ? Long.toString(options.getLowerBound()) : "") + ".." + (options.hasUpperBound() ? Long.toString(options.getUpperBound()) : "") + "]");
            }
        }
    }
    // Field level aliases
    for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) {
        String alias = e.getKey();
        String name = e.getValue();
        addIndexAlias(alias, name);
    }
    if (field.usesStructOrMap()) {
        for (ImmutableSDField structField : field.getStructFields()) {
            // Recursion
            derive(structField, search);
        }
    }
    if (field.getDataType().equals(PositionDataType.INSTANCE) || field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) {
        addIndexCommand(field.getName(), CMD_DEFAULT_POSITION);
    }
    // List the indices
    addIndexCommand(field, CMD_INDEX);
    if (field.doesIndexing() || field.doesLowerCasing()) {
        addIndexCommand(field, CMD_LOWERCASE);
    }
    if (field.getDataType().isMultivalue()) {
        addIndexCommand(field, CMD_MULTIVALUE);
    }
    if (field.doesAttributing() && !field.doesIndexing()) {
        addIndexCommand(field.getName(), CMD_ATTRIBUTE);
        Attribute attribute = field.getAttributes().get(field.getName());
        if (attribute != null && attribute.isFastSearch())
            addIndexCommand(field.getName(), CMD_FAST_SEARCH);
    } else if (field.doesIndexing()) {
        if (stemSomehow(field, search)) {
            addIndexCommand(field, stemCmd(field, search), new StemmingOverrider(this, search));
        }
        if (normalizeAccents(field)) {
            addIndexCommand(field, CMD_NORMALIZE);
        }
        if (field.getMatching() == null || field.getMatching().getType().equals(Matching.Type.TEXT)) {
            addIndexCommand(field, CMD_PLAIN_TOKENS);
        }
    }
    if (isUriField(field)) {
        addUriIndexCommands(field);
    }
    if (field.getDataType() instanceof NumericDataType) {
        addIndexCommand(field, CMD_NUMERICAL);
    }
    // Explicit commands
    for (String command : field.getQueryCommands()) {
        addIndexCommand(field, command);
    }
}

Example 4 with Index

use of com.yahoo.searchdefinition.Index in project vespa by vespa-engine.

the class PredicateProcessor method process.

@Override
public void process(boolean validate) {
    for (SDField field : search.allConcreteFields()) {
        if (field.getDataType() == DataType.PREDICATE) {
            if (validate && field.doesIndexing()) {
                fail(search, field, "Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded.");
            }
            if (field.doesAttributing()) {
                Attribute attribute = field.getAttributes().get(field.getName());
                for (Index index : field.getIndices().values()) {
                    BooleanIndexDefinition booleanDefinition = index.getBooleanIndexDefiniton();
                    if (validate && (booleanDefinition == null || !booleanDefinition.hasArity())) {
                        fail(search, field, "Missing arity value in predicate field.");
                    }
                    if (validate && (booleanDefinition.getArity() < 2)) {
                        fail(search, field, "Invalid arity value in predicate field, must be greater than 1.");
                    }
                    double threshold = booleanDefinition.getDensePostingListThreshold();
                    if (validate && (threshold <= 0 || threshold > 1)) {
                        fail(search, field, "Invalid dense-posting-list-threshold value in predicate field. " + "Value must be in range (0..1].");
                    }
                    attribute.setArity(booleanDefinition.getArity());
                    attribute.setLowerBound(booleanDefinition.getLowerBound());
                    attribute.setUpperBound(booleanDefinition.getUpperBound());
                    attribute.setDensePostingListThreshold(threshold);
                    addPredicateOptimizationIlScript(field, booleanDefinition);
                }
                DocumentSummary summary = search.getSummary("attributeprefetch");
                if (summary != null) {
                    summary.remove(attribute.getName());
                }
                for (SummaryField summaryField : search.getSummaryFields(field).values()) {
                    summaryField.setTransform(SummaryTransform.NONE);
                }
            }
        } else if (validate && field.getDataType().getPrimitiveType() == DataType.PREDICATE) {
            fail(search, field, "Collections of predicates are not allowed.");
        } else if (validate && field.getDataType() == DataType.RAW && field.doesIndexing()) {
            fail(search, field, "Indexing of RAW fields is not supported.");
        } else if (validate) {
            // if field is not a predicate, disallow predicate-related index parameters
            for (Index index : field.getIndices().values()) {
                if (index.getBooleanIndexDefiniton() != null) {
                    BooleanIndexDefinition def = index.getBooleanIndexDefiniton();
                    if (def.hasArity()) {
                        fail(search, field, "Arity parameter is used only for predicate type fields.");
                    } else if (def.hasLowerBound() || def.hasUpperBound()) {
                        fail(search, field, "Parameters lower-bound and upper-bound are used only for predicate type fields.");
                    } else if (def.hasDensePostingListThreshold()) {
                        fail(search, field, "Parameter dense-posting-list-threshold is used only for predicate type fields.");
                    }
                }
            }
        }
    }
}

Example 5 with Index

use of com.yahoo.searchdefinition.Index in project vespa by vespa-engine.

the class Processor method addField.

/**
 * Convenience method for adding a no-strings-attached implementation field for a regular field
 *
 * @param search       the search definition in question
 * @param field        the field to add an implementation field for
 * @param suffix       the suffix of the added implementation field (without the underscore)
 * @param indexing     the indexing statement of the field
 * @param queryCommand the query command of the original field, or null if none
 * @return the implementation field which is added to the search
 */
protected SDField addField(Search search, SDField field, String suffix, String indexing, String queryCommand) {
    SDField implementationField = search.getConcreteField(field.getName() + "_" + suffix);
    if (implementationField != null) {
        deployLogger.log(Level.WARNING, "Implementation field " + implementationField + " added twice");
    } else {
        implementationField = new SDField(search.getDocument(), field.getName() + "_" + suffix, DataType.STRING);
    }
    implementationField.setRankType(RankType.EMPTY);
    implementationField.setStemming(Stemming.NONE);
    implementationField.getNormalizing().inferCodepoint();
    implementationField.parseIndexingScript(indexing);
    for (Iterator i = field.getFieldNameAsIterator(); i.hasNext(); ) {
        String indexName = (String) i.next();
        String implementationIndexName = indexName + "_" + suffix;
        Index implementationIndex = new Index(implementationIndexName);
        search.addIndex(implementationIndex);
    }
    if (queryCommand != null) {
        field.addQueryCommand(queryCommand);
    }
    search.addExtraField(implementationField);
    search.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, implementationField.getName());
    return implementationField;
}