Examples with DataType com.yahoo.document.DataType used on opensource projects

Example 31 with DataType

use of com.yahoo.document.DataType in project vespa by vespa-engine.

the class CreatePositionZCurve method process.

@Override
public void process(boolean validate) {
    for (SDField field : search.allConcreteFields()) {
        DataType fieldType = field.getDataType();
        if (!isSupportedPositionType(fieldType))
            continue;
        if (validate && field.doesIndexing()) {
            fail(search, field, "Indexing of data type '" + fieldType.getName() + "' is not supported, " + "replace 'index' statement with 'attribute'.");
        }
        if (!field.doesAttributing())
            continue;
        boolean doesSummary = field.doesSummarying();
        String fieldName = field.getName();
        field.getAttributes().remove(fieldName);
        String zName = PositionDataType.getZCurveFieldName(fieldName);
        SDField zCurveField = createZCurveField(field, zName, validate);
        search.addExtraField(zCurveField);
        search.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, zCurveField.getName());
        // configure summary
        Collection<String> summaryTo = removeSummaryTo(field);
        ensureCompatibleSummary(field, zName, PositionDataType.getPositionSummaryFieldName(fieldName), // will become "xmlstring"
        DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, summaryTo, validate);
        ensureCompatibleSummary(field, zName, PositionDataType.getDistanceSummaryFieldName(fieldName), DataType.INT, SummaryTransform.DISTANCE, summaryTo, validate);
        // clear indexing script
        field.setIndexingScript(null);
        SDField posX = field.getStructField(PositionDataType.FIELD_X);
        if (posX != null) {
            posX.setIndexingScript(null);
        }
        SDField posY = field.getStructField(PositionDataType.FIELD_Y);
        if (posY != null) {
            posY.setIndexingScript(null);
        }
        if (doesSummary)
            ensureCompatibleSummary(field, zName, field.getName(), field.getDataType(), SummaryTransform.GEOPOS, summaryTo, validate);
    }
}

Example 32 with DataType

use of com.yahoo.document.DataType in project vespa by vespa-engine.

the class IndexingOutputs method fillSummaryToFromSummaryField.

private void fillSummaryToFromSummaryField(Search search, SDField field, SummaryField summaryField, Set<String> dynamicSummary, Set<String> staticSummary) {
    SummaryTransform summaryTransform = summaryField.getTransform();
    String summaryName = summaryField.getName();
    if (summaryTransform.isDynamic() && summaryField.getSourceCount() > 2) {
        // summary rewriter in the search core.
        return;
    }
    if (summaryTransform.isDynamic()) {
        DataType fieldType = field.getDataType();
        if (fieldType != DataType.URI && fieldType != DataType.STRING) {
            warn(search, field, "Dynamic summaries are only supported for fields of type " + "string, ignoring summary field '" + summaryField.getName() + "' for sd field '" + field.getName() + "' of type " + fieldType.getName() + ".");
            return;
        }
        dynamicSummary.add(summaryName);
    } else if (summaryTransform != SummaryTransform.ATTRIBUTE) {
        staticSummary.add(summaryName);
    }
}

Example 33 with DataType

use of com.yahoo.document.DataType in project vespa by vespa-engine.

the class SearchField method validate.

@SuppressWarnings({ "deprecation" })
private void validate() {
    if (attribute || !indexed) {
        return;
    }
    DataType fieldType = getDataType();
    DataType primiType = fieldType.getPrimitiveType();
    if (DataType.STRING.equals(primiType) || DataType.URI.equals(primiType)) {
        return;
    }
    throw new IllegalStateException("Expected type " + DataType.STRING.getName() + " for indexed field '" + getName() + "', got " + fieldType.getName() + ".");
}

Example 34 with DataType

use of com.yahoo.document.DataType in project vespa by vespa-engine.

the class ValidateFieldTypes method process.

@Override
public void process(boolean validate) {
    if (!validate)
        return;
    String searchName = search.getName();
    Map<String, DataType> seenFields = new HashMap<>();
    search.allFields().forEach(field -> {
        checkFieldType(searchName, "index field", field.getName(), field.getDataType(), seenFields);
        for (Map.Entry<String, Attribute> entry : field.getAttributes().entrySet()) {
            checkFieldType(searchName, "attribute", entry.getKey(), entry.getValue().getDataType(), seenFields);
        }
    });
    for (DocumentSummary summary : search.getSummaries().values()) {
        for (SummaryField field : summary.getSummaryFields()) {
            checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), seenFields);
        }
    }
}

Example 35 with DataType

use of com.yahoo.document.DataType in project vespa by vespa-engine.

the class JsonReaderTestCase method setUp.

@Before
public void setUp() throws Exception {
    parserFactory = new JsonFactory();
    types = new DocumentTypeManager();
    {
        DocumentType x = new DocumentType("smoke");
        x.addField(new Field("something", DataType.STRING));
        x.addField(new Field("nalle", DataType.STRING));
        x.addField(new Field("int1", DataType.INT));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("mirrors");
        StructDataType woo = new StructDataType("woo");
        woo.addField(new Field("sandra", DataType.STRING));
        woo.addField(new Field("cloud", DataType.STRING));
        x.addField(new Field("skuggsjaa", woo));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testarray");
        DataType d = new ArrayDataType(DataType.STRING);
        x.addField(new Field("actualarray", d));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testset");
        DataType d = new WeightedSetDataType(DataType.STRING, true, true);
        x.addField(new Field("actualset", d));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testmap");
        DataType d = new MapDataType(DataType.STRING, DataType.STRING);
        x.addField(new Field("actualmap", d));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testraw");
        DataType d = DataType.RAW;
        x.addField(new Field("actualraw", d));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testMapStringToArrayOfInt");
        DataType value = new ArrayDataType(DataType.INT);
        DataType d = new MapDataType(DataType.STRING, value);
        x.addField(new Field("actualMapStringToArrayOfInt", d));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testsinglepos");
        DataType d = PositionDataType.INSTANCE;
        x.addField(new Field("singlepos", d));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testtensor");
        x.addField(new Field("mappedtensorfield", new TensorDataType(new TensorType.Builder().mapped("x").mapped("y").build())));
        x.addField(new Field("indexedtensorfield", new TensorDataType(new TensorType.Builder().indexed("x").indexed("y").build())));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testpredicate");
        x.addField(new Field("boolean", DataType.PREDICATE));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testint");
        x.addField(new Field("integerfield", DataType.INT));
        types.registerDocumentType(x);
    }
}