Examples with Mapper org.opensearch.index.mapper.Mapper used on opensource projects

Example 6 with Mapper

use of org.opensearch.index.mapper.Mapper in project k-NN by opensearch-project.

the class KNNVectorFieldMapperTests method testKNNVectorFieldMapper_merge_fromModel.

public void testKNNVectorFieldMapper_merge_fromModel() throws IOException {
    String fieldName = "test-field-name";
    String indexName = "test-index-name";
    Settings settings = Settings.builder().put(settings(CURRENT).build()).build();
    String modelId = "test-id";
    int dimension = 133;
    ModelDao mockModelDao = mock(ModelDao.class);
    ModelMetadata mockModelMetadata = new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, dimension, ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "");
    when(mockModelDao.getMetadata(modelId)).thenReturn(mockModelMetadata);
    KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> mockModelDao);
    XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().field("type", "knn_vector").field(MODEL_ID, modelId).endObject();
    KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilder), buildParserContext(indexName, settings));
    Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath());
    KNNVectorFieldMapper knnVectorFieldMapper1 = builder.build(builderContext);
    // merge with itself - should be successful
    KNNVectorFieldMapper knnVectorFieldMapperMerge1 = (KNNVectorFieldMapper) knnVectorFieldMapper1.merge(knnVectorFieldMapper1);
    assertEquals(knnVectorFieldMapper1.modelId, knnVectorFieldMapperMerge1.modelId);
    // merge with another mapper of the same field with same context
    KNNVectorFieldMapper knnVectorFieldMapper2 = builder.build(builderContext);
    KNNVectorFieldMapper knnVectorFieldMapperMerge2 = (KNNVectorFieldMapper) knnVectorFieldMapper1.merge(knnVectorFieldMapper2);
    assertEquals(knnVectorFieldMapper1.modelId, knnVectorFieldMapperMerge2.modelId);
    // merge with another mapper of the same field with different context
    xContentBuilder = XContentFactory.jsonBuilder().startObject().field("type", "knn_vector").field("dimension", dimension).startObject(KNN_METHOD).field(NAME, METHOD_HNSW).endObject().endObject();
    builder = (KNNVectorFieldMapper.Builder) typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilder), buildParserContext(indexName, settings));
    KNNVectorFieldMapper knnVectorFieldMapper3 = builder.build(builderContext);
    expectThrows(IllegalArgumentException.class, () -> knnVectorFieldMapper1.merge(knnVectorFieldMapper3));
}

Example 7 with Mapper

use of org.opensearch.index.mapper.Mapper in project OpenSearch by opensearch-project.

the class TransportGetFieldMappingsIndexAction method shardOperation.

@Override
protected GetFieldMappingsResponse shardOperation(final GetFieldMappingsIndexRequest request, ShardId shardId) {
    assert shardId != null;
    IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
    Version indexCreatedVersion = indexService.mapperService().getIndexSettings().getIndexVersionCreated();
    Predicate<String> metadataFieldPredicate = (f) -> indicesService.isMetadataField(indexCreatedVersion, f);
    Predicate<String> fieldPredicate = metadataFieldPredicate.or(indicesService.getFieldFilter().apply(shardId.getIndexName()));
    DocumentMapper mapper = indexService.mapperService().documentMapper();
    Collection<String> typeIntersection;
    if (request.types().length == 0) {
        typeIntersection = mapper == null ? Collections.emptySet() : Collections.singleton(mapper.type());
    } else {
        typeIntersection = mapper != null && Regex.simpleMatch(request.types(), mapper.type()) ? Collections.singleton(mapper.type()) : Collections.emptySet();
        if (typeIntersection.isEmpty()) {
            throw new TypeMissingException(shardId.getIndex(), request.types());
        }
    }
    Map<String, Map<String, FieldMappingMetadata>> typeMappings = new HashMap<>();
    for (String type : typeIntersection) {
        DocumentMapper documentMapper = indexService.mapperService().documentMapper(type);
        Map<String, FieldMappingMetadata> fieldMapping = findFieldMappingsByType(fieldPredicate, documentMapper, request);
        if (!fieldMapping.isEmpty()) {
            typeMappings.put(type, fieldMapping);
        }
    }
    return new GetFieldMappingsResponse(singletonMap(shardId.getIndexName(), Collections.unmodifiableMap(typeMappings)));
}

Example 8 with Mapper

use of org.opensearch.index.mapper.Mapper in project OpenSearch by opensearch-project.

the class ShardGetService method innerGetLoadFromStoredFields.

private GetResult innerGetLoadFromStoredFields(String id, String[] storedFields, FetchSourceContext fetchSourceContext, Engine.GetResult get, MapperService mapperService) {
    assert get.exists() : "method should only be called if document could be retrieved";
    // check first if stored fields to be loaded don't contain an object field
    DocumentMapper docMapper = mapperService.documentMapper();
    if (storedFields != null) {
        for (String field : storedFields) {
            Mapper fieldMapper = docMapper.mappers().getMapper(field);
            if (fieldMapper == null) {
                if (docMapper.objectMappers().get(field) != null) {
                    // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
                    throw new IllegalArgumentException("field [" + field + "] isn't a leaf field");
                }
            }
        }
    }
    Map<String, DocumentField> documentFields = null;
    Map<String, DocumentField> metadataFields = null;
    BytesReference source = null;
    DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
    // force fetching source if we read from translog and need to recreate stored fields
    boolean forceSourceForComputingTranslogStoredFields = get.isFromTranslog() && storedFields != null && Stream.of(storedFields).anyMatch(f -> TranslogLeafReader.ALL_FIELD_NAMES.contains(f) == false);
    FieldsVisitor fieldVisitor = buildFieldsVisitors(storedFields, forceSourceForComputingTranslogStoredFields ? FetchSourceContext.FETCH_SOURCE : fetchSourceContext);
    if (fieldVisitor != null) {
        try {
            docIdAndVersion.reader.document(docIdAndVersion.docId, fieldVisitor);
        } catch (IOException e) {
            throw new OpenSearchException("Failed to get id [" + id + "]", e);
        }
        source = fieldVisitor.source();
        // in case we read from translog, some extra steps are needed to make _source consistent and to load stored fields
        if (get.isFromTranslog()) {
            // just make source consistent by reapplying source filters from mapping (possibly also nulling the source)
            if (forceSourceForComputingTranslogStoredFields == false) {
                try {
                    source = indexShard.mapperService().documentMapper().sourceMapper().applyFilters(source, null);
                } catch (IOException e) {
                    throw new OpenSearchException("Failed to reapply filters for [" + id + "] after reading from translog", e);
                }
            } else {
                // Slow path: recreate stored fields from original source
                assert source != null : "original source in translog must exist";
                SourceToParse sourceToParse = new SourceToParse(shardId.getIndexName(), MapperService.SINGLE_MAPPING_NAME, id, source, XContentHelper.xContentType(source), fieldVisitor.routing());
                ParsedDocument doc = indexShard.mapperService().documentMapper().parse(sourceToParse);
                assert doc.dynamicMappingsUpdate() == null : "mapping updates should not be required on already-indexed doc";
                // update special fields
                doc.updateSeqID(docIdAndVersion.seqNo, docIdAndVersion.primaryTerm);
                doc.version().setLongValue(docIdAndVersion.version);
                // retrieve stored fields from parsed doc
                fieldVisitor = buildFieldsVisitors(storedFields, fetchSourceContext);
                for (IndexableField indexableField : doc.rootDoc().getFields()) {
                    IndexableFieldType fieldType = indexableField.fieldType();
                    if (fieldType.stored()) {
                        FieldInfo fieldInfo = new FieldInfo(indexableField.name(), 0, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false);
                        StoredFieldVisitor.Status status = fieldVisitor.needsField(fieldInfo);
                        if (status == StoredFieldVisitor.Status.YES) {
                            if (indexableField.numericValue() != null) {
                                fieldVisitor.objectField(fieldInfo, indexableField.numericValue());
                            } else if (indexableField.binaryValue() != null) {
                                fieldVisitor.binaryField(fieldInfo, indexableField.binaryValue());
                            } else if (indexableField.stringValue() != null) {
                                fieldVisitor.objectField(fieldInfo, indexableField.stringValue());
                            }
                        } else if (status == StoredFieldVisitor.Status.STOP) {
                            break;
                        }
                    }
                }
                // retrieve source (with possible transformations, e.g. source filters
                source = fieldVisitor.source();
            }
        }
        // put stored fields into result objects
        if (!fieldVisitor.fields().isEmpty()) {
            fieldVisitor.postProcess(mapperService);
            documentFields = new HashMap<>();
            metadataFields = new HashMap<>();
            for (Map.Entry<String, List<Object>> entry : fieldVisitor.fields().entrySet()) {
                if (mapperService.isMetadataField(entry.getKey())) {
                    metadataFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
                } else {
                    documentFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
                }
            }
        }
    }
    if (source != null) {
        // apply request-level source filtering
        if (fetchSourceContext.fetchSource() == false) {
            source = null;
        } else if (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0) {
            Map<String, Object> sourceAsMap;
            // TODO: The source might be parsed and available in the sourceLookup but that one uses unordered maps so different.
            // Do we care?
            Tuple<XContentType, Map<String, Object>> typeMapTuple = XContentHelper.convertToMap(source, true);
            XContentType sourceContentType = typeMapTuple.v1();
            sourceAsMap = typeMapTuple.v2();
            sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes());
            try {
                source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap));
            } catch (IOException e) {
                throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e);
            }
        }
    }
    if (!fetchSourceContext.fetchSource()) {
        source = null;
    }
    if (source != null && get.isFromTranslog()) {
        // reapply source filters from mapping (possibly also nulling the source)
        try {
            source = docMapper.sourceMapper().applyFilters(source, null);
        } catch (IOException e) {
            throw new OpenSearchException("Failed to reapply filters for [" + id + "] after reading from translog", e);
        }
    }
    if (source != null && (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0)) {
        Map<String, Object> sourceAsMap;
        // TODO: The source might parsed and available in the sourceLookup but that one uses unordered maps so different. Do we care?
        Tuple<XContentType, Map<String, Object>> typeMapTuple = XContentHelper.convertToMap(source, true);
        XContentType sourceContentType = typeMapTuple.v1();
        sourceAsMap = typeMapTuple.v2();
        sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes());
        try {
            source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap));
        } catch (IOException e) {
            throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e);
        }
    }
    return new GetResult(shardId.getIndexName(), id, get.docIdAndVersion().seqNo, get.docIdAndVersion().primaryTerm, get.version(), get.exists(), source, documentFields, metadataFields);
}

Example 9 with Mapper

use of org.opensearch.index.mapper.Mapper in project OpenSearch by opensearch-project.

the class CategoryContextMappingTests method testIndexingWithMixedTypeContextList.

public void testIndexingWithMixedTypeContextList() throws Exception {
    String mapping = Strings.toString(jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("completion").field("type", "completion").startArray("contexts").startObject().field("name", "ctx").field("type", "category").endObject().endArray().endObject().endObject().endObject().endObject());
    DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
    Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
    ParsedDocument parsedDocument = defaultMapper.parse(new SourceToParse("test", "type1", "1", BytesReference.bytes(jsonBuilder().startObject().startObject("completion").array("input", "suggestion5", "suggestion6", "suggestion7").startObject("contexts").array("ctx", "ctx1", true, 100).endObject().field("weight", 5).endObject().endObject()), XContentType.JSON));
    IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
    assertContextSuggestFields(fields, 3);
}

Example 10 with Mapper

use of org.opensearch.index.mapper.Mapper in project OpenSearch by opensearch-project.

the class CategoryContextMappingTests method testIndexingWithNoContexts.

public void testIndexingWithNoContexts() throws Exception {
    String mapping = Strings.toString(jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("completion").field("type", "completion").startArray("contexts").startObject().field("name", "ctx").field("type", "category").endObject().endArray().endObject().endObject().endObject().endObject());
    DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
    Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
    ParsedDocument parsedDocument = defaultMapper.parse(new SourceToParse("test", "type1", "1", BytesReference.bytes(jsonBuilder().startObject().startArray("completion").startObject().array("input", "suggestion1", "suggestion2").field("weight", 3).endObject().startObject().array("input", "suggestion3", "suggestion4").field("weight", 4).endObject().startObject().array("input", "suggestion5", "suggestion6", "suggestion7").field("weight", 5).endObject().endArray().endObject()), XContentType.JSON));
    IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
    assertContextSuggestFields(fields, 7);
}