Examples with Mapper org.opensearch.index.mapper.Mapper used on opensource projects

Example 11 with Mapper

use of org.opensearch.index.mapper.Mapper in project OpenSearch by opensearch-project.

the class CategoryContextMappingTests method testIndexingWithMixedTypeContextList.

public void testIndexingWithMixedTypeContextList() throws Exception {
    String mapping = Strings.toString(jsonBuilder().startObject().startObject("properties").startObject("completion").field("type", "completion").startArray("contexts").startObject().field("name", "ctx").field("type", "category").endObject().endArray().endObject().endObject().endObject());
    DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
    Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
    ParsedDocument parsedDocument = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(jsonBuilder().startObject().startObject("completion").array("input", "suggestion5", "suggestion6", "suggestion7").startObject("contexts").array("ctx", "ctx1", true, 100).endObject().field("weight", 5).endObject().endObject()), XContentType.JSON));
    IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
    assertContextSuggestFields(fields, 3);
}

Example 12 with Mapper

use of org.opensearch.index.mapper.Mapper in project OpenSearch by opensearch-project.

the class ShardGetService method innerGetLoadFromStoredFields.

private GetResult innerGetLoadFromStoredFields(String id, String[] storedFields, FetchSourceContext fetchSourceContext, Engine.GetResult get, MapperService mapperService) {
    assert get.exists() : "method should only be called if document could be retrieved";
    // check first if stored fields to be loaded don't contain an object field
    DocumentMapper docMapper = mapperService.documentMapper();
    if (storedFields != null) {
        for (String field : storedFields) {
            Mapper fieldMapper = docMapper.mappers().getMapper(field);
            if (fieldMapper == null) {
                if (docMapper.objectMappers().get(field) != null) {
                    // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
                    throw new IllegalArgumentException("field [" + field + "] isn't a leaf field");
                }
            }
        }
    }
    Map<String, DocumentField> documentFields = null;
    Map<String, DocumentField> metadataFields = null;
    BytesReference source = null;
    DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
    // force fetching source if we read from translog and need to recreate stored fields
    boolean forceSourceForComputingTranslogStoredFields = get.isFromTranslog() && storedFields != null && Stream.of(storedFields).anyMatch(f -> TranslogLeafReader.ALL_FIELD_NAMES.contains(f) == false);
    FieldsVisitor fieldVisitor = buildFieldsVisitors(storedFields, forceSourceForComputingTranslogStoredFields ? FetchSourceContext.FETCH_SOURCE : fetchSourceContext);
    if (fieldVisitor != null) {
        try {
            docIdAndVersion.reader.document(docIdAndVersion.docId, fieldVisitor);
        } catch (IOException e) {
            throw new OpenSearchException("Failed to get id [" + id + "]", e);
        }
        source = fieldVisitor.source();
        // in case we read from translog, some extra steps are needed to make _source consistent and to load stored fields
        if (get.isFromTranslog()) {
            // just make source consistent by reapplying source filters from mapping (possibly also nulling the source)
            if (forceSourceForComputingTranslogStoredFields == false) {
                try {
                    source = indexShard.mapperService().documentMapper().sourceMapper().applyFilters(source, null);
                } catch (IOException e) {
                    throw new OpenSearchException("Failed to reapply filters for [" + id + "] after reading from translog", e);
                }
            } else {
                // Slow path: recreate stored fields from original source
                assert source != null : "original source in translog must exist";
                SourceToParse sourceToParse = new SourceToParse(shardId.getIndexName(), id, source, XContentHelper.xContentType(source), fieldVisitor.routing());
                ParsedDocument doc = indexShard.mapperService().documentMapper().parse(sourceToParse);
                assert doc.dynamicMappingsUpdate() == null : "mapping updates should not be required on already-indexed doc";
                // update special fields
                doc.updateSeqID(docIdAndVersion.seqNo, docIdAndVersion.primaryTerm);
                doc.version().setLongValue(docIdAndVersion.version);
                // retrieve stored fields from parsed doc
                fieldVisitor = buildFieldsVisitors(storedFields, fetchSourceContext);
                for (IndexableField indexableField : doc.rootDoc().getFields()) {
                    IndexableFieldType fieldType = indexableField.fieldType();
                    if (fieldType.stored()) {
                        FieldInfo fieldInfo = new FieldInfo(indexableField.name(), 0, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorSimilarityFunction.EUCLIDEAN, false);
                        StoredFieldVisitor.Status status = fieldVisitor.needsField(fieldInfo);
                        if (status == StoredFieldVisitor.Status.YES) {
                            if (indexableField.numericValue() != null) {
                                fieldVisitor.objectField(fieldInfo, indexableField.numericValue());
                            } else if (indexableField.binaryValue() != null) {
                                fieldVisitor.binaryField(fieldInfo, indexableField.binaryValue());
                            } else if (indexableField.stringValue() != null) {
                                fieldVisitor.objectField(fieldInfo, indexableField.stringValue());
                            }
                        } else if (status == StoredFieldVisitor.Status.STOP) {
                            break;
                        }
                    }
                }
                // retrieve source (with possible transformations, e.g. source filters
                source = fieldVisitor.source();
            }
        }
        // put stored fields into result objects
        if (!fieldVisitor.fields().isEmpty()) {
            fieldVisitor.postProcess(mapperService::fieldType);
            documentFields = new HashMap<>();
            metadataFields = new HashMap<>();
            for (Map.Entry<String, List<Object>> entry : fieldVisitor.fields().entrySet()) {
                if (mapperService.isMetadataField(entry.getKey())) {
                    metadataFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
                } else {
                    documentFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
                }
            }
        }
    }
    if (source != null) {
        // apply request-level source filtering
        if (fetchSourceContext.fetchSource() == false) {
            source = null;
        } else if (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0) {
            Map<String, Object> sourceAsMap;
            // TODO: The source might be parsed and available in the sourceLookup but that one uses unordered maps so different.
            // Do we care?
            Tuple<XContentType, Map<String, Object>> typeMapTuple = XContentHelper.convertToMap(source, true);
            XContentType sourceContentType = typeMapTuple.v1();
            sourceAsMap = typeMapTuple.v2();
            sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes());
            try {
                source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap));
            } catch (IOException e) {
                throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e);
            }
        }
    }
    if (!fetchSourceContext.fetchSource()) {
        source = null;
    }
    if (source != null && get.isFromTranslog()) {
        // reapply source filters from mapping (possibly also nulling the source)
        try {
            source = docMapper.sourceMapper().applyFilters(source, null);
        } catch (IOException e) {
            throw new OpenSearchException("Failed to reapply filters for [" + id + "] after reading from translog", e);
        }
    }
    if (source != null && (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0)) {
        Map<String, Object> sourceAsMap;
        // TODO: The source might parsed and available in the sourceLookup but that one uses unordered maps so different. Do we care?
        Tuple<XContentType, Map<String, Object>> typeMapTuple = XContentHelper.convertToMap(source, true);
        XContentType sourceContentType = typeMapTuple.v1();
        sourceAsMap = typeMapTuple.v2();
        sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes());
        try {
            source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap));
        } catch (IOException e) {
            throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e);
        }
    }
    return new GetResult(shardId.getIndexName(), id, get.docIdAndVersion().seqNo, get.docIdAndVersion().primaryTerm, get.version(), get.exists(), source, documentFields, metadataFields);
}

Example 13 with Mapper

use of org.opensearch.index.mapper.Mapper in project OpenSearch by opensearch-project.

the class TransportGetFieldMappingsIndexAction method findFieldMappings.

private static Map<String, FieldMappingMetadata> findFieldMappings(Predicate<String> fieldPredicate, DocumentMapper documentMapper, GetFieldMappingsIndexRequest request) {
    if (documentMapper == null) {
        return Collections.emptyMap();
    }
    Map<String, FieldMappingMetadata> fieldMappings = new HashMap<>();
    final MappingLookup allFieldMappers = documentMapper.mappers();
    for (String field : request.fields()) {
        if (Regex.isMatchAllPattern(field)) {
            for (Mapper fieldMapper : allFieldMappers) {
                addFieldMapper(fieldPredicate, fieldMapper.name(), fieldMapper, fieldMappings, request.includeDefaults());
            }
        } else if (Regex.isSimpleMatchPattern(field)) {
            for (Mapper fieldMapper : allFieldMappers) {
                if (Regex.simpleMatch(field, fieldMapper.name())) {
                    addFieldMapper(fieldPredicate, fieldMapper.name(), fieldMapper, fieldMappings, request.includeDefaults());
                }
            }
        } else {
            // not a pattern
            Mapper fieldMapper = allFieldMappers.getMapper(field);
            if (fieldMapper != null) {
                addFieldMapper(fieldPredicate, field, fieldMapper, fieldMappings, request.includeDefaults());
            }
        }
    }
    return Collections.unmodifiableMap(fieldMappings);
}